Merge pull request #41 from vulcanize/bump

bump patch version

Makefile

@@ -51,11 +51,13 @@ CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?ssl
 #Test
 TEST_DB = vulcanize_testing
 TEST_CONNECT_STRING = postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING_LOCAL = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
 
 .PHONY: test
 test: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
+	export PGPASSWORD=$(DATABASE_PASSWORD)
 	dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
 	createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
 	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
@@ -65,10 +67,31 @@ test: | $(GINKGO) $(GOOSE)
 integrationtest: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	#dropdb -h $(DATABASE_HOSTNAME) -p $(PORT) -U $(USER) -W --if-exists $(TEST_DB)
+	export PGPASSWORD=$(DATABASE_PASSWORD)
-	#createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) -W $(TEST_DB)
+	dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up
+	createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
+	$(GINKGO) -r integration_test/
+
+.PHONY: test_local
+test_local: | $(GINKGO) $(GOOSE)
+	go vet ./...
+	go fmt ./...
+	dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
+	createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
+	make migrate NAME=$(TEST_DB)
+	$(GINKGO) -r --skipPackage=integration_tests,integration
+
+.PHONY: integrationtest_local
+integrationtest_local: | $(GINKGO) $(GOOSE)
+	go vet ./...
+	go fmt ./...
+	dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
+	createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
 	make migrate NAME=$(TEST_DB)
 	$(GINKGO) -r integration_test/
 

cmd/proxy.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/gap-filler/pkg/mux"
+)
+
+var ErrNoRpcEndpoints = errors.New("no rpc endpoints is available")
+
+// proxyCmd represents the proxy command
+var proxyCmd = &cobra.Command{
+	Use:   "proxy",
+	Short: "serve chain data from PG-IPFS or proxy geths",
+	Long: `This command configures a VulcanizeDB ipld-eth-server graphql server.
+
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *logrus.WithField("SubCommand", subCommand)
+		proxy()
+	},
+}
+
+func proxy() {
+	gqlDefaultAddr, err := url.Parse(viper.GetString("gql.default"))
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	gqlTracingAPIAddr, err := url.Parse(viper.GetString("gql.tracing"))
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	rpcClients, err := parseRpcAddresses(viper.GetString("rpc.eth"))
+	if err != nil {
+		logrus.Error("bad rpc.eth addresses")
+		logWithCommand.Fatal(err)
+	}
+
+	tracingClients, err := parseRpcAddresses(viper.GetString("rpc.tracing"))
+	if err != nil {
+		logrus.Error("bad rpc.tracing addresses")
+		logWithCommand.Fatal(err)
+	}
+
+	router, err := mux.NewServeMux(&mux.Options{
+		BasePath:       viper.GetString("http.path"),
+		EnableGraphiQL: viper.GetBool("gql.gui"),
+		Postgraphile: mux.PostgraphileOptions{
+			Default:    gqlDefaultAddr,
+			TracingAPI: gqlTracingAPIAddr,
+		},
+		RPC: mux.RPCOptions{
+			DefaultClients: rpcClients,
+			TracingClients: tracingClients,
+		},
+	})
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	addr := fmt.Sprintf("%s:%s", viper.GetString("http.host"), viper.GetString("http.port"))
+	if err := http.ListenAndServe(addr, router); err != nil {
+		logWithCommand.Fatal(err)
+	}
+}
+
+func parseRpcAddresses(value string) ([]*rpc.Client, error) {
+	rpcAddresses := strings.Split(value, ",")
+	rpcClients := make([]*rpc.Client, 0, len(rpcAddresses))
+	for _, address := range rpcAddresses {
+		rpcClient, err := rpc.Dial(address)
+		if err != nil {
+			logWithCommand.Errorf("couldn't connect to %s. Error: %s", address, err)
+			continue
+		}
+
+		rpcClients = append(rpcClients, rpcClient)
+	}
+
+	if len(rpcClients) == 0 {
+		logWithCommand.Error(ErrNoRpcEndpoints)
+		return nil, ErrNoRpcEndpoints
+	}
+
+	return rpcClients, nil
+}
+
+func init() {
+	rootCmd.AddCommand(proxyCmd)
+
+	// flags
+	proxyCmd.PersistentFlags().String("http-host", "127.0.0.1", "http host")
+	proxyCmd.PersistentFlags().String("http-port", "8083", "http port")
+	proxyCmd.PersistentFlags().String("http-path", "/", "http base path")
+
+	proxyCmd.PersistentFlags().String("rpc-eth", "http://127.0.0.1:8545", "comma separated ethereum rpc addresses. Example http://127.0.0.1:8545,http://127.0.0.2:8545")
+	proxyCmd.PersistentFlags().String("rpc-tracing", "http://127.0.0.1:8000", "comma separated traicing api addresses")
+
+	proxyCmd.PersistentFlags().String("gql-default", "http://127.0.0.1:5020/graphql", "postgraphile address")
+	proxyCmd.PersistentFlags().String("gql-tracing", "http://127.0.0.1:5020/graphql", "tracing api postgraphile address")
+	proxyCmd.PersistentFlags().Bool("gql-gui", false, "enable graphiql interface")
+
+	// and their .toml config bindings
+	viper.BindPFlag("http.host", proxyCmd.PersistentFlags().Lookup("http-host"))
+	viper.BindPFlag("http.port", proxyCmd.PersistentFlags().Lookup("http-port"))
+	viper.BindPFlag("http.path", proxyCmd.PersistentFlags().Lookup("http-path"))
+
+	viper.BindPFlag("rpc.eth", proxyCmd.PersistentFlags().Lookup("rpc-eth"))
+	viper.BindPFlag("rpc.tracing", proxyCmd.PersistentFlags().Lookup("rpc-tracing"))
+
+	viper.BindPFlag("gql.default", proxyCmd.PersistentFlags().Lookup("gql-default"))
+	viper.BindPFlag("gql.tracing", proxyCmd.PersistentFlags().Lookup("gql-tracing"))
+	viper.BindPFlag("gql.gui", proxyCmd.PersistentFlags().Lookup("gql-gui"))
+}

cmd/serve.go

@@ -137,6 +137,8 @@ func init() {
 	serveCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
 	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
 	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
+	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
+	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether or not the proxy ethereum client supports statediffing endpoints")
 
 	// and their bindings
 	viper.BindPFlag("server.graphql", serveCmd.PersistentFlags().Lookup("server-graphql"))
@@ -153,4 +155,6 @@ func init() {
 	viper.BindPFlag("ethereum.chainID", serveCmd.PersistentFlags().Lookup("eth-chain-id"))
 	viper.BindPFlag("ethereum.defaultSender", serveCmd.PersistentFlags().Lookup("eth-default-sender"))
 	viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap"))
+	viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config"))
+	viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff"))
 }

db/migrations/00002_create_nodes_table.sql

@@ -5,7 +5,8 @@ CREATE TABLE nodes (
   genesis_block VARCHAR(66),
   network_id    VARCHAR,
   node_id       VARCHAR(128),
-  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
+  chain_id      INTEGER DEFAULT 1,
+  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id, chain_id)
 );
 
 -- +goose Down

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -8,7 +8,6 @@ CREATE TABLE eth.transaction_cids (
   mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   dst                   VARCHAR(66) NOT NULL,
   src                   VARCHAR(66) NOT NULL,
-  deployment            BOOL NOT NULL,
   tx_data               BYTEA,
   UNIQUE (header_id, tx_hash)
 );

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -11,6 +11,8 @@ CREATE TABLE eth.receipt_cids (
   topic2s               VARCHAR(66)[],
   topic3s               VARCHAR(66)[],
   log_contracts         VARCHAR(66)[],
+  post_state            VARCHAR(66),
+  post_status           INTEGER,
   UNIQUE (tx_id)
 );
 

db/migrations/00008_create_eth_state_cids_table.sql

@@ -1,6 +1,6 @@
 -- +goose Up
 CREATE TABLE eth.state_cids (
-  id                    SERIAL PRIMARY KEY,
+  id                    BIGSERIAL PRIMARY KEY,
   header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   state_leaf_key        VARCHAR(66),
   cid                   TEXT NOT NULL,

db/migrations/00009_create_eth_storage_cids_table.sql

@@ -1,7 +1,7 @@
 -- +goose Up
 CREATE TABLE eth.storage_cids (
-  id                    SERIAL PRIMARY KEY,
+  id                    BIGSERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  state_id              BIGINT NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   storage_leaf_key      VARCHAR(66),
   cid                   TEXT NOT NULL,
   mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,

db/migrations/00010_create_eth_state_accouts_table.sql

@@ -1,7 +1,7 @@
 -- +goose Up
 CREATE TABLE eth.state_accounts (
   id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  state_id              BIGINT NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   balance               NUMERIC NOT NULL,
   nonce                 INTEGER NOT NULL,
   code_hash             BYTEA NOT NULL,

db/migrations/00012_add_chain_id_to_nodes.sql

@@ -1,21 +0,0 @@
--- +goose Up
-ALTER TABLE public.nodes
-ADD COLUMN chain_id INTEGER DEFAULT 1;
-
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id, chain_id);
-
--- +goose Down
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id);
-
-ALTER TABLE public.nodes
-DROP COLUMN chain_id;

db/migrations/00014_create_stored_functions.sql

@@ -1,4 +1,39 @@
 -- +goose Up
+-- +goose StatementBegin
+-- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
+CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
+AS $$
+SELECT exists(SELECT 1
+              FROM eth.storage_cids
+                       INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
+                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+              WHERE storage_path = path
+                AND block_number > height
+                AND block_number <= (SELECT block_number
+                                     FROM eth.header_cids
+                                     WHERE block_hash = hash)
+                AND storage_cids.node_type = 3
+              LIMIT 1);
+$$ LANGUAGE SQL;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+-- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
+CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
+AS $$
+SELECT exists(SELECT 1
+              FROM eth.state_cids
+                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+              WHERE state_path = path
+                AND block_number > height
+                AND block_number <= (SELECT block_number
+                                     FROM eth.header_cids
+                                     WHERE block_hash = hash)
+                AND state_cids.node_type = 3
+              LIMIT 1);
+$$ LANGUAGE SQL;
+-- +goose StatementEnd
+
 -- +goose StatementBegin
 CREATE TYPE child_result AS (
     has_child BOOLEAN,
@@ -115,6 +150,8 @@ LANGUAGE 'plpgsql';
 -- +goose StatementEnd
 
 -- +goose Down
+DROP FUNCTION was_storage_removed;
+DROP FUNCTION was_state_removed;
 DROP FUNCTION canonical_header_id;
 DROP FUNCTION canonical_header_from_array;
 DROP FUNCTION has_child;

db/migrations/00015_create_canonical_hash_finder_functions.sql

@@ -1,48 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns the number of child headers that reference backwards to the header with the provided hash
-CREATE OR REPLACE FUNCTION header_weight(hash VARCHAR(66)) RETURNS BIGINT
-AS $$
-  WITH RECURSIVE validator AS (
-          SELECT block_hash, parent_hash, block_number
-          FROM eth.header_cids
-          WHERE block_hash = hash
-      UNION
-          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
-          FROM eth.header_cids
-          INNER JOIN validator
-            ON eth.header_cids.parent_hash = validator.block_hash
-            AND eth.header_cids.block_number = validator.block_number + 1
-  )
-  SELECT COUNT(*) FROM validator;
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns the id for the header at the provided height which is heaviest
-CREATE OR REPLACE FUNCTION canonical_header(height BIGINT) RETURNS INT AS
-$BODY$
-DECLARE
-  current_weight INT;
-  heaviest_weight INT DEFAULT 0;
-  heaviest_id INT;
-  r eth.header_cids%ROWTYPE;
-BEGIN
-  FOR r IN SELECT * FROM eth.header_cids
-  WHERE block_number = height
-  LOOP
-    SELECT INTO current_weight * FROM header_weight(r.block_hash);
-    IF current_weight > heaviest_weight THEN
-        heaviest_weight := current_weight;
-        heaviest_id := r.id;
-    END IF;
-  END LOOP;
-  RETURN heaviest_id;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION header_weight;
-DROP FUNCTION canonical_header;

db/migrations/00016_remove_deployment_from_tx_cids.sql

@@ -1,7 +0,0 @@
--- +goose Up
-ALTER TABLE eth.transaction_cids
-DROP COLUMN deployment;
-
--- +goose Down
-ALTER TABLE eth.transaction_cids
-ADD COLUMN deployment BOOL NOT NULL DEFAULT FALSE;

db/migrations/00017_state_and_storage_ids_use_big_serial.sql

@@ -1,25 +0,0 @@
--- +goose Up
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE BIGINT;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE BIGINT;
-
--- +goose Down
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE INTEGER;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE INTEGER;

db/migrations/00018_create_check_if_removed_functions.sql

@@ -1,39 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-  SELECT exists(SELECT 1
-					FROM eth.storage_cids
-            INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE storage_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND storage_cids.node_type = 3
-					LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-  SELECT exists(SELECT 1
-					FROM eth.state_cids
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE state_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND state_cids.node_type = 3
-					LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION was_storage_removed;
-DROP FUNCTION was_state_removed;

db/migrations/00020_add_poststate_and_status_to_receipt_cids.sql

@@ -1,13 +0,0 @@
--- +goose Up
-ALTER TABLE eth.receipt_cids
-ADD COLUMN post_state VARCHAR(66);
-
-ALTER TABLE eth.receipt_cids
-ADD COLUMN post_status INTEGER;
-
--- +goose Down
-ALTER TABLE eth.receipt_cids
-DROP COLUMN post_status;
-
-ALTER TABLE eth.receipt_cids
-DROP COLUMN post_state;

db/schema.sql

@@ -103,33 +103,6 @@ end;
 $_$;
 
 
---
--- Name: canonical_header(bigint); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.canonical_header(height bigint) RETURNS integer
-    LANGUAGE plpgsql
-    AS $$
-DECLARE
-  current_weight INT;
-  heaviest_weight INT DEFAULT 0;
-  heaviest_id INT;
-  r eth.header_cids%ROWTYPE;
-BEGIN
-  FOR r IN SELECT * FROM eth.header_cids
-  WHERE block_number = height
-  LOOP
-    SELECT INTO current_weight * FROM header_weight(r.block_hash);
-    IF current_weight > heaviest_weight THEN
-        heaviest_weight := current_weight;
-        heaviest_id := r.id;
-    END IF;
-  END LOOP;
-  RETURN heaviest_id;
-END
-$$;
-
-
 --
 -- Name: canonical_header_from_array(eth.header_cids[]); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -249,28 +222,6 @@ END
 $$;
 
 
---
--- Name: header_weight(character varying); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.header_weight(hash character varying) RETURNS bigint
-    LANGUAGE sql
-    AS $$
-  WITH RECURSIVE validator AS (
-          SELECT block_hash, parent_hash, block_number
-          FROM eth.header_cids
-          WHERE block_hash = hash
-      UNION
-          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
-          FROM eth.header_cids
-          INNER JOIN validator
-            ON eth.header_cids.parent_hash = validator.block_hash
-            AND eth.header_cids.block_number = validator.block_number + 1
-  )
-  SELECT COUNT(*) FROM validator;
-$$;
-
-
 --
 -- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -347,7 +298,9 @@ CREATE TABLE eth.receipt_cids (
     topic1s character varying(66)[],
     topic2s character varying(66)[],
     topic3s character varying(66)[],
-    log_contracts character varying(66)[]
+    log_contracts character varying(66)[],
+    post_state character varying(66),
+    post_status integer
 );
 
 
@@ -426,7 +379,6 @@ CREATE TABLE eth.state_cids (
 --
 
 CREATE SEQUENCE eth.state_cids_id_seq
-    AS integer
     START WITH 1
     INCREMENT BY 1
     NO MINVALUE
@@ -462,7 +414,6 @@ CREATE TABLE eth.storage_cids (
 --
 
 CREATE SEQUENCE eth.storage_cids_id_seq
-    AS integer
     START WITH 1
     INCREMENT BY 1
     NO MINVALUE

environments/example.toml

@@ -20,6 +20,7 @@
     defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
     rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
     httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
     nodeID = "arch1" # $ETH_NODE_ID
     clientName = "Geth" # $ETH_CLIENT_NAME
     genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK

go.mod

@@ -16,13 +16,15 @@ require (
 	github.com/onsi/ginkgo v1.15.0
 	github.com/onsi/gomega v1.10.1
 	github.com/prometheus/client_golang v1.5.1
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.0.0
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/ipld-eth-indexer v0.7.0-alpha
+	github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha
+	github.com/vulcanize/gap-filler v0.3.1
 	github.com/vulcanize/ipfs-ethdb v0.0.2-alpha
 	golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14
+replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.15
+
 replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha

go.sum

@@ -154,6 +154,8 @@ github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepB
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
+github.com/friendsofgo/graphiql v0.2.2 h1:ccnuxpjgIkB+Lr9YB2ZouiZm7wvciSfqwpa9ugWzmn0=
+github.com/friendsofgo/graphiql v0.2.2/go.mod h1:8Y2kZ36AoTGWs78+VRpvATyt3LJBx0SZXmay80ZTRWo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -237,6 +239,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e h1:IpssFbpfPSx/3c7x601Npx+UOQ4tqd0Rk4sObCQ+zlQ=
 github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/graphql-go/graphql v0.7.9 h1:5Va/Rt4l5g3YjwDnid3vFfn43faaQBq7rMcIZ0VnV34=
+github.com/graphql-go/graphql v0.7.9/go.mod h1:k6yrAYQaSP59DC5UVxbgxESlmVyojThKdORUqGDGmrI=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -449,6 +453,8 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1C
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jinzhu/copier v0.2.4 h1:dT3tI+8GzU8DjJFCj9mLYtjfRtUmK7edauduQdcZCpI=
+github.com/jinzhu/copier v0.2.4/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
@@ -929,6 +935,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
@@ -952,10 +960,14 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
@@ -989,18 +1001,28 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
+github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/vulcanize/gap-filler v0.3.0 h1:SaBrOxlxCDPbSA902K/46iELH8yM3BqhvBuXK1ktFDI=
+github.com/vulcanize/gap-filler v0.3.0/go.mod h1:4odsXyckNU1xdXk37qXY2SbxUM6oDpQrpWZtEHfq+0w=
+github.com/vulcanize/gap-filler v0.3.1 h1:N5d+jCJo/VTWFvBSbTD7biRhK/OqDZzi1tgA85SIBKs=
+github.com/vulcanize/gap-filler v0.3.1/go.mod h1:qowG1cgshVpBqMokiWro/1xhh0uypw7oAu8FQ42JMy4=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5 h1:U+BqhjRLR22e9OEm8cgWC3Eq3bh8G6azjNpXeenfCG4=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8 h1:7TK52k55uvSl+1SCKYYFelzH1NrpvEcDrpeU9nUDIpI=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
 github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14 h1:DFN3WmdStbNWfTx5qOGDlZkoUwVxlcs4fGqPwzczE+8=
 github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14/go.mod h1:q5X5BpZkLp+OgWByNdZ8NudGkq0Vq4LjdrtmyyICUw4=
+github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.15 h1:f/frl9tMTKCcu2yh1tOgwXeQn3GJ2+21Zp0lSZzl1v0=
+github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.15/go.mod h1:q5X5BpZkLp+OgWByNdZ8NudGkq0Vq4LjdrtmyyICUw4=
 github.com/vulcanize/ipfs-ethdb v0.0.2 h1:Ogp9k+OMKIn2xo9Y9VoNTjd5v1kXcjL+k3cDQwqpUPU=
 github.com/vulcanize/ipfs-ethdb v0.0.2/go.mod h1:9yDJ8PCVKK4auwM8KgVLVkGM05Atp2OITJvt6cSvsHs=
 github.com/vulcanize/ipld-eth-indexer v0.7.0-alpha h1:VuvKC2qLajohczJX74lwBrpr/X8ZNSyC23jIdN5Ppxg=
 github.com/vulcanize/ipld-eth-indexer v0.7.0-alpha/go.mod h1:LYN2c28N9ceDf66tppc+TpaLqs4UcUlLTvUvg3Awccc=
+github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha h1:By8Yyt8IjneVhL+xne/V6VLumKcUx06iE5Kq2rFphzA=
+github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha/go.mod h1:LYN2c28N9ceDf66tppc+TpaLqs4UcUlLTvUvg3Awccc=
 github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha h1:wvhpjrRuxjudAael5zhNwGymce/ggvOhHo1q9VJ0NjU=
 github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha/go.mod h1:WIceyu15uxQ4JanLxzKXn4iAyJJSmHAJNWux7qYXGEQ=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
@@ -1334,6 +1356,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=

pkg/eth/api.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
@@ -34,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
@@ -50,19 +52,21 @@ type PublicEthAPI struct {
 	// Local db backend
 	B *Backend
 
-	// Remote node for forwarding cache misses
+	// Proxy node for forwarding cache misses
+	supportsStateDiff bool // Whether or not the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss
 	rpc               *rpc.Client
 	ethClient         *ethclient.Client
 }
 
 // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
-func NewPublicEthAPI(b *Backend, client *rpc.Client) *PublicEthAPI {
+func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff bool) *PublicEthAPI {
 	var ethClient *ethclient.Client
 	if client != nil {
 		ethClient = ethclient.NewClient(client)
 	}
 	return &PublicEthAPI{
 		B:                 b,
+		supportsStateDiff: supportsStateDiff,
 		rpc:               client,
 		ethClient:         ethClient,
 	}
@@ -84,6 +88,7 @@ func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.Block
 	}
 	if pea.ethClient != nil {
 		if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil {
+			go pea.writeStateDiffAt(number.Int64())
 			return pea.rpcMarshalHeader(header)
 		}
 	}
@@ -100,6 +105,7 @@ func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash)
 	}
 	if pea.ethClient != nil {
 		if header, err := pea.ethClient.HeaderByHash(ctx, hash); header != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			if res, err := pea.rpcMarshalHeader(header); err != nil {
 				return res
 			}
@@ -137,6 +143,7 @@ func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockN
 	}
 	if pea.ethClient != nil {
 		if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil {
+			go pea.writeStateDiffAt(number.Int64())
 			return pea.rpcMarshalBlock(block, true, fullTx)
 		}
 	}
@@ -152,6 +159,7 @@ func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, f
 	}
 	if pea.ethClient != nil {
 		if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			return pea.rpcMarshalBlock(block, true, fullTx)
 		}
 	}
@@ -179,6 +187,7 @@ func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, bloc
 	}
 	if pea.rpc != nil {
 		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockNumberAndIndex", blockNr, index); uncle != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
 		}
 	}
@@ -200,6 +209,7 @@ func (pea *PublicEthAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockH
 	}
 	if pea.rpc != nil {
 		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockHashAndIndex", blockHash, index); uncle != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
 		}
 	}
@@ -215,6 +225,7 @@ func (pea *PublicEthAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockNumber", blockNr); num != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return num
 		}
 	}
@@ -230,6 +241,7 @@ func (pea *PublicEthAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockHash", blockHash); num != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return num
 		}
 	}
@@ -251,6 +263,7 @@ func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common
 	if pea.rpc != nil {
 		var num *hexutil.Uint64
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getTransactionCount", address, blockNrOrHash); num != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return num, nil
 		}
 	}
@@ -275,6 +288,7 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, b
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByNumber", blockNr); num != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return num
 		}
 	}
@@ -290,6 +304,7 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByHash(ctx context.Context, blo
 	if pea.rpc != nil {
 		var num *hexutil.Uint
 		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByHash", blockHash); num != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return num
 		}
 	}
@@ -304,6 +319,7 @@ func (pea *PublicEthAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context
 	if pea.rpc != nil {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return tx
 		}
 	}
@@ -318,6 +334,7 @@ func (pea *PublicEthAPI) GetTransactionByBlockHashAndIndex(ctx context.Context,
 	if pea.rpc != nil {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return tx
 		}
 	}
@@ -332,6 +349,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Cont
 	if pea.rpc != nil {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
 			return tx
 		}
 	}
@@ -346,6 +364,7 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Contex
 	if pea.rpc != nil {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
 			return tx
 		}
 	}
@@ -362,6 +381,7 @@ func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.H
 	if pea.rpc != nil {
 		var tx *RPCTransaction
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); tx != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			return tx, nil
 		}
 	}
@@ -378,6 +398,7 @@ func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash commo
 	if pea.rpc != nil {
 		var tx hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); tx != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
 			return tx, nil
 		}
 	}
@@ -398,6 +419,7 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.
 	}
 	if pea.rpc != nil {
 		if receipt := pea.remoteGetTransactionReceipt(ctx, hash); receipt != nil {
+			go pea.writeStateDiffFor(hash)
 			return receipt, nil
 		}
 	}
@@ -489,6 +511,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
 		if arg, err := toFilterArg(crit); err == nil {
 			var res []*types.Log
 			if err := pea.rpc.CallContext(ctx, &res, "eth_getLogs", arg); err == nil {
+				go pea.writeStateDiffWithCriteria(crit)
 				return res, nil
 			}
 		}
@@ -601,6 +624,7 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address,
 	if pea.rpc != nil {
 		var res *hexutil.Big
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -626,6 +650,7 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres
 	if pea.rpc != nil {
 		var res hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -641,6 +666,7 @@ func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, bl
 	if pea.rpc != nil {
 		var res hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -656,6 +682,7 @@ func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, s
 	if pea.rpc != nil {
 		var res *AccountResult
 		if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return res, nil
 		}
 	}
@@ -726,6 +753,7 @@ func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash
 	if (failed || err != nil) && pea.rpc != nil {
 		var hex hexutil.Bytes
 		if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
 			return hex, nil
 		}
 	}
@@ -845,6 +873,89 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl
 	return result.Return(), result.UsedGas, result.Failed(), err
 }
 
+// writeStateDiffAtOrFor calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffAtOrFor(blockNrOrHash rpc.BlockNumberOrHash) {
+	// short circuit right away if the proxy doesn't support diffing
+	if !pea.supportsStateDiff {
+		return
+	}
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		pea.writeStateDiffAt(blockNr.Int64())
+		return
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		pea.writeStateDiffFor(hash)
+	}
+}
+
+// writeStateDiffWithCriteria calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffWithCriteria(crit ethereum.FilterQuery) {
+	// short circuit right away if the proxy doesn't support diffing
+	if !pea.supportsStateDiff {
+		return
+	}
+	if crit.BlockHash != nil {
+		pea.writeStateDiffFor(*crit.BlockHash)
+		return
+	}
+	var start, end int64
+	if crit.FromBlock != nil {
+		start = crit.FromBlock.Int64()
+	}
+	if crit.ToBlock != nil {
+		end = crit.ToBlock.Int64()
+	} else {
+		end = start
+	}
+	for i := start; i <= end; i++ {
+		pea.writeStateDiffAt(i)
+	}
+}
+
+// writeStateDiffAt calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffAt(height int64) {
+	if !pea.supportsStateDiff {
+		return
+	}
+	// we use a separate context than the one provided by the client
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
+	defer cancel()
+	var data json.RawMessage
+	params := statediff.Params{
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		IncludeBlock:             true,
+		IncludeReceipts:          true,
+		IncludeTD:                true,
+		IncludeCode:              true,
+	}
+	if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffAt", uint64(height), params); err != nil {
+		logrus.Errorf("writeStateDiffAt %d faild with err %s", height, err.Error())
+	}
+}
+
+// writeStateDiffFor calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffFor(blockHash common.Hash) {
+	if !pea.supportsStateDiff {
+		return
+	}
+	// we use a separate context than the one provided by the client
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
+	defer cancel()
+	var data json.RawMessage
+	params := statediff.Params{
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		IncludeBlock:             true,
+		IncludeReceipts:          true,
+		IncludeTD:                true,
+		IncludeCode:              true,
+	}
+	if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffFor", blockHash, params); err != nil {
+		logrus.Errorf("writeStateDiffFor %s faild with err %s", blockHash.Hex(), err.Error())
+	}
+}
+
 // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field
 func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
 	fields, err := RPCMarshalBlock(b, inclTx, fullTx)

pkg/eth/api_test.go

@@ -189,7 +189,7 @@ var _ = Describe("API", func() {
 		indexAndPublisher := eth2.NewIPLDPublisher(db)
 		backend, err := eth.NewEthBackend(db, &eth.Config{})
 		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil)
+		api = eth.NewPublicEthAPI(backend, nil, false)
 		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
 		Expect(err).ToNot(HaveOccurred())
 		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)

pkg/eth/backend.go

@@ -20,12 +20,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/ethereum/go-ethereum/trie"
 	"math/big"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
-
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
@@ -40,8 +38,9 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/vulcanize/ipfs-ethdb"
+	"github.com/ethereum/go-ethereum/trie"
 
+	"github.com/vulcanize/ipfs-ethdb"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 	shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 

pkg/eth/cid_retriever_test.go

@@ -17,9 +17,10 @@
 package eth_test
 
 import (
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"

pkg/eth/eth_state_test.go

@@ -19,7 +19,6 @@ package eth_test
 import (
 	"bytes"
 	"context"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 	"io/ioutil"
 	"math/big"
 
@@ -38,6 +37,7 @@ import (
 
 	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
@@ -83,7 +83,7 @@ var _ = Describe("eth state reading tests", func() {
 			RPCGasCap:   big.NewInt(10000000000),
 		})
 		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil)
+		api = eth.NewPublicEthAPI(backend, nil, false)
 
 		// make the test blockchain (and state)
 		blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen)
@@ -153,7 +153,7 @@ var _ = Describe("eth state reading tests", func() {
 
 		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
 		indexAndPublisher := eth2.NewIPLDPublisher(db)
-		api = eth.NewPublicEthAPI(backend, nil)
+		api = eth.NewPublicEthAPI(backend, nil, false)
 		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
 		Expect(err).ToNot(HaveOccurred())
 		// The non-canonical header has a child

pkg/eth/filterer.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"bytes"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	"github.com/ethereum/go-ethereum/common"

pkg/eth/filterer_test.go

@@ -18,6 +18,7 @@ package eth_test
 
 import (
 	"bytes"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	. "github.com/onsi/ginkgo"

pkg/eth/helpers.go

@@ -17,7 +17,12 @@
 package eth
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	log "github.com/sirupsen/logrus"
+	"os"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	"github.com/ethereum/go-ethereum/params"
@@ -38,6 +43,28 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType {
 	}
 }
 
+// LoadConfig loads chain config from json file
+func LoadConfig(chainConfigPath string) (*params.ChainConfig, error) {
+	file, err := os.Open(chainConfigPath)
+	if err != nil {
+		utils.Fatalf("Failed to read chain config file: %v", err)
+
+		return nil, err
+	}
+	defer file.Close()
+
+	chainConfig := new(params.ChainConfig)
+	if err := json.NewDecoder(file).Decode(chainConfig); err != nil {
+		utils.Fatalf("invalid chain config file: %v", err)
+
+		return nil, err
+	}
+
+	log.Infof("Using chain config from %s file. Content %+v", chainConfigPath, chainConfig)
+
+	return chainConfig, nil
+}
+
 // ChainConfig returns the appropriate ethereum chain config for the provided chain id
 func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
 	switch chainID {

pkg/eth/test_helpers/test_data.go

@@ -20,9 +20,10 @@ import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"math/big"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/ethereum/go-ethereum/trie"
-	"math/big"
 
 	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 

pkg/graphql/service.go

@@ -18,11 +18,11 @@ package graphql
 
 import (
 	"fmt"
-	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/node"
 	"net"
 	"net/http"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/graph-gophers/graphql-go"

pkg/rpc/http.go

@@ -18,6 +18,7 @@ package rpc
 
 import (
 	"fmt"
+
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"

pkg/rpc/ws.go

@@ -17,12 +17,13 @@
 package rpc
 
 import (
-	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/node"
 	"net"
 	"net/http"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
+
 	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 )
 

pkg/serve/config.go

@@ -48,6 +48,8 @@ const (
 
 	ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR"
 	ETH_RPC_GAS_CAP         = "ETH_RPC_GAS_CAP"
+	ETH_CHAIN_CONFIG        = "ETH_CHAIN_CONFIG"
+	ETH_SUPPORTS_STATEDIFF  = "ETH_SUPPORTS_STATEDIFF"
 )
 
 // Config struct
@@ -61,6 +63,7 @@ type Config struct {
 	DefaultSender    *common.Address
 	RPCGasCap        *big.Int
 	Client           *rpc.Client
+	SupportStateDiff bool
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -74,6 +77,8 @@ func NewConfig() (*Config, error) {
 	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
 	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
+	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
 
 	c.DBConfig.Init()
 
@@ -83,6 +88,7 @@ func NewConfig() (*Config, error) {
 		return nil, err
 	}
 	c.Client = cli
+	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
 
 	wsPath := viper.GetString("server.wsPath")
 	if wsPath == "" {
@@ -104,7 +110,7 @@ func NewConfig() (*Config, error) {
 	}
 	c.HTTPEndpoint = httpPath
 	overrideDBConnConfig(&c.DBConfig)
-	serveDB := utils.LoadPostgres(c.DBConfig, nodeInfo)
+	serveDB := utils.LoadPostgres(c.DBConfig, nodeInfo, false)
 	prom.RegisterDBCollector(c.DBConfig.Name, serveDB.DB)
 	c.DB = &serveDB
 
@@ -119,7 +125,12 @@ func NewConfig() (*Config, error) {
 			c.RPCGasCap = rpcGasCap
 		}
 	}
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	if chainConfigPath != "" {
+		c.ChainConfig, err = eth.LoadConfig(chainConfigPath)
+	} else {
 		c.ChainConfig, err = eth.ChainConfig(nodeInfo.ChainID)
+	}
 	return c, err
 }
 

pkg/serve/service.go

@@ -80,6 +80,8 @@ type Service struct {
 	serveWg *sync.WaitGroup
 	// rpc client for forwarding cache misses
 	client *rpc.Client
+	// whether the proxied client supports state diffing
+	supportsStateDiffing bool
 	// backend for the server
 	backend *eth.Backend
 }
@@ -94,6 +96,8 @@ func NewServer(settings *Config) (Server, error) {
 	sap.QuitChan = make(chan bool)
 	sap.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
 	sap.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
+	sap.client = settings.Client
+	sap.supportsStateDiffing = settings.SupportStateDiff
 	var err error
 	sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{
 		ChainConfig:   settings.ChainConfig,
@@ -141,7 +145,7 @@ func (sap *Service) APIs() []rpc.API {
 	return append(apis, rpc.API{
 		Namespace: eth.APIName,
 		Version:   eth.APIVersion,
-		Service:   eth.NewPublicEthAPI(sap.backend, sap.client),
+		Service:   eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing),
 		Public:    true,
 	})
 }

version/version.go

@@ -21,7 +21,7 @@ import "fmt"
 const (
 	Major = 0       // Major version component of the current release
 	Minor = 3       // Minor version component of the current release
-	Patch = 0       // Patch version component of the current release
+	Patch = 4       // Patch version component of the current release
 	Meta  = "alpha" // Version metadata to append to the version string
 )