Merge pull request #41 from vulcanize/bump

bump patch version

cmd/proxy.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulcanize/gap-filler/pkg/mux"
+)
+
+var ErrNoRpcEndpoints = errors.New("no rpc endpoints is available")
+
+// proxyCmd represents the proxy command
+var proxyCmd = &cobra.Command{
+	Use:   "proxy",
+	Short: "serve chain data from PG-IPFS or proxy geths",
+	Long: `This command configures a VulcanizeDB ipld-eth-server graphql server.
+
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *logrus.WithField("SubCommand", subCommand)
+		proxy()
+	},
+}
+
+func proxy() {
+	gqlDefaultAddr, err := url.Parse(viper.GetString("gql.default"))
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	gqlTracingAPIAddr, err := url.Parse(viper.GetString("gql.tracing"))
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	rpcClients, err := parseRpcAddresses(viper.GetString("rpc.eth"))
+	if err != nil {
+		logrus.Error("bad rpc.eth addresses")
+		logWithCommand.Fatal(err)
+	}
+
+	tracingClients, err := parseRpcAddresses(viper.GetString("rpc.tracing"))
+	if err != nil {
+		logrus.Error("bad rpc.tracing addresses")
+		logWithCommand.Fatal(err)
+	}
+
+	router, err := mux.NewServeMux(&mux.Options{
+		BasePath:       viper.GetString("http.path"),
+		EnableGraphiQL: viper.GetBool("gql.gui"),
+		Postgraphile: mux.PostgraphileOptions{
+			Default:    gqlDefaultAddr,
+			TracingAPI: gqlTracingAPIAddr,
+		},
+		RPC: mux.RPCOptions{
+			DefaultClients: rpcClients,
+			TracingClients: tracingClients,
+		},
+	})
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	addr := fmt.Sprintf("%s:%s", viper.GetString("http.host"), viper.GetString("http.port"))
+	if err := http.ListenAndServe(addr, router); err != nil {
+		logWithCommand.Fatal(err)
+	}
+}
+
+func parseRpcAddresses(value string) ([]*rpc.Client, error) {
+	rpcAddresses := strings.Split(value, ",")
+	rpcClients := make([]*rpc.Client, 0, len(rpcAddresses))
+	for _, address := range rpcAddresses {
+		rpcClient, err := rpc.Dial(address)
+		if err != nil {
+			logWithCommand.Errorf("couldn't connect to %s. Error: %s", address, err)
+			continue
+		}
+
+		rpcClients = append(rpcClients, rpcClient)
+	}
+
+	if len(rpcClients) == 0 {
+		logWithCommand.Error(ErrNoRpcEndpoints)
+		return nil, ErrNoRpcEndpoints
+	}
+
+	return rpcClients, nil
+}
+
+func init() {
+	rootCmd.AddCommand(proxyCmd)
+
+	// flags
+	proxyCmd.PersistentFlags().String("http-host", "127.0.0.1", "http host")
+	proxyCmd.PersistentFlags().String("http-port", "8083", "http port")
+	proxyCmd.PersistentFlags().String("http-path", "/", "http base path")
+
+	proxyCmd.PersistentFlags().String("rpc-eth", "http://127.0.0.1:8545", "comma separated ethereum rpc addresses. Example http://127.0.0.1:8545,http://127.0.0.2:8545")
+	proxyCmd.PersistentFlags().String("rpc-tracing", "http://127.0.0.1:8000", "comma separated traicing api addresses")
+
+	proxyCmd.PersistentFlags().String("gql-default", "http://127.0.0.1:5020/graphql", "postgraphile address")
+	proxyCmd.PersistentFlags().String("gql-tracing", "http://127.0.0.1:5020/graphql", "tracing api postgraphile address")
+	proxyCmd.PersistentFlags().Bool("gql-gui", false, "enable graphiql interface")
+
+	// and their .toml config bindings
+	viper.BindPFlag("http.host", proxyCmd.PersistentFlags().Lookup("http-host"))
+	viper.BindPFlag("http.port", proxyCmd.PersistentFlags().Lookup("http-port"))
+	viper.BindPFlag("http.path", proxyCmd.PersistentFlags().Lookup("http-path"))
+
+	viper.BindPFlag("rpc.eth", proxyCmd.PersistentFlags().Lookup("rpc-eth"))
+	viper.BindPFlag("rpc.tracing", proxyCmd.PersistentFlags().Lookup("rpc-tracing"))
+
+	viper.BindPFlag("gql.default", proxyCmd.PersistentFlags().Lookup("gql-default"))
+	viper.BindPFlag("gql.tracing", proxyCmd.PersistentFlags().Lookup("gql-tracing"))
+	viper.BindPFlag("gql.gui", proxyCmd.PersistentFlags().Lookup("gql-gui"))
+}

cmd/serve.go

@@ -137,6 +137,8 @@ func init() {
 	serveCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
 	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
 	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
+	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
+	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether or not the proxy ethereum client supports statediffing endpoints")
 
 	// and their bindings
 	viper.BindPFlag("server.graphql", serveCmd.PersistentFlags().Lookup("server-graphql"))
@@ -153,4 +155,6 @@ func init() {
 	viper.BindPFlag("ethereum.chainID", serveCmd.PersistentFlags().Lookup("eth-chain-id"))
 	viper.BindPFlag("ethereum.defaultSender", serveCmd.PersistentFlags().Lookup("eth-default-sender"))
 	viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap"))
+	viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config"))
+	viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff"))
 }

db/migrations/00002_create_nodes_table.sql

@@ -5,7 +5,8 @@ CREATE TABLE nodes (
   genesis_block VARCHAR(66),
   network_id    VARCHAR,
   node_id       VARCHAR(128),
-  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
+  chain_id      INTEGER DEFAULT 1,
+  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id, chain_id)
 );
 
 -- +goose Down

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -8,7 +8,6 @@ CREATE TABLE eth.transaction_cids (
   mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   dst                   VARCHAR(66) NOT NULL,
   src                   VARCHAR(66) NOT NULL,
-  deployment            BOOL NOT NULL,
   tx_data               BYTEA,
   UNIQUE (header_id, tx_hash)
 );

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -11,6 +11,8 @@ CREATE TABLE eth.receipt_cids (
   topic2s               VARCHAR(66)[],
   topic3s               VARCHAR(66)[],
   log_contracts         VARCHAR(66)[],
+  post_state            VARCHAR(66),
+  post_status           INTEGER,
   UNIQUE (tx_id)
 );
 

db/migrations/00008_create_eth_state_cids_table.sql

@@ -1,6 +1,6 @@
 -- +goose Up
 CREATE TABLE eth.state_cids (
-  id                    SERIAL PRIMARY KEY,
+  id                    BIGSERIAL PRIMARY KEY,
   header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   state_leaf_key        VARCHAR(66),
   cid                   TEXT NOT NULL,

db/migrations/00009_create_eth_storage_cids_table.sql

@@ -1,7 +1,7 @@
 -- +goose Up
 CREATE TABLE eth.storage_cids (
-  id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  id                    BIGSERIAL PRIMARY KEY,
+  state_id              BIGINT NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   storage_leaf_key      VARCHAR(66),
   cid                   TEXT NOT NULL,
   mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,

db/migrations/00010_create_eth_state_accouts_table.sql

@@ -1,7 +1,7 @@
 -- +goose Up
 CREATE TABLE eth.state_accounts (
   id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  state_id              BIGINT NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   balance               NUMERIC NOT NULL,
   nonce                 INTEGER NOT NULL,
   code_hash             BYTEA NOT NULL,

db/migrations/00012_add_chain_id_to_nodes.sql

@@ -1,21 +0,0 @@
--- +goose Up
-ALTER TABLE public.nodes
-ADD COLUMN chain_id INTEGER DEFAULT 1;
-
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id, chain_id);
-
--- +goose Down
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id);
-
-ALTER TABLE public.nodes
-DROP COLUMN chain_id;

db/migrations/00014_create_stored_functions.sql

@@ -1,34 +1,69 @@
 -- +goose Up
+-- +goose StatementBegin
+-- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
+CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
+AS $$
+SELECT exists(SELECT 1
+              FROM eth.storage_cids
+                       INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
+                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+              WHERE storage_path = path
+                AND block_number > height
+                AND block_number <= (SELECT block_number
+                                     FROM eth.header_cids
+                                     WHERE block_hash = hash)
+                AND storage_cids.node_type = 3
+              LIMIT 1);
+$$ LANGUAGE SQL;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+-- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
+CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
+AS $$
+SELECT exists(SELECT 1
+              FROM eth.state_cids
+                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+              WHERE state_path = path
+                AND block_number > height
+                AND block_number <= (SELECT block_number
+                                     FROM eth.header_cids
+                                     WHERE block_hash = hash)
+                AND state_cids.node_type = 3
+              LIMIT 1);
+$$ LANGUAGE SQL;
+-- +goose StatementEnd
+
 -- +goose StatementBegin
 CREATE TYPE child_result AS (
-  has_child BOOLEAN,
-  children eth.header_cids[]
+    has_child BOOLEAN,
+    children eth.header_cids[]
 );
 
 CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
 $BODY$
 DECLARE
-  child_height INT;
+child_height INT;
   temp_child eth.header_cids;
   new_child_result child_result;
 BEGIN
   child_height = height + 1;
   -- short circuit if there are no children
-  SELECT exists(SELECT 1
-                FROM eth.header_cids
-                WHERE parent_hash = hash
+SELECT exists(SELECT 1
+              FROM eth.header_cids
+              WHERE parent_hash = hash
                 AND block_number = child_height
-                LIMIT 1)
-  INTO new_child_result.has_child;
-  -- collect all the children for this header
-  IF new_child_result.has_child THEN
+              LIMIT 1)
+INTO new_child_result.has_child;
+-- collect all the children for this header
+IF new_child_result.has_child THEN
     FOR temp_child IN
-      SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
+SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
     LOOP
       new_child_result.children = array_append(new_child_result.children, temp_child);
-    END LOOP;
-  END IF;
-  RETURN new_child_result;
+END LOOP;
+END IF;
+RETURN new_child_result;
 END
 $BODY$
 LANGUAGE 'plpgsql';
@@ -38,7 +73,7 @@ LANGUAGE 'plpgsql';
 CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
 $BODY$
 DECLARE
-  canonical_header eth.header_cids;
+canonical_header eth.header_cids;
   canonical_child eth.header_cids;
   header eth.header_cids;
   current_child_result child_result;
@@ -57,25 +92,25 @@ BEGIN
       current_header_with_child = header;
       -- and add the children to the growing set of child headers
       child_headers = array_cat(child_headers, current_child_result.children);
-    END IF;
-  END LOOP;
+END IF;
+END LOOP;
   -- if none of the headers had children, none is more canonical than the other
   IF has_children_count = 0 THEN
     -- return the first one selected
-    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
-  -- if only one header had children, it can be considered the heaviest/canonical header of the set
-  ELSIF has_children_count = 1 THEN
+SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
+-- if only one header had children, it can be considered the heaviest/canonical header of the set
+ELSIF has_children_count = 1 THEN
     -- return the only header with a child
     canonical_header = current_header_with_child;
   -- if there are multiple headers with children
-  ELSE
+ELSE
     -- find the canonical header from the child set
     canonical_child = canonical_header_from_array(child_headers);
     -- the header that is parent to this header, is the canonical header at this level
-    SELECT * INTO canonical_header FROM unnest(headers)
-    WHERE block_hash = canonical_child.parent_hash;
-  END IF;
-  RETURN canonical_header;
+SELECT * INTO canonical_header FROM unnest(headers)
+WHERE block_hash = canonical_child.parent_hash;
+END IF;
+RETURN canonical_header;
 END
 $BODY$
 LANGUAGE 'plpgsql';
@@ -85,17 +120,17 @@ LANGUAGE 'plpgsql';
 CREATE OR REPLACE FUNCTION canonical_header_id(height BIGINT) RETURNS INTEGER AS
 $BODY$
 DECLARE
-  canonical_header eth.header_cids;
+canonical_header eth.header_cids;
   headers eth.header_cids[];
   header_count INT;
   temp_header eth.header_cids;
 BEGIN
   -- collect all headers at this height
-  FOR temp_header IN
-    SELECT * FROM eth.header_cids WHERE block_number = height
-  LOOP
+FOR temp_header IN
+SELECT * FROM eth.header_cids WHERE block_number = height
+    LOOP
     headers = array_append(headers, temp_header);
-  END LOOP;
+END LOOP;
   -- count the number of headers collected
   header_count = array_length(headers, 1);
   -- if we have less than 1 header, return NULL
@@ -105,17 +140,19 @@ BEGIN
   ELSIF header_count = 1 THEN
     RETURN headers[1].id;
   -- if we have multiple headers we need to determine which one is canonical
-  ELSE
+ELSE
     canonical_header = canonical_header_from_array(headers);
-    RETURN canonical_header.id;
-  END IF;
+RETURN canonical_header.id;
+END IF;
 END;
 $BODY$
 LANGUAGE 'plpgsql';
 -- +goose StatementEnd
 
 -- +goose Down
+DROP FUNCTION was_storage_removed;
+DROP FUNCTION was_state_removed;
 DROP FUNCTION canonical_header_id;
 DROP FUNCTION canonical_header_from_array;
 DROP FUNCTION has_child;
-DROP TYPE child_result;
+DROP TYPE child_result;

db/migrations/00015_create_canonical_hash_finder_functions.sql

@@ -1,48 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns the number of child headers that reference backwards to the header with the provided hash
-CREATE OR REPLACE FUNCTION header_weight(hash VARCHAR(66)) RETURNS BIGINT
-AS $$
-  WITH RECURSIVE validator AS (
-          SELECT block_hash, parent_hash, block_number
-          FROM eth.header_cids
-          WHERE block_hash = hash
-      UNION
-          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
-          FROM eth.header_cids
-          INNER JOIN validator
-            ON eth.header_cids.parent_hash = validator.block_hash
-            AND eth.header_cids.block_number = validator.block_number + 1
-  )
-  SELECT COUNT(*) FROM validator;
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns the id for the header at the provided height which is heaviest
-CREATE OR REPLACE FUNCTION canonical_header(height BIGINT) RETURNS INT AS
-$BODY$
-DECLARE
-  current_weight INT;
-  heaviest_weight INT DEFAULT 0;
-  heaviest_id INT;
-  r eth.header_cids%ROWTYPE;
-BEGIN
-  FOR r IN SELECT * FROM eth.header_cids
-  WHERE block_number = height
-  LOOP
-    SELECT INTO current_weight * FROM header_weight(r.block_hash);
-    IF current_weight > heaviest_weight THEN
-        heaviest_weight := current_weight;
-        heaviest_id := r.id;
-    END IF;
-  END LOOP;
-  RETURN heaviest_id;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION header_weight;
-DROP FUNCTION canonical_header;

db/migrations/00016_remove_deployment_from_tx_cids.sql

@@ -1,7 +0,0 @@
--- +goose Up
-ALTER TABLE eth.transaction_cids
-DROP COLUMN deployment;
-
--- +goose Down
-ALTER TABLE eth.transaction_cids
-ADD COLUMN deployment BOOL NOT NULL DEFAULT FALSE;

db/migrations/00017_state_and_storage_ids_use_big_serial.sql

@@ -1,25 +0,0 @@
--- +goose Up
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE BIGINT;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE BIGINT;
-
--- +goose Down
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE INTEGER;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE INTEGER;

db/migrations/00018_create_check_if_removed_functions.sql

@@ -1,39 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-  SELECT exists(SELECT 1
-					FROM eth.storage_cids
-            INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE storage_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND storage_cids.node_type = 3
-					LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-  SELECT exists(SELECT 1
-					FROM eth.state_cids
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE state_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND state_cids.node_type = 3
-					LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION was_storage_removed;
-DROP FUNCTION was_state_removed;

db/migrations/00020_add_poststate_and_status_to_receipt_cids.sql

@@ -1,13 +0,0 @@
--- +goose Up
-ALTER TABLE eth.receipt_cids
-ADD COLUMN post_state VARCHAR(66);
-
-ALTER TABLE eth.receipt_cids
-ADD COLUMN post_status INTEGER;
-
--- +goose Down
-ALTER TABLE eth.receipt_cids
-DROP COLUMN post_status;
-
-ALTER TABLE eth.receipt_cids
-DROP COLUMN post_state;

db/schema.sql

@@ -103,33 +103,6 @@ end;
 $_$;
 
 
---
--- Name: canonical_header(bigint); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.canonical_header(height bigint) RETURNS integer
-    LANGUAGE plpgsql
-    AS $$
-DECLARE
-  current_weight INT;
-  heaviest_weight INT DEFAULT 0;
-  heaviest_id INT;
-  r eth.header_cids%ROWTYPE;
-BEGIN
-  FOR r IN SELECT * FROM eth.header_cids
-  WHERE block_number = height
-  LOOP
-    SELECT INTO current_weight * FROM header_weight(r.block_hash);
-    IF current_weight > heaviest_weight THEN
-        heaviest_weight := current_weight;
-        heaviest_id := r.id;
-    END IF;
-  END LOOP;
-  RETURN heaviest_id;
-END
-$$;
-
-
 --
 -- Name: canonical_header_from_array(eth.header_cids[]); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -138,7 +111,7 @@ CREATE FUNCTION public.canonical_header_from_array(headers eth.header_cids[]) RE
     LANGUAGE plpgsql
     AS $$
 DECLARE
-  canonical_header eth.header_cids;
+canonical_header eth.header_cids;
   canonical_child eth.header_cids;
   header eth.header_cids;
   current_child_result child_result;
@@ -157,25 +130,25 @@ BEGIN
       current_header_with_child = header;
       -- and add the children to the growing set of child headers
       child_headers = array_cat(child_headers, current_child_result.children);
-    END IF;
-  END LOOP;
+END IF;
+END LOOP;
   -- if none of the headers had children, none is more canonical than the other
   IF has_children_count = 0 THEN
     -- return the first one selected
-    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
-  -- if only one header had children, it can be considered the heaviest/canonical header of the set
-  ELSIF has_children_count = 1 THEN
+SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
+-- if only one header had children, it can be considered the heaviest/canonical header of the set
+ELSIF has_children_count = 1 THEN
     -- return the only header with a child
     canonical_header = current_header_with_child;
   -- if there are multiple headers with children
-  ELSE
+ELSE
     -- find the canonical header from the child set
     canonical_child = canonical_header_from_array(child_headers);
     -- the header that is parent to this header, is the canonical header at this level
-    SELECT * INTO canonical_header FROM unnest(headers)
-    WHERE block_hash = canonical_child.parent_hash;
-  END IF;
-  RETURN canonical_header;
+SELECT * INTO canonical_header FROM unnest(headers)
+WHERE block_hash = canonical_child.parent_hash;
+END IF;
+RETURN canonical_header;
 END
 $$;
 
@@ -188,17 +161,17 @@ CREATE FUNCTION public.canonical_header_id(height bigint) RETURNS integer
     LANGUAGE plpgsql
     AS $$
 DECLARE
-  canonical_header eth.header_cids;
+canonical_header eth.header_cids;
   headers eth.header_cids[];
   header_count INT;
   temp_header eth.header_cids;
 BEGIN
   -- collect all headers at this height
-  FOR temp_header IN
-    SELECT * FROM eth.header_cids WHERE block_number = height
-  LOOP
+FOR temp_header IN
+SELECT * FROM eth.header_cids WHERE block_number = height
+    LOOP
     headers = array_append(headers, temp_header);
-  END LOOP;
+END LOOP;
   -- count the number of headers collected
   header_count = array_length(headers, 1);
   -- if we have less than 1 header, return NULL
@@ -208,10 +181,10 @@ BEGIN
   ELSIF header_count = 1 THEN
     RETURN headers[1].id;
   -- if we have multiple headers we need to determine which one is canonical
-  ELSE
+ELSE
     canonical_header = canonical_header_from_array(headers);
-    RETURN canonical_header.id;
-  END IF;
+RETURN canonical_header.id;
+END IF;
 END;
 $$;
 
@@ -224,53 +197,31 @@ CREATE FUNCTION public.has_child(hash character varying, height bigint) RETURNS
     LANGUAGE plpgsql
     AS $$
 DECLARE
-  child_height INT;
+child_height INT;
   temp_child eth.header_cids;
   new_child_result child_result;
 BEGIN
   child_height = height + 1;
   -- short circuit if there are no children
-  SELECT exists(SELECT 1
-                FROM eth.header_cids
-                WHERE parent_hash = hash
+SELECT exists(SELECT 1
+              FROM eth.header_cids
+              WHERE parent_hash = hash
                 AND block_number = child_height
-                LIMIT 1)
-  INTO new_child_result.has_child;
-  -- collect all the children for this header
-  IF new_child_result.has_child THEN
+              LIMIT 1)
+INTO new_child_result.has_child;
+-- collect all the children for this header
+IF new_child_result.has_child THEN
     FOR temp_child IN
-      SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
+SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
     LOOP
       new_child_result.children = array_append(new_child_result.children, temp_child);
-    END LOOP;
-  END IF;
-  RETURN new_child_result;
+END LOOP;
+END IF;
+RETURN new_child_result;
 END
 $$;
 
 
---
--- Name: header_weight(character varying); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.header_weight(hash character varying) RETURNS bigint
-    LANGUAGE sql
-    AS $$
-  WITH RECURSIVE validator AS (
-          SELECT block_hash, parent_hash, block_number
-          FROM eth.header_cids
-          WHERE block_hash = hash
-      UNION
-          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
-          FROM eth.header_cids
-          INNER JOIN validator
-            ON eth.header_cids.parent_hash = validator.block_hash
-            AND eth.header_cids.block_number = validator.block_number + 1
-  )
-  SELECT COUNT(*) FROM validator;
-$$;
-
-
 --
 -- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -278,16 +229,16 @@ $$;
 CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean
     LANGUAGE sql
     AS $$
-  SELECT exists(SELECT 1
-					FROM eth.state_cids
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE state_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND state_cids.node_type = 3
-					LIMIT 1);
+SELECT exists(SELECT 1
+              FROM eth.state_cids
+                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+              WHERE state_path = path
+                AND block_number > height
+                AND block_number <= (SELECT block_number
+                                     FROM eth.header_cids
+                                     WHERE block_hash = hash)
+                AND state_cids.node_type = 3
+              LIMIT 1);
 $$;
 
 
@@ -298,17 +249,17 @@ $$;
 CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean
     LANGUAGE sql
     AS $$
-  SELECT exists(SELECT 1
-					FROM eth.storage_cids
-            INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE storage_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND storage_cids.node_type = 3
-					LIMIT 1);
+SELECT exists(SELECT 1
+              FROM eth.storage_cids
+                       INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
+                       INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+              WHERE storage_path = path
+                AND block_number > height
+                AND block_number <= (SELECT block_number
+                                     FROM eth.header_cids
+                                     WHERE block_hash = hash)
+                AND storage_cids.node_type = 3
+              LIMIT 1);
 $$;
 
 
@@ -428,7 +379,6 @@ CREATE TABLE eth.state_cids (
 --
 
 CREATE SEQUENCE eth.state_cids_id_seq
-    AS integer
     START WITH 1
     INCREMENT BY 1
     NO MINVALUE
@@ -464,7 +414,6 @@ CREATE TABLE eth.storage_cids (
 --
 
 CREATE SEQUENCE eth.storage_cids_id_seq
-    AS integer
     START WITH 1
     INCREMENT BY 1
     NO MINVALUE

go.mod

@@ -16,10 +16,11 @@ require (
 	github.com/onsi/ginkgo v1.15.0
 	github.com/onsi/gomega v1.10.1
 	github.com/prometheus/client_golang v1.5.1
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/cobra v1.0.0
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.0
 	github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha
+	github.com/vulcanize/gap-filler v0.3.1
 	github.com/vulcanize/ipfs-ethdb v0.0.2-alpha
 	golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
 )

go.sum

@@ -154,6 +154,8 @@ github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepB
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
+github.com/friendsofgo/graphiql v0.2.2 h1:ccnuxpjgIkB+Lr9YB2ZouiZm7wvciSfqwpa9ugWzmn0=
+github.com/friendsofgo/graphiql v0.2.2/go.mod h1:8Y2kZ36AoTGWs78+VRpvATyt3LJBx0SZXmay80ZTRWo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -237,6 +239,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e h1:IpssFbpfPSx/3c7x601Npx+UOQ4tqd0Rk4sObCQ+zlQ=
 github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/graphql-go/graphql v0.7.9 h1:5Va/Rt4l5g3YjwDnid3vFfn43faaQBq7rMcIZ0VnV34=
+github.com/graphql-go/graphql v0.7.9/go.mod h1:k6yrAYQaSP59DC5UVxbgxESlmVyojThKdORUqGDGmrI=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -449,6 +453,8 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1C
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jinzhu/copier v0.2.4 h1:dT3tI+8GzU8DjJFCj9mLYtjfRtUmK7edauduQdcZCpI=
+github.com/jinzhu/copier v0.2.4/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
@@ -929,6 +935,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
@@ -952,10 +960,14 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
@@ -989,8 +1001,14 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
+github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/vulcanize/gap-filler v0.3.0 h1:SaBrOxlxCDPbSA902K/46iELH8yM3BqhvBuXK1ktFDI=
+github.com/vulcanize/gap-filler v0.3.0/go.mod h1:4odsXyckNU1xdXk37qXY2SbxUM6oDpQrpWZtEHfq+0w=
+github.com/vulcanize/gap-filler v0.3.1 h1:N5d+jCJo/VTWFvBSbTD7biRhK/OqDZzi1tgA85SIBKs=
+github.com/vulcanize/gap-filler v0.3.1/go.mod h1:qowG1cgshVpBqMokiWro/1xhh0uypw7oAu8FQ42JMy4=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5 h1:U+BqhjRLR22e9OEm8cgWC3Eq3bh8G6azjNpXeenfCG4=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8 h1:7TK52k55uvSl+1SCKYYFelzH1NrpvEcDrpeU9nUDIpI=
@@ -1338,6 +1356,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=

pkg/eth/helpers.go

@@ -17,7 +17,11 @@
 package eth
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	log "github.com/sirupsen/logrus"
+	"os"
 
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
@@ -39,6 +43,28 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType {
 	}
 }
 
+// LoadConfig loads chain config from json file
+func LoadConfig(chainConfigPath string) (*params.ChainConfig, error) {
+	file, err := os.Open(chainConfigPath)
+	if err != nil {
+		utils.Fatalf("Failed to read chain config file: %v", err)
+
+		return nil, err
+	}
+	defer file.Close()
+
+	chainConfig := new(params.ChainConfig)
+	if err := json.NewDecoder(file).Decode(chainConfig); err != nil {
+		utils.Fatalf("invalid chain config file: %v", err)
+
+		return nil, err
+	}
+
+	log.Infof("Using chain config from %s file. Content %+v", chainConfigPath, chainConfig)
+
+	return chainConfig, nil
+}
+
 // ChainConfig returns the appropriate ethereum chain config for the provided chain id
 func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
 	switch chainID {

pkg/serve/config.go

@@ -48,6 +48,7 @@ const (
 
 	ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR"
 	ETH_RPC_GAS_CAP         = "ETH_RPC_GAS_CAP"
+	ETH_CHAIN_CONFIG        = "ETH_CHAIN_CONFIG"
 	ETH_SUPPORTS_STATEDIFF  = "ETH_SUPPORTS_STATEDIFF"
 )
 
@@ -76,6 +77,7 @@ func NewConfig() (*Config, error) {
 	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
 	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
 	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
 
 	c.DBConfig.Init()
@@ -123,7 +125,12 @@ func NewConfig() (*Config, error) {
 			c.RPCGasCap = rpcGasCap
 		}
 	}
-	c.ChainConfig, err = eth.ChainConfig(nodeInfo.ChainID)
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	if chainConfigPath != "" {
+		c.ChainConfig, err = eth.LoadConfig(chainConfigPath)
+	} else {
+		c.ChainConfig, err = eth.ChainConfig(nodeInfo.ChainID)
+	}
 	return c, err
 }
 

version/version.go

@@ -21,7 +21,7 @@ import "fmt"
 const (
 	Major = 0       // Major version component of the current release
 	Minor = 3       // Minor version component of the current release
-	Patch = 1       // Patch version component of the current release
+	Patch = 4       // Patch version component of the current release
 	Meta  = "alpha" // Version metadata to append to the version string
 )