final touches, test canonicty functions and get tests passing again

db/migrations/00018_create_check_if_removed_functions.sql

@@ -3,7 +3,7 @@
 -- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
 CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
 AS $$
-  SELECT exists(SELECT *
+  SELECT exists(SELECT 1
 					FROM eth.storage_cids
             INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
             INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
@@ -12,7 +12,8 @@ AS $$
 					AND block_number <= (SELECT block_number
 										FROM eth.header_cids
 										WHERE block_hash = hash)
-					AND storage_cids.node_type = 3);
+					AND storage_cids.node_type = 3
+					LIMIT 1);
 $$ LANGUAGE SQL;
 -- +goose StatementEnd
 
@@ -20,7 +21,7 @@ $$ LANGUAGE SQL;
 -- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
 CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
 AS $$
-  SELECT exists(SELECT *
+  SELECT exists(SELECT 1
 					FROM eth.state_cids
             INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
 					WHERE state_path = path
@@ -28,7 +29,8 @@ AS $$
 					AND block_number <= (SELECT block_number
 										FROM eth.header_cids
 										WHERE block_hash = hash)
-					AND state_cids.node_type = 3);
+					AND state_cids.node_type = 3
+					LIMIT 1);
 $$ LANGUAGE SQL;
 -- +goose StatementEnd
 

db/migrations/00019_updated_canonical_hash_finder_functions.sql

@@ -1,30 +1,32 @@
 -- +goose Up
-ALTER FUNCTION canonical_header RENAME TO canonical_header_id;
-
 -- +goose StatementBegin
 CREATE TYPE child_result AS (
   has_child BOOLEAN,
   children eth.header_cids[]
 );
 
-CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66)) RETURNS child_result AS
+CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
 $BODY$
 DECLARE
+  child_height INT;
+  temp_child eth.header_cids;
   new_child_result child_result;
 BEGIN
+  child_height = height + 1;
   -- short circuit if there are no children
   SELECT exists(SELECT 1
                 FROM eth.header_cids
                 WHERE parent_hash = hash
+                AND block_number = child_height
                 LIMIT 1)
   INTO new_child_result.has_child;
   -- collect all the children for this header
   IF new_child_result.has_child THEN
-    SELECT *
+    FOR temp_child IN
-    INTO new_child_result.children
+      SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
-    FROM eth.header_cids
+    LOOP
-    WHERE parent_hash = hash;
+      new_child_result.children = array_append(new_child_result.children, temp_child);
-  ELSE
+    END LOOP;
   END IF;
   RETURN new_child_result;
 END
@@ -33,7 +35,7 @@ LANGUAGE 'plpgsql';
 -- +goose StatementEnd
 
 -- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header_from_set(headers eth.header_cids[]) RETURNS eth.header_cids AS
+CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
 $BODY$
 DECLARE
   canonical_header eth.header_cids;
@@ -45,10 +47,10 @@ DECLARE
   has_children_count INT DEFAULT 0;
 BEGIN
   -- for each header in the provided set
-  FOR header IN SELECT * FROM headers
+  FOREACH header IN ARRAY headers
   LOOP
     -- check if it has any children
-    SELECT * INTO current_child_result FROM has_child(header.block_hash);
+    current_child_result = has_child(header.block_hash, header.block_number);
     IF current_child_result.has_child THEN
       -- if it does, take note
       has_children_count = has_children_count + 1;
@@ -60,17 +62,17 @@ BEGIN
   -- if none of the headers had children, none is more canonical than the other
   IF has_children_count = 0 THEN
     -- return the first one selected
-    SELECT * INTO canonical_header FROM headers LIMIT 1;
+    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
   -- if only one header had children, it can be considered the heaviest/canonical header of the set
   ELSIF has_children_count = 1 THEN
     -- return the only header with a child
     canonical_header = current_header_with_child;
   -- if there are multiple headers with children
   ELSE
-    -- find the canonical the canonical header from the child set
+    -- find the canonical header from the child set
-    canonical_child = canonical_header_from_set(child_headers);
+    canonical_child = canonical_header_from_array(child_headers);
     -- the header that is parent to this header, is the canonical header at this level
-    SELECT * INTO canonical_header FROM headers
+    SELECT * INTO canonical_header FROM unnest(headers)
     WHERE block_hash = canonical_child.parent_hash;
   END IF;
   RETURN canonical_header;
@@ -80,31 +82,40 @@ LANGUAGE 'plpgsql';
 -- +goose StatementEnd
 
 -- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header(height BIGINT) RETURNS eth.header_cids AS
+CREATE OR REPLACE FUNCTION canonical_header_id(height BIGINT) RETURNS INTEGER AS
 $BODY$
 DECLARE
-  header_count INT;
+  canonical_header eth.header_cids;
   headers eth.header_cids[];
+  header_count INT;
+  temp_header eth.header_cids;
 BEGIN
-  -- collect every header at this height, noting how many are collected
+  -- collect all headers at this height
-  SELECT *, count(*)
+  FOR temp_header IN
-  INTO headers, header_count
+    SELECT * FROM eth.header_cids WHERE block_number = height
-  FROM eth.header_cids
+  LOOP
-  WHERE block_number = height;
+    headers = array_append(headers, temp_header);
-  -- if only one header is present, it can be considered canonical (if no header is present we will throw an error)
+  END LOOP;
-  IF header_count = 1 THEN
+  -- count the number of headers collected
-    RETURN headers[0];
+  header_count = array_length(headers, 1);
+  -- if we have less than 1 header, return NULL
+  IF header_count IS NULL OR header_count < 1 THEN
+    RETURN NULL;
+  -- if we have one header, return its id
+  ELSIF header_count = 1 THEN
+    RETURN headers[1].id;
+  -- if we have multiple headers we need to determine which one is canonical
+  ELSE
+    canonical_header = canonical_header_from_array(headers);
+    RETURN canonical_header.id;
   END IF;
-  -- otherwise, if there are multiple headers at this height, we need to determine which is canonical
+END;
-  RETURN canonical_header_from_set(headers);
-END
 $BODY$
 LANGUAGE 'plpgsql';
 -- +goose StatementEnd
 
 -- +goose Down
-DROP FUNCTION canonical_header;
+DROP FUNCTION canonical_header_id;
-DROP FUNCTION canonical_header_from_set;
+DROP FUNCTION canonical_header_from_array;
 DROP FUNCTION has_child;
 DROP TYPE child_result;
-ALTER FUNCTION canonical_header_id RENAME TO canonical_header;

db/schema.sql

@@ -107,83 +107,7 @@ $_$;
 -- Name: canonical_header(bigint); Type: FUNCTION; Schema: public; Owner: -
 --
 
-CREATE FUNCTION public.canonical_header(height bigint) RETURNS eth.header_cids
+CREATE FUNCTION public.canonical_header(height bigint) RETURNS integer
-    LANGUAGE plpgsql
-    AS $$
-DECLARE
-  header_count INT;
-  headers eth.header_cids[];
-BEGIN
-  -- collect every header at this height, noting how many are collected
-  SELECT *, count(*)
-  INTO headers, header_count
-  FROM eth.header_cids
-  WHERE block_number = height;
-  -- if only one header is present, it can be considered canonical (if no header is present we will throw an error)
-  IF header_count = 1 THEN
-    RETURN headers[0];
-  END IF;
-  -- otherwise, if there are multiple headers at this height, we need to determine which is canonical
-  RETURN canonical_header_from_set(headers);
-END
-$$;
-
-
---
--- Name: canonical_header_from_set(eth.header_cids[]); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.canonical_header_from_set(headers eth.header_cids[]) RETURNS eth.header_cids
-    LANGUAGE plpgsql
-    AS $$
-DECLARE
-  canonical_header eth.header_cids;
-  canonical_child eth.header_cids;
-  header eth.header_cids;
-  current_child_result child_result;
-  child_headers eth.header_cids[];
-  current_header_with_child eth.header_cids;
-  has_children_count INT DEFAULT 0;
-BEGIN
-  -- for each header in the provided set
-  FOR header IN SELECT * FROM headers
-  LOOP
-    -- check if it has any children
-    SELECT * INTO current_child_result FROM has_child(header.block_hash);
-    IF current_child_result.has_child THEN
-      -- if it does, take note
-      has_children_count = has_children_count + 1;
-      current_header_with_child = header;
-      -- and add the children to the growing set of child headers
-      child_headers = array_cat(child_headers, current_child_result.children);
-    END IF;
-  END LOOP;
-  -- if none of the headers had children, none is more canonical than the other
-  IF has_children_count = 0 THEN
-    -- return the first one selected
-    SELECT * INTO canonical_header FROM headers LIMIT 1;
-  -- if only one header had children, it can be considered the heaviest/canonical header of the set
-  ELSIF has_children_count = 1 THEN
-    -- return the only header with a child
-    canonical_header = current_header_with_child;
-  -- if there are multiple headers with children
-  ELSE
-    -- find the canonical the canonical header from the child set
-    canonical_child = canonical_header_from_set(child_headers);
-    -- the header that is parent to this header, is the canonical header at this level
-    SELECT * INTO canonical_header FROM headers
-    WHERE block_hash = canonical_child.parent_hash;
-  END IF;
-  RETURN canonical_header;
-END
-$$;
-
-
---
--- Name: canonical_header_id(bigint); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.canonical_header_id(height bigint) RETURNS integer
     LANGUAGE plpgsql
     AS $$
 DECLARE
@@ -207,28 +131,118 @@ $$;
 
 
 --
--- Name: has_child(character varying); Type: FUNCTION; Schema: public; Owner: -
+-- Name: canonical_header_from_array(eth.header_cids[]); Type: FUNCTION; Schema: public; Owner: -
 --
 
-CREATE FUNCTION public.has_child(hash character varying) RETURNS public.child_result
+CREATE FUNCTION public.canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids
     LANGUAGE plpgsql
     AS $$
 DECLARE
+  canonical_header eth.header_cids;
+  canonical_child eth.header_cids;
+  header eth.header_cids;
+  current_child_result child_result;
+  child_headers eth.header_cids[];
+  current_header_with_child eth.header_cids;
+  has_children_count INT DEFAULT 0;
+BEGIN
+  -- for each header in the provided set
+  FOREACH header IN ARRAY headers
+  LOOP
+    -- check if it has any children
+    current_child_result = has_child(header.block_hash, header.block_number);
+    IF current_child_result.has_child THEN
+      -- if it does, take note
+      has_children_count = has_children_count + 1;
+      current_header_with_child = header;
+      -- and add the children to the growing set of child headers
+      child_headers = array_cat(child_headers, current_child_result.children);
+    END IF;
+  END LOOP;
+  -- if none of the headers had children, none is more canonical than the other
+  IF has_children_count = 0 THEN
+    -- return the first one selected
+    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
+  -- if only one header had children, it can be considered the heaviest/canonical header of the set
+  ELSIF has_children_count = 1 THEN
+    -- return the only header with a child
+    canonical_header = current_header_with_child;
+  -- if there are multiple headers with children
+  ELSE
+    -- find the canonical header from the child set
+    canonical_child = canonical_header_from_array(child_headers);
+    -- the header that is parent to this header, is the canonical header at this level
+    SELECT * INTO canonical_header FROM unnest(headers)
+    WHERE block_hash = canonical_child.parent_hash;
+  END IF;
+  RETURN canonical_header;
+END
+$$;
+
+
+--
+-- Name: canonical_header_id(bigint); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.canonical_header_id(height bigint) RETURNS integer
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  canonical_header eth.header_cids;
+  headers eth.header_cids[];
+  header_count INT;
+  temp_header eth.header_cids;
+BEGIN
+  -- collect all headers at this height
+  FOR temp_header IN
+    SELECT * FROM eth.header_cids WHERE block_number = height
+  LOOP
+    headers = array_append(headers, temp_header);
+  END LOOP;
+  -- count the number of headers collected
+  header_count = array_length(headers, 1);
+  -- if we have less than 1 header, return NULL
+  IF header_count IS NULL OR header_count < 1 THEN
+    RETURN NULL;
+  -- if we have one header, return its id
+  ELSIF header_count = 1 THEN
+    RETURN headers[1].id;
+  -- if we have multiple headers we need to determine which one is canonical
+  ELSE
+    canonical_header = canonical_header_from_array(headers);
+    RETURN canonical_header.id;
+  END IF;
+END;
+$$;
+
+
+--
+-- Name: has_child(character varying, bigint); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.has_child(hash character varying, height bigint) RETURNS public.child_result
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  child_height INT;
+  temp_child eth.header_cids;
   new_child_result child_result;
 BEGIN
+  child_height = height + 1;
   -- short circuit if there are no children
   SELECT exists(SELECT 1
                 FROM eth.header_cids
                 WHERE parent_hash = hash
+                AND block_number = child_height
                 LIMIT 1)
   INTO new_child_result.has_child;
   -- collect all the children for this header
   IF new_child_result.has_child THEN
-    SELECT *
+    FOR temp_child IN
-    INTO new_child_result.children
+      SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
-    FROM eth.header_cids
+    LOOP
-    WHERE parent_hash = hash;
+      new_child_result.children = array_append(new_child_result.children, temp_child);
-  ELSE
+    END LOOP;
   END IF;
   RETURN new_child_result;
 END
@@ -264,7 +278,7 @@ $$;
 CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean
     LANGUAGE sql
     AS $$
-  SELECT exists(SELECT *
+  SELECT exists(SELECT 1
 					FROM eth.state_cids
             INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
 					WHERE state_path = path
@@ -272,7 +286,8 @@ CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash charact
 					AND block_number <= (SELECT block_number
 										FROM eth.header_cids
 										WHERE block_hash = hash)
-					AND state_cids.node_type = 3);
+					AND state_cids.node_type = 3
+					LIMIT 1);
 $$;
 
 
@@ -283,7 +298,7 @@ $$;
 CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean
     LANGUAGE sql
     AS $$
-  SELECT exists(SELECT *
+  SELECT exists(SELECT 1
 					FROM eth.storage_cids
             INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
             INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
@@ -292,7 +307,8 @@ CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash chara
 					AND block_number <= (SELECT block_number
 										FROM eth.header_cids
 										WHERE block_hash = hash)
-					AND storage_cids.node_type = 3);
+					AND storage_cids.node_type = 3
+					LIMIT 1);
 $$;
 
 

pkg/eth/eth_state_test.go

@@ -69,6 +69,7 @@ var _ = Describe("eth state reading tests", func() {
 		backend                 *eth.Backend
 		chainConfig             = params.TestChainConfig
 		mockTD                  = big.NewInt(1337)
+		expectedCanonicalHeader map[string]interface{}
 	)
 	It("test init", func() {
 		// db and type initializations
@@ -90,6 +91,27 @@ var _ = Describe("eth state reading tests", func() {
 			IntermediateStateNodes:   true,
 			IntermediateStorageNodes: true,
 		}
+		canonicalHeader := blocks[1].Header()
+		expectedCanonicalHeader = map[string]interface{}{
+			"number":           (*hexutil.Big)(canonicalHeader.Number),
+			"hash":             canonicalHeader.Hash(),
+			"parentHash":       canonicalHeader.ParentHash,
+			"nonce":            canonicalHeader.Nonce,
+			"mixHash":          canonicalHeader.MixDigest,
+			"sha3Uncles":       canonicalHeader.UncleHash,
+			"logsBloom":        canonicalHeader.Bloom,
+			"stateRoot":        canonicalHeader.Root,
+			"miner":            canonicalHeader.Coinbase,
+			"difficulty":       (*hexutil.Big)(canonicalHeader.Difficulty),
+			"extraData":        hexutil.Bytes([]byte{}),
+			"size":             hexutil.Uint64(canonicalHeader.Size()),
+			"gasLimit":         hexutil.Uint64(canonicalHeader.GasLimit),
+			"gasUsed":          hexutil.Uint64(canonicalHeader.GasUsed),
+			"timestamp":        hexutil.Uint64(canonicalHeader.Time),
+			"transactionsRoot": canonicalHeader.TxHash,
+			"receiptsRoot":     canonicalHeader.ReceiptHash,
+			"totalDifficulty":  (*hexutil.Big)(mockTD),
+		}
 		// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
 		builder := statediff.NewBuilder(chain.StateCache())
 		for i, block := range blocks {
@@ -128,6 +150,17 @@ var _ = Describe("eth state reading tests", func() {
 			_, err = transformer.Transform(0, payload)
 			Expect(err).ToNot(HaveOccurred())
 		}
+
+		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+		indexAndPublisher := eth2.NewIPLDPublisher(db)
+		api = eth.NewPublicEthAPI(backend, nil)
+		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
+		Expect(err).ToNot(HaveOccurred())
+		// The non-canonical header has a child
+		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayloadForChild)
+		Expect(err).ToNot(HaveOccurred())
+		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
+		Expect(err).ToNot(HaveOccurred())
 	})
 	defer It("test teardown", func() {
 		eth.TearDownDB(db)
@@ -423,4 +456,12 @@ var _ = Describe("eth state reading tests", func() {
 			Expect(val).To(Equal(hexutil.Bytes{}))
 		})
 	})
+
+	Describe("eth_getHeaderByNumber", func() {
+		It("Finds the canonical header based on the header's weight relative to others at the provided height", func() {
+			header, err := api.GetHeaderByNumber(ctx, number)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header).To(Equal(expectedCanonicalHeader))
+		})
+	})
 })

pkg/eth/test_helpers/test_data.go

@@ -80,6 +80,18 @@ var (
 	ReceiptsRlp, _   = rlp.EncodeToBytes(MockReceipts)
 	MockBlock        = types.NewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts)
 	MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
+	MockChildHeader  = types.Header{
+		Time:        0,
+		Number:      new(big.Int).Add(BlockNumber, common.Big1),
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+		Difficulty:  big.NewInt(5000001),
+		Extra:       []byte{},
+		ParentHash:  MockBlock.Header().Hash(),
+	}
+	MockChild            = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts)
+	MockChildRlp, _      = rlp.EncodeToBytes(MockChild.Header())
 	Address              = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
 	AnotherAddress       = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
 	ContractAddress      = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
@@ -351,6 +363,15 @@ var (
 		StorageNodes:    MockStorageNodes,
 		StateNodes:      MockStateNodes,
 	}
+	MockConvertedPayloadForChild = eth.ConvertedPayload{
+		TotalDifficulty: MockChild.Difficulty(),
+		Block:           MockChild,
+		Receipts:        MockReceipts,
+		TxMetaData:      MockTrxMeta,
+		ReceiptMetaData: MockRctMeta,
+		StorageNodes:    MockStorageNodes,
+		StateNodes:      MockStateNodes,
+	}
 
 	MockCIDWrapper = &eth2.CIDWrapper{
 		BlockNumber: new(big.Int).Set(BlockNumber),