state account support and add roots and additional metadata to header

index

db/migrations/00029_update_header_cids.sql

@@ -0,0 +1,37 @@
+-- +goose Up
+ALTER TABLE eth.header_cids
+ADD COLUMN state_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN tx_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN receipt_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN uncle_root VARCHAR(66);
+
+ALTER TABLE eth.header_cids
+ADD COLUMN bloom BYTEA;
+
+ALTER TABLE eth.header_cids
+ADD COLUMN timestamp NUMERIC;
+
+-- +goose Down
+ALTER TABLE eth.header_cids
+DROP COLUMN timestamp;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN bloom;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN uncle_root;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN receipt_root;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN tx_root;
+
+ALTER TABLE eth.header_cids
+DROP COLUMN state_root;

db/migrations/00030_create_eth_state_accouts_table.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+CREATE TABLE eth.state_accounts (
+  id                    SERIAL PRIMARY KEY,
+  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE,
+  balance               NUMERIC NOT NULL,
+  nonce                 INTEGER NOT NULL,
+  code_hash             BYTEA NOT NULL,
+  storage_root          VARCHAR(66) NOT NULL,
+  UNIQUE (state_id)
+);
+
+-- +goose Down
+DROP TABLE eth.state_accounts;

db/schema.sql

@@ -71,6 +71,13 @@ CREATE TABLE btc.header_cids (
 COMMENT ON TABLE btc.header_cids IS '@name BtcHeaderCids';
 
 
+--
+-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: btc; Owner: -
+--
+
+COMMENT ON COLUMN btc.header_cids.node_id IS '@name BtcNodeID';
+
+
 --
 -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
 --
@@ -254,7 +261,13 @@ CREATE TABLE eth.header_cids (
     cid text NOT NULL,
     td numeric NOT NULL,
     node_id integer NOT NULL,
-    reward numeric NOT NULL
+    reward numeric NOT NULL,
+    state_root character varying(66),
+    tx_root character varying(66),
+    receipt_root character varying(66),
+    uncle_root character varying(66),
+    bloom bytea,
+    "timestamp" numeric
 );
 
 
@@ -265,6 +278,13 @@ CREATE TABLE eth.header_cids (
 COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids';
 
 
+--
+-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: eth; Owner: -
+--
+
+COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID';
+
+
 --
 -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
 --
@@ -359,6 +379,40 @@ CREATE SEQUENCE eth.receipt_cids_id_seq
 ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id;
 
 
+--
+-- Name: state_accounts; Type: TABLE; Schema: eth; Owner: -
+--
+
+CREATE TABLE eth.state_accounts (
+    id integer NOT NULL,
+    state_id integer NOT NULL,
+    balance numeric NOT NULL,
+    nonce integer NOT NULL,
+    code_hash bytea NOT NULL,
+    storage_root character varying(66) NOT NULL
+);
+
+
+--
+-- Name: state_accounts_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
+--
+
+CREATE SEQUENCE eth.state_accounts_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: state_accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
+--
+
+ALTER SEQUENCE eth.state_accounts_id_seq OWNED BY eth.state_accounts.id;
+
+
 --
 -- Name: state_cids; Type: TABLE; Schema: eth; Owner: -
 --
@@ -369,7 +423,7 @@ CREATE TABLE eth.state_cids (
     state_key character varying(66),
     cid text NOT NULL,
     state_path bytea,
-    node_type integer NOT NULL
+    node_type integer
 );
 
 
@@ -403,7 +457,7 @@ CREATE TABLE eth.storage_cids (
     storage_key character varying(66),
     cid text NOT NULL,
     storage_path bytea,
-    node_type integer NOT NULL
+    node_type integer
 );
 
 
@@ -740,6 +794,20 @@ CREATE TABLE public.headers (
 );
 
 
+--
+-- Name: TABLE headers; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON TABLE public.headers IS '@name EthHeaders';
+
+
+--
+-- Name: COLUMN headers.node_id; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.headers.node_id IS '@name EthNodeID';
+
+
 --
 -- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
@@ -777,7 +845,14 @@ CREATE TABLE public.nodes (
 -- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
 --
 
-COMMENT ON TABLE public.nodes IS '@name NodeIDs';
+COMMENT ON TABLE public.nodes IS '@name NodeInfo';
+
+
+--
+-- Name: COLUMN nodes.node_id; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.node_id IS '@name ChainNodeID';
 
 
 --
@@ -951,6 +1026,13 @@ ALTER TABLE ONLY eth.queue_data ALTER COLUMN id SET DEFAULT nextval('eth.queue_d
 ALTER TABLE ONLY eth.receipt_cids ALTER COLUMN id SET DEFAULT nextval('eth.receipt_cids_id_seq'::regclass);
 
 
+--
+-- Name: state_accounts id; Type: DEFAULT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.state_accounts ALTER COLUMN id SET DEFAULT nextval('eth.state_accounts_id_seq'::regclass);
+
+
 --
 -- Name: state_cids id; Type: DEFAULT; Schema: eth; Owner: -
 --
@@ -1176,6 +1258,22 @@ ALTER TABLE ONLY eth.receipt_cids
     ADD CONSTRAINT receipt_cids_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: state_accounts state_accounts_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.state_accounts
+    ADD CONSTRAINT state_accounts_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: state_accounts state_accounts_state_id_key; Type: CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.state_accounts
+    ADD CONSTRAINT state_accounts_state_id_key UNIQUE (state_id);
+
+
 --
 -- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
 --
@@ -1498,6 +1596,14 @@ ALTER TABLE ONLY eth.receipt_cids
     ADD CONSTRAINT receipt_cids_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
 
 
+--
+-- Name: state_accounts state_accounts_state_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.state_accounts
+    ADD CONSTRAINT state_accounts_state_id_fkey FOREIGN KEY (state_id) REFERENCES eth.state_cids(id) ON DELETE CASCADE;
+
+
 --
 -- Name: state_cids state_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
 --

pkg/super_node/eth/indexer.go

@@ -88,10 +88,12 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
 
 func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
 	var headerID int64
-	err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward) VALUES ($1, $2, $3, $4, $5, $6, $7)
+	err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp) 
-								ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward) = ($3, $4, $5, $6, $7)
+								VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+								ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
 								RETURNING id`,
-		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward).Scan(&headerID)
+		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward, header.StateRoot, header.TxRoot,
+		header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp).Scan(&headerID)
 	return headerID, err
 }
 
@@ -138,15 +140,31 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
 		if err != nil {
 			return err
 		}
-		for _, storageCID := range payload.StorageNodeCIDs[crypto.Keccak256Hash(stateCID.Path)] {
+		// If we have a state leaf node, index the associated account and storage nodes
-			if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
+		if stateCID.NodeType == 2 {
-				return err
+			pathKey := crypto.Keccak256Hash(stateCID.Path)
+			for _, storageCID := range payload.StorageNodeCIDs[pathKey] {
+				if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
+					return err
+				}
+			}
+			if stateAccount, ok := payload.StateAccounts[pathKey]; ok {
+				if err := in.indexStateAccount(tx, stateAccount, stateID); err != nil {
+					return err
+				}
 			}
 		}
 	}
 	return nil
 }
 
+func (in *CIDIndexer) indexStateAccount(tx *sqlx.Tx, stateAccount StateAccountModel, stateID int64) error {
+	_, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5)
+								ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`,
+		stateID, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot)
+	return err
+}
+
 func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error {
 	_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5) 
 								   ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_key, cid, node_type) = ($2, $3, $5)`,

pkg/super_node/eth/models.go

@@ -28,6 +28,12 @@ type HeaderModel struct {
 	TotalDifficulty string `db:"td"`
 	NodeID          int64  `db:"node_id"`
 	Reward          string `db:"reward"`
+	StateRoot       string `db:"state_root"`
+	UncleRoot       string `db:"uncle_root"`
+	TxRoot          string `db:"tx_root"`
+	RctRoot         string `db:"receipt_root"`
+	Bloom           []byte `db:"bloom"`
+	Timestamp       uint64 `db:"timestamp"`
 }
 
 // UncleModel is the db model for eth.uncle_cids
@@ -93,3 +99,13 @@ type StorageNodeWithStateKeyModel struct {
 	NodeType   int    `db:"node_type"`
 	CID        string `db:"cid"`
 }
+
+// StateAccountModel is a db model for an eth state account (decoded value of state leaf node)
+type StateAccountModel struct {
+	ID          int64  `db:"id"`
+	StateID     int64  `db:"state_id"`
+	Balance     string `db:"balance"`
+	Nonce       uint64 `db:"nonce"`
+	CodeHash    []byte `db:"code_hash"`
+	StorageRoot string `db:"storage_root"`
+}

pkg/super_node/eth/publisher.go

@@ -19,8 +19,13 @@ package eth
 import (
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/statediff"
 
 	common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
@@ -82,6 +87,12 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
 		BlockHash:       ipldPayload.Block.Hash().String(),
 		TotalDifficulty: ipldPayload.TotalDifficulty.String(),
 		Reward:          reward.String(),
+		Bloom:           ipldPayload.Block.Bloom().Bytes(),
+		StateRoot:       ipldPayload.Block.Root().String(),
+		RctRoot:         ipldPayload.Block.ReceiptHash().String(),
+		TxRoot:          ipldPayload.Block.TxHash().String(),
+		UncleRoot:       ipldPayload.Block.UncleHash().String(),
+		Timestamp:       ipldPayload.Block.Time(),
 	}
 
 	// Process and publish uncles
@@ -113,7 +124,7 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
 	}
 
 	// Process and publish state leafs
-	stateNodeCids, err := pub.publishStateNodes(ipldPayload.StateNodes)
+	stateNodeCids, stateAccounts, err := pub.publishStateNodes(ipldPayload.StateNodes)
 	if err != nil {
 		return nil, err
 	}
@@ -132,6 +143,7 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
 		ReceiptCIDs:     receiptsCids,
 		StateNodeCIDs:   stateNodeCids,
 		StorageNodeCIDs: storageNodeCids,
+		StateAccounts:   stateAccounts,
 	}, nil
 }
 
@@ -193,16 +205,17 @@ func (pub *IPLDPublisher) publishReceipts(receipts []*ipld.EthReceipt, receiptTr
 	return rctCids, nil
 }
 
-func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) {
+func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, map[common.Hash]StateAccountModel, error) {
 	stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
+	stateAccounts := make(map[common.Hash]StateAccountModel)
 	for _, stateNode := range stateNodes {
 		node, err := ipld.FromStateTrieRLP(stateNode.Value)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		cid, err := pub.StatePutter.DagPut(node)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		stateNodeCids = append(stateNodeCids, StateNodeModel{
 			Path:     stateNode.Path,
@@ -210,8 +223,30 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
 			CID:      cid,
 			NodeType: ResolveFromNodeType(stateNode.Type),
 		})
+		// If we have a leaf, decode the account to extract additional metadata for indexing
+		if stateNode.Type == statediff.Leaf {
+			var i []interface{}
+			if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil {
+				return nil, nil, err
+			}
+			if len(i) != 2 {
+				return nil, nil, fmt.Errorf("IPLDPublisher expected state leaf node rlp to decode into two elements")
+			}
+			var account state.Account
+			if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
+				return nil, nil, err
+			}
+			// Map state account to the state path hash
+			statePathHash := crypto.Keccak256Hash(stateNode.Path)
+			stateAccounts[statePathHash] = StateAccountModel{
+				Balance:     account.Balance.String(),
+				Nonce:       account.Nonce,
+				CodeHash:    account.CodeHash,
+				StorageRoot: account.Root.String(),
+			}
+		}
 	}
-	return stateNodeCids, nil
+	return stateNodeCids, stateAccounts, nil
 }
 
 func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
@@ -227,7 +262,7 @@ func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]Tri
 			if err != nil {
 				return nil, err
 			}
-			// Map storage node cids to their path hashes
+			// Map storage node cids to the state path hash
 			storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{
 				Path:       storageNode.Path,
 				StorageKey: storageNode.LeafKey.Hex(),

pkg/super_node/eth/types.go

@@ -62,6 +62,7 @@ type CIDPayload struct {
 	TransactionCIDs []TxModel
 	ReceiptCIDs     map[common.Hash]ReceiptModel
 	StateNodeCIDs   []StateNodeModel
+	StateAccounts   map[common.Hash]StateAccountModel
 	StorageNodeCIDs map[common.Hash][]StorageNodeModel
 }