From ba5d6f9b9f54f9be87ce936f9f591639d2ff7bb7 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sat, 21 Mar 2020 19:50:12 -0500 Subject: [PATCH] state account support and add roots and additional metadata to header index --- db/migrations/00029_update_header_cids.sql | 37 ++++++ .../00030_create_eth_state_accouts_table.sql | 13 ++ db/schema.sql | 114 +++++++++++++++++- pkg/super_node/eth/indexer.go | 30 ++++- pkg/super_node/eth/models.go | 16 +++ pkg/super_node/eth/publisher.go | 47 +++++++- pkg/super_node/eth/types.go | 1 + 7 files changed, 242 insertions(+), 16 deletions(-) create mode 100644 db/migrations/00029_update_header_cids.sql create mode 100644 db/migrations/00030_create_eth_state_accouts_table.sql diff --git a/db/migrations/00029_update_header_cids.sql b/db/migrations/00029_update_header_cids.sql new file mode 100644 index 00000000..1c69c11c --- /dev/null +++ b/db/migrations/00029_update_header_cids.sql @@ -0,0 +1,37 @@ +-- +goose Up +ALTER TABLE eth.header_cids +ADD COLUMN state_root VARCHAR(66); + +ALTER TABLE eth.header_cids +ADD COLUMN tx_root VARCHAR(66); + +ALTER TABLE eth.header_cids +ADD COLUMN receipt_root VARCHAR(66); + +ALTER TABLE eth.header_cids +ADD COLUMN uncle_root VARCHAR(66); + +ALTER TABLE eth.header_cids +ADD COLUMN bloom BYTEA; + +ALTER TABLE eth.header_cids +ADD COLUMN timestamp NUMERIC; + +-- +goose Down +ALTER TABLE eth.header_cids +DROP COLUMN timestamp; + +ALTER TABLE eth.header_cids +DROP COLUMN bloom; + +ALTER TABLE eth.header_cids +DROP COLUMN uncle_root; + +ALTER TABLE eth.header_cids +DROP COLUMN receipt_root; + +ALTER TABLE eth.header_cids +DROP COLUMN tx_root; + +ALTER TABLE eth.header_cids +DROP COLUMN state_root; \ No newline at end of file diff --git a/db/migrations/00030_create_eth_state_accouts_table.sql b/db/migrations/00030_create_eth_state_accouts_table.sql new file mode 100644 index 00000000..322d948f --- /dev/null +++ b/db/migrations/00030_create_eth_state_accouts_table.sql @@ -0,0 +1,13 @@ +-- +goose Up +CREATE TABLE eth.state_accounts ( + id SERIAL PRIMARY KEY, + state_id INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE, + balance NUMERIC NOT NULL, + nonce INTEGER NOT NULL, + code_hash BYTEA NOT NULL, + storage_root VARCHAR(66) NOT NULL, + UNIQUE (state_id) +); + +-- +goose Down +DROP TABLE eth.state_accounts; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 332f6589..580ed65d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -71,6 +71,13 @@ CREATE TABLE btc.header_cids ( COMMENT ON TABLE btc.header_cids IS '@name BtcHeaderCids'; +-- +-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON COLUMN btc.header_cids.node_id IS '@name BtcNodeID'; + + -- -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: - -- @@ -254,7 +261,13 @@ CREATE TABLE eth.header_cids ( cid text NOT NULL, td numeric NOT NULL, node_id integer NOT NULL, - reward numeric NOT NULL + reward numeric NOT NULL, + state_root character varying(66), + tx_root character varying(66), + receipt_root character varying(66), + uncle_root character varying(66), + bloom bytea, + "timestamp" numeric ); @@ -265,6 +278,13 @@ CREATE TABLE eth.header_cids ( COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids'; +-- +-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID'; + + -- -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - -- @@ -359,6 +379,40 @@ CREATE SEQUENCE eth.receipt_cids_id_seq ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id; +-- +-- Name: state_accounts; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.state_accounts ( + id integer NOT NULL, + state_id integer NOT NULL, + balance numeric NOT NULL, + nonce integer NOT NULL, + code_hash bytea NOT NULL, + storage_root character varying(66) NOT NULL +); + + +-- +-- Name: state_accounts_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.state_accounts_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: state_accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.state_accounts_id_seq OWNED BY eth.state_accounts.id; + + -- -- Name: state_cids; Type: TABLE; Schema: eth; Owner: - -- @@ -369,7 +423,7 @@ CREATE TABLE eth.state_cids ( state_key character varying(66), cid text NOT NULL, state_path bytea, - node_type integer NOT NULL + node_type integer ); @@ -403,7 +457,7 @@ CREATE TABLE eth.storage_cids ( storage_key character varying(66), cid text NOT NULL, storage_path bytea, - node_type integer NOT NULL + node_type integer ); @@ -740,6 +794,20 @@ CREATE TABLE public.headers ( ); +-- +-- Name: TABLE headers; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.headers IS '@name EthHeaders'; + + +-- +-- Name: COLUMN headers.node_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.headers.node_id IS '@name EthNodeID'; + + -- -- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -777,7 +845,14 @@ CREATE TABLE public.nodes ( -- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: - -- -COMMENT ON TABLE public.nodes IS '@name NodeIDs'; +COMMENT ON TABLE public.nodes IS '@name NodeInfo'; + + +-- +-- Name: COLUMN nodes.node_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.nodes.node_id IS '@name ChainNodeID'; -- @@ -951,6 +1026,13 @@ ALTER TABLE ONLY eth.queue_data ALTER COLUMN id SET DEFAULT nextval('eth.queue_d ALTER TABLE ONLY eth.receipt_cids ALTER COLUMN id SET DEFAULT nextval('eth.receipt_cids_id_seq'::regclass); +-- +-- Name: state_accounts id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts ALTER COLUMN id SET DEFAULT nextval('eth.state_accounts_id_seq'::regclass); + + -- -- Name: state_cids id; Type: DEFAULT; Schema: eth; Owner: - -- @@ -1176,6 +1258,22 @@ ALTER TABLE ONLY eth.receipt_cids ADD CONSTRAINT receipt_cids_pkey PRIMARY KEY (id); +-- +-- Name: state_accounts state_accounts_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts + ADD CONSTRAINT state_accounts_pkey PRIMARY KEY (id); + + +-- +-- Name: state_accounts state_accounts_state_id_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts + ADD CONSTRAINT state_accounts_state_id_key UNIQUE (state_id); + + -- -- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: - -- @@ -1498,6 +1596,14 @@ ALTER TABLE ONLY eth.receipt_cids ADD CONSTRAINT receipt_cids_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +-- +-- Name: state_accounts state_accounts_state_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts + ADD CONSTRAINT state_accounts_state_id_fkey FOREIGN KEY (state_id) REFERENCES eth.state_cids(id) ON DELETE CASCADE; + + -- -- Name: state_cids state_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - -- diff --git a/pkg/super_node/eth/indexer.go b/pkg/super_node/eth/indexer.go index f9755ff3..7d7fdb9d 100644 --- a/pkg/super_node/eth/indexer.go +++ b/pkg/super_node/eth/indexer.go @@ -88,10 +88,12 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error { func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) { var headerID int64 - err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward) = ($3, $4, $5, $6, $7) + err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id`, - header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward).Scan(&headerID) + header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward, header.StateRoot, header.TxRoot, + header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp).Scan(&headerID) return headerID, err } @@ -138,15 +140,31 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, if err != nil { return err } - for _, storageCID := range payload.StorageNodeCIDs[crypto.Keccak256Hash(stateCID.Path)] { - if err := in.indexStorageCID(tx, storageCID, stateID); err != nil { - return err + // If we have a state leaf node, index the associated account and storage nodes + if stateCID.NodeType == 2 { + pathKey := crypto.Keccak256Hash(stateCID.Path) + for _, storageCID := range payload.StorageNodeCIDs[pathKey] { + if err := in.indexStorageCID(tx, storageCID, stateID); err != nil { + return err + } + } + if stateAccount, ok := payload.StateAccounts[pathKey]; ok { + if err := in.indexStateAccount(tx, stateAccount, stateID); err != nil { + return err + } } } } return nil } +func (in *CIDIndexer) indexStateAccount(tx *sqlx.Tx, stateAccount StateAccountModel, stateID int64) error { + _, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`, + stateID, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot) + return err +} + func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error { _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_key, cid, node_type) = ($2, $3, $5)`, diff --git a/pkg/super_node/eth/models.go b/pkg/super_node/eth/models.go index e0fbc72a..6a01c753 100644 --- a/pkg/super_node/eth/models.go +++ b/pkg/super_node/eth/models.go @@ -28,6 +28,12 @@ type HeaderModel struct { TotalDifficulty string `db:"td"` NodeID int64 `db:"node_id"` Reward string `db:"reward"` + StateRoot string `db:"state_root"` + UncleRoot string `db:"uncle_root"` + TxRoot string `db:"tx_root"` + RctRoot string `db:"receipt_root"` + Bloom []byte `db:"bloom"` + Timestamp uint64 `db:"timestamp"` } // UncleModel is the db model for eth.uncle_cids @@ -93,3 +99,13 @@ type StorageNodeWithStateKeyModel struct { NodeType int `db:"node_type"` CID string `db:"cid"` } + +// StateAccountModel is a db model for an eth state account (decoded value of state leaf node) +type StateAccountModel struct { + ID int64 `db:"id"` + StateID int64 `db:"state_id"` + Balance string `db:"balance"` + Nonce uint64 `db:"nonce"` + CodeHash []byte `db:"code_hash"` + StorageRoot string `db:"storage_root"` +} diff --git a/pkg/super_node/eth/publisher.go b/pkg/super_node/eth/publisher.go index b2c76a79..0d09c403 100644 --- a/pkg/super_node/eth/publisher.go +++ b/pkg/super_node/eth/publisher.go @@ -19,8 +19,13 @@ package eth import ( "fmt" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff" common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common" "github.com/vulcanize/vulcanizedb/pkg/ipfs" @@ -82,6 +87,12 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI BlockHash: ipldPayload.Block.Hash().String(), TotalDifficulty: ipldPayload.TotalDifficulty.String(), Reward: reward.String(), + Bloom: ipldPayload.Block.Bloom().Bytes(), + StateRoot: ipldPayload.Block.Root().String(), + RctRoot: ipldPayload.Block.ReceiptHash().String(), + TxRoot: ipldPayload.Block.TxHash().String(), + UncleRoot: ipldPayload.Block.UncleHash().String(), + Timestamp: ipldPayload.Block.Time(), } // Process and publish uncles @@ -113,7 +124,7 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI } // Process and publish state leafs - stateNodeCids, err := pub.publishStateNodes(ipldPayload.StateNodes) + stateNodeCids, stateAccounts, err := pub.publishStateNodes(ipldPayload.StateNodes) if err != nil { return nil, err } @@ -132,6 +143,7 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI ReceiptCIDs: receiptsCids, StateNodeCIDs: stateNodeCids, StorageNodeCIDs: storageNodeCids, + StateAccounts: stateAccounts, }, nil } @@ -193,16 +205,17 @@ func (pub *IPLDPublisher) publishReceipts(receipts []*ipld.EthReceipt, receiptTr return rctCids, nil } -func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) { +func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, map[common.Hash]StateAccountModel, error) { stateNodeCids := make([]StateNodeModel, 0, len(stateNodes)) + stateAccounts := make(map[common.Hash]StateAccountModel) for _, stateNode := range stateNodes { node, err := ipld.FromStateTrieRLP(stateNode.Value) if err != nil { - return nil, err + return nil, nil, err } cid, err := pub.StatePutter.DagPut(node) if err != nil { - return nil, err + return nil, nil, err } stateNodeCids = append(stateNodeCids, StateNodeModel{ Path: stateNode.Path, @@ -210,8 +223,30 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM CID: cid, NodeType: ResolveFromNodeType(stateNode.Type), }) + // If we have a leaf, decode the account to extract additional metadata for indexing + if stateNode.Type == statediff.Leaf { + var i []interface{} + if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil { + return nil, nil, err + } + if len(i) != 2 { + return nil, nil, fmt.Errorf("IPLDPublisher expected state leaf node rlp to decode into two elements") + } + var account state.Account + if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { + return nil, nil, err + } + // Map state account to the state path hash + statePathHash := crypto.Keccak256Hash(stateNode.Path) + stateAccounts[statePathHash] = StateAccountModel{ + Balance: account.Balance.String(), + Nonce: account.Nonce, + CodeHash: account.CodeHash, + StorageRoot: account.Root.String(), + } + } } - return stateNodeCids, nil + return stateNodeCids, stateAccounts, nil } func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) { @@ -227,7 +262,7 @@ func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]Tri if err != nil { return nil, err } - // Map storage node cids to their path hashes + // Map storage node cids to the state path hash storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{ Path: storageNode.Path, StorageKey: storageNode.LeafKey.Hex(), diff --git a/pkg/super_node/eth/types.go b/pkg/super_node/eth/types.go index 904cc7c3..03244814 100644 --- a/pkg/super_node/eth/types.go +++ b/pkg/super_node/eth/types.go @@ -62,6 +62,7 @@ type CIDPayload struct { TransactionCIDs []TxModel ReceiptCIDs map[common.Hash]ReceiptModel StateNodeCIDs []StateNodeModel + StateAccounts map[common.Hash]StateAccountModel StorageNodeCIDs map[common.Hash][]StorageNodeModel }