deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

state account support and add roots and additional metadata to header

Browse Source 
index
This commit is contained in:
Ian Norden 2020-03-21 19:50:12 -05:00
parent 57bdcca43c
commit ba5d6f9b9f
7 changed files with 242 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
db/migrations/00029_update_header_cids.sql Normal file
View File

@ -0,0 +1,37 @@
-- +goose Up
ALTER TABLE eth.header_cids
ADD COLUMN state_root VARCHAR(66);
ALTER TABLE eth.header_cids
ADD COLUMN tx_root VARCHAR(66);
ALTER TABLE eth.header_cids
ADD COLUMN receipt_root VARCHAR(66);
ALTER TABLE eth.header_cids
ADD COLUMN uncle_root VARCHAR(66);
ALTER TABLE eth.header_cids
ADD COLUMN bloom BYTEA;
ALTER TABLE eth.header_cids
ADD COLUMN timestamp NUMERIC;
-- +goose Down
ALTER TABLE eth.header_cids
DROP COLUMN timestamp;
ALTER TABLE eth.header_cids
DROP COLUMN bloom;
ALTER TABLE eth.header_cids
DROP COLUMN uncle_root;
ALTER TABLE eth.header_cids
DROP COLUMN receipt_root;
ALTER TABLE eth.header_cids
DROP COLUMN tx_root;
ALTER TABLE eth.header_cids
DROP COLUMN state_root;

13
db/migrations/00030_create_eth_state_accouts_table.sql Normal file
View File

@ -0,0 +1,13 @@
-- +goose Up
CREATE TABLE eth.state_accounts (
id SERIAL PRIMARY KEY,
state_id INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE,
balance NUMERIC NOT NULL,
nonce INTEGER NOT NULL,
code_hash BYTEA NOT NULL,
storage_root VARCHAR(66) NOT NULL,
UNIQUE (state_id)
);
-- +goose Down
DROP TABLE eth.state_accounts;

114
db/schema.sql
View File

@ -71,6 +71,13 @@ CREATE TABLE btc.header_cids (
COMMENT ON TABLE btc.header_cids IS '@name BtcHeaderCids';
--
-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: btc; Owner: -
--
COMMENT ON COLUMN btc.header_cids.node_id IS '@name BtcNodeID';
--
-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
--
@ -254,7 +261,13 @@ CREATE TABLE eth.header_cids (
cid text NOT NULL,
td numeric NOT NULL,
node_id integer NOT NULL,
reward numeric NOT NULL
reward numeric NOT NULL,
state_root character varying(66),
tx_root character varying(66),
receipt_root character varying(66),
uncle_root character varying(66),
bloom bytea,
"timestamp" numeric
);
@ -265,6 +278,13 @@ CREATE TABLE eth.header_cids (
COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids';
--
-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: eth; Owner: -
--
COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID';
--
-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
--
@ -359,6 +379,40 @@ CREATE SEQUENCE eth.receipt_cids_id_seq
ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id;
--
-- Name: state_accounts; Type: TABLE; Schema: eth; Owner: -
--
CREATE TABLE eth.state_accounts (
id integer NOT NULL,
state_id integer NOT NULL,
balance numeric NOT NULL,
nonce integer NOT NULL,
code_hash bytea NOT NULL,
storage_root character varying(66) NOT NULL
);
--
-- Name: state_accounts_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
--
CREATE SEQUENCE eth.state_accounts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: state_accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
--
ALTER SEQUENCE eth.state_accounts_id_seq OWNED BY eth.state_accounts.id;
--
-- Name: state_cids; Type: TABLE; Schema: eth; Owner: -
--
@ -369,7 +423,7 @@ CREATE TABLE eth.state_cids (
state_key character varying(66),
cid text NOT NULL,
state_path bytea,
node_type integer NOT NULL
node_type integer
);
@ -403,7 +457,7 @@ CREATE TABLE eth.storage_cids (
storage_key character varying(66),
cid text NOT NULL,
storage_path bytea,
node_type integer NOT NULL
node_type integer
);
@ -740,6 +794,20 @@ CREATE TABLE public.headers (
);
--
-- Name: TABLE headers; Type: COMMENT; Schema: public; Owner: -
--
COMMENT ON TABLE public.headers IS '@name EthHeaders';
--
-- Name: COLUMN headers.node_id; Type: COMMENT; Schema: public; Owner: -
--
COMMENT ON COLUMN public.headers.node_id IS '@name EthNodeID';
--
-- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@ -777,7 +845,14 @@ CREATE TABLE public.nodes (
-- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
--
COMMENT ON TABLE public.nodes IS '@name NodeIDs';
COMMENT ON TABLE public.nodes IS '@name NodeInfo';
--
-- Name: COLUMN nodes.node_id; Type: COMMENT; Schema: public; Owner: -
--
COMMENT ON COLUMN public.nodes.node_id IS '@name ChainNodeID';
--
@ -951,6 +1026,13 @@ ALTER TABLE ONLY eth.queue_data ALTER COLUMN id SET DEFAULT nextval('eth.queue_d
ALTER TABLE ONLY eth.receipt_cids ALTER COLUMN id SET DEFAULT nextval('eth.receipt_cids_id_seq'::regclass);
--
-- Name: state_accounts id; Type: DEFAULT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.state_accounts ALTER COLUMN id SET DEFAULT nextval('eth.state_accounts_id_seq'::regclass);
--
-- Name: state_cids id; Type: DEFAULT; Schema: eth; Owner: -
--
@ -1176,6 +1258,22 @@ ALTER TABLE ONLY eth.receipt_cids
ADD CONSTRAINT receipt_cids_pkey PRIMARY KEY (id);
--
-- Name: state_accounts state_accounts_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.state_accounts
ADD CONSTRAINT state_accounts_pkey PRIMARY KEY (id);
--
-- Name: state_accounts state_accounts_state_id_key; Type: CONSTRAINT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.state_accounts
ADD CONSTRAINT state_accounts_state_id_key UNIQUE (state_id);
--
-- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
--
@ -1498,6 +1596,14 @@ ALTER TABLE ONLY eth.receipt_cids
ADD CONSTRAINT receipt_cids_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
--
-- Name: state_accounts state_accounts_state_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.state_accounts
ADD CONSTRAINT state_accounts_state_id_fkey FOREIGN KEY (state_id) REFERENCES eth.state_cids(id) ON DELETE CASCADE;
--
-- Name: state_cids state_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
--

30
pkg/super_node/eth/indexer.go
View File

@ -88,10 +88,12 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward) = ($3, $4, $5, $6, $7)
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
RETURNING id`,
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward).Scan(&headerID)
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward, header.StateRoot, header.TxRoot,
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp).Scan(&headerID)
return headerID, err
}
@ -138,15 +140,31 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
if err != nil {
return err
}
for _, storageCID := range payload.StorageNodeCIDs[crypto.Keccak256Hash(stateCID.Path)] {
if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
return err
// If we have a state leaf node, index the associated account and storage nodes
if stateCID.NodeType == 2 {
pathKey := crypto.Keccak256Hash(stateCID.Path)
for _, storageCID := range payload.StorageNodeCIDs[pathKey] {
if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
return err
}
}
if stateAccount, ok := payload.StateAccounts[pathKey]; ok {
if err := in.indexStateAccount(tx, stateAccount, stateID); err != nil {
return err
}
}
}
}
return nil
}
func (in *CIDIndexer) indexStateAccount(tx *sqlx.Tx, stateAccount StateAccountModel, stateID int64) error {
_, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`,
stateID, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot)
return err
}
func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error {
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_key, cid, node_type) = ($2, $3, $5)`,

16
pkg/super_node/eth/models.go
View File

@ -28,6 +28,12 @@ type HeaderModel struct {
TotalDifficulty string `db:"td"`
NodeID int64 `db:"node_id"`
Reward string `db:"reward"`
StateRoot string `db:"state_root"`
UncleRoot string `db:"uncle_root"`
TxRoot string `db:"tx_root"`
RctRoot string `db:"receipt_root"`
Bloom []byte `db:"bloom"`
Timestamp uint64 `db:"timestamp"`
}
// UncleModel is the db model for eth.uncle_cids
@ -93,3 +99,13 @@ type StorageNodeWithStateKeyModel struct {
NodeType int `db:"node_type"`
CID string `db:"cid"`
}
// StateAccountModel is a db model for an eth state account (decoded value of state leaf node)
type StateAccountModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Balance string `db:"balance"`
Nonce uint64 `db:"nonce"`
CodeHash []byte `db:"code_hash"`
StorageRoot string `db:"storage_root"`
}

47
pkg/super_node/eth/publisher.go
View File

@ -19,8 +19,13 @@ package eth
import (
"fmt"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/statediff"
common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
@ -82,6 +87,12 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
BlockHash: ipldPayload.Block.Hash().String(),
TotalDifficulty: ipldPayload.TotalDifficulty.String(),
Reward: reward.String(),
Bloom: ipldPayload.Block.Bloom().Bytes(),
StateRoot: ipldPayload.Block.Root().String(),
RctRoot: ipldPayload.Block.ReceiptHash().String(),
TxRoot: ipldPayload.Block.TxHash().String(),
UncleRoot: ipldPayload.Block.UncleHash().String(),
Timestamp: ipldPayload.Block.Time(),
}
// Process and publish uncles
@ -113,7 +124,7 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
}
// Process and publish state leafs
stateNodeCids, err := pub.publishStateNodes(ipldPayload.StateNodes)
stateNodeCids, stateAccounts, err := pub.publishStateNodes(ipldPayload.StateNodes)
if err != nil {
return nil, err
}
@ -132,6 +143,7 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
ReceiptCIDs: receiptsCids,
StateNodeCIDs: stateNodeCids,
StorageNodeCIDs: storageNodeCids,
StateAccounts: stateAccounts,
}, nil
}
@ -193,16 +205,17 @@ func (pub *IPLDPublisher) publishReceipts(receipts []*ipld.EthReceipt, receiptTr
return rctCids, nil
}
func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) {
func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, map[common.Hash]StateAccountModel, error) {
stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
stateAccounts := make(map[common.Hash]StateAccountModel)
for _, stateNode := range stateNodes {
node, err := ipld.FromStateTrieRLP(stateNode.Value)
if err != nil {
return nil, err
return nil, nil, err
}
cid, err := pub.StatePutter.DagPut(node)
if err != nil {
return nil, err
return nil, nil, err
}
stateNodeCids = append(stateNodeCids, StateNodeModel{
Path: stateNode.Path,
@ -210,8 +223,30 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
CID: cid,
NodeType: ResolveFromNodeType(stateNode.Type),
})
// If we have a leaf, decode the account to extract additional metadata for indexing
if stateNode.Type == statediff.Leaf {
var i []interface{}
if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil {
return nil, nil, err
}
if len(i) != 2 {
return nil, nil, fmt.Errorf("IPLDPublisher expected state leaf node rlp to decode into two elements")
}
var account state.Account
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
return nil, nil, err
}
// Map state account to the state path hash
statePathHash := crypto.Keccak256Hash(stateNode.Path)
stateAccounts[statePathHash] = StateAccountModel{
Balance: account.Balance.String(),
Nonce: account.Nonce,
CodeHash: account.CodeHash,
StorageRoot: account.Root.String(),
}
}
}
return stateNodeCids, nil
return stateNodeCids, stateAccounts, nil
}
func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
@ -227,7 +262,7 @@ func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]Tri
if err != nil {
return nil, err
}
// Map storage node cids to their path hashes
// Map storage node cids to the state path hash
storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{
Path: storageNode.Path,
StorageKey: storageNode.LeafKey.Hex(),

1
pkg/super_node/eth/types.go
View File

@ -62,6 +62,7 @@ type CIDPayload struct {
TransactionCIDs []TxModel
ReceiptCIDs map[common.Hash]ReceiptModel
StateNodeCIDs []StateNodeModel
StateAccounts map[common.Hash]StateAccountModel
StorageNodeCIDs map[common.Hash][]StorageNodeModel
}