adhjustments to work with statediffing geth v1.10-alpha.2
This commit is contained in:
parent
be875c0100
commit
e5c5422edc
@ -139,12 +139,12 @@ func streamEthSubscription() {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("Account for key %s, and root %s, with balance %s\n",
|
||||
stateNode.StateTrieKey.Hex(), acct.Root.Hex(), acct.Balance.String())
|
||||
stateNode.StateLeafKey.Hex(), acct.Root.Hex(), acct.Balance.String())
|
||||
fmt.Printf("state account: %+v\n", acct)
|
||||
}
|
||||
for _, storageNode := range ethData.StorageNodes {
|
||||
fmt.Printf("Storage for state key %s ", storageNode.StateTrieKey.Hex())
|
||||
fmt.Printf("with storage key %s\n", storageNode.StorageTrieKey.Hex())
|
||||
fmt.Printf("Storage for state key %s ", storageNode.StateLeafKey.Hex())
|
||||
fmt.Printf("with storage key %s\n", storageNode.StorageLeafKey.Hex())
|
||||
var i []interface{}
|
||||
err := rlp.DecodeBytes(storageNode.IPLD.Data, &i)
|
||||
if err != nil {
|
||||
@ -158,7 +158,7 @@ func streamEthSubscription() {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
|
||||
storageNode.StorageTrieKey.Hex(), common.BytesToHash(valueBytes).Hex())
|
||||
storageNode.StorageLeafKey.Hex(), common.BytesToHash(valueBytes).Hex())
|
||||
}
|
||||
}
|
||||
case err = <-sub.Err():
|
||||
|
37
db/migrations/00027_update_state_cids.sql
Normal file
37
db/migrations/00027_update_state_cids.sql
Normal file
@ -0,0 +1,37 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE eth.state_cids
|
||||
ADD COLUMN state_path BYTEA;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
DROP COLUMN leaf;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
ADD COLUMN node_type INTEGER NOT NULL;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
ALTER COLUMN state_key DROP NOT NULL;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
DROP CONSTRAINT state_cids_header_id_state_key_key;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path);
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE eth.state_cids
|
||||
ADD CONSTRAINT state_cids_header_id_state_key_key UNIQUE (header_id, state_key);
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
DROP CONSTRAINT state_cids_header_id_state_path_key;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
ALTER COLUMN state_key SET NOT NULL;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
DROP COLUMN node_type;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
ADD COLUMN leaf BOOLEAN NOT NULL;
|
||||
|
||||
ALTER TABLE eth.state_cids
|
||||
DROP COLUMN state_path;
|
37
db/migrations/00028_update_storage_cids.sql
Normal file
37
db/migrations/00028_update_storage_cids.sql
Normal file
@ -0,0 +1,37 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE eth.storage_cids
|
||||
ADD COLUMN storage_path BYTEA;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
DROP COLUMN leaf;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
ADD COLUMN node_type INTEGER NOT NULL;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
ALTER COLUMN storage_key DROP NOT NULL;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
DROP CONSTRAINT storage_cids_state_id_storage_key_key;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path);
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE eth.storage_cids
|
||||
DROP CONSTRAINT storage_cids_state_id_storage_path_key;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
ADD CONSTRAINT storage_cids_state_id_storage_key_key UNIQUE (state_id, storage_key);
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
ALTER COLUMN storage_key SET NOT NULL;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
DROP COLUMN node_type;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
ADD COLUMN leaf BOOLEAN NOT NULL;
|
||||
|
||||
ALTER TABLE eth.storage_cids
|
||||
DROP COLUMN storage_path;
|
@ -3,7 +3,7 @@
|
||||
--
|
||||
|
||||
-- Dumped from database version 10.10
|
||||
-- Dumped by pg_dump version 12.1
|
||||
-- Dumped by pg_dump version 10.10
|
||||
|
||||
SET statement_timeout = 0;
|
||||
SET lock_timeout = 0;
|
||||
@ -30,8 +30,24 @@ CREATE SCHEMA btc;
|
||||
CREATE SCHEMA eth;
|
||||
|
||||
|
||||
--
|
||||
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
|
||||
|
||||
|
||||
--
|
||||
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
|
||||
|
||||
|
||||
SET default_tablespace = '';
|
||||
|
||||
SET default_with_oids = false;
|
||||
|
||||
--
|
||||
-- Name: header_cids; Type: TABLE; Schema: btc; Owner: -
|
||||
--
|
||||
@ -48,6 +64,13 @@ CREATE TABLE btc.header_cids (
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: TABLE header_cids; Type: COMMENT; Schema: btc; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON TABLE btc.header_cids IS '@name BtcHeaderCids';
|
||||
|
||||
|
||||
--
|
||||
-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
|
||||
--
|
||||
@ -79,6 +102,13 @@ CREATE TABLE btc.queue_data (
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: TABLE queue_data; Type: COMMENT; Schema: btc; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON TABLE btc.queue_data IS '@name BtcQueueData';
|
||||
|
||||
|
||||
--
|
||||
-- Name: queue_data_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
|
||||
--
|
||||
@ -114,6 +144,13 @@ CREATE TABLE btc.transaction_cids (
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: TABLE transaction_cids; Type: COMMENT; Schema: btc; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON TABLE btc.transaction_cids IS '@name BtcTransactionCids';
|
||||
|
||||
|
||||
--
|
||||
-- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
|
||||
--
|
||||
@ -221,6 +258,13 @@ CREATE TABLE eth.header_cids (
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: TABLE header_cids; Type: COMMENT; Schema: eth; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids';
|
||||
|
||||
|
||||
--
|
||||
-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
|
||||
--
|
||||
@ -252,6 +296,13 @@ CREATE TABLE eth.queue_data (
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: TABLE queue_data; Type: COMMENT; Schema: eth; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON TABLE eth.queue_data IS '@name EthQueueData';
|
||||
|
||||
|
||||
--
|
||||
-- Name: queue_data_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
|
||||
--
|
||||
@ -315,9 +366,10 @@ ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id;
|
||||
CREATE TABLE eth.state_cids (
|
||||
id integer NOT NULL,
|
||||
header_id integer NOT NULL,
|
||||
state_key character varying(66) NOT NULL,
|
||||
leaf boolean NOT NULL,
|
||||
cid text NOT NULL
|
||||
state_key character varying(66),
|
||||
cid text NOT NULL,
|
||||
state_path bytea,
|
||||
node_type integer NOT NULL
|
||||
);
|
||||
|
||||
|
||||
@ -348,9 +400,10 @@ ALTER SEQUENCE eth.state_cids_id_seq OWNED BY eth.state_cids.id;
|
||||
CREATE TABLE eth.storage_cids (
|
||||
id integer NOT NULL,
|
||||
state_id integer NOT NULL,
|
||||
storage_key character varying(66) NOT NULL,
|
||||
leaf boolean NOT NULL,
|
||||
cid text NOT NULL
|
||||
storage_key character varying(66),
|
||||
cid text NOT NULL,
|
||||
storage_path bytea,
|
||||
node_type integer NOT NULL
|
||||
);
|
||||
|
||||
|
||||
@ -389,6 +442,13 @@ CREATE TABLE eth.transaction_cids (
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: TABLE transaction_cids; Type: COMMENT; Schema: eth; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON TABLE eth.transaction_cids IS '@name EthTransactionCids';
|
||||
|
||||
|
||||
--
|
||||
-- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
|
||||
--
|
||||
@ -713,6 +773,13 @@ CREATE TABLE public.nodes (
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON TABLE public.nodes IS '@name NodeIDs';
|
||||
|
||||
|
||||
--
|
||||
-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
||||
--
|
||||
@ -1110,11 +1177,11 @@ ALTER TABLE ONLY eth.receipt_cids
|
||||
|
||||
|
||||
--
|
||||
-- Name: state_cids state_cids_header_id_state_key_key; Type: CONSTRAINT; Schema: eth; Owner: -
|
||||
-- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY eth.state_cids
|
||||
ADD CONSTRAINT state_cids_header_id_state_key_key UNIQUE (header_id, state_key);
|
||||
ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path);
|
||||
|
||||
|
||||
--
|
||||
@ -1134,11 +1201,11 @@ ALTER TABLE ONLY eth.storage_cids
|
||||
|
||||
|
||||
--
|
||||
-- Name: storage_cids storage_cids_state_id_storage_key_key; Type: CONSTRAINT; Schema: eth; Owner: -
|
||||
-- Name: storage_cids storage_cids_state_id_storage_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY eth.storage_cids
|
||||
ADD CONSTRAINT storage_cids_state_id_storage_key_key UNIQUE (state_id, storage_key);
|
||||
ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path);
|
||||
|
||||
|
||||
--
|
||||
|
2
go.mod
2
go.mod
@ -97,4 +97,4 @@ replace github.com/ipfs/go-ipfs v0.4.22 => github.com/vulcanize/go-ipfs v0.4.22-
|
||||
|
||||
replace github.com/ipfs/go-ipfs-config v0.0.3 => github.com/vulcanize/go-ipfs-config v0.0.8-alpha
|
||||
|
||||
replace github.com/ethereum/go-ethereum v1.9.1 => github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8
|
||||
replace github.com/ethereum/go-ethereum v1.9.1 => github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290
|
||||
|
7
go.sum
7
go.sum
@ -37,6 +37,7 @@ github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:
|
||||
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0=
|
||||
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d h1:QgeLLoPD3kRVmeu/1al9iIpIANMi9O1zXFm8BnYGCJg=
|
||||
@ -105,7 +106,9 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
|
||||
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
|
||||
@ -134,6 +137,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
|
||||
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
@ -327,6 +331,7 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj
|
||||
github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
|
||||
github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmoiron/sqlx v0.0.0-20190426154859-38398a30ed85 h1:+LZtdhpMITOXE+MztQPPcwUl+eqYjwlXXLHrd0yWlxw=
|
||||
github.com/jmoiron/sqlx v0.0.0-20190426154859-38398a30ed85/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
@ -723,6 +728,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8 h1:BHt0OW0rTgndFjSju7brF3dPceXWQuEV0IdtY8BjjT8=
|
||||
github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo=
|
||||
github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290 h1:uMWt+x6JhVT7GyL983weZSxv1zDBxvGlI9HNkcTnUeg=
|
||||
github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
|
||||
github.com/vulcanize/go-ipfs v0.4.22-alpha h1:W+6njT14KWllMhABRFtPndqHw8SHCt5SqD4YX528kxM=
|
||||
github.com/vulcanize/go-ipfs v0.4.22-alpha/go.mod h1:uaekWWeoaA0A9Dv1LObOKCSh9kIzTpZ5RbKW4g5CQHE=
|
||||
github.com/vulcanize/go-ipfs-config v0.0.8-alpha h1:peaFvbEcPShF6ymOd8flqKkFz4YfcrNr/UOO7FmbWoQ=
|
||||
|
@ -65,11 +65,11 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
|
||||
accounts := utils.GetAccountsFromDiff(*stateDiff)
|
||||
logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
|
||||
for _, account := range accounts {
|
||||
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.Key).Hex()))
|
||||
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex()))
|
||||
for _, storage := range account.Storage {
|
||||
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
|
||||
if formatErr != nil {
|
||||
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.Key), "from account with key: ", common.BytesToHash(account.Key))
|
||||
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey))
|
||||
errs <- formatErr
|
||||
continue
|
||||
}
|
||||
|
@ -127,11 +127,11 @@ func (bf *backFiller) backFillRange(blockHeights []uint64, diffChan chan utils.S
|
||||
}
|
||||
accounts := utils.GetAccountsFromDiff(*stateDiff)
|
||||
for _, account := range accounts {
|
||||
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.Key).Hex()))
|
||||
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex()))
|
||||
for _, storage := range account.Storage {
|
||||
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
|
||||
if formatErr != nil {
|
||||
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.Key), "from account with key: ", common.BytesToHash(account.Key))
|
||||
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey))
|
||||
errChan <- formatErr
|
||||
continue
|
||||
}
|
||||
|
@ -59,16 +59,16 @@ func FromParityCsvRow(csvRow []string) (StorageDiffInput, error) {
|
||||
|
||||
func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiffInput, error) {
|
||||
var decodedValue []byte
|
||||
err := rlp.DecodeBytes(storage.Value, &decodedValue)
|
||||
err := rlp.DecodeBytes(storage.NodeValue, &decodedValue)
|
||||
if err != nil {
|
||||
return StorageDiffInput{}, err
|
||||
}
|
||||
|
||||
return StorageDiffInput{
|
||||
HashedAddress: common.BytesToHash(account.Key),
|
||||
HashedAddress: common.BytesToHash(account.LeafKey),
|
||||
BlockHash: stateDiff.BlockHash,
|
||||
BlockHeight: int(stateDiff.BlockNumber.Int64()),
|
||||
StorageKey: common.BytesToHash(storage.Key),
|
||||
StorageKey: common.BytesToHash(storage.LeafKey),
|
||||
StorageValue: common.BytesToHash(decodedValue),
|
||||
}, nil
|
||||
}
|
||||
|
@ -67,7 +67,7 @@ var _ = Describe("Storage row parsing", func() {
|
||||
|
||||
Describe("FromGethStateDiff", func() {
|
||||
var (
|
||||
accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
|
||||
accountDiff = statediff.AccountDiff{LeafKey: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
|
||||
stateDiff = &statediff.StateDiff{
|
||||
BlockNumber: big.NewInt(rand.Int63()),
|
||||
BlockHash: fakes.FakeHash,
|
||||
@ -80,19 +80,20 @@ var _ = Describe("Storage row parsing", func() {
|
||||
Expect(encodeErr).NotTo(HaveOccurred())
|
||||
|
||||
storageDiff := statediff.StorageDiff{
|
||||
Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
|
||||
Value: storageValueRlp,
|
||||
LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
|
||||
NodeValue: storageValueRlp,
|
||||
NodeType: statediff.Leaf,
|
||||
}
|
||||
|
||||
result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
expectedAddress := common.BytesToHash(accountDiff.Key)
|
||||
expectedAddress := common.BytesToHash(accountDiff.LeafKey)
|
||||
Expect(result.HashedAddress).To(Equal(expectedAddress))
|
||||
Expect(result.BlockHash).To(Equal(fakes.FakeHash))
|
||||
expectedBlockHeight := int(stateDiff.BlockNumber.Int64())
|
||||
Expect(result.BlockHeight).To(Equal(expectedBlockHeight))
|
||||
expectedStorageKey := common.BytesToHash(storageDiff.Key)
|
||||
expectedStorageKey := common.BytesToHash(storageDiff.LeafKey)
|
||||
Expect(result.StorageKey).To(Equal(expectedStorageKey))
|
||||
expectedStorageValue := common.BytesToHash(storageValueBytes)
|
||||
Expect(result.StorageValue).To(Equal(expectedStorageValue))
|
||||
@ -104,8 +105,9 @@ var _ = Describe("Storage row parsing", func() {
|
||||
Expect(encodeErr).NotTo(HaveOccurred())
|
||||
|
||||
storageDiff := statediff.StorageDiff{
|
||||
Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
|
||||
Value: storageValueRlp,
|
||||
LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
|
||||
NodeValue: storageValueRlp,
|
||||
NodeType: statediff.Leaf,
|
||||
}
|
||||
|
||||
result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
|
||||
|
@ -41,22 +41,24 @@ var (
|
||||
SmallStorageValue = common.Hex2Bytes("03")
|
||||
SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue)
|
||||
storageWithSmallValue = []statediff.StorageDiff{{
|
||||
Key: StorageKey,
|
||||
Value: SmallStorageValueRlp,
|
||||
LeafKey: StorageKey,
|
||||
NodeValue: SmallStorageValueRlp,
|
||||
NodeType: statediff.Leaf,
|
||||
Path: StoragePath,
|
||||
Proof: [][]byte{},
|
||||
}}
|
||||
LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000")
|
||||
LargeStorageValueRlp, _ = rlp.EncodeToBytes(LargeStorageValue)
|
||||
storageWithLargeValue = []statediff.StorageDiff{{
|
||||
Key: StorageKey,
|
||||
Value: LargeStorageValueRlp,
|
||||
LeafKey: StorageKey,
|
||||
NodeValue: LargeStorageValueRlp,
|
||||
Path: StoragePath,
|
||||
Proof: [][]byte{},
|
||||
NodeType: statediff.Leaf,
|
||||
}}
|
||||
StorageWithBadValue = statediff.StorageDiff{
|
||||
Key: StorageKey,
|
||||
Value: []byte{0, 1, 2},
|
||||
LeafKey: StorageKey,
|
||||
NodeValue: []byte{0, 1, 2},
|
||||
NodeType: statediff.Leaf,
|
||||
Path: StoragePath,
|
||||
// this storage value will fail to be decoded as an RLP with the following error message:
|
||||
// "input contains more than one value"
|
||||
}
|
||||
@ -74,26 +76,26 @@ var (
|
||||
valueBytes, _ = rlp.EncodeToBytes(testAccount)
|
||||
CreatedAccountDiffs = []statediff.AccountDiff{
|
||||
{
|
||||
Key: ContractLeafKey.Bytes(),
|
||||
Value: valueBytes,
|
||||
LeafKey: ContractLeafKey.Bytes(),
|
||||
NodeValue: valueBytes,
|
||||
Storage: storageWithSmallValue,
|
||||
},
|
||||
}
|
||||
|
||||
UpdatedAccountDiffs = []statediff.AccountDiff{{
|
||||
Key: AnotherContractLeafKey.Bytes(),
|
||||
Value: valueBytes,
|
||||
LeafKey: AnotherContractLeafKey.Bytes(),
|
||||
NodeValue: valueBytes,
|
||||
Storage: storageWithLargeValue,
|
||||
}}
|
||||
UpdatedAccountDiffs2 = []statediff.AccountDiff{{
|
||||
Key: AnotherContractLeafKey.Bytes(),
|
||||
Value: valueBytes,
|
||||
LeafKey: AnotherContractLeafKey.Bytes(),
|
||||
NodeValue: valueBytes,
|
||||
Storage: storageWithSmallValue,
|
||||
}}
|
||||
|
||||
DeletedAccountDiffs = []statediff.AccountDiff{{
|
||||
Key: AnotherContractLeafKey.Bytes(),
|
||||
Value: valueBytes,
|
||||
LeafKey: AnotherContractLeafKey.Bytes(),
|
||||
NodeValue: valueBytes,
|
||||
Storage: storageWithSmallValue,
|
||||
}}
|
||||
|
||||
|
@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
@ -123,47 +124,53 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
|
||||
return nil, err
|
||||
}
|
||||
for _, createdAccount := range stateDiff.CreatedAccounts {
|
||||
hashKey := common.BytesToHash(createdAccount.Key)
|
||||
statePathHash := crypto.Keccak256Hash(createdAccount.Path)
|
||||
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
|
||||
Key: hashKey,
|
||||
Value: createdAccount.Value,
|
||||
Leaf: createdAccount.Leaf,
|
||||
Path: createdAccount.Path,
|
||||
Value: createdAccount.NodeValue,
|
||||
Type: createdAccount.NodeType,
|
||||
LeafKey: common.BytesToHash(createdAccount.LeafKey),
|
||||
})
|
||||
for _, storageDiff := range createdAccount.Storage {
|
||||
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
|
||||
Key: common.BytesToHash(storageDiff.Key),
|
||||
Value: storageDiff.Value,
|
||||
Leaf: storageDiff.Leaf,
|
||||
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
|
||||
Path: storageDiff.Path,
|
||||
Value: storageDiff.NodeValue,
|
||||
Type: storageDiff.NodeType,
|
||||
LeafKey: common.BytesToHash(storageDiff.LeafKey),
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, deletedAccount := range stateDiff.DeletedAccounts {
|
||||
hashKey := common.BytesToHash(deletedAccount.Key)
|
||||
statePathHash := crypto.Keccak256Hash(deletedAccount.Path)
|
||||
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
|
||||
Key: hashKey,
|
||||
Value: deletedAccount.Value,
|
||||
Leaf: deletedAccount.Leaf,
|
||||
Path: deletedAccount.Path,
|
||||
Value: deletedAccount.NodeValue,
|
||||
Type: deletedAccount.NodeType,
|
||||
LeafKey: common.BytesToHash(deletedAccount.LeafKey),
|
||||
})
|
||||
for _, storageDiff := range deletedAccount.Storage {
|
||||
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
|
||||
Key: common.BytesToHash(storageDiff.Key),
|
||||
Value: storageDiff.Value,
|
||||
Leaf: storageDiff.Leaf,
|
||||
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
|
||||
Path: storageDiff.Path,
|
||||
Value: storageDiff.NodeValue,
|
||||
Type: storageDiff.NodeType,
|
||||
LeafKey: common.BytesToHash(storageDiff.LeafKey),
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, updatedAccount := range stateDiff.UpdatedAccounts {
|
||||
hashKey := common.BytesToHash(updatedAccount.Key)
|
||||
statePathHash := crypto.Keccak256Hash(updatedAccount.Path)
|
||||
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
|
||||
Key: hashKey,
|
||||
Value: updatedAccount.Value,
|
||||
Leaf: updatedAccount.Leaf,
|
||||
Path: updatedAccount.Path,
|
||||
Value: updatedAccount.NodeValue,
|
||||
Type: updatedAccount.NodeType,
|
||||
LeafKey: common.BytesToHash(updatedAccount.LeafKey),
|
||||
})
|
||||
for _, storageDiff := range updatedAccount.Storage {
|
||||
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
|
||||
Key: common.BytesToHash(storageDiff.Key),
|
||||
Value: storageDiff.Value,
|
||||
Leaf: storageDiff.Leaf,
|
||||
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
|
||||
Path: storageDiff.Path,
|
||||
Value: storageDiff.NodeValue,
|
||||
Type: storageDiff.NodeType,
|
||||
LeafKey: common.BytesToHash(storageDiff.LeafKey),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -20,6 +20,8 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
@ -66,10 +68,7 @@ func (s *ResponseFilterer) Filter(filter shared.SubscriptionSettings, payload sh
|
||||
if err := s.filerReceipts(ethFilters.ReceiptFilter, response, ethPayload, filterTxs); err != nil {
|
||||
return IPLDs{}, err
|
||||
}
|
||||
if err := s.filterState(ethFilters.StateFilter, response, ethPayload); err != nil {
|
||||
return IPLDs{}, err
|
||||
}
|
||||
if err := s.filterStorage(ethFilters.StorageFilter, response, ethPayload); err != nil {
|
||||
if err := s.filterStateAndStorage(ethFilters.StateFilter, ethFilters.StorageFilter, response, ethPayload); err != nil {
|
||||
return IPLDs{}, err
|
||||
}
|
||||
response.BlockNumber = ethPayload.Block.Number()
|
||||
@ -255,27 +254,56 @@ func slicesShareString(slice1, slice2 []string) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *ResponseFilterer) filterState(stateFilter StateFilter, response *IPLDs, payload ConvertedPayload) error {
|
||||
if !stateFilter.Off {
|
||||
// filterStateAndStorage filters state and storage nodes into the response according to the provided filters
|
||||
func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
|
||||
response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
|
||||
keyFilters := make([]common.Hash, len(stateFilter.Addresses))
|
||||
response.StorageNodes = make([]StorageNode, 0)
|
||||
stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
|
||||
for i, addr := range stateFilter.Addresses {
|
||||
keyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
|
||||
stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
|
||||
}
|
||||
storageAddressFilters := make([]common.Hash, len(storageFilter.Addresses))
|
||||
for i, addr := range storageFilter.Addresses {
|
||||
storageAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
|
||||
}
|
||||
storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
|
||||
for i, store := range storageFilter.StorageKeys {
|
||||
storageKeyFilters[i] = common.HexToHash(store)
|
||||
}
|
||||
for _, stateNode := range payload.StateNodes {
|
||||
if checkNodeKeys(keyFilters, stateNode.Key) {
|
||||
if stateNode.Leaf || stateFilter.IntermediateNodes {
|
||||
if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
|
||||
if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes {
|
||||
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.StateNodes = append(response.StateNodes, StateNode{
|
||||
StateTrieKey: stateNode.Key,
|
||||
StateLeafKey: stateNode.LeafKey,
|
||||
Path: stateNode.Path,
|
||||
IPLD: ipfs.BlockModel{
|
||||
Data: stateNode.Value,
|
||||
CID: cid.String(),
|
||||
},
|
||||
Leaf: stateNode.Leaf,
|
||||
Type: stateNode.Type,
|
||||
})
|
||||
}
|
||||
}
|
||||
if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
|
||||
for _, storageNode := range payload.StorageNodes[crypto.Keccak256Hash(stateNode.Path)] {
|
||||
if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
|
||||
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.StorageNodes = append(response.StorageNodes, StorageNode{
|
||||
StateLeafKey: stateNode.LeafKey,
|
||||
StorageLeafKey: storageNode.LeafKey,
|
||||
IPLD: ipfs.BlockModel{
|
||||
Data: storageNode.Value,
|
||||
CID: cid.String(),
|
||||
},
|
||||
Type: storageNode.Type,
|
||||
Path: storageNode.Path,
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -296,39 +324,3 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *ResponseFilterer) filterStorage(storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
|
||||
if !storageFilter.Off {
|
||||
response.StorageNodes = make([]StorageNode, 0)
|
||||
stateKeyFilters := make([]common.Hash, len(storageFilter.Addresses))
|
||||
for i, addr := range storageFilter.Addresses {
|
||||
stateKeyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
|
||||
}
|
||||
storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
|
||||
for i, store := range storageFilter.StorageKeys {
|
||||
storageKeyFilters[i] = common.HexToHash(store)
|
||||
}
|
||||
for stateKey, storageNodes := range payload.StorageNodes {
|
||||
if checkNodeKeys(stateKeyFilters, stateKey) {
|
||||
for _, storageNode := range storageNodes {
|
||||
if checkNodeKeys(storageKeyFilters, storageNode.Key) {
|
||||
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.StorageNodes = append(response.StorageNodes, StorageNode{
|
||||
StateTrieKey: stateKey,
|
||||
StorageTrieKey: storageNode.Key,
|
||||
IPLD: ipfs.BlockModel{
|
||||
Data: storageNode.Value,
|
||||
CID: cid.String(),
|
||||
},
|
||||
Leaf: storageNode.Leaf,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
@ -17,11 +17,13 @@
|
||||
package eth_test
|
||||
|
||||
import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
"bytes"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
|
||||
@ -54,19 +56,19 @@ var _ = Describe("Filterer", func() {
|
||||
Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
|
||||
Expect(len(iplds.StateNodes)).To(Equal(2))
|
||||
for _, stateNode := range iplds.StateNodes {
|
||||
Expect(stateNode.Leaf).To(BeTrue())
|
||||
if stateNode.StateTrieKey == mocks.ContractLeafKey {
|
||||
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
|
||||
Data: mocks.State1IPLD.RawData(),
|
||||
CID: mocks.State1IPLD.Cid().String(),
|
||||
}))
|
||||
}
|
||||
if stateNode.StateTrieKey == mocks.AnotherContractLeafKey {
|
||||
Expect(stateNode.Type).To(Equal(statediff.Leaf))
|
||||
if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.AccountLeafKey) {
|
||||
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
|
||||
Data: mocks.State2IPLD.RawData(),
|
||||
CID: mocks.State2IPLD.Cid().String(),
|
||||
}))
|
||||
}
|
||||
if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.ContractLeafKey) {
|
||||
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
|
||||
Data: mocks.State1IPLD.RawData(),
|
||||
CID: mocks.State1IPLD.Cid().String(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
|
||||
})
|
||||
@ -180,10 +182,10 @@ var _ = Describe("Filterer", func() {
|
||||
Expect(len(iplds7.StorageNodes)).To(Equal(0))
|
||||
Expect(len(iplds7.Receipts)).To(Equal(0))
|
||||
Expect(len(iplds7.StateNodes)).To(Equal(1))
|
||||
Expect(iplds7.StateNodes[0].StateTrieKey).To(Equal(mocks.ContractLeafKey))
|
||||
Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks.AccountLeafKey))
|
||||
Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
|
||||
Data: mocks.State1IPLD.RawData(),
|
||||
CID: mocks.State1IPLD.Cid().String(),
|
||||
Data: mocks.State2IPLD.RawData(),
|
||||
CID: mocks.State2IPLD.Cid().String(),
|
||||
}))
|
||||
|
||||
payload8, err := filterer.Filter(rctTopicsAndContractFilterFail, mocks.MockConvertedPayload)
|
||||
|
45
pkg/super_node/eth/helpers.go
Normal file
45
pkg/super_node/eth/helpers.go
Normal file
@ -0,0 +1,45 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package eth
|
||||
|
||||
import "github.com/ethereum/go-ethereum/statediff"
|
||||
|
||||
func ResolveFromNodeType(nodeType statediff.NodeType) int {
|
||||
switch nodeType {
|
||||
case statediff.Branch:
|
||||
return 0
|
||||
case statediff.Extension:
|
||||
return 1
|
||||
case statediff.Leaf:
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
func ResolveToNodeType(nodeType int) statediff.NodeType {
|
||||
switch nodeType {
|
||||
case 0:
|
||||
return statediff.Branch
|
||||
case 1:
|
||||
return statediff.Extension
|
||||
case 2:
|
||||
return statediff.Leaf
|
||||
default:
|
||||
return statediff.Unknown
|
||||
}
|
||||
}
|
@ -19,6 +19,8 @@ package eth
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
@ -129,14 +131,14 @@ func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID in
|
||||
func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
|
||||
for _, stateCID := range payload.StateNodeCIDs {
|
||||
var stateID int64
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_key, cid, node_type) = ($2, $3, $5)
|
||||
RETURNING id`,
|
||||
headerID, stateCID.StateKey, stateCID.CID, stateCID.Leaf).Scan(&stateID)
|
||||
headerID, stateCID.StateKey, stateCID.CID, stateCID.Path, stateCID.NodeType).Scan(&stateID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, storageCID := range payload.StorageNodeCIDs[common.HexToHash(stateCID.StateKey)] {
|
||||
for _, storageCID := range payload.StorageNodeCIDs[crypto.Keccak256Hash(stateCID.Path)] {
|
||||
if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -146,8 +148,8 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
|
||||
}
|
||||
|
||||
func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error {
|
||||
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
|
||||
stateID, storageCID.StorageKey, storageCID.CID, storageCID.Leaf)
|
||||
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_key, cid, node_type) = ($2, $3, $5)`,
|
||||
stateID, storageCID.StorageKey, storageCID.CID, storageCID.Path, storageCID.NodeType)
|
||||
return err
|
||||
}
|
||||
|
@ -17,6 +17,7 @@
|
||||
package eth_test
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
@ -45,13 +46,15 @@ var _ = Describe("Indexer", func() {
|
||||
It("Indexes CIDs and related metadata into vulcanizedb", func() {
|
||||
err = repo.Index(mocks.MockCIDPayload)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
pgStr := `SELECT cid, td, reward FROM eth.header_cids
|
||||
pgStr := `SELECT cid, td, reward, id
|
||||
FROM eth.header_cids
|
||||
WHERE block_number = $1`
|
||||
// check header was properly indexed
|
||||
type res struct {
|
||||
CID string
|
||||
TD string
|
||||
Reward string
|
||||
ID int
|
||||
}
|
||||
headers := new(res)
|
||||
err = db.QueryRowx(pgStr, 1).StructScan(headers)
|
||||
@ -81,24 +84,28 @@ var _ = Describe("Indexer", func() {
|
||||
Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
|
||||
// check that state nodes were properly indexed
|
||||
stateNodes := make([]eth.StateNodeModel, 0)
|
||||
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
|
||||
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
|
||||
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
|
||||
WHERE header_cids.block_number = $1`
|
||||
err = db.Select(&stateNodes, pgStr, 1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(stateNodes)).To(Equal(2))
|
||||
for _, stateNode := range stateNodes {
|
||||
if stateNode.CID == mocks.State1CID.String() {
|
||||
Expect(stateNode.Leaf).To(Equal(true))
|
||||
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
|
||||
Expect(stateNode.NodeType).To(Equal(2))
|
||||
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
|
||||
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
|
||||
}
|
||||
if stateNode.CID == mocks.State2CID.String() {
|
||||
Expect(stateNode.Leaf).To(Equal(true))
|
||||
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
|
||||
Expect(stateNode.NodeType).To(Equal(2))
|
||||
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
|
||||
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
|
||||
}
|
||||
}
|
||||
// check that storage nodes were properly indexed
|
||||
storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
|
||||
pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.leaf FROM eth.storage_cids, eth.state_cids, eth.header_cids
|
||||
pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.node_type, storage_cids.storage_path
|
||||
FROM eth.storage_cids, eth.state_cids, eth.header_cids
|
||||
WHERE storage_cids.state_id = state_cids.id
|
||||
AND state_cids.header_id = header_cids.id
|
||||
AND header_cids.block_number = $1`
|
||||
@ -107,9 +114,10 @@ var _ = Describe("Indexer", func() {
|
||||
Expect(len(storageNodes)).To(Equal(1))
|
||||
Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
|
||||
CID: mocks.StorageCID.String(),
|
||||
Leaf: true,
|
||||
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
|
||||
StateKey: mocks.ContractLeafKey.Hex(),
|
||||
NodeType: 2,
|
||||
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
|
||||
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
|
||||
Path: []byte{},
|
||||
}))
|
||||
})
|
||||
})
|
||||
|
@ -216,8 +216,9 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
|
||||
Data: state.RawData(),
|
||||
CID: state.Cid().String(),
|
||||
},
|
||||
StateTrieKey: common.HexToHash(stateNode.StateKey),
|
||||
Leaf: stateNode.Leaf,
|
||||
StateLeafKey: common.HexToHash(stateNode.StateKey),
|
||||
Type: ResolveToNodeType(stateNode.NodeType),
|
||||
Path: stateNode.Path,
|
||||
}
|
||||
}
|
||||
return stateNodes, nil
|
||||
@ -246,9 +247,10 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
|
||||
Data: storage.RawData(),
|
||||
CID: storage.Cid().String(),
|
||||
},
|
||||
StateTrieKey: common.HexToHash(storageNode.StateKey),
|
||||
StorageTrieKey: common.HexToHash(storageNode.StorageKey),
|
||||
Leaf: storageNode.Leaf,
|
||||
StateLeafKey: common.HexToHash(storageNode.StateKey),
|
||||
StorageLeafKey: common.HexToHash(storageNode.StorageKey),
|
||||
Type: ResolveToNodeType(storageNode.NodeType),
|
||||
Path: storageNode.Path,
|
||||
}
|
||||
}
|
||||
return storageNodes, nil
|
||||
|
@ -20,13 +20,13 @@ import (
|
||||
"bytes"
|
||||
"math/big"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
"github.com/ipfs/go-block-format"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
|
||||
)
|
||||
@ -71,18 +71,18 @@ var (
|
||||
},
|
||||
StateNodes: []eth.StateNodeModel{{
|
||||
CID: mockStateBlock.Cid().String(),
|
||||
Leaf: true,
|
||||
NodeType: 2,
|
||||
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
|
||||
}},
|
||||
StorageNodes: []eth.StorageNodeWithStateKeyModel{{
|
||||
CID: mockStorageBlock1.Cid().String(),
|
||||
Leaf: true,
|
||||
NodeType: 2,
|
||||
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
|
||||
StorageKey: "0000000000000000000000000000000000000000000000000000000000000001",
|
||||
},
|
||||
{
|
||||
CID: mockStorageBlock2.Cid().String(),
|
||||
Leaf: true,
|
||||
NodeType: 2,
|
||||
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
|
||||
StorageKey: "0000000000000000000000000000000000000000000000000000000000000002",
|
||||
}},
|
||||
@ -127,23 +127,23 @@ var _ = Describe("Fetcher", func() {
|
||||
CID: mockReceiptBlock.Cid().String(),
|
||||
}))
|
||||
Expect(len(iplds.StateNodes)).To(Equal(1))
|
||||
Expect(iplds.StateNodes[0].StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
|
||||
Expect(iplds.StateNodes[0].Leaf).To(BeTrue())
|
||||
Expect(iplds.StateNodes[0].StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
|
||||
Expect(iplds.StateNodes[0].Type).To(Equal(statediff.Leaf))
|
||||
Expect(iplds.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
|
||||
Data: mockStateBlock.RawData(),
|
||||
CID: mockStateBlock.Cid().String(),
|
||||
}))
|
||||
Expect(len(iplds.StorageNodes)).To(Equal(2))
|
||||
for _, storage := range iplds.StorageNodes {
|
||||
Expect(storage.Leaf).To(BeTrue())
|
||||
Expect(storage.StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
|
||||
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
|
||||
Expect(storage.Type).To(Equal(statediff.Leaf))
|
||||
Expect(storage.StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
|
||||
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
|
||||
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
|
||||
Data: mockStorageBlock1.RawData(),
|
||||
CID: mockStorageBlock1.Cid().String(),
|
||||
}))
|
||||
}
|
||||
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
|
||||
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
|
||||
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
|
||||
Data: mockStorageBlock2.RawData(),
|
||||
CID: mockStorageBlock2.Cid().String(),
|
||||
|
@ -21,12 +21,6 @@ import (
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"math/big"
|
||||
rand2 "math/rand"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
@ -35,9 +29,13 @@ import (
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
"github.com/ethereum/go-ethereum/statediff/testhelpers"
|
||||
"github.com/ipfs/go-block-format"
|
||||
"github.com/multiformats/go-multihash"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
|
||||
eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
|
||||
)
|
||||
@ -79,10 +77,9 @@ var (
|
||||
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
|
||||
Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
|
||||
Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
|
||||
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256)
|
||||
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256)
|
||||
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256)
|
||||
|
||||
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
|
||||
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
|
||||
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
|
||||
MockTrxMeta = []eth.TxModel{
|
||||
{
|
||||
CID: "", // This is empty until we go to publish to ipfs
|
||||
@ -161,51 +158,71 @@ var (
|
||||
}
|
||||
|
||||
// statediff data
|
||||
CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
|
||||
NonceValue = rand2.Uint64()
|
||||
anotherNonceValue = rand2.Uint64()
|
||||
BalanceValue = rand2.Int63()
|
||||
anotherBalanceValue = rand2.Int63()
|
||||
ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
|
||||
StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
|
||||
StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
|
||||
StorageValue = common.Hex2Bytes("0x03")
|
||||
storage = []statediff.StorageDiff{{
|
||||
Key: StorageKey,
|
||||
Value: StorageValue,
|
||||
Path: StoragePath,
|
||||
Proof: [][]byte{},
|
||||
Leaf: true,
|
||||
}}
|
||||
emptyStorage = make([]statediff.StorageDiff, 0)
|
||||
ContractLeafKey = crypto.Keccak256Hash(Address.Bytes())
|
||||
AnotherContractLeafKey = crypto.Keccak256Hash(AnotherAddress.Bytes())
|
||||
testAccount = state.Account{
|
||||
Nonce: NonceValue,
|
||||
Balance: big.NewInt(BalanceValue),
|
||||
Root: ContractRoot,
|
||||
CodeHash: CodeHash,
|
||||
}
|
||||
anotherTestAccount = state.Account{
|
||||
Nonce: anotherNonceValue,
|
||||
Balance: big.NewInt(anotherBalanceValue),
|
||||
Root: common.HexToHash("0x"),
|
||||
CodeHash: nil,
|
||||
}
|
||||
ValueBytes, _ = rlp.EncodeToBytes(testAccount)
|
||||
AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount)
|
||||
storageLocation = common.HexToHash("0")
|
||||
StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes()
|
||||
StorageValue = common.Hex2Bytes("01")
|
||||
StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
|
||||
StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
|
||||
StoragePartialPath,
|
||||
StorageValue,
|
||||
})
|
||||
|
||||
nonce1 = uint64(1)
|
||||
contractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
|
||||
contractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
|
||||
contractPathHash = crypto.Keccak256Hash([]byte{'\x06'})
|
||||
ContractAddress = common.HexToAddress("0x703c4b2bD70c169f5717101CaeE543299Fc946C7")
|
||||
ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress)
|
||||
ContractAccount, _ = rlp.EncodeToBytes(state.Account{
|
||||
Nonce: nonce1,
|
||||
Balance: big.NewInt(0),
|
||||
CodeHash: contractCodeHash.Bytes(),
|
||||
Root: common.HexToHash(contractRoot),
|
||||
})
|
||||
ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
|
||||
ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
|
||||
ContractPartialPath,
|
||||
ContractAccount,
|
||||
})
|
||||
|
||||
nonce0 = uint64(0)
|
||||
accountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
|
||||
accountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
|
||||
AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
|
||||
AccountLeafKey = testhelpers.Account2LeafKey
|
||||
Account, _ = rlp.EncodeToBytes(state.Account{
|
||||
Nonce: nonce0,
|
||||
Balance: big.NewInt(1000),
|
||||
CodeHash: accountCodeHash.Bytes(),
|
||||
Root: common.HexToHash(accountRoot),
|
||||
})
|
||||
AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
|
||||
AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
|
||||
AccountPartialPath,
|
||||
Account,
|
||||
})
|
||||
|
||||
CreatedAccountDiffs = []statediff.AccountDiff{
|
||||
{
|
||||
Key: ContractLeafKey.Bytes(),
|
||||
Value: ValueBytes,
|
||||
Storage: storage,
|
||||
Leaf: true,
|
||||
Path: []byte{'\x06'},
|
||||
NodeType: statediff.Leaf,
|
||||
LeafKey: ContractLeafKey,
|
||||
NodeValue: ContractLeafNode,
|
||||
Storage: []statediff.StorageDiff{
|
||||
{
|
||||
Path: []byte{},
|
||||
NodeType: statediff.Leaf,
|
||||
LeafKey: StorageLeafKey,
|
||||
NodeValue: StorageLeafNode,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: AnotherContractLeafKey.Bytes(),
|
||||
Value: AnotherValueBytes,
|
||||
Storage: emptyStorage,
|
||||
Leaf: true,
|
||||
Path: []byte{'\x0c'},
|
||||
NodeType: statediff.Leaf,
|
||||
LeafKey: AccountLeafKey,
|
||||
NodeValue: AccountLeafNode,
|
||||
Storage: []statediff.StorageDiff{},
|
||||
},
|
||||
}
|
||||
|
||||
@ -217,34 +234,39 @@ var (
|
||||
MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
|
||||
MockStateNodes = []eth.TrieNode{
|
||||
{
|
||||
Key: ContractLeafKey,
|
||||
Value: ValueBytes,
|
||||
Leaf: true,
|
||||
LeafKey: common.BytesToHash(ContractLeafKey),
|
||||
Path: []byte{'\x06'},
|
||||
Value: ContractLeafNode,
|
||||
Type: statediff.Leaf,
|
||||
},
|
||||
{
|
||||
Key: AnotherContractLeafKey,
|
||||
Value: AnotherValueBytes,
|
||||
Leaf: true,
|
||||
LeafKey: common.BytesToHash(AccountLeafKey),
|
||||
Path: []byte{'\x0c'},
|
||||
Value: AccountLeafNode,
|
||||
Type: statediff.Leaf,
|
||||
},
|
||||
}
|
||||
MockStateMetaPostPublish = []eth.StateNodeModel{
|
||||
{
|
||||
CID: State1CID.String(),
|
||||
Leaf: true,
|
||||
StateKey: ContractLeafKey.String(),
|
||||
Path: []byte{'\x06'},
|
||||
NodeType: 2,
|
||||
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
|
||||
},
|
||||
{
|
||||
CID: State2CID.String(),
|
||||
Leaf: true,
|
||||
StateKey: AnotherContractLeafKey.String(),
|
||||
Path: []byte{'\x0c'},
|
||||
NodeType: 2,
|
||||
StateKey: common.BytesToHash(AccountLeafKey).Hex(),
|
||||
},
|
||||
}
|
||||
MockStorageNodes = map[common.Hash][]eth.TrieNode{
|
||||
ContractLeafKey: {
|
||||
contractPathHash: {
|
||||
{
|
||||
Key: common.BytesToHash(StorageKey),
|
||||
Value: StorageValue,
|
||||
Leaf: true,
|
||||
LeafKey: common.BytesToHash(StorageLeafKey),
|
||||
Value: StorageLeafNode,
|
||||
Type: statediff.Leaf,
|
||||
Path: []byte{},
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -284,11 +306,12 @@ var (
|
||||
},
|
||||
StateNodeCIDs: MockStateMetaPostPublish,
|
||||
StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
|
||||
ContractLeafKey: {
|
||||
contractPathHash: {
|
||||
{
|
||||
CID: StorageCID.String(),
|
||||
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
|
||||
Leaf: true,
|
||||
Path: []byte{},
|
||||
StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
|
||||
NodeType: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -310,10 +333,11 @@ var (
|
||||
StateNodes: MockStateMetaPostPublish,
|
||||
StorageNodes: []eth.StorageNodeWithStateKeyModel{
|
||||
{
|
||||
Path: []byte{},
|
||||
CID: StorageCID.String(),
|
||||
Leaf: true,
|
||||
StateKey: ContractLeafKey.Hex(),
|
||||
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
|
||||
NodeType: 2,
|
||||
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
|
||||
StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -323,9 +347,9 @@ var (
|
||||
Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
|
||||
Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
|
||||
Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
|
||||
State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, State1CID)
|
||||
State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, State2CID)
|
||||
StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, StorageCID)
|
||||
State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
|
||||
State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
|
||||
StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
|
||||
|
||||
MockIPLDs = eth.IPLDs{
|
||||
BlockNumber: big.NewInt(1),
|
||||
@ -355,31 +379,34 @@ var (
|
||||
},
|
||||
StateNodes: []eth2.StateNode{
|
||||
{
|
||||
StateTrieKey: ContractLeafKey,
|
||||
Leaf: true,
|
||||
StateLeafKey: common.BytesToHash(ContractLeafKey),
|
||||
Type: statediff.Leaf,
|
||||
IPLD: ipfs.BlockModel{
|
||||
Data: State1IPLD.RawData(),
|
||||
CID: State1IPLD.Cid().String(),
|
||||
},
|
||||
Path: []byte{'\x06'},
|
||||
},
|
||||
{
|
||||
StateTrieKey: AnotherContractLeafKey,
|
||||
Leaf: true,
|
||||
StateLeafKey: common.BytesToHash(AccountLeafKey),
|
||||
Type: statediff.Leaf,
|
||||
IPLD: ipfs.BlockModel{
|
||||
Data: State2IPLD.RawData(),
|
||||
CID: State2IPLD.Cid().String(),
|
||||
},
|
||||
Path: []byte{'\x0c'},
|
||||
},
|
||||
},
|
||||
StorageNodes: []eth2.StorageNode{
|
||||
{
|
||||
StateTrieKey: ContractLeafKey,
|
||||
StorageTrieKey: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"),
|
||||
Leaf: true,
|
||||
StateLeafKey: common.BytesToHash(ContractLeafKey),
|
||||
StorageLeafKey: common.BytesToHash(StorageLeafKey),
|
||||
Type: statediff.Leaf,
|
||||
IPLD: ipfs.BlockModel{
|
||||
Data: StorageIPLD.RawData(),
|
||||
CID: StorageIPLD.Cid().String(),
|
||||
},
|
||||
Path: []byte{},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -67,8 +67,9 @@ type ReceiptModel struct {
|
||||
type StateNodeModel struct {
|
||||
ID int64 `db:"id"`
|
||||
HeaderID int64 `db:"header_id"`
|
||||
Path []byte `db:"state_path"`
|
||||
StateKey string `db:"state_key"`
|
||||
Leaf bool `db:"leaf"`
|
||||
NodeType int `db:"node_type"`
|
||||
CID string `db:"cid"`
|
||||
}
|
||||
|
||||
@ -76,8 +77,9 @@ type StateNodeModel struct {
|
||||
type StorageNodeModel struct {
|
||||
ID int64 `db:"id"`
|
||||
StateID int64 `db:"state_id"`
|
||||
Path []byte `db:"storage_path"`
|
||||
StorageKey string `db:"storage_key"`
|
||||
Leaf bool `db:"leaf"`
|
||||
NodeType int `db:"node_type"`
|
||||
CID string `db:"cid"`
|
||||
}
|
||||
|
||||
@ -85,8 +87,9 @@ type StorageNodeModel struct {
|
||||
type StorageNodeWithStateKeyModel struct {
|
||||
ID int64 `db:"id"`
|
||||
StateID int64 `db:"state_id"`
|
||||
Path []byte `db:"storage_path"`
|
||||
StateKey string `db:"state_key"`
|
||||
StorageKey string `db:"storage_key"`
|
||||
Leaf bool `db:"leaf"`
|
||||
NodeType int `db:"node_type"`
|
||||
CID string `db:"cid"`
|
||||
}
|
||||
|
@ -185,9 +185,10 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
|
||||
return nil, err
|
||||
}
|
||||
stateNodeCids = append(stateNodeCids, StateNodeModel{
|
||||
StateKey: node.Key.String(),
|
||||
Path: node.Path,
|
||||
StateKey: node.LeafKey.String(),
|
||||
CID: cids[0],
|
||||
Leaf: node.Leaf,
|
||||
NodeType: ResolveFromNodeType(node.Type),
|
||||
})
|
||||
}
|
||||
return stateNodeCids, nil
|
||||
@ -195,18 +196,19 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
|
||||
|
||||
func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
|
||||
storageLeafCids := make(map[common.Hash][]StorageNodeModel)
|
||||
for addrKey, storageTrie := range storageNodes {
|
||||
storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie))
|
||||
for pathHash, storageTrie := range storageNodes {
|
||||
storageLeafCids[pathHash] = make([]StorageNodeModel, 0, len(storageTrie))
|
||||
for _, node := range storageTrie {
|
||||
cids, err := pub.StoragePutter.DagPut(node.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Map storage node cids to their state key hashes
|
||||
storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{
|
||||
StorageKey: node.Key.Hex(),
|
||||
// Map storage node cids to their path hashes
|
||||
storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{
|
||||
Path: node.Path,
|
||||
StorageKey: node.LeafKey.Hex(),
|
||||
CID: cids[0],
|
||||
Leaf: node.Leaf,
|
||||
NodeType: ResolveFromNodeType(node.Type),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -391,7 +391,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
|
||||
log.Debug("retrieving state cids for header id ", headerID)
|
||||
args := make([]interface{}, 0, 2)
|
||||
pgStr := `SELECT state_cids.id, state_cids.header_id,
|
||||
state_cids.state_key, state_cids.leaf, state_cids.cid
|
||||
state_cids.state_key, state_cids.node_type, state_cids.cid, state_cids.state_path
|
||||
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
|
||||
WHERE header_cids.id = $1`
|
||||
args = append(args, headerID)
|
||||
@ -405,7 +405,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
|
||||
args = append(args, pq.Array(keys))
|
||||
}
|
||||
if !stateFilter.IntermediateNodes {
|
||||
pgStr += ` AND state_cids.leaf = TRUE`
|
||||
pgStr += ` AND state_cids.node_type = 2`
|
||||
}
|
||||
stateNodeCIDs := make([]StateNodeModel, 0)
|
||||
return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
|
||||
@ -416,7 +416,8 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
|
||||
log.Debug("retrieving storage cids for header id ", headerID)
|
||||
args := make([]interface{}, 0, 3)
|
||||
pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
|
||||
storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM eth.storage_cids, eth.state_cids, eth.header_cids
|
||||
storage_cids.node_type, storage_cids.cid, storage_cids.storage_path, state_cids.state_key
|
||||
FROM eth.storage_cids, eth.state_cids, eth.header_cids
|
||||
WHERE storage_cids.state_id = state_cids.id
|
||||
AND state_cids.header_id = header_cids.id
|
||||
AND header_cids.id = $1`
|
||||
@ -437,7 +438,7 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
|
||||
args = append(args, pq.Array(storageFilter.StorageKeys))
|
||||
}
|
||||
if !storageFilter.IntermediateNodes {
|
||||
pgStr += ` AND storage_cids.leaf = TRUE`
|
||||
pgStr += ` AND storage_cids.node_type = 2`
|
||||
}
|
||||
storageNodeCIDs := make([]StorageNodeWithStateKeyModel, 0)
|
||||
return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
|
||||
|
@ -19,6 +19,8 @@ package eth_test
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
@ -196,7 +198,7 @@ var (
|
||||
Off: true,
|
||||
},
|
||||
StateFilter: eth.StateFilter{
|
||||
Addresses: []string{mocks.Address.Hex()},
|
||||
Addresses: []string{mocks.AccountAddresss.Hex()},
|
||||
},
|
||||
StorageFilter: eth.StorageFilter{
|
||||
Off: true,
|
||||
@ -247,12 +249,14 @@ var _ = Describe("Retriever", func() {
|
||||
Expect(len(cidWrapper.StateNodes)).To(Equal(2))
|
||||
for _, stateNode := range cidWrapper.StateNodes {
|
||||
if stateNode.CID == mocks.State1CID.String() {
|
||||
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
|
||||
Expect(stateNode.Leaf).To(Equal(true))
|
||||
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
|
||||
Expect(stateNode.NodeType).To(Equal(2))
|
||||
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
|
||||
}
|
||||
if stateNode.CID == mocks.State2CID.String() {
|
||||
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
|
||||
Expect(stateNode.Leaf).To(Equal(true))
|
||||
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
|
||||
Expect(stateNode.NodeType).To(Equal(2))
|
||||
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
|
||||
}
|
||||
}
|
||||
Expect(len(cidWrapper.StorageNodes)).To(Equal(1))
|
||||
@ -384,9 +388,10 @@ var _ = Describe("Retriever", func() {
|
||||
Expect(cidWrapper7.StateNodes[0]).To(Equal(eth.StateNodeModel{
|
||||
ID: cidWrapper7.StateNodes[0].ID,
|
||||
HeaderID: cidWrapper7.StateNodes[0].HeaderID,
|
||||
Leaf: true,
|
||||
StateKey: mocks.ContractLeafKey.Hex(),
|
||||
CID: mocks.State1CID.String(),
|
||||
NodeType: 2,
|
||||
StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(),
|
||||
CID: mocks.State2CID.String(),
|
||||
Path: []byte{'\x0c'},
|
||||
}))
|
||||
|
||||
_, empty, err = retriever.Retrieve(rctTopicsAndContractFilterFail, 1)
|
||||
|
@ -19,6 +19,8 @@ package eth
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
@ -45,9 +47,10 @@ func (i ConvertedPayload) Height() int64 {
|
||||
|
||||
// Trie struct used to flag node as leaf or not
|
||||
type TrieNode struct {
|
||||
Key common.Hash
|
||||
Path []byte
|
||||
LeafKey common.Hash
|
||||
Value []byte
|
||||
Leaf bool
|
||||
Type statediff.NodeType
|
||||
}
|
||||
|
||||
// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres
|
||||
@ -94,14 +97,16 @@ func (i IPLDs) Height() int64 {
|
||||
}
|
||||
|
||||
type StateNode struct {
|
||||
StateTrieKey common.Hash
|
||||
Type statediff.NodeType
|
||||
StateLeafKey common.Hash
|
||||
Path []byte
|
||||
IPLD ipfs.BlockModel
|
||||
Leaf bool
|
||||
}
|
||||
|
||||
type StorageNode struct {
|
||||
StateTrieKey common.Hash
|
||||
StorageTrieKey common.Hash
|
||||
Type statediff.NodeType
|
||||
StateLeafKey common.Hash
|
||||
StorageLeafKey common.Hash
|
||||
Path []byte
|
||||
IPLD ipfs.BlockModel
|
||||
Leaf bool
|
||||
}
|
||||
|
@ -150,16 +150,16 @@ func (pc *WatcherConverter) Convert(ethIPLDs eth.IPLDs) (*eth.CIDPayload, error)
|
||||
for i, stateIPLD := range ethIPLDs.StateNodes {
|
||||
cids.StateNodeCIDs[i] = eth.StateNodeModel{
|
||||
CID: stateIPLD.IPLD.CID,
|
||||
Leaf: stateIPLD.Leaf,
|
||||
StateKey: stateIPLD.StateTrieKey.String(),
|
||||
NodeType: eth.ResolveFromNodeType(stateIPLD.Type),
|
||||
StateKey: stateIPLD.StateLeafKey.String(),
|
||||
}
|
||||
}
|
||||
// Storage data
|
||||
for _, storageIPLD := range ethIPLDs.StorageNodes {
|
||||
cids.StorageNodeCIDs[storageIPLD.StateTrieKey] = append(cids.StorageNodeCIDs[storageIPLD.StateTrieKey], eth.StorageNodeModel{
|
||||
cids.StorageNodeCIDs[storageIPLD.StateLeafKey] = append(cids.StorageNodeCIDs[storageIPLD.StateLeafKey], eth.StorageNodeModel{
|
||||
CID: storageIPLD.IPLD.CID,
|
||||
Leaf: storageIPLD.Leaf,
|
||||
StorageKey: storageIPLD.StorageTrieKey.String(),
|
||||
NodeType: eth.ResolveFromNodeType(storageIPLD.Type),
|
||||
StorageKey: storageIPLD.StorageLeafKey.String(),
|
||||
})
|
||||
}
|
||||
return cids, nil
|
||||
|
Loading…
Reference in New Issue
Block a user