From e5c5422edce585deb64e10e9bcf65b6173d65582 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 11 Mar 2020 13:41:59 -0500 Subject: [PATCH] adhjustments to work with statediffing geth v1.10-alpha.2 --- cmd/streamEthSubscribe.go | 8 +- db/migrations/00027_update_state_cids.sql | 37 ++++ db/migrations/00028_update_storage_cids.sql | 37 ++++ db/schema.sql | 89 +++++++- go.mod | 2 +- go.sum | 7 + .../fetcher/geth_rpc_storage_fetcher.go | 4 +- libraries/shared/storage/backfiller.go | 4 +- libraries/shared/storage/utils/diff.go | 6 +- libraries/shared/storage/utils/diff_test.go | 16 +- libraries/shared/test_data/statediff.go | 46 +++-- pkg/super_node/eth/converter.go | 55 ++--- pkg/super_node/eth/filterer.go | 100 +++++---- pkg/super_node/eth/filterer_test.go | 26 +-- pkg/super_node/eth/helpers.go | 45 ++++ pkg/super_node/eth/indexer.go | 16 +- pkg/super_node/eth/indexer_test.go | 28 ++- pkg/super_node/eth/ipld_fetcher.go | 12 +- pkg/super_node/eth/ipld_fetcher_test.go | 22 +- pkg/super_node/eth/mocks/test_data.go | 193 ++++++++++-------- pkg/super_node/eth/models.go | 9 +- pkg/super_node/eth/publisher.go | 18 +- pkg/super_node/eth/retriever.go | 9 +- pkg/super_node/eth/retriever_test.go | 21 +- pkg/super_node/eth/types.go | 21 +- pkg/watcher/eth/converter.go | 10 +- 26 files changed, 547 insertions(+), 294 deletions(-) create mode 100644 db/migrations/00027_update_state_cids.sql create mode 100644 db/migrations/00028_update_storage_cids.sql create mode 100644 pkg/super_node/eth/helpers.go diff --git a/cmd/streamEthSubscribe.go b/cmd/streamEthSubscribe.go index 586ed49f..e46aee52 100644 --- a/cmd/streamEthSubscribe.go +++ b/cmd/streamEthSubscribe.go @@ -139,12 +139,12 @@ func streamEthSubscription() { continue } fmt.Printf("Account for key %s, and root %s, with balance %s\n", - stateNode.StateTrieKey.Hex(), acct.Root.Hex(), acct.Balance.String()) + stateNode.StateLeafKey.Hex(), acct.Root.Hex(), acct.Balance.String()) fmt.Printf("state account: %+v\n", acct) } for _, storageNode := range ethData.StorageNodes { - fmt.Printf("Storage for state key %s ", storageNode.StateTrieKey.Hex()) - fmt.Printf("with storage key %s\n", storageNode.StorageTrieKey.Hex()) + fmt.Printf("Storage for state key %s ", storageNode.StateLeafKey.Hex()) + fmt.Printf("with storage key %s\n", storageNode.StorageLeafKey.Hex()) var i []interface{} err := rlp.DecodeBytes(storageNode.IPLD.Data, &i) if err != nil { @@ -158,7 +158,7 @@ func streamEthSubscription() { continue } fmt.Printf("Storage leaf key: %s, and value hash: %s\n", - storageNode.StorageTrieKey.Hex(), common.BytesToHash(valueBytes).Hex()) + storageNode.StorageLeafKey.Hex(), common.BytesToHash(valueBytes).Hex()) } } case err = <-sub.Err(): diff --git a/db/migrations/00027_update_state_cids.sql b/db/migrations/00027_update_state_cids.sql new file mode 100644 index 00000000..e4fdaa1b --- /dev/null +++ b/db/migrations/00027_update_state_cids.sql @@ -0,0 +1,37 @@ +-- +goose Up +ALTER TABLE eth.state_cids +ADD COLUMN state_path BYTEA; + +ALTER TABLE eth.state_cids +DROP COLUMN leaf; + +ALTER TABLE eth.state_cids +ADD COLUMN node_type INTEGER NOT NULL; + +ALTER TABLE eth.state_cids +ALTER COLUMN state_key DROP NOT NULL; + +ALTER TABLE eth.state_cids +DROP CONSTRAINT state_cids_header_id_state_key_key; + +ALTER TABLE eth.state_cids +ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path); + +-- +goose Down +ALTER TABLE eth.state_cids +ADD CONSTRAINT state_cids_header_id_state_key_key UNIQUE (header_id, state_key); + +ALTER TABLE eth.state_cids +DROP CONSTRAINT state_cids_header_id_state_path_key; + +ALTER TABLE eth.state_cids +ALTER COLUMN state_key SET NOT NULL; + +ALTER TABLE eth.state_cids +DROP COLUMN node_type; + +ALTER TABLE eth.state_cids +ADD COLUMN leaf BOOLEAN NOT NULL; + +ALTER TABLE eth.state_cids +DROP COLUMN state_path; \ No newline at end of file diff --git a/db/migrations/00028_update_storage_cids.sql b/db/migrations/00028_update_storage_cids.sql new file mode 100644 index 00000000..937c22a6 --- /dev/null +++ b/db/migrations/00028_update_storage_cids.sql @@ -0,0 +1,37 @@ +-- +goose Up +ALTER TABLE eth.storage_cids +ADD COLUMN storage_path BYTEA; + +ALTER TABLE eth.storage_cids +DROP COLUMN leaf; + +ALTER TABLE eth.storage_cids +ADD COLUMN node_type INTEGER NOT NULL; + +ALTER TABLE eth.storage_cids +ALTER COLUMN storage_key DROP NOT NULL; + +ALTER TABLE eth.storage_cids +DROP CONSTRAINT storage_cids_state_id_storage_key_key; + +ALTER TABLE eth.storage_cids +ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path); + +-- +goose Down +ALTER TABLE eth.storage_cids +DROP CONSTRAINT storage_cids_state_id_storage_path_key; + +ALTER TABLE eth.storage_cids +ADD CONSTRAINT storage_cids_state_id_storage_key_key UNIQUE (state_id, storage_key); + +ALTER TABLE eth.storage_cids +ALTER COLUMN storage_key SET NOT NULL; + +ALTER TABLE eth.storage_cids +DROP COLUMN node_type; + +ALTER TABLE eth.storage_cids +ADD COLUMN leaf BOOLEAN NOT NULL; + +ALTER TABLE eth.storage_cids +DROP COLUMN storage_path; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 367674d3..332f6589 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -3,7 +3,7 @@ -- -- Dumped from database version 10.10 --- Dumped by pg_dump version 12.1 +-- Dumped by pg_dump version 10.10 SET statement_timeout = 0; SET lock_timeout = 0; @@ -30,8 +30,24 @@ CREATE SCHEMA btc; CREATE SCHEMA eth; +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + SET default_tablespace = ''; +SET default_with_oids = false; + -- -- Name: header_cids; Type: TABLE; Schema: btc; Owner: - -- @@ -48,6 +64,13 @@ CREATE TABLE btc.header_cids ( ); +-- +-- Name: TABLE header_cids; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON TABLE btc.header_cids IS '@name BtcHeaderCids'; + + -- -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: - -- @@ -79,6 +102,13 @@ CREATE TABLE btc.queue_data ( ); +-- +-- Name: TABLE queue_data; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON TABLE btc.queue_data IS '@name BtcQueueData'; + + -- -- Name: queue_data_id_seq; Type: SEQUENCE; Schema: btc; Owner: - -- @@ -114,6 +144,13 @@ CREATE TABLE btc.transaction_cids ( ); +-- +-- Name: TABLE transaction_cids; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON TABLE btc.transaction_cids IS '@name BtcTransactionCids'; + + -- -- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: - -- @@ -221,6 +258,13 @@ CREATE TABLE eth.header_cids ( ); +-- +-- Name: TABLE header_cids; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids'; + + -- -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - -- @@ -252,6 +296,13 @@ CREATE TABLE eth.queue_data ( ); +-- +-- Name: TABLE queue_data; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON TABLE eth.queue_data IS '@name EthQueueData'; + + -- -- Name: queue_data_id_seq; Type: SEQUENCE; Schema: eth; Owner: - -- @@ -315,9 +366,10 @@ ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id; CREATE TABLE eth.state_cids ( id integer NOT NULL, header_id integer NOT NULL, - state_key character varying(66) NOT NULL, - leaf boolean NOT NULL, - cid text NOT NULL + state_key character varying(66), + cid text NOT NULL, + state_path bytea, + node_type integer NOT NULL ); @@ -348,9 +400,10 @@ ALTER SEQUENCE eth.state_cids_id_seq OWNED BY eth.state_cids.id; CREATE TABLE eth.storage_cids ( id integer NOT NULL, state_id integer NOT NULL, - storage_key character varying(66) NOT NULL, - leaf boolean NOT NULL, - cid text NOT NULL + storage_key character varying(66), + cid text NOT NULL, + storage_path bytea, + node_type integer NOT NULL ); @@ -389,6 +442,13 @@ CREATE TABLE eth.transaction_cids ( ); +-- +-- Name: TABLE transaction_cids; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON TABLE eth.transaction_cids IS '@name EthTransactionCids'; + + -- -- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - -- @@ -713,6 +773,13 @@ CREATE TABLE public.nodes ( ); +-- +-- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.nodes IS '@name NodeIDs'; + + -- -- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -1110,11 +1177,11 @@ ALTER TABLE ONLY eth.receipt_cids -- --- Name: state_cids state_cids_header_id_state_key_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: - -- ALTER TABLE ONLY eth.state_cids - ADD CONSTRAINT state_cids_header_id_state_key_key UNIQUE (header_id, state_key); + ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path); -- @@ -1134,11 +1201,11 @@ ALTER TABLE ONLY eth.storage_cids -- --- Name: storage_cids storage_cids_state_id_storage_key_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- Name: storage_cids storage_cids_state_id_storage_path_key; Type: CONSTRAINT; Schema: eth; Owner: - -- ALTER TABLE ONLY eth.storage_cids - ADD CONSTRAINT storage_cids_state_id_storage_key_key UNIQUE (state_id, storage_key); + ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path); -- diff --git a/go.mod b/go.mod index 0ece4815..17360377 100644 --- a/go.mod +++ b/go.mod @@ -97,4 +97,4 @@ replace github.com/ipfs/go-ipfs v0.4.22 => github.com/vulcanize/go-ipfs v0.4.22- replace github.com/ipfs/go-ipfs-config v0.0.3 => github.com/vulcanize/go-ipfs-config v0.0.8-alpha -replace github.com/ethereum/go-ethereum v1.9.1 => github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8 +replace github.com/ethereum/go-ethereum v1.9.1 => github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290 diff --git a/go.sum b/go.sum index 0ecbfd21..dae0ac07 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,7 @@ github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1: github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0= github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d h1:QgeLLoPD3kRVmeu/1al9iIpIANMi9O1zXFm8BnYGCJg= @@ -105,7 +106,9 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -134,6 +137,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= @@ -327,6 +331,7 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmoiron/sqlx v0.0.0-20190426154859-38398a30ed85 h1:+LZtdhpMITOXE+MztQPPcwUl+eqYjwlXXLHrd0yWlxw= github.com/jmoiron/sqlx v0.0.0-20190426154859-38398a30ed85/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -723,6 +728,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8 h1:BHt0OW0rTgndFjSju7brF3dPceXWQuEV0IdtY8BjjT8= github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= +github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290 h1:uMWt+x6JhVT7GyL983weZSxv1zDBxvGlI9HNkcTnUeg= +github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= github.com/vulcanize/go-ipfs v0.4.22-alpha h1:W+6njT14KWllMhABRFtPndqHw8SHCt5SqD4YX528kxM= github.com/vulcanize/go-ipfs v0.4.22-alpha/go.mod h1:uaekWWeoaA0A9Dv1LObOKCSh9kIzTpZ5RbKW4g5CQHE= github.com/vulcanize/go-ipfs-config v0.0.8-alpha h1:peaFvbEcPShF6ymOd8flqKkFz4YfcrNr/UOO7FmbWoQ= diff --git a/libraries/shared/fetcher/geth_rpc_storage_fetcher.go b/libraries/shared/fetcher/geth_rpc_storage_fetcher.go index b4809cf6..f535c410 100644 --- a/libraries/shared/fetcher/geth_rpc_storage_fetcher.go +++ b/libraries/shared/fetcher/geth_rpc_storage_fetcher.go @@ -65,11 +65,11 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD accounts := utils.GetAccountsFromDiff(*stateDiff) logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber)) for _, account := range accounts { - logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.Key).Hex())) + logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex())) for _, storage := range account.Storage { diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage) if formatErr != nil { - logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.Key), "from account with key: ", common.BytesToHash(account.Key)) + logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey)) errs <- formatErr continue } diff --git a/libraries/shared/storage/backfiller.go b/libraries/shared/storage/backfiller.go index e9cc6aa5..6584e38d 100644 --- a/libraries/shared/storage/backfiller.go +++ b/libraries/shared/storage/backfiller.go @@ -127,11 +127,11 @@ func (bf *backFiller) backFillRange(blockHeights []uint64, diffChan chan utils.S } accounts := utils.GetAccountsFromDiff(*stateDiff) for _, account := range accounts { - logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.Key).Hex())) + logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex())) for _, storage := range account.Storage { diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage) if formatErr != nil { - logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.Key), "from account with key: ", common.BytesToHash(account.Key)) + logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey)) errChan <- formatErr continue } diff --git a/libraries/shared/storage/utils/diff.go b/libraries/shared/storage/utils/diff.go index 331ebf96..e6ad4c78 100644 --- a/libraries/shared/storage/utils/diff.go +++ b/libraries/shared/storage/utils/diff.go @@ -59,16 +59,16 @@ func FromParityCsvRow(csvRow []string) (StorageDiffInput, error) { func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiffInput, error) { var decodedValue []byte - err := rlp.DecodeBytes(storage.Value, &decodedValue) + err := rlp.DecodeBytes(storage.NodeValue, &decodedValue) if err != nil { return StorageDiffInput{}, err } return StorageDiffInput{ - HashedAddress: common.BytesToHash(account.Key), + HashedAddress: common.BytesToHash(account.LeafKey), BlockHash: stateDiff.BlockHash, BlockHeight: int(stateDiff.BlockNumber.Int64()), - StorageKey: common.BytesToHash(storage.Key), + StorageKey: common.BytesToHash(storage.LeafKey), StorageValue: common.BytesToHash(decodedValue), }, nil } diff --git a/libraries/shared/storage/utils/diff_test.go b/libraries/shared/storage/utils/diff_test.go index 32e6b437..ad47a6a7 100644 --- a/libraries/shared/storage/utils/diff_test.go +++ b/libraries/shared/storage/utils/diff_test.go @@ -67,7 +67,7 @@ var _ = Describe("Storage row parsing", func() { Describe("FromGethStateDiff", func() { var ( - accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} + accountDiff = statediff.AccountDiff{LeafKey: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} stateDiff = &statediff.StateDiff{ BlockNumber: big.NewInt(rand.Int63()), BlockHash: fakes.FakeHash, @@ -80,19 +80,20 @@ var _ = Describe("Storage row parsing", func() { Expect(encodeErr).NotTo(HaveOccurred()) storageDiff := statediff.StorageDiff{ - Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Value: storageValueRlp, + LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, + NodeValue: storageValueRlp, + NodeType: statediff.Leaf, } result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff) Expect(err).NotTo(HaveOccurred()) - expectedAddress := common.BytesToHash(accountDiff.Key) + expectedAddress := common.BytesToHash(accountDiff.LeafKey) Expect(result.HashedAddress).To(Equal(expectedAddress)) Expect(result.BlockHash).To(Equal(fakes.FakeHash)) expectedBlockHeight := int(stateDiff.BlockNumber.Int64()) Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) - expectedStorageKey := common.BytesToHash(storageDiff.Key) + expectedStorageKey := common.BytesToHash(storageDiff.LeafKey) Expect(result.StorageKey).To(Equal(expectedStorageKey)) expectedStorageValue := common.BytesToHash(storageValueBytes) Expect(result.StorageValue).To(Equal(expectedStorageValue)) @@ -104,8 +105,9 @@ var _ = Describe("Storage row parsing", func() { Expect(encodeErr).NotTo(HaveOccurred()) storageDiff := statediff.StorageDiff{ - Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Value: storageValueRlp, + LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, + NodeValue: storageValueRlp, + NodeType: statediff.Leaf, } result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff) diff --git a/libraries/shared/test_data/statediff.go b/libraries/shared/test_data/statediff.go index bc54431f..a7f15758 100644 --- a/libraries/shared/test_data/statediff.go +++ b/libraries/shared/test_data/statediff.go @@ -41,22 +41,24 @@ var ( SmallStorageValue = common.Hex2Bytes("03") SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue) storageWithSmallValue = []statediff.StorageDiff{{ - Key: StorageKey, - Value: SmallStorageValueRlp, - Path: StoragePath, - Proof: [][]byte{}, + LeafKey: StorageKey, + NodeValue: SmallStorageValueRlp, + NodeType: statediff.Leaf, + Path: StoragePath, }} LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000") LargeStorageValueRlp, _ = rlp.EncodeToBytes(LargeStorageValue) storageWithLargeValue = []statediff.StorageDiff{{ - Key: StorageKey, - Value: LargeStorageValueRlp, - Path: StoragePath, - Proof: [][]byte{}, + LeafKey: StorageKey, + NodeValue: LargeStorageValueRlp, + Path: StoragePath, + NodeType: statediff.Leaf, }} StorageWithBadValue = statediff.StorageDiff{ - Key: StorageKey, - Value: []byte{0, 1, 2}, + LeafKey: StorageKey, + NodeValue: []byte{0, 1, 2}, + NodeType: statediff.Leaf, + Path: StoragePath, // this storage value will fail to be decoded as an RLP with the following error message: // "input contains more than one value" } @@ -74,27 +76,27 @@ var ( valueBytes, _ = rlp.EncodeToBytes(testAccount) CreatedAccountDiffs = []statediff.AccountDiff{ { - Key: ContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storageWithSmallValue, + LeafKey: ContractLeafKey.Bytes(), + NodeValue: valueBytes, + Storage: storageWithSmallValue, }, } UpdatedAccountDiffs = []statediff.AccountDiff{{ - Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storageWithLargeValue, + LeafKey: AnotherContractLeafKey.Bytes(), + NodeValue: valueBytes, + Storage: storageWithLargeValue, }} UpdatedAccountDiffs2 = []statediff.AccountDiff{{ - Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storageWithSmallValue, + LeafKey: AnotherContractLeafKey.Bytes(), + NodeValue: valueBytes, + Storage: storageWithSmallValue, }} DeletedAccountDiffs = []statediff.AccountDiff{{ - Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storageWithSmallValue, + LeafKey: AnotherContractLeafKey.Bytes(), + NodeValue: valueBytes, + Storage: storageWithSmallValue, }} MockStateDiff = statediff.StateDiff{ diff --git a/pkg/super_node/eth/converter.go b/pkg/super_node/eth/converter.go index 7b4eb19f..61a7785b 100644 --- a/pkg/super_node/eth/converter.go +++ b/pkg/super_node/eth/converter.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" @@ -123,47 +124,53 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert return nil, err } for _, createdAccount := range stateDiff.CreatedAccounts { - hashKey := common.BytesToHash(createdAccount.Key) + statePathHash := crypto.Keccak256Hash(createdAccount.Path) convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{ - Key: hashKey, - Value: createdAccount.Value, - Leaf: createdAccount.Leaf, + Path: createdAccount.Path, + Value: createdAccount.NodeValue, + Type: createdAccount.NodeType, + LeafKey: common.BytesToHash(createdAccount.LeafKey), }) for _, storageDiff := range createdAccount.Storage { - convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{ - Key: common.BytesToHash(storageDiff.Key), - Value: storageDiff.Value, - Leaf: storageDiff.Leaf, + convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{ + Path: storageDiff.Path, + Value: storageDiff.NodeValue, + Type: storageDiff.NodeType, + LeafKey: common.BytesToHash(storageDiff.LeafKey), }) } } for _, deletedAccount := range stateDiff.DeletedAccounts { - hashKey := common.BytesToHash(deletedAccount.Key) + statePathHash := crypto.Keccak256Hash(deletedAccount.Path) convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{ - Key: hashKey, - Value: deletedAccount.Value, - Leaf: deletedAccount.Leaf, + Path: deletedAccount.Path, + Value: deletedAccount.NodeValue, + Type: deletedAccount.NodeType, + LeafKey: common.BytesToHash(deletedAccount.LeafKey), }) for _, storageDiff := range deletedAccount.Storage { - convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{ - Key: common.BytesToHash(storageDiff.Key), - Value: storageDiff.Value, - Leaf: storageDiff.Leaf, + convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{ + Path: storageDiff.Path, + Value: storageDiff.NodeValue, + Type: storageDiff.NodeType, + LeafKey: common.BytesToHash(storageDiff.LeafKey), }) } } for _, updatedAccount := range stateDiff.UpdatedAccounts { - hashKey := common.BytesToHash(updatedAccount.Key) + statePathHash := crypto.Keccak256Hash(updatedAccount.Path) convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{ - Key: hashKey, - Value: updatedAccount.Value, - Leaf: updatedAccount.Leaf, + Path: updatedAccount.Path, + Value: updatedAccount.NodeValue, + Type: updatedAccount.NodeType, + LeafKey: common.BytesToHash(updatedAccount.LeafKey), }) for _, storageDiff := range updatedAccount.Storage { - convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{ - Key: common.BytesToHash(storageDiff.Key), - Value: storageDiff.Value, - Leaf: storageDiff.Leaf, + convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{ + Path: storageDiff.Path, + Value: storageDiff.NodeValue, + Type: storageDiff.NodeType, + LeafKey: common.BytesToHash(storageDiff.LeafKey), }) } } diff --git a/pkg/super_node/eth/filterer.go b/pkg/super_node/eth/filterer.go index 01b63dd9..eb941ae7 100644 --- a/pkg/super_node/eth/filterer.go +++ b/pkg/super_node/eth/filterer.go @@ -20,6 +20,8 @@ import ( "bytes" "fmt" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -66,10 +68,7 @@ func (s *ResponseFilterer) Filter(filter shared.SubscriptionSettings, payload sh if err := s.filerReceipts(ethFilters.ReceiptFilter, response, ethPayload, filterTxs); err != nil { return IPLDs{}, err } - if err := s.filterState(ethFilters.StateFilter, response, ethPayload); err != nil { - return IPLDs{}, err - } - if err := s.filterStorage(ethFilters.StorageFilter, response, ethPayload); err != nil { + if err := s.filterStateAndStorage(ethFilters.StateFilter, ethFilters.StorageFilter, response, ethPayload); err != nil { return IPLDs{}, err } response.BlockNumber = ethPayload.Block.Number() @@ -255,27 +254,56 @@ func slicesShareString(slice1, slice2 []string) int { return 0 } -func (s *ResponseFilterer) filterState(stateFilter StateFilter, response *IPLDs, payload ConvertedPayload) error { - if !stateFilter.Off { - response.StateNodes = make([]StateNode, 0, len(payload.StateNodes)) - keyFilters := make([]common.Hash, len(stateFilter.Addresses)) - for i, addr := range stateFilter.Addresses { - keyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()) +// filterStateAndStorage filters state and storage nodes into the response according to the provided filters +func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error { + response.StateNodes = make([]StateNode, 0, len(payload.StateNodes)) + response.StorageNodes = make([]StorageNode, 0) + stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses)) + for i, addr := range stateFilter.Addresses { + stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()) + } + storageAddressFilters := make([]common.Hash, len(storageFilter.Addresses)) + for i, addr := range storageFilter.Addresses { + storageAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()) + } + storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys)) + for i, store := range storageFilter.StorageKeys { + storageKeyFilters[i] = common.HexToHash(store) + } + for _, stateNode := range payload.StateNodes { + if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) { + if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes { + cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256) + if err != nil { + return err + } + response.StateNodes = append(response.StateNodes, StateNode{ + StateLeafKey: stateNode.LeafKey, + Path: stateNode.Path, + IPLD: ipfs.BlockModel{ + Data: stateNode.Value, + CID: cid.String(), + }, + Type: stateNode.Type, + }) + } } - for _, stateNode := range payload.StateNodes { - if checkNodeKeys(keyFilters, stateNode.Key) { - if stateNode.Leaf || stateFilter.IntermediateNodes { - cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256) + if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) { + for _, storageNode := range payload.StorageNodes[crypto.Keccak256Hash(stateNode.Path)] { + if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) { + cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256) if err != nil { return err } - response.StateNodes = append(response.StateNodes, StateNode{ - StateTrieKey: stateNode.Key, + response.StorageNodes = append(response.StorageNodes, StorageNode{ + StateLeafKey: stateNode.LeafKey, + StorageLeafKey: storageNode.LeafKey, IPLD: ipfs.BlockModel{ - Data: stateNode.Value, + Data: storageNode.Value, CID: cid.String(), }, - Leaf: stateNode.Leaf, + Type: storageNode.Type, + Path: storageNode.Path, }) } } @@ -296,39 +324,3 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool { } return false } - -func (s *ResponseFilterer) filterStorage(storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error { - if !storageFilter.Off { - response.StorageNodes = make([]StorageNode, 0) - stateKeyFilters := make([]common.Hash, len(storageFilter.Addresses)) - for i, addr := range storageFilter.Addresses { - stateKeyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()) - } - storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys)) - for i, store := range storageFilter.StorageKeys { - storageKeyFilters[i] = common.HexToHash(store) - } - for stateKey, storageNodes := range payload.StorageNodes { - if checkNodeKeys(stateKeyFilters, stateKey) { - for _, storageNode := range storageNodes { - if checkNodeKeys(storageKeyFilters, storageNode.Key) { - cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256) - if err != nil { - return err - } - response.StorageNodes = append(response.StorageNodes, StorageNode{ - StateTrieKey: stateKey, - StorageTrieKey: storageNode.Key, - IPLD: ipfs.BlockModel{ - Data: storageNode.Value, - CID: cid.String(), - }, - Leaf: storageNode.Leaf, - }) - } - } - } - } - } - return nil -} diff --git a/pkg/super_node/eth/filterer_test.go b/pkg/super_node/eth/filterer_test.go index 98ef3c09..568f0e2e 100644 --- a/pkg/super_node/eth/filterer_test.go +++ b/pkg/super_node/eth/filterer_test.go @@ -17,11 +17,13 @@ package eth_test import ( - "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "bytes" + "github.com/ethereum/go-ethereum/statediff" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/ipfs" "github.com/vulcanize/vulcanizedb/pkg/super_node/eth" "github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks" "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" @@ -54,19 +56,19 @@ var _ = Describe("Filterer", func() { Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue()) Expect(len(iplds.StateNodes)).To(Equal(2)) for _, stateNode := range iplds.StateNodes { - Expect(stateNode.Leaf).To(BeTrue()) - if stateNode.StateTrieKey == mocks.ContractLeafKey { - Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{ - Data: mocks.State1IPLD.RawData(), - CID: mocks.State1IPLD.Cid().String(), - })) - } - if stateNode.StateTrieKey == mocks.AnotherContractLeafKey { + Expect(stateNode.Type).To(Equal(statediff.Leaf)) + if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.AccountLeafKey) { Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{ Data: mocks.State2IPLD.RawData(), CID: mocks.State2IPLD.Cid().String(), })) } + if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.ContractLeafKey) { + Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{ + Data: mocks.State1IPLD.RawData(), + CID: mocks.State1IPLD.Cid().String(), + })) + } } Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes)) }) @@ -180,10 +182,10 @@ var _ = Describe("Filterer", func() { Expect(len(iplds7.StorageNodes)).To(Equal(0)) Expect(len(iplds7.Receipts)).To(Equal(0)) Expect(len(iplds7.StateNodes)).To(Equal(1)) - Expect(iplds7.StateNodes[0].StateTrieKey).To(Equal(mocks.ContractLeafKey)) + Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks.AccountLeafKey)) Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{ - Data: mocks.State1IPLD.RawData(), - CID: mocks.State1IPLD.Cid().String(), + Data: mocks.State2IPLD.RawData(), + CID: mocks.State2IPLD.Cid().String(), })) payload8, err := filterer.Filter(rctTopicsAndContractFilterFail, mocks.MockConvertedPayload) diff --git a/pkg/super_node/eth/helpers.go b/pkg/super_node/eth/helpers.go new file mode 100644 index 00000000..c46cf430 --- /dev/null +++ b/pkg/super_node/eth/helpers.go @@ -0,0 +1,45 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package eth + +import "github.com/ethereum/go-ethereum/statediff" + +func ResolveFromNodeType(nodeType statediff.NodeType) int { + switch nodeType { + case statediff.Branch: + return 0 + case statediff.Extension: + return 1 + case statediff.Leaf: + return 2 + default: + return -1 + } +} + +func ResolveToNodeType(nodeType int) statediff.NodeType { + switch nodeType { + case 0: + return statediff.Branch + case 1: + return statediff.Extension + case 2: + return statediff.Leaf + default: + return statediff.Unknown + } +} diff --git a/pkg/super_node/eth/indexer.go b/pkg/super_node/eth/indexer.go index 77637677..f9755ff3 100644 --- a/pkg/super_node/eth/indexer.go +++ b/pkg/super_node/eth/indexer.go @@ -19,6 +19,8 @@ package eth import ( "fmt" + "github.com/ethereum/go-ethereum/crypto" + "github.com/vulcanize/vulcanizedb/pkg/super_node/shared" "github.com/ethereum/go-ethereum/common" @@ -129,14 +131,14 @@ func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID in func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error { for _, stateCID := range payload.StateNodeCIDs { var stateID int64 - err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4) - ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4) + err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (header_id, state_path) DO UPDATE SET (state_key, cid, node_type) = ($2, $3, $5) RETURNING id`, - headerID, stateCID.StateKey, stateCID.CID, stateCID.Leaf).Scan(&stateID) + headerID, stateCID.StateKey, stateCID.CID, stateCID.Path, stateCID.NodeType).Scan(&stateID) if err != nil { return err } - for _, storageCID := range payload.StorageNodeCIDs[common.HexToHash(stateCID.StateKey)] { + for _, storageCID := range payload.StorageNodeCIDs[crypto.Keccak256Hash(stateCID.Path)] { if err := in.indexStorageCID(tx, storageCID, stateID); err != nil { return err } @@ -146,8 +148,8 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, } func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error { - _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4) - ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`, - stateID, storageCID.StorageKey, storageCID.CID, storageCID.Leaf) + _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_key, cid, node_type) = ($2, $3, $5)`, + stateID, storageCID.StorageKey, storageCID.CID, storageCID.Path, storageCID.NodeType) return err } diff --git a/pkg/super_node/eth/indexer_test.go b/pkg/super_node/eth/indexer_test.go index 4d61d298..e061786e 100644 --- a/pkg/super_node/eth/indexer_test.go +++ b/pkg/super_node/eth/indexer_test.go @@ -17,6 +17,7 @@ package eth_test import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -45,13 +46,15 @@ var _ = Describe("Indexer", func() { It("Indexes CIDs and related metadata into vulcanizedb", func() { err = repo.Index(mocks.MockCIDPayload) Expect(err).ToNot(HaveOccurred()) - pgStr := `SELECT cid, td, reward FROM eth.header_cids + pgStr := `SELECT cid, td, reward, id + FROM eth.header_cids WHERE block_number = $1` // check header was properly indexed type res struct { CID string TD string Reward string + ID int } headers := new(res) err = db.QueryRowx(pgStr, 1).StructScan(headers) @@ -81,24 +84,28 @@ var _ = Describe("Indexer", func() { Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue()) // check that state nodes were properly indexed stateNodes := make([]eth.StateNodeModel, 0) - pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.node_type, state_cids.state_path, state_cids.header_id + FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) WHERE header_cids.block_number = $1` err = db.Select(&stateNodes, pgStr, 1) Expect(err).ToNot(HaveOccurred()) Expect(len(stateNodes)).To(Equal(2)) for _, stateNode := range stateNodes { if stateNode.CID == mocks.State1CID.String() { - Expect(stateNode.Leaf).To(Equal(true)) - Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex())) + Expect(stateNode.NodeType).To(Equal(2)) + Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex())) + Expect(stateNode.Path).To(Equal([]byte{'\x06'})) } if stateNode.CID == mocks.State2CID.String() { - Expect(stateNode.Leaf).To(Equal(true)) - Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex())) + Expect(stateNode.NodeType).To(Equal(2)) + Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex())) + Expect(stateNode.Path).To(Equal([]byte{'\x0c'})) } } // check that storage nodes were properly indexed storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0) - pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.leaf FROM eth.storage_cids, eth.state_cids, eth.header_cids + pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.node_type, storage_cids.storage_path + FROM eth.storage_cids, eth.state_cids, eth.header_cids WHERE storage_cids.state_id = state_cids.id AND state_cids.header_id = header_cids.id AND header_cids.block_number = $1` @@ -107,9 +114,10 @@ var _ = Describe("Indexer", func() { Expect(len(storageNodes)).To(Equal(1)) Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{ CID: mocks.StorageCID.String(), - Leaf: true, - StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001", - StateKey: mocks.ContractLeafKey.Hex(), + NodeType: 2, + StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{}, })) }) }) diff --git a/pkg/super_node/eth/ipld_fetcher.go b/pkg/super_node/eth/ipld_fetcher.go index c14fc732..64596b3a 100644 --- a/pkg/super_node/eth/ipld_fetcher.go +++ b/pkg/super_node/eth/ipld_fetcher.go @@ -216,8 +216,9 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) { Data: state.RawData(), CID: state.Cid().String(), }, - StateTrieKey: common.HexToHash(stateNode.StateKey), - Leaf: stateNode.Leaf, + StateLeafKey: common.HexToHash(stateNode.StateKey), + Type: ResolveToNodeType(stateNode.NodeType), + Path: stateNode.Path, } } return stateNodes, nil @@ -246,9 +247,10 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora Data: storage.RawData(), CID: storage.Cid().String(), }, - StateTrieKey: common.HexToHash(storageNode.StateKey), - StorageTrieKey: common.HexToHash(storageNode.StorageKey), - Leaf: storageNode.Leaf, + StateLeafKey: common.HexToHash(storageNode.StateKey), + StorageLeafKey: common.HexToHash(storageNode.StorageKey), + Type: ResolveToNodeType(storageNode.NodeType), + Path: storageNode.Path, } } return storageNodes, nil diff --git a/pkg/super_node/eth/ipld_fetcher_test.go b/pkg/super_node/eth/ipld_fetcher_test.go index e76480fb..5192685d 100644 --- a/pkg/super_node/eth/ipld_fetcher_test.go +++ b/pkg/super_node/eth/ipld_fetcher_test.go @@ -20,13 +20,13 @@ import ( "bytes" "math/big" - "github.com/vulcanize/vulcanizedb/pkg/ipfs" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff" "github.com/ipfs/go-block-format" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/ipfs" "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" "github.com/vulcanize/vulcanizedb/pkg/super_node/eth" ) @@ -71,18 +71,18 @@ var ( }, StateNodes: []eth.StateNodeModel{{ CID: mockStateBlock.Cid().String(), - Leaf: true, + NodeType: 2, StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", }}, StorageNodes: []eth.StorageNodeWithStateKeyModel{{ CID: mockStorageBlock1.Cid().String(), - Leaf: true, + NodeType: 2, StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", StorageKey: "0000000000000000000000000000000000000000000000000000000000000001", }, { CID: mockStorageBlock2.Cid().String(), - Leaf: true, + NodeType: 2, StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", StorageKey: "0000000000000000000000000000000000000000000000000000000000000002", }}, @@ -127,23 +127,23 @@ var _ = Describe("Fetcher", func() { CID: mockReceiptBlock.Cid().String(), })) Expect(len(iplds.StateNodes)).To(Equal(1)) - Expect(iplds.StateNodes[0].StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"))) - Expect(iplds.StateNodes[0].Leaf).To(BeTrue()) + Expect(iplds.StateNodes[0].StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"))) + Expect(iplds.StateNodes[0].Type).To(Equal(statediff.Leaf)) Expect(iplds.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{ Data: mockStateBlock.RawData(), CID: mockStateBlock.Cid().String(), })) Expect(len(iplds.StorageNodes)).To(Equal(2)) for _, storage := range iplds.StorageNodes { - Expect(storage.Leaf).To(BeTrue()) - Expect(storage.StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"))) - if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) { + Expect(storage.Type).To(Equal(statediff.Leaf)) + Expect(storage.StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"))) + if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) { Expect(storage.IPLD).To(Equal(ipfs.BlockModel{ Data: mockStorageBlock1.RawData(), CID: mockStorageBlock1.Cid().String(), })) } - if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) { + if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) { Expect(storage.IPLD).To(Equal(ipfs.BlockModel{ Data: mockStorageBlock2.RawData(), CID: mockStorageBlock2.Cid().String(), diff --git a/pkg/super_node/eth/mocks/test_data.go b/pkg/super_node/eth/mocks/test_data.go index 80065e7b..10d74ae7 100644 --- a/pkg/super_node/eth/mocks/test_data.go +++ b/pkg/super_node/eth/mocks/test_data.go @@ -21,12 +21,6 @@ import ( "crypto/elliptic" "crypto/rand" "math/big" - rand2 "math/rand" - - "github.com/vulcanize/vulcanizedb/pkg/ipfs" - - "github.com/multiformats/go-multihash" - "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" @@ -35,9 +29,13 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" "github.com/ipfs/go-block-format" + "github.com/multiformats/go-multihash" log "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld" "github.com/vulcanize/vulcanizedb/pkg/super_node/eth" eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth" ) @@ -79,11 +77,10 @@ var ( Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256) Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256) Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256) - State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256) - State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256) - StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256) - - MockTrxMeta = []eth.TxModel{ + State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) + State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) + StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) + MockTrxMeta = []eth.TxModel{ { CID: "", // This is empty until we go to publish to ipfs Src: senderAddr.Hex(), @@ -161,51 +158,71 @@ var ( } // statediff data - CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") - NonceValue = rand2.Uint64() - anotherNonceValue = rand2.Uint64() - BalanceValue = rand2.Int63() - anotherBalanceValue = rand2.Int63() - ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() - StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() - StorageValue = common.Hex2Bytes("0x03") - storage = []statediff.StorageDiff{{ - Key: StorageKey, - Value: StorageValue, - Path: StoragePath, - Proof: [][]byte{}, - Leaf: true, - }} - emptyStorage = make([]statediff.StorageDiff, 0) - ContractLeafKey = crypto.Keccak256Hash(Address.Bytes()) - AnotherContractLeafKey = crypto.Keccak256Hash(AnotherAddress.Bytes()) - testAccount = state.Account{ - Nonce: NonceValue, - Balance: big.NewInt(BalanceValue), - Root: ContractRoot, - CodeHash: CodeHash, - } - anotherTestAccount = state.Account{ - Nonce: anotherNonceValue, - Balance: big.NewInt(anotherBalanceValue), - Root: common.HexToHash("0x"), - CodeHash: nil, - } - ValueBytes, _ = rlp.EncodeToBytes(testAccount) - AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount) - CreatedAccountDiffs = []statediff.AccountDiff{ + storageLocation = common.HexToHash("0") + StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() + StorageValue = common.Hex2Bytes("01") + StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") + StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + StoragePartialPath, + StorageValue, + }) + + nonce1 = uint64(1) + contractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + contractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea") + contractPathHash = crypto.Keccak256Hash([]byte{'\x06'}) + ContractAddress = common.HexToAddress("0x703c4b2bD70c169f5717101CaeE543299Fc946C7") + ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress) + ContractAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: contractCodeHash.Bytes(), + Root: common.HexToHash(contractRoot), + }) + ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45") + ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + ContractPartialPath, + ContractAccount, + }) + + nonce0 = uint64(0) + accountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + accountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e") + AccountLeafKey = testhelpers.Account2LeafKey + Account, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(1000), + CodeHash: accountCodeHash.Bytes(), + Root: common.HexToHash(accountRoot), + }) + AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45") + AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + AccountPartialPath, + Account, + }) + + CreatedAccountDiffs = []statediff.AccountDiff{ { - Key: ContractLeafKey.Bytes(), - Value: ValueBytes, - Storage: storage, - Leaf: true, + Path: []byte{'\x06'}, + NodeType: statediff.Leaf, + LeafKey: ContractLeafKey, + NodeValue: ContractLeafNode, + Storage: []statediff.StorageDiff{ + { + Path: []byte{}, + NodeType: statediff.Leaf, + LeafKey: StorageLeafKey, + NodeValue: StorageLeafNode, + }, + }, }, { - Key: AnotherContractLeafKey.Bytes(), - Value: AnotherValueBytes, - Storage: emptyStorage, - Leaf: true, + Path: []byte{'\x0c'}, + NodeType: statediff.Leaf, + LeafKey: AccountLeafKey, + NodeValue: AccountLeafNode, + Storage: []statediff.StorageDiff{}, }, } @@ -217,34 +234,39 @@ var ( MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff) MockStateNodes = []eth.TrieNode{ { - Key: ContractLeafKey, - Value: ValueBytes, - Leaf: true, + LeafKey: common.BytesToHash(ContractLeafKey), + Path: []byte{'\x06'}, + Value: ContractLeafNode, + Type: statediff.Leaf, }, { - Key: AnotherContractLeafKey, - Value: AnotherValueBytes, - Leaf: true, + LeafKey: common.BytesToHash(AccountLeafKey), + Path: []byte{'\x0c'}, + Value: AccountLeafNode, + Type: statediff.Leaf, }, } MockStateMetaPostPublish = []eth.StateNodeModel{ { CID: State1CID.String(), - Leaf: true, - StateKey: ContractLeafKey.String(), + Path: []byte{'\x06'}, + NodeType: 2, + StateKey: common.BytesToHash(ContractLeafKey).Hex(), }, { CID: State2CID.String(), - Leaf: true, - StateKey: AnotherContractLeafKey.String(), + Path: []byte{'\x0c'}, + NodeType: 2, + StateKey: common.BytesToHash(AccountLeafKey).Hex(), }, } MockStorageNodes = map[common.Hash][]eth.TrieNode{ - ContractLeafKey: { + contractPathHash: { { - Key: common.BytesToHash(StorageKey), - Value: StorageValue, - Leaf: true, + LeafKey: common.BytesToHash(StorageLeafKey), + Value: StorageLeafNode, + Type: statediff.Leaf, + Path: []byte{}, }, }, } @@ -284,11 +306,12 @@ var ( }, StateNodeCIDs: MockStateMetaPostPublish, StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{ - ContractLeafKey: { + contractPathHash: { { CID: StorageCID.String(), - StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001", - Leaf: true, + Path: []byte{}, + StorageKey: common.BytesToHash(StorageLeafKey).Hex(), + NodeType: 2, }, }, }, @@ -310,10 +333,11 @@ var ( StateNodes: MockStateMetaPostPublish, StorageNodes: []eth.StorageNodeWithStateKeyModel{ { + Path: []byte{}, CID: StorageCID.String(), - Leaf: true, - StateKey: ContractLeafKey.Hex(), - StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001", + NodeType: 2, + StateKey: common.BytesToHash(ContractLeafKey).Hex(), + StorageKey: common.BytesToHash(StorageLeafKey).Hex(), }, }, } @@ -323,9 +347,9 @@ var ( Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID) Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID) Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID) - State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, State1CID) - State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, State2CID) - StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, StorageCID) + State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID) + State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID) + StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID) MockIPLDs = eth.IPLDs{ BlockNumber: big.NewInt(1), @@ -355,31 +379,34 @@ var ( }, StateNodes: []eth2.StateNode{ { - StateTrieKey: ContractLeafKey, - Leaf: true, + StateLeafKey: common.BytesToHash(ContractLeafKey), + Type: statediff.Leaf, IPLD: ipfs.BlockModel{ Data: State1IPLD.RawData(), CID: State1IPLD.Cid().String(), }, + Path: []byte{'\x06'}, }, { - StateTrieKey: AnotherContractLeafKey, - Leaf: true, + StateLeafKey: common.BytesToHash(AccountLeafKey), + Type: statediff.Leaf, IPLD: ipfs.BlockModel{ Data: State2IPLD.RawData(), CID: State2IPLD.Cid().String(), }, + Path: []byte{'\x0c'}, }, }, StorageNodes: []eth2.StorageNode{ { - StateTrieKey: ContractLeafKey, - StorageTrieKey: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"), - Leaf: true, + StateLeafKey: common.BytesToHash(ContractLeafKey), + StorageLeafKey: common.BytesToHash(StorageLeafKey), + Type: statediff.Leaf, IPLD: ipfs.BlockModel{ Data: StorageIPLD.RawData(), CID: StorageIPLD.Cid().String(), }, + Path: []byte{}, }, }, } diff --git a/pkg/super_node/eth/models.go b/pkg/super_node/eth/models.go index 7846bcc3..e0fbc72a 100644 --- a/pkg/super_node/eth/models.go +++ b/pkg/super_node/eth/models.go @@ -67,8 +67,9 @@ type ReceiptModel struct { type StateNodeModel struct { ID int64 `db:"id"` HeaderID int64 `db:"header_id"` + Path []byte `db:"state_path"` StateKey string `db:"state_key"` - Leaf bool `db:"leaf"` + NodeType int `db:"node_type"` CID string `db:"cid"` } @@ -76,8 +77,9 @@ type StateNodeModel struct { type StorageNodeModel struct { ID int64 `db:"id"` StateID int64 `db:"state_id"` + Path []byte `db:"storage_path"` StorageKey string `db:"storage_key"` - Leaf bool `db:"leaf"` + NodeType int `db:"node_type"` CID string `db:"cid"` } @@ -85,8 +87,9 @@ type StorageNodeModel struct { type StorageNodeWithStateKeyModel struct { ID int64 `db:"id"` StateID int64 `db:"state_id"` + Path []byte `db:"storage_path"` StateKey string `db:"state_key"` StorageKey string `db:"storage_key"` - Leaf bool `db:"leaf"` + NodeType int `db:"node_type"` CID string `db:"cid"` } diff --git a/pkg/super_node/eth/publisher.go b/pkg/super_node/eth/publisher.go index ce4d5da3..60b0d76e 100644 --- a/pkg/super_node/eth/publisher.go +++ b/pkg/super_node/eth/publisher.go @@ -185,9 +185,10 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM return nil, err } stateNodeCids = append(stateNodeCids, StateNodeModel{ - StateKey: node.Key.String(), + Path: node.Path, + StateKey: node.LeafKey.String(), CID: cids[0], - Leaf: node.Leaf, + NodeType: ResolveFromNodeType(node.Type), }) } return stateNodeCids, nil @@ -195,18 +196,19 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) { storageLeafCids := make(map[common.Hash][]StorageNodeModel) - for addrKey, storageTrie := range storageNodes { - storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie)) + for pathHash, storageTrie := range storageNodes { + storageLeafCids[pathHash] = make([]StorageNodeModel, 0, len(storageTrie)) for _, node := range storageTrie { cids, err := pub.StoragePutter.DagPut(node.Value) if err != nil { return nil, err } - // Map storage node cids to their state key hashes - storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{ - StorageKey: node.Key.Hex(), + // Map storage node cids to their path hashes + storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{ + Path: node.Path, + StorageKey: node.LeafKey.Hex(), CID: cids[0], - Leaf: node.Leaf, + NodeType: ResolveFromNodeType(node.Type), }) } } diff --git a/pkg/super_node/eth/retriever.go b/pkg/super_node/eth/retriever.go index c56dc514..16d4a191 100644 --- a/pkg/super_node/eth/retriever.go +++ b/pkg/super_node/eth/retriever.go @@ -391,7 +391,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, log.Debug("retrieving state cids for header id ", headerID) args := make([]interface{}, 0, 2) pgStr := `SELECT state_cids.id, state_cids.header_id, - state_cids.state_key, state_cids.leaf, state_cids.cid + state_cids.state_key, state_cids.node_type, state_cids.cid, state_cids.state_path FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) WHERE header_cids.id = $1` args = append(args, headerID) @@ -405,7 +405,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, args = append(args, pq.Array(keys)) } if !stateFilter.IntermediateNodes { - pgStr += ` AND state_cids.leaf = TRUE` + pgStr += ` AND state_cids.node_type = 2` } stateNodeCIDs := make([]StateNodeModel, 0) return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...) @@ -416,7 +416,8 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF log.Debug("retrieving storage cids for header id ", headerID) args := make([]interface{}, 0, 3) pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key, - storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM eth.storage_cids, eth.state_cids, eth.header_cids + storage_cids.node_type, storage_cids.cid, storage_cids.storage_path, state_cids.state_key + FROM eth.storage_cids, eth.state_cids, eth.header_cids WHERE storage_cids.state_id = state_cids.id AND state_cids.header_id = header_cids.id AND header_cids.id = $1` @@ -437,7 +438,7 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF args = append(args, pq.Array(storageFilter.StorageKeys)) } if !storageFilter.IntermediateNodes { - pgStr += ` AND storage_cids.leaf = TRUE` + pgStr += ` AND storage_cids.node_type = 2` } storageNodeCIDs := make([]StorageNodeWithStateKeyModel, 0) return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...) diff --git a/pkg/super_node/eth/retriever_test.go b/pkg/super_node/eth/retriever_test.go index 45813569..d2de0e46 100644 --- a/pkg/super_node/eth/retriever_test.go +++ b/pkg/super_node/eth/retriever_test.go @@ -19,6 +19,8 @@ package eth_test import ( "math/big" + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -196,7 +198,7 @@ var ( Off: true, }, StateFilter: eth.StateFilter{ - Addresses: []string{mocks.Address.Hex()}, + Addresses: []string{mocks.AccountAddresss.Hex()}, }, StorageFilter: eth.StorageFilter{ Off: true, @@ -247,12 +249,14 @@ var _ = Describe("Retriever", func() { Expect(len(cidWrapper.StateNodes)).To(Equal(2)) for _, stateNode := range cidWrapper.StateNodes { if stateNode.CID == mocks.State1CID.String() { - Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex())) - Expect(stateNode.Leaf).To(Equal(true)) + Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex())) + Expect(stateNode.NodeType).To(Equal(2)) + Expect(stateNode.Path).To(Equal([]byte{'\x06'})) } if stateNode.CID == mocks.State2CID.String() { - Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex())) - Expect(stateNode.Leaf).To(Equal(true)) + Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex())) + Expect(stateNode.NodeType).To(Equal(2)) + Expect(stateNode.Path).To(Equal([]byte{'\x0c'})) } } Expect(len(cidWrapper.StorageNodes)).To(Equal(1)) @@ -384,9 +388,10 @@ var _ = Describe("Retriever", func() { Expect(cidWrapper7.StateNodes[0]).To(Equal(eth.StateNodeModel{ ID: cidWrapper7.StateNodes[0].ID, HeaderID: cidWrapper7.StateNodes[0].HeaderID, - Leaf: true, - StateKey: mocks.ContractLeafKey.Hex(), - CID: mocks.State1CID.String(), + NodeType: 2, + StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(), + CID: mocks.State2CID.String(), + Path: []byte{'\x0c'}, })) _, empty, err = retriever.Retrieve(rctTopicsAndContractFilterFail, 1) diff --git a/pkg/super_node/eth/types.go b/pkg/super_node/eth/types.go index cd295368..904cc7c3 100644 --- a/pkg/super_node/eth/types.go +++ b/pkg/super_node/eth/types.go @@ -19,6 +19,8 @@ package eth import ( "math/big" + "github.com/ethereum/go-ethereum/statediff" + "github.com/vulcanize/vulcanizedb/pkg/ipfs" "github.com/ethereum/go-ethereum/common" @@ -45,9 +47,10 @@ func (i ConvertedPayload) Height() int64 { // Trie struct used to flag node as leaf or not type TrieNode struct { - Key common.Hash - Value []byte - Leaf bool + Path []byte + LeafKey common.Hash + Value []byte + Type statediff.NodeType } // CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres @@ -94,14 +97,16 @@ func (i IPLDs) Height() int64 { } type StateNode struct { - StateTrieKey common.Hash + Type statediff.NodeType + StateLeafKey common.Hash + Path []byte IPLD ipfs.BlockModel - Leaf bool } type StorageNode struct { - StateTrieKey common.Hash - StorageTrieKey common.Hash + Type statediff.NodeType + StateLeafKey common.Hash + StorageLeafKey common.Hash + Path []byte IPLD ipfs.BlockModel - Leaf bool } diff --git a/pkg/watcher/eth/converter.go b/pkg/watcher/eth/converter.go index 91a7ccf4..98e4702b 100644 --- a/pkg/watcher/eth/converter.go +++ b/pkg/watcher/eth/converter.go @@ -150,16 +150,16 @@ func (pc *WatcherConverter) Convert(ethIPLDs eth.IPLDs) (*eth.CIDPayload, error) for i, stateIPLD := range ethIPLDs.StateNodes { cids.StateNodeCIDs[i] = eth.StateNodeModel{ CID: stateIPLD.IPLD.CID, - Leaf: stateIPLD.Leaf, - StateKey: stateIPLD.StateTrieKey.String(), + NodeType: eth.ResolveFromNodeType(stateIPLD.Type), + StateKey: stateIPLD.StateLeafKey.String(), } } // Storage data for _, storageIPLD := range ethIPLDs.StorageNodes { - cids.StorageNodeCIDs[storageIPLD.StateTrieKey] = append(cids.StorageNodeCIDs[storageIPLD.StateTrieKey], eth.StorageNodeModel{ + cids.StorageNodeCIDs[storageIPLD.StateLeafKey] = append(cids.StorageNodeCIDs[storageIPLD.StateLeafKey], eth.StorageNodeModel{ CID: storageIPLD.IPLD.CID, - Leaf: storageIPLD.Leaf, - StorageKey: storageIPLD.StorageTrieKey.String(), + NodeType: eth.ResolveFromNodeType(storageIPLD.Type), + StorageKey: storageIPLD.StorageLeafKey.String(), }) } return cids, nil