cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

adhjustments to work with statediffing geth v1.10-alpha.2

Browse Source
This commit is contained in:
Ian Norden 2020-03-11 13:41:59 -05:00
parent be875c0100
commit e5c5422edc
26 changed files with 547 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/streamEthSubscribe.go
View File

@ -139,12 +139,12 @@ func streamEthSubscription() {
continue
}
fmt.Printf("Account for key %s, and root %s, with balance %s\n",
stateNode.StateTrieKey.Hex(), acct.Root.Hex(), acct.Balance.String())
stateNode.StateLeafKey.Hex(), acct.Root.Hex(), acct.Balance.String())
fmt.Printf("state account: %+v\n", acct)
}
for _, storageNode := range ethData.StorageNodes {
fmt.Printf("Storage for state key %s ", storageNode.StateTrieKey.Hex())
fmt.Printf("with storage key %s\n", storageNode.StorageTrieKey.Hex())
fmt.Printf("Storage for state key %s ", storageNode.StateLeafKey.Hex())
fmt.Printf("with storage key %s\n", storageNode.StorageLeafKey.Hex())
var i []interface{}
err := rlp.DecodeBytes(storageNode.IPLD.Data, &i)
if err != nil {
@ -158,7 +158,7 @@ func streamEthSubscription() {
continue
}
fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
storageNode.StorageTrieKey.Hex(), common.BytesToHash(valueBytes).Hex())
storageNode.StorageLeafKey.Hex(), common.BytesToHash(valueBytes).Hex())
}
}
case err = <-sub.Err():

37
db/migrations/00027_update_state_cids.sql Normal file
View File

@ -0,0 +1,37 @@
-- +goose Up
ALTER TABLE eth.state_cids
ADD COLUMN state_path BYTEA;
ALTER TABLE eth.state_cids
DROP COLUMN leaf;
ALTER TABLE eth.state_cids
ADD COLUMN node_type INTEGER NOT NULL;
ALTER TABLE eth.state_cids
ALTER COLUMN state_key DROP NOT NULL;
ALTER TABLE eth.state_cids
DROP CONSTRAINT state_cids_header_id_state_key_key;
ALTER TABLE eth.state_cids
ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path);
-- +goose Down
ALTER TABLE eth.state_cids
ADD CONSTRAINT state_cids_header_id_state_key_key UNIQUE (header_id, state_key);
ALTER TABLE eth.state_cids
DROP CONSTRAINT state_cids_header_id_state_path_key;
ALTER TABLE eth.state_cids
ALTER COLUMN state_key SET NOT NULL;
ALTER TABLE eth.state_cids
DROP COLUMN node_type;
ALTER TABLE eth.state_cids
ADD COLUMN leaf BOOLEAN NOT NULL;
ALTER TABLE eth.state_cids
DROP COLUMN state_path;

37
db/migrations/00028_update_storage_cids.sql Normal file
View File

@ -0,0 +1,37 @@
-- +goose Up
ALTER TABLE eth.storage_cids
ADD COLUMN storage_path BYTEA;
ALTER TABLE eth.storage_cids
DROP COLUMN leaf;
ALTER TABLE eth.storage_cids
ADD COLUMN node_type INTEGER NOT NULL;
ALTER TABLE eth.storage_cids
ALTER COLUMN storage_key DROP NOT NULL;
ALTER TABLE eth.storage_cids
DROP CONSTRAINT storage_cids_state_id_storage_key_key;
ALTER TABLE eth.storage_cids
ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path);
-- +goose Down
ALTER TABLE eth.storage_cids
DROP CONSTRAINT storage_cids_state_id_storage_path_key;
ALTER TABLE eth.storage_cids
ADD CONSTRAINT storage_cids_state_id_storage_key_key UNIQUE (state_id, storage_key);
ALTER TABLE eth.storage_cids
ALTER COLUMN storage_key SET NOT NULL;
ALTER TABLE eth.storage_cids
DROP COLUMN node_type;
ALTER TABLE eth.storage_cids
ADD COLUMN leaf BOOLEAN NOT NULL;
ALTER TABLE eth.storage_cids
DROP COLUMN storage_path;

89
db/schema.sql
View File

@ -3,7 +3,7 @@
--
-- Dumped from database version 10.10
-- Dumped by pg_dump version 12.1
-- Dumped by pg_dump version 10.10
SET statement_timeout = 0;
SET lock_timeout = 0;
@ -30,8 +30,24 @@ CREATE SCHEMA btc;
CREATE SCHEMA eth;
--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
--
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
SET default_tablespace = '';
SET default_with_oids = false;
--
-- Name: header_cids; Type: TABLE; Schema: btc; Owner: -
--
@ -48,6 +64,13 @@ CREATE TABLE btc.header_cids (
);
--
-- Name: TABLE header_cids; Type: COMMENT; Schema: btc; Owner: -
--
COMMENT ON TABLE btc.header_cids IS '@name BtcHeaderCids';
--
-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
--
@ -79,6 +102,13 @@ CREATE TABLE btc.queue_data (
);
--
-- Name: TABLE queue_data; Type: COMMENT; Schema: btc; Owner: -
--
COMMENT ON TABLE btc.queue_data IS '@name BtcQueueData';
--
-- Name: queue_data_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
--
@ -114,6 +144,13 @@ CREATE TABLE btc.transaction_cids (
);
--
-- Name: TABLE transaction_cids; Type: COMMENT; Schema: btc; Owner: -
--
COMMENT ON TABLE btc.transaction_cids IS '@name BtcTransactionCids';
--
-- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: -
--
@ -221,6 +258,13 @@ CREATE TABLE eth.header_cids (
);
--
-- Name: TABLE header_cids; Type: COMMENT; Schema: eth; Owner: -
--
COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids';
--
-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
--
@ -252,6 +296,13 @@ CREATE TABLE eth.queue_data (
);
--
-- Name: TABLE queue_data; Type: COMMENT; Schema: eth; Owner: -
--
COMMENT ON TABLE eth.queue_data IS '@name EthQueueData';
--
-- Name: queue_data_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
--
@ -315,9 +366,10 @@ ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id;
CREATE TABLE eth.state_cids (
id integer NOT NULL,
header_id integer NOT NULL,
state_key character varying(66) NOT NULL,
leaf boolean NOT NULL,
cid text NOT NULL
state_key character varying(66),
cid text NOT NULL,
state_path bytea,
node_type integer NOT NULL
);
@ -348,9 +400,10 @@ ALTER SEQUENCE eth.state_cids_id_seq OWNED BY eth.state_cids.id;
CREATE TABLE eth.storage_cids (
id integer NOT NULL,
state_id integer NOT NULL,
storage_key character varying(66) NOT NULL,
leaf boolean NOT NULL,
cid text NOT NULL
storage_key character varying(66),
cid text NOT NULL,
storage_path bytea,
node_type integer NOT NULL
);
@ -389,6 +442,13 @@ CREATE TABLE eth.transaction_cids (
);
--
-- Name: TABLE transaction_cids; Type: COMMENT; Schema: eth; Owner: -
--
COMMENT ON TABLE eth.transaction_cids IS '@name EthTransactionCids';
--
-- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
--
@ -713,6 +773,13 @@ CREATE TABLE public.nodes (
);
--
-- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
--
COMMENT ON TABLE public.nodes IS '@name NodeIDs';
--
-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@ -1110,11 +1177,11 @@ ALTER TABLE ONLY eth.receipt_cids
--
-- Name: state_cids state_cids_header_id_state_key_key; Type: CONSTRAINT; Schema: eth; Owner: -
-- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.state_cids
ADD CONSTRAINT state_cids_header_id_state_key_key UNIQUE (header_id, state_key);
ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path);
--
@ -1134,11 +1201,11 @@ ALTER TABLE ONLY eth.storage_cids
--
-- Name: storage_cids storage_cids_state_id_storage_key_key; Type: CONSTRAINT; Schema: eth; Owner: -
-- Name: storage_cids storage_cids_state_id_storage_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.storage_cids
ADD CONSTRAINT storage_cids_state_id_storage_key_key UNIQUE (state_id, storage_key);
ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path);
--

2
go.mod
View File

@ -97,4 +97,4 @@ replace github.com/ipfs/go-ipfs v0.4.22 => github.com/vulcanize/go-ipfs v0.4.22-
replace github.com/ipfs/go-ipfs-config v0.0.3 => github.com/vulcanize/go-ipfs-config v0.0.8-alpha
replace github.com/ethereum/go-ethereum v1.9.1 => github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8
replace github.com/ethereum/go-ethereum v1.9.1 => github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290

7
go.sum
View File

@ -37,6 +37,7 @@ github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0=
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d h1:QgeLLoPD3kRVmeu/1al9iIpIANMi9O1zXFm8BnYGCJg=
@ -105,7 +106,9 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
@ -134,6 +137,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
@ -327,6 +331,7 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj
github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmoiron/sqlx v0.0.0-20190426154859-38398a30ed85 h1:+LZtdhpMITOXE+MztQPPcwUl+eqYjwlXXLHrd0yWlxw=
github.com/jmoiron/sqlx v0.0.0-20190426154859-38398a30ed85/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@ -723,6 +728,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8 h1:BHt0OW0rTgndFjSju7brF3dPceXWQuEV0IdtY8BjjT8=
github.com/vulcanize/go-ethereum v1.5.10-0.20200116224441-2a980ec3dcb8/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo=
github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290 h1:uMWt+x6JhVT7GyL983weZSxv1zDBxvGlI9HNkcTnUeg=
github.com/vulcanize/go-ethereum v1.5.10-0.20200311182536-d07dc803d290/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
github.com/vulcanize/go-ipfs v0.4.22-alpha h1:W+6njT14KWllMhABRFtPndqHw8SHCt5SqD4YX528kxM=
github.com/vulcanize/go-ipfs v0.4.22-alpha/go.mod h1:uaekWWeoaA0A9Dv1LObOKCSh9kIzTpZ5RbKW4g5CQHE=
github.com/vulcanize/go-ipfs-config v0.0.8-alpha h1:peaFvbEcPShF6ymOd8flqKkFz4YfcrNr/UOO7FmbWoQ=

4
libraries/shared/fetcher/geth_rpc_storage_fetcher.go
View File

@ -65,11 +65,11 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
accounts := utils.GetAccountsFromDiff(*stateDiff)
logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
for _, account := range accounts {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.Key).Hex()))
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex()))
for _, storage := range account.Storage {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
if formatErr != nil {
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.Key), "from account with key: ", common.BytesToHash(account.Key))
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey))
errs <- formatErr
continue
}

4
libraries/shared/storage/backfiller.go
View File

@ -127,11 +127,11 @@ func (bf *backFiller) backFillRange(blockHeights []uint64, diffChan chan utils.S
}
accounts := utils.GetAccountsFromDiff(*stateDiff)
for _, account := range accounts {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.Key).Hex()))
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex()))
for _, storage := range account.Storage {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
if formatErr != nil {
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.Key), "from account with key: ", common.BytesToHash(account.Key))
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey))
errChan <- formatErr
continue
}

6
libraries/shared/storage/utils/diff.go
View File

@ -59,16 +59,16 @@ func FromParityCsvRow(csvRow []string) (StorageDiffInput, error) {
func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiffInput, error) {
var decodedValue []byte
err := rlp.DecodeBytes(storage.Value, &decodedValue)
err := rlp.DecodeBytes(storage.NodeValue, &decodedValue)
if err != nil {
return StorageDiffInput{}, err
}
return StorageDiffInput{
HashedAddress: common.BytesToHash(account.Key),
HashedAddress: common.BytesToHash(account.LeafKey),
BlockHash: stateDiff.BlockHash,
BlockHeight: int(stateDiff.BlockNumber.Int64()),
StorageKey: common.BytesToHash(storage.Key),
StorageKey: common.BytesToHash(storage.LeafKey),
StorageValue: common.BytesToHash(decodedValue),
}, nil
}

16
libraries/shared/storage/utils/diff_test.go
View File

@ -67,7 +67,7 @@ var _ = Describe("Storage row parsing", func() {
Describe("FromGethStateDiff", func() {
var (
accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
accountDiff = statediff.AccountDiff{LeafKey: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
stateDiff = &statediff.StateDiff{
BlockNumber: big.NewInt(rand.Int63()),
BlockHash: fakes.FakeHash,
@ -80,19 +80,20 @@ var _ = Describe("Storage row parsing", func() {
Expect(encodeErr).NotTo(HaveOccurred())
storageDiff := statediff.StorageDiff{
Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
Value: storageValueRlp,
LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
NodeValue: storageValueRlp,
NodeType: statediff.Leaf,
}
result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
Expect(err).NotTo(HaveOccurred())
expectedAddress := common.BytesToHash(accountDiff.Key)
expectedAddress := common.BytesToHash(accountDiff.LeafKey)
Expect(result.HashedAddress).To(Equal(expectedAddress))
Expect(result.BlockHash).To(Equal(fakes.FakeHash))
expectedBlockHeight := int(stateDiff.BlockNumber.Int64())
Expect(result.BlockHeight).To(Equal(expectedBlockHeight))
expectedStorageKey := common.BytesToHash(storageDiff.Key)
expectedStorageKey := common.BytesToHash(storageDiff.LeafKey)
Expect(result.StorageKey).To(Equal(expectedStorageKey))
expectedStorageValue := common.BytesToHash(storageValueBytes)
Expect(result.StorageValue).To(Equal(expectedStorageValue))
@ -104,8 +105,9 @@ var _ = Describe("Storage row parsing", func() {
Expect(encodeErr).NotTo(HaveOccurred())
storageDiff := statediff.StorageDiff{
Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
Value: storageValueRlp,
LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
NodeValue: storageValueRlp,
NodeType: statediff.Leaf,
}
result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)

46
libraries/shared/test_data/statediff.go
View File

@ -41,22 +41,24 @@ var (
SmallStorageValue = common.Hex2Bytes("03")
SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue)
storageWithSmallValue = []statediff.StorageDiff{{
Key: StorageKey,
Value: SmallStorageValueRlp,
Path: StoragePath,
Proof: [][]byte{},
LeafKey: StorageKey,
NodeValue: SmallStorageValueRlp,
NodeType: statediff.Leaf,
Path: StoragePath,
}}
LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000")
LargeStorageValueRlp, _ = rlp.EncodeToBytes(LargeStorageValue)
storageWithLargeValue = []statediff.StorageDiff{{
Key: StorageKey,
Value: LargeStorageValueRlp,
Path: StoragePath,
Proof: [][]byte{},
LeafKey: StorageKey,
NodeValue: LargeStorageValueRlp,
Path: StoragePath,
NodeType: statediff.Leaf,
}}
StorageWithBadValue = statediff.StorageDiff{
Key: StorageKey,
Value: []byte{0, 1, 2},
LeafKey: StorageKey,
NodeValue: []byte{0, 1, 2},
NodeType: statediff.Leaf,
Path: StoragePath,
// this storage value will fail to be decoded as an RLP with the following error message:
// "input contains more than one value"
}
@ -74,27 +76,27 @@ var (
valueBytes, _ = rlp.EncodeToBytes(testAccount)
CreatedAccountDiffs = []statediff.AccountDiff{
{
Key: ContractLeafKey.Bytes(),
Value: valueBytes,
Storage: storageWithSmallValue,
LeafKey: ContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithSmallValue,
},
}
UpdatedAccountDiffs = []statediff.AccountDiff{{
Key: AnotherContractLeafKey.Bytes(),
Value: valueBytes,
Storage: storageWithLargeValue,
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithLargeValue,
}}
UpdatedAccountDiffs2 = []statediff.AccountDiff{{
Key: AnotherContractLeafKey.Bytes(),
Value: valueBytes,
Storage: storageWithSmallValue,
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithSmallValue,
}}
DeletedAccountDiffs = []statediff.AccountDiff{{
Key: AnotherContractLeafKey.Bytes(),
Value: valueBytes,
Storage: storageWithSmallValue,
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithSmallValue,
}}
MockStateDiff = statediff.StateDiff{

55
pkg/super_node/eth/converter.go
View File

@ -21,6 +21,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
@ -123,47 +124,53 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
return nil, err
}
for _, createdAccount := range stateDiff.CreatedAccounts {
hashKey := common.BytesToHash(createdAccount.Key)
statePathHash := crypto.Keccak256Hash(createdAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Key: hashKey,
Value: createdAccount.Value,
Leaf: createdAccount.Leaf,
Path: createdAccount.Path,
Value: createdAccount.NodeValue,
Type: createdAccount.NodeType,
LeafKey: common.BytesToHash(createdAccount.LeafKey),
})
for _, storageDiff := range createdAccount.Storage {
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
Key: common.BytesToHash(storageDiff.Key),
Value: storageDiff.Value,
Leaf: storageDiff.Leaf,
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}
for _, deletedAccount := range stateDiff.DeletedAccounts {
hashKey := common.BytesToHash(deletedAccount.Key)
statePathHash := crypto.Keccak256Hash(deletedAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Key: hashKey,
Value: deletedAccount.Value,
Leaf: deletedAccount.Leaf,
Path: deletedAccount.Path,
Value: deletedAccount.NodeValue,
Type: deletedAccount.NodeType,
LeafKey: common.BytesToHash(deletedAccount.LeafKey),
})
for _, storageDiff := range deletedAccount.Storage {
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
Key: common.BytesToHash(storageDiff.Key),
Value: storageDiff.Value,
Leaf: storageDiff.Leaf,
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}
for _, updatedAccount := range stateDiff.UpdatedAccounts {
hashKey := common.BytesToHash(updatedAccount.Key)
statePathHash := crypto.Keccak256Hash(updatedAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Key: hashKey,
Value: updatedAccount.Value,
Leaf: updatedAccount.Leaf,
Path: updatedAccount.Path,
Value: updatedAccount.NodeValue,
Type: updatedAccount.NodeType,
LeafKey: common.BytesToHash(updatedAccount.LeafKey),
})
for _, storageDiff := range updatedAccount.Storage {
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
Key: common.BytesToHash(storageDiff.Key),
Value: storageDiff.Value,
Leaf: storageDiff.Leaf,
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}

100
pkg/super_node/eth/filterer.go
View File

@ -20,6 +20,8 @@ import (
"bytes"
"fmt"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
@ -66,10 +68,7 @@ func (s *ResponseFilterer) Filter(filter shared.SubscriptionSettings, payload sh
if err := s.filerReceipts(ethFilters.ReceiptFilter, response, ethPayload, filterTxs); err != nil {
return IPLDs{}, err
}
if err := s.filterState(ethFilters.StateFilter, response, ethPayload); err != nil {
return IPLDs{}, err
}
if err := s.filterStorage(ethFilters.StorageFilter, response, ethPayload); err != nil {
if err := s.filterStateAndStorage(ethFilters.StateFilter, ethFilters.StorageFilter, response, ethPayload); err != nil {
return IPLDs{}, err
}
response.BlockNumber = ethPayload.Block.Number()
@ -255,27 +254,56 @@ func slicesShareString(slice1, slice2 []string) int {
return 0
}
func (s *ResponseFilterer) filterState(stateFilter StateFilter, response *IPLDs, payload ConvertedPayload) error {
if !stateFilter.Off {
response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
keyFilters := make([]common.Hash, len(stateFilter.Addresses))
for i, addr := range stateFilter.Addresses {
keyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
// filterStateAndStorage filters state and storage nodes into the response according to the provided filters
func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
response.StorageNodes = make([]StorageNode, 0)
stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
for i, addr := range stateFilter.Addresses {
stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
}
storageAddressFilters := make([]common.Hash, len(storageFilter.Addresses))
for i, addr := range storageFilter.Addresses {
storageAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
}
storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
for i, store := range storageFilter.StorageKeys {
storageKeyFilters[i] = common.HexToHash(store)
}
for _, stateNode := range payload.StateNodes {
if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StateNodes = append(response.StateNodes, StateNode{
StateLeafKey: stateNode.LeafKey,
Path: stateNode.Path,
IPLD: ipfs.BlockModel{
Data: stateNode.Value,
CID: cid.String(),
},
Type: stateNode.Type,
})
}
}
for _, stateNode := range payload.StateNodes {
if checkNodeKeys(keyFilters, stateNode.Key) {
if stateNode.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
for _, storageNode := range payload.StorageNodes[crypto.Keccak256Hash(stateNode.Path)] {
if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StateNodes = append(response.StateNodes, StateNode{
StateTrieKey: stateNode.Key,
response.StorageNodes = append(response.StorageNodes, StorageNode{
StateLeafKey: stateNode.LeafKey,
StorageLeafKey: storageNode.LeafKey,
IPLD: ipfs.BlockModel{
Data: stateNode.Value,
Data: storageNode.Value,
CID: cid.String(),
},
Leaf: stateNode.Leaf,
Type: storageNode.Type,
Path: storageNode.Path,
})
}
}
@ -296,39 +324,3 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
}
return false
}
func (s *ResponseFilterer) filterStorage(storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
if !storageFilter.Off {
response.StorageNodes = make([]StorageNode, 0)
stateKeyFilters := make([]common.Hash, len(storageFilter.Addresses))
for i, addr := range storageFilter.Addresses {
stateKeyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
}
storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
for i, store := range storageFilter.StorageKeys {
storageKeyFilters[i] = common.HexToHash(store)
}
for stateKey, storageNodes := range payload.StorageNodes {
if checkNodeKeys(stateKeyFilters, stateKey) {
for _, storageNode := range storageNodes {
if checkNodeKeys(storageKeyFilters, storageNode.Key) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StorageNodes = append(response.StorageNodes, StorageNode{
StateTrieKey: stateKey,
StorageTrieKey: storageNode.Key,
IPLD: ipfs.BlockModel{
Data: storageNode.Value,
CID: cid.String(),
},
Leaf: storageNode.Leaf,
})
}
}
}
}
}
return nil
}

26
pkg/super_node/eth/filterer_test.go
View File

@ -17,11 +17,13 @@
package eth_test
import (
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"bytes"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
@ -54,19 +56,19 @@ var _ = Describe("Filterer", func() {
Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
Expect(len(iplds.StateNodes)).To(Equal(2))
for _, stateNode := range iplds.StateNodes {
Expect(stateNode.Leaf).To(BeTrue())
if stateNode.StateTrieKey == mocks.ContractLeafKey {
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
}
if stateNode.StateTrieKey == mocks.AnotherContractLeafKey {
Expect(stateNode.Type).To(Equal(statediff.Leaf))
if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.AccountLeafKey) {
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State2IPLD.RawData(),
CID: mocks.State2IPLD.Cid().String(),
}))
}
if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.ContractLeafKey) {
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
}
}
Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
})
@ -180,10 +182,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds7.StorageNodes)).To(Equal(0))
Expect(len(iplds7.Receipts)).To(Equal(0))
Expect(len(iplds7.StateNodes)).To(Equal(1))
Expect(iplds7.StateNodes[0].StateTrieKey).To(Equal(mocks.ContractLeafKey))
Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks.AccountLeafKey))
Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
Data: mocks.State2IPLD.RawData(),
CID: mocks.State2IPLD.Cid().String(),
}))
payload8, err := filterer.Filter(rctTopicsAndContractFilterFail, mocks.MockConvertedPayload)

45
pkg/super_node/eth/helpers.go Normal file
View File

@ -0,0 +1,45 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth
import "github.com/ethereum/go-ethereum/statediff"
func ResolveFromNodeType(nodeType statediff.NodeType) int {
switch nodeType {
case statediff.Branch:
return 0
case statediff.Extension:
return 1
case statediff.Leaf:
return 2
default:
return -1
}
}
func ResolveToNodeType(nodeType int) statediff.NodeType {
switch nodeType {
case 0:
return statediff.Branch
case 1:
return statediff.Extension
case 2:
return statediff.Leaf
default:
return statediff.Unknown
}
}

16
pkg/super_node/eth/indexer.go
View File

@ -19,6 +19,8 @@ package eth
import (
"fmt"
"github.com/ethereum/go-ethereum/crypto"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/ethereum/go-ethereum/common"
@ -129,14 +131,14 @@ func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID in
func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
for _, stateCID := range payload.StateNodeCIDs {
var stateID int64
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_key, cid, node_type) = ($2, $3, $5)
RETURNING id`,
headerID, stateCID.StateKey, stateCID.CID, stateCID.Leaf).Scan(&stateID)
headerID, stateCID.StateKey, stateCID.CID, stateCID.Path, stateCID.NodeType).Scan(&stateID)
if err != nil {
return err
}
for _, storageCID := range payload.StorageNodeCIDs[common.HexToHash(stateCID.StateKey)] {
for _, storageCID := range payload.StorageNodeCIDs[crypto.Keccak256Hash(stateCID.Path)] {
if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
return err
}
@ -146,8 +148,8 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
}
func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error {
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
stateID, storageCID.StorageKey, storageCID.CID, storageCID.Leaf)
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_key, cid, node_type) = ($2, $3, $5)`,
stateID, storageCID.StorageKey, storageCID.CID, storageCID.Path, storageCID.NodeType)
return err
}

28
pkg/super_node/eth/indexer_test.go
View File

@ -17,6 +17,7 @@
package eth_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -45,13 +46,15 @@ var _ = Describe("Indexer", func() {
It("Indexes CIDs and related metadata into vulcanizedb", func() {
err = repo.Index(mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
pgStr := `SELECT cid, td, reward FROM eth.header_cids
pgStr := `SELECT cid, td, reward, id
FROM eth.header_cids
WHERE block_number = $1`
// check header was properly indexed
type res struct {
CID string
TD string
Reward string
ID int
}
headers := new(res)
err = db.QueryRowx(pgStr, 1).StructScan(headers)
@ -81,24 +84,28 @@ var _ = Describe("Indexer", func() {
Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
// check that state nodes were properly indexed
stateNodes := make([]eth.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&stateNodes, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(stateNodes)).To(Equal(2))
for _, stateNode := range stateNodes {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
}
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
}
}
// check that storage nodes were properly indexed
storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.leaf FROM eth.storage_cids, eth.state_cids, eth.header_cids
pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
@ -107,9 +114,10 @@ var _ = Describe("Indexer", func() {
Expect(len(storageNodes)).To(Equal(1))
Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
CID: mocks.StorageCID.String(),
Leaf: true,
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
StateKey: mocks.ContractLeafKey.Hex(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{},
}))
})
})

12
pkg/super_node/eth/ipld_fetcher.go
View File

@ -216,8 +216,9 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
Data: state.RawData(),
CID: state.Cid().String(),
},
StateTrieKey: common.HexToHash(stateNode.StateKey),
Leaf: stateNode.Leaf,
StateLeafKey: common.HexToHash(stateNode.StateKey),
Type: ResolveToNodeType(stateNode.NodeType),
Path: stateNode.Path,
}
}
return stateNodes, nil
@ -246,9 +247,10 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
Data: storage.RawData(),
CID: storage.Cid().String(),
},
StateTrieKey: common.HexToHash(storageNode.StateKey),
StorageTrieKey: common.HexToHash(storageNode.StorageKey),
Leaf: storageNode.Leaf,
StateLeafKey: common.HexToHash(storageNode.StateKey),
StorageLeafKey: common.HexToHash(storageNode.StorageKey),
Type: ResolveToNodeType(storageNode.NodeType),
Path: storageNode.Path,
}
}
return storageNodes, nil

22
pkg/super_node/eth/ipld_fetcher_test.go
View File

@ -20,13 +20,13 @@ import (
"bytes"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ipfs/go-block-format"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
)
@ -71,18 +71,18 @@ var (
},
StateNodes: []eth.StateNodeModel{{
CID: mockStateBlock.Cid().String(),
Leaf: true,
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
}},
StorageNodes: []eth.StorageNodeWithStateKeyModel{{
CID: mockStorageBlock1.Cid().String(),
Leaf: true,
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
StorageKey: "0000000000000000000000000000000000000000000000000000000000000001",
},
{
CID: mockStorageBlock2.Cid().String(),
Leaf: true,
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
StorageKey: "0000000000000000000000000000000000000000000000000000000000000002",
}},
@ -127,23 +127,23 @@ var _ = Describe("Fetcher", func() {
CID: mockReceiptBlock.Cid().String(),
}))
Expect(len(iplds.StateNodes)).To(Equal(1))
Expect(iplds.StateNodes[0].StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
Expect(iplds.StateNodes[0].Leaf).To(BeTrue())
Expect(iplds.StateNodes[0].StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
Expect(iplds.StateNodes[0].Type).To(Equal(statediff.Leaf))
Expect(iplds.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mockStateBlock.RawData(),
CID: mockStateBlock.Cid().String(),
}))
Expect(len(iplds.StorageNodes)).To(Equal(2))
for _, storage := range iplds.StorageNodes {
Expect(storage.Leaf).To(BeTrue())
Expect(storage.StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
Expect(storage.Type).To(Equal(statediff.Leaf))
Expect(storage.StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock1.RawData(),
CID: mockStorageBlock1.Cid().String(),
}))
}
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock2.RawData(),
CID: mockStorageBlock2.Cid().String(),

193
pkg/super_node/eth/mocks/test_data.go
View File

@ -21,12 +21,6 @@ import (
"crypto/elliptic"
"crypto/rand"
"math/big"
rand2 "math/rand"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
@ -35,9 +29,13 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/statediff/testhelpers"
"github.com/ipfs/go-block-format"
"github.com/multiformats/go-multihash"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
)
@ -79,11 +77,10 @@ var (
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256)
MockTrxMeta = []eth.TxModel{
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
MockTrxMeta = []eth.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
Src: senderAddr.Hex(),
@ -161,51 +158,71 @@ var (
}
// statediff data
CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
NonceValue = rand2.Uint64()
anotherNonceValue = rand2.Uint64()
BalanceValue = rand2.Int63()
anotherBalanceValue = rand2.Int63()
ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
StorageValue = common.Hex2Bytes("0x03")
storage = []statediff.StorageDiff{{
Key: StorageKey,
Value: StorageValue,
Path: StoragePath,
Proof: [][]byte{},
Leaf: true,
}}
emptyStorage = make([]statediff.StorageDiff, 0)
ContractLeafKey = crypto.Keccak256Hash(Address.Bytes())
AnotherContractLeafKey = crypto.Keccak256Hash(AnotherAddress.Bytes())
testAccount = state.Account{
Nonce: NonceValue,
Balance: big.NewInt(BalanceValue),
Root: ContractRoot,
CodeHash: CodeHash,
}
anotherTestAccount = state.Account{
Nonce: anotherNonceValue,
Balance: big.NewInt(anotherBalanceValue),
Root: common.HexToHash("0x"),
CodeHash: nil,
}
ValueBytes, _ = rlp.EncodeToBytes(testAccount)
AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount)
CreatedAccountDiffs = []statediff.AccountDiff{
storageLocation = common.HexToHash("0")
StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes()
StorageValue = common.Hex2Bytes("01")
StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
StoragePartialPath,
StorageValue,
})
nonce1 = uint64(1)
contractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
contractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
contractPathHash = crypto.Keccak256Hash([]byte{'\x06'})
ContractAddress = common.HexToAddress("0x703c4b2bD70c169f5717101CaeE543299Fc946C7")
ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress)
ContractAccount, _ = rlp.EncodeToBytes(state.Account{
Nonce: nonce1,
Balance: big.NewInt(0),
CodeHash: contractCodeHash.Bytes(),
Root: common.HexToHash(contractRoot),
})
ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
ContractPartialPath,
ContractAccount,
})
nonce0 = uint64(0)
accountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
accountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
AccountLeafKey = testhelpers.Account2LeafKey
Account, _ = rlp.EncodeToBytes(state.Account{
Nonce: nonce0,
Balance: big.NewInt(1000),
CodeHash: accountCodeHash.Bytes(),
Root: common.HexToHash(accountRoot),
})
AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
AccountPartialPath,
Account,
})
CreatedAccountDiffs = []statediff.AccountDiff{
{
Key: ContractLeafKey.Bytes(),
Value: ValueBytes,
Storage: storage,
Leaf: true,
Path: []byte{'\x06'},
NodeType: statediff.Leaf,
LeafKey: ContractLeafKey,
NodeValue: ContractLeafNode,
Storage: []statediff.StorageDiff{
{
Path: []byte{},
NodeType: statediff.Leaf,
LeafKey: StorageLeafKey,
NodeValue: StorageLeafNode,
},
},
},
{
Key: AnotherContractLeafKey.Bytes(),
Value: AnotherValueBytes,
Storage: emptyStorage,
Leaf: true,
Path: []byte{'\x0c'},
NodeType: statediff.Leaf,
LeafKey: AccountLeafKey,
NodeValue: AccountLeafNode,
Storage: []statediff.StorageDiff{},
},
}
@ -217,34 +234,39 @@ var (
MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
MockStateNodes = []eth.TrieNode{
{
Key: ContractLeafKey,
Value: ValueBytes,
Leaf: true,
LeafKey: common.BytesToHash(ContractLeafKey),
Path: []byte{'\x06'},
Value: ContractLeafNode,
Type: statediff.Leaf,
},
{
Key: AnotherContractLeafKey,
Value: AnotherValueBytes,
Leaf: true,
LeafKey: common.BytesToHash(AccountLeafKey),
Path: []byte{'\x0c'},
Value: AccountLeafNode,
Type: statediff.Leaf,
},
}
MockStateMetaPostPublish = []eth.StateNodeModel{
{
CID: State1CID.String(),
Leaf: true,
StateKey: ContractLeafKey.String(),
Path: []byte{'\x06'},
NodeType: 2,
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
},
{
CID: State2CID.String(),
Leaf: true,
StateKey: AnotherContractLeafKey.String(),
Path: []byte{'\x0c'},
NodeType: 2,
StateKey: common.BytesToHash(AccountLeafKey).Hex(),
},
}
MockStorageNodes = map[common.Hash][]eth.TrieNode{
ContractLeafKey: {
contractPathHash: {
{
Key: common.BytesToHash(StorageKey),
Value: StorageValue,
Leaf: true,
LeafKey: common.BytesToHash(StorageLeafKey),
Value: StorageLeafNode,
Type: statediff.Leaf,
Path: []byte{},
},
},
}
@ -284,11 +306,12 @@ var (
},
StateNodeCIDs: MockStateMetaPostPublish,
StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
ContractLeafKey: {
contractPathHash: {
{
CID: StorageCID.String(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
Leaf: true,
Path: []byte{},
StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
NodeType: 2,
},
},
},
@ -310,10 +333,11 @@ var (
StateNodes: MockStateMetaPostPublish,
StorageNodes: []eth.StorageNodeWithStateKeyModel{
{
Path: []byte{},
CID: StorageCID.String(),
Leaf: true,
StateKey: ContractLeafKey.Hex(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
NodeType: 2,
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
},
},
}
@ -323,9 +347,9 @@ var (
Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, StorageCID)
State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
MockIPLDs = eth.IPLDs{
BlockNumber: big.NewInt(1),
@ -355,31 +379,34 @@ var (
},
StateNodes: []eth2.StateNode{
{
StateTrieKey: ContractLeafKey,
Leaf: true,
StateLeafKey: common.BytesToHash(ContractLeafKey),
Type: statediff.Leaf,
IPLD: ipfs.BlockModel{
Data: State1IPLD.RawData(),
CID: State1IPLD.Cid().String(),
},
Path: []byte{'\x06'},
},
{
StateTrieKey: AnotherContractLeafKey,
Leaf: true,
StateLeafKey: common.BytesToHash(AccountLeafKey),
Type: statediff.Leaf,
IPLD: ipfs.BlockModel{
Data: State2IPLD.RawData(),
CID: State2IPLD.Cid().String(),
},
Path: []byte{'\x0c'},
},
},
StorageNodes: []eth2.StorageNode{
{
StateTrieKey: ContractLeafKey,
StorageTrieKey: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"),
Leaf: true,
StateLeafKey: common.BytesToHash(ContractLeafKey),
StorageLeafKey: common.BytesToHash(StorageLeafKey),
Type: statediff.Leaf,
IPLD: ipfs.BlockModel{
Data: StorageIPLD.RawData(),
CID: StorageIPLD.Cid().String(),
},
Path: []byte{},
},
},
}

9
pkg/super_node/eth/models.go
View File

@ -67,8 +67,9 @@ type ReceiptModel struct {
type StateNodeModel struct {
ID int64 `db:"id"`
HeaderID int64 `db:"header_id"`
Path []byte `db:"state_path"`
StateKey string `db:"state_key"`
Leaf bool `db:"leaf"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
}
@ -76,8 +77,9 @@ type StateNodeModel struct {
type StorageNodeModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Path []byte `db:"storage_path"`
StorageKey string `db:"storage_key"`
Leaf bool `db:"leaf"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
}
@ -85,8 +87,9 @@ type StorageNodeModel struct {
type StorageNodeWithStateKeyModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Path []byte `db:"storage_path"`
StateKey string `db:"state_key"`
StorageKey string `db:"storage_key"`
Leaf bool `db:"leaf"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
}

18
pkg/super_node/eth/publisher.go
View File

@ -185,9 +185,10 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
return nil, err
}
stateNodeCids = append(stateNodeCids, StateNodeModel{
StateKey: node.Key.String(),
Path: node.Path,
StateKey: node.LeafKey.String(),
CID: cids[0],
Leaf: node.Leaf,
NodeType: ResolveFromNodeType(node.Type),
})
}
return stateNodeCids, nil
@ -195,18 +196,19 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
storageLeafCids := make(map[common.Hash][]StorageNodeModel)
for addrKey, storageTrie := range storageNodes {
storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie))
for pathHash, storageTrie := range storageNodes {
storageLeafCids[pathHash] = make([]StorageNodeModel, 0, len(storageTrie))
for _, node := range storageTrie {
cids, err := pub.StoragePutter.DagPut(node.Value)
if err != nil {
return nil, err
}
// Map storage node cids to their state key hashes
storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{
StorageKey: node.Key.Hex(),
// Map storage node cids to their path hashes
storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{
Path: node.Path,
StorageKey: node.LeafKey.Hex(),
CID: cids[0],
Leaf: node.Leaf,
NodeType: ResolveFromNodeType(node.Type),
})
}
}

9
pkg/super_node/eth/retriever.go
View File

@ -391,7 +391,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
log.Debug("retrieving state cids for header id ", headerID)
args := make([]interface{}, 0, 2)
pgStr := `SELECT state_cids.id, state_cids.header_id,
state_cids.state_key, state_cids.leaf, state_cids.cid
state_cids.state_key, state_cids.node_type, state_cids.cid, state_cids.state_path
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.id = $1`
args = append(args, headerID)
@ -405,7 +405,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
args = append(args, pq.Array(keys))
}
if !stateFilter.IntermediateNodes {
pgStr += ` AND state_cids.leaf = TRUE`
pgStr += ` AND state_cids.node_type = 2`
}
stateNodeCIDs := make([]StateNodeModel, 0)
return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
@ -416,7 +416,8 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
log.Debug("retrieving storage cids for header id ", headerID)
args := make([]interface{}, 0, 3)
pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM eth.storage_cids, eth.state_cids, eth.header_cids
storage_cids.node_type, storage_cids.cid, storage_cids.storage_path, state_cids.state_key
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.id = $1`
@ -437,7 +438,7 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
args = append(args, pq.Array(storageFilter.StorageKeys))
}
if !storageFilter.IntermediateNodes {
pgStr += ` AND storage_cids.leaf = TRUE`
pgStr += ` AND storage_cids.node_type = 2`
}
storageNodeCIDs := make([]StorageNodeWithStateKeyModel, 0)
return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)

21
pkg/super_node/eth/retriever_test.go
View File

@ -19,6 +19,8 @@ package eth_test
import (
"math/big"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -196,7 +198,7 @@ var (
Off: true,
},
StateFilter: eth.StateFilter{
Addresses: []string{mocks.Address.Hex()},
Addresses: []string{mocks.AccountAddresss.Hex()},
},
StorageFilter: eth.StorageFilter{
Off: true,
@ -247,12 +249,14 @@ var _ = Describe("Retriever", func() {
Expect(len(cidWrapper.StateNodes)).To(Equal(2))
for _, stateNode := range cidWrapper.StateNodes {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
}
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
}
}
Expect(len(cidWrapper.StorageNodes)).To(Equal(1))
@ -384,9 +388,10 @@ var _ = Describe("Retriever", func() {
Expect(cidWrapper7.StateNodes[0]).To(Equal(eth.StateNodeModel{
ID: cidWrapper7.StateNodes[0].ID,
HeaderID: cidWrapper7.StateNodes[0].HeaderID,
Leaf: true,
StateKey: mocks.ContractLeafKey.Hex(),
CID: mocks.State1CID.String(),
NodeType: 2,
StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(),
CID: mocks.State2CID.String(),
Path: []byte{'\x0c'},
}))
_, empty, err = retriever.Retrieve(rctTopicsAndContractFilterFail, 1)

21
pkg/super_node/eth/types.go
View File

@ -19,6 +19,8 @@ package eth
import (
"math/big"
"github.com/ethereum/go-ethereum/statediff"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
@ -45,9 +47,10 @@ func (i ConvertedPayload) Height() int64 {
// Trie struct used to flag node as leaf or not
type TrieNode struct {
Key common.Hash
Value []byte
Leaf bool
Path []byte
LeafKey common.Hash
Value []byte
Type statediff.NodeType
}
// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres
@ -94,14 +97,16 @@ func (i IPLDs) Height() int64 {
}
type StateNode struct {
StateTrieKey common.Hash
Type statediff.NodeType
StateLeafKey common.Hash
Path []byte
IPLD ipfs.BlockModel
Leaf bool
}
type StorageNode struct {
StateTrieKey common.Hash
StorageTrieKey common.Hash
Type statediff.NodeType
StateLeafKey common.Hash
StorageLeafKey common.Hash
Path []byte
IPLD ipfs.BlockModel
Leaf bool
}

10
pkg/watcher/eth/converter.go
View File

@ -150,16 +150,16 @@ func (pc *WatcherConverter) Convert(ethIPLDs eth.IPLDs) (*eth.CIDPayload, error)
for i, stateIPLD := range ethIPLDs.StateNodes {
cids.StateNodeCIDs[i] = eth.StateNodeModel{
CID: stateIPLD.IPLD.CID,
Leaf: stateIPLD.Leaf,
StateKey: stateIPLD.StateTrieKey.String(),
NodeType: eth.ResolveFromNodeType(stateIPLD.Type),
StateKey: stateIPLD.StateLeafKey.String(),
}
}
// Storage data
for _, storageIPLD := range ethIPLDs.StorageNodes {
cids.StorageNodeCIDs[storageIPLD.StateTrieKey] = append(cids.StorageNodeCIDs[storageIPLD.StateTrieKey], eth.StorageNodeModel{
cids.StorageNodeCIDs[storageIPLD.StateLeafKey] = append(cids.StorageNodeCIDs[storageIPLD.StateLeafKey], eth.StorageNodeModel{
CID: storageIPLD.IPLD.CID,
Leaf: storageIPLD.Leaf,
StorageKey: storageIPLD.StorageTrieKey.String(),
NodeType: eth.ResolveFromNodeType(storageIPLD.Type),
StorageKey: storageIPLD.StorageLeafKey.String(),
})
}
return cids, nil