cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

adhjustments to work with statediffing geth v1.10-alpha.2

Browse Source
This commit is contained in:
Ian Norden
2020-03-11 13:41:59 -05:00
parent be875c0100
commit e5c5422edc
26 changed files with 547 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/super_node/eth/converter.go
+31 -24
View File
@@ -21,6 +21,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
@@ -123,47 +124,53 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
return nil, err
}
for _, createdAccount := range stateDiff.CreatedAccounts {
hashKey := common.BytesToHash(createdAccount.Key)
statePathHash := crypto.Keccak256Hash(createdAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Key: hashKey,
Value: createdAccount.Value,
Leaf: createdAccount.Leaf,
Path: createdAccount.Path,
Value: createdAccount.NodeValue,
Type: createdAccount.NodeType,
LeafKey: common.BytesToHash(createdAccount.LeafKey),
})
for _, storageDiff := range createdAccount.Storage {
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
Key: common.BytesToHash(storageDiff.Key),
Value: storageDiff.Value,
Leaf: storageDiff.Leaf,
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}
for _, deletedAccount := range stateDiff.DeletedAccounts {
hashKey := common.BytesToHash(deletedAccount.Key)
statePathHash := crypto.Keccak256Hash(deletedAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Key: hashKey,
Value: deletedAccount.Value,
Leaf: deletedAccount.Leaf,
Path: deletedAccount.Path,
Value: deletedAccount.NodeValue,
Type: deletedAccount.NodeType,
LeafKey: common.BytesToHash(deletedAccount.LeafKey),
})
for _, storageDiff := range deletedAccount.Storage {
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
Key: common.BytesToHash(storageDiff.Key),
Value: storageDiff.Value,
Leaf: storageDiff.Leaf,
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}
for _, updatedAccount := range stateDiff.UpdatedAccounts {
hashKey := common.BytesToHash(updatedAccount.Key)
statePathHash := crypto.Keccak256Hash(updatedAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Key: hashKey,
Value: updatedAccount.Value,
Leaf: updatedAccount.Leaf,
Path: updatedAccount.Path,
Value: updatedAccount.NodeValue,
Type: updatedAccount.NodeType,
LeafKey: common.BytesToHash(updatedAccount.LeafKey),
})
for _, storageDiff := range updatedAccount.Storage {
convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
Key: common.BytesToHash(storageDiff.Key),
Value: storageDiff.Value,
Leaf: storageDiff.Leaf,
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}
pkg/super_node/eth/filterer.go
+46 -54
View File
@@ -20,6 +20,8 @@ import (
"bytes"
"fmt"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
@@ -66,10 +68,7 @@ func (s *ResponseFilterer) Filter(filter shared.SubscriptionSettings, payload sh
if err := s.filerReceipts(ethFilters.ReceiptFilter, response, ethPayload, filterTxs); err != nil {
return IPLDs{}, err
}
if err := s.filterState(ethFilters.StateFilter, response, ethPayload); err != nil {
return IPLDs{}, err
}
if err := s.filterStorage(ethFilters.StorageFilter, response, ethPayload); err != nil {
if err := s.filterStateAndStorage(ethFilters.StateFilter, ethFilters.StorageFilter, response, ethPayload); err != nil {
return IPLDs{}, err
}
response.BlockNumber = ethPayload.Block.Number()
@@ -255,27 +254,56 @@ func slicesShareString(slice1, slice2 []string) int {
return 0
}
func (s *ResponseFilterer) filterState(stateFilter StateFilter, response *IPLDs, payload ConvertedPayload) error {
if !stateFilter.Off {
response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
keyFilters := make([]common.Hash, len(stateFilter.Addresses))
for i, addr := range stateFilter.Addresses {
keyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
// filterStateAndStorage filters state and storage nodes into the response according to the provided filters
func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
response.StorageNodes = make([]StorageNode, 0)
stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
for i, addr := range stateFilter.Addresses {
stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
}
storageAddressFilters := make([]common.Hash, len(storageFilter.Addresses))
for i, addr := range storageFilter.Addresses {
storageAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
}
storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
for i, store := range storageFilter.StorageKeys {
storageKeyFilters[i] = common.HexToHash(store)
}
for _, stateNode := range payload.StateNodes {
if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StateNodes = append(response.StateNodes, StateNode{
StateLeafKey: stateNode.LeafKey,
Path: stateNode.Path,
IPLD: ipfs.BlockModel{
Data: stateNode.Value,
CID: cid.String(),
},
Type: stateNode.Type,
})
}
}
for _, stateNode := range payload.StateNodes {
if checkNodeKeys(keyFilters, stateNode.Key) {
if stateNode.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
for _, storageNode := range payload.StorageNodes[crypto.Keccak256Hash(stateNode.Path)] {
if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StateNodes = append(response.StateNodes, StateNode{
StateTrieKey: stateNode.Key,
response.StorageNodes = append(response.StorageNodes, StorageNode{
StateLeafKey: stateNode.LeafKey,
StorageLeafKey: storageNode.LeafKey,
IPLD: ipfs.BlockModel{
Data: stateNode.Value,
Data: storageNode.Value,
CID: cid.String(),
},
Leaf: stateNode.Leaf,
Type: storageNode.Type,
Path: storageNode.Path,
})
}
}
@@ -296,39 +324,3 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
}
return false
}
func (s *ResponseFilterer) filterStorage(storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
if !storageFilter.Off {
response.StorageNodes = make([]StorageNode, 0)
stateKeyFilters := make([]common.Hash, len(storageFilter.Addresses))
for i, addr := range storageFilter.Addresses {
stateKeyFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
}
storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
for i, store := range storageFilter.StorageKeys {
storageKeyFilters[i] = common.HexToHash(store)
}
for stateKey, storageNodes := range payload.StorageNodes {
if checkNodeKeys(stateKeyFilters, stateKey) {
for _, storageNode := range storageNodes {
if checkNodeKeys(storageKeyFilters, storageNode.Key) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StorageNodes = append(response.StorageNodes, StorageNode{
StateTrieKey: stateKey,
StorageTrieKey: storageNode.Key,
IPLD: ipfs.BlockModel{
Data: storageNode.Value,
CID: cid.String(),
},
Leaf: storageNode.Leaf,
})
}
}
}
}
}
return nil
}
pkg/super_node/eth/filterer_test.go
+14 -12
View File
@@ -17,11 +17,13 @@
package eth_test
import (
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"bytes"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
@@ -54,19 +56,19 @@ var _ = Describe("Filterer", func() {
Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
Expect(len(iplds.StateNodes)).To(Equal(2))
for _, stateNode := range iplds.StateNodes {
Expect(stateNode.Leaf).To(BeTrue())
if stateNode.StateTrieKey == mocks.ContractLeafKey {
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
}
if stateNode.StateTrieKey == mocks.AnotherContractLeafKey {
Expect(stateNode.Type).To(Equal(statediff.Leaf))
if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.AccountLeafKey) {
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State2IPLD.RawData(),
CID: mocks.State2IPLD.Cid().String(),
}))
}
if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.ContractLeafKey) {
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
}
}
Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
})
@@ -180,10 +182,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds7.StorageNodes)).To(Equal(0))
Expect(len(iplds7.Receipts)).To(Equal(0))
Expect(len(iplds7.StateNodes)).To(Equal(1))
Expect(iplds7.StateNodes[0].StateTrieKey).To(Equal(mocks.ContractLeafKey))
Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks.AccountLeafKey))
Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
Data: mocks.State2IPLD.RawData(),
CID: mocks.State2IPLD.Cid().String(),
}))
payload8, err := filterer.Filter(rctTopicsAndContractFilterFail, mocks.MockConvertedPayload)
pkg/super_node/eth/helpers.go
+45
View File
@@ -0,0 +1,45 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth
import "github.com/ethereum/go-ethereum/statediff"
func ResolveFromNodeType(nodeType statediff.NodeType) int {
switch nodeType {
case statediff.Branch:
return 0
case statediff.Extension:
return 1
case statediff.Leaf:
return 2
default:
return -1
}
}
func ResolveToNodeType(nodeType int) statediff.NodeType {
switch nodeType {
case 0:
return statediff.Branch
case 1:
return statediff.Extension
case 2:
return statediff.Leaf
default:
return statediff.Unknown
}
}
pkg/super_node/eth/indexer.go
+9 -7
View File
@@ -19,6 +19,8 @@ package eth
import (
"fmt"
"github.com/ethereum/go-ethereum/crypto"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/ethereum/go-ethereum/common"
@@ -129,14 +131,14 @@ func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID in
func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
for _, stateCID := range payload.StateNodeCIDs {
var stateID int64
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_key, cid, node_type) = ($2, $3, $5)
RETURNING id`,
headerID, stateCID.StateKey, stateCID.CID, stateCID.Leaf).Scan(&stateID)
headerID, stateCID.StateKey, stateCID.CID, stateCID.Path, stateCID.NodeType).Scan(&stateID)
if err != nil {
return err
}
for _, storageCID := range payload.StorageNodeCIDs[common.HexToHash(stateCID.StateKey)] {
for _, storageCID := range payload.StorageNodeCIDs[crypto.Keccak256Hash(stateCID.Path)] {
if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
return err
}
@@ -146,8 +148,8 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
}
func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error {
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
stateID, storageCID.StorageKey, storageCID.CID, storageCID.Leaf)
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_key, cid, node_type) = ($2, $3, $5)`,
stateID, storageCID.StorageKey, storageCID.CID, storageCID.Path, storageCID.NodeType)
return err
}
pkg/super_node/eth/indexer_test.go
+18 -10
View File
@@ -17,6 +17,7 @@
package eth_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -45,13 +46,15 @@ var _ = Describe("Indexer", func() {
It("Indexes CIDs and related metadata into vulcanizedb", func() {
err = repo.Index(mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
pgStr := `SELECT cid, td, reward FROM eth.header_cids
pgStr := `SELECT cid, td, reward, id
FROM eth.header_cids
WHERE block_number = $1`
// check header was properly indexed
type res struct {
CID string
TD string
Reward string
ID int
}
headers := new(res)
err = db.QueryRowx(pgStr, 1).StructScan(headers)
@@ -81,24 +84,28 @@ var _ = Describe("Indexer", func() {
Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
// check that state nodes were properly indexed
stateNodes := make([]eth.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&stateNodes, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(stateNodes)).To(Equal(2))
for _, stateNode := range stateNodes {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
}
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
}
}
// check that storage nodes were properly indexed
storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.leaf FROM eth.storage_cids, eth.state_cids, eth.header_cids
pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
@@ -107,9 +114,10 @@ var _ = Describe("Indexer", func() {
Expect(len(storageNodes)).To(Equal(1))
Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
CID: mocks.StorageCID.String(),
Leaf: true,
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
StateKey: mocks.ContractLeafKey.Hex(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{},
}))
})
})
pkg/super_node/eth/ipld_fetcher.go
+7 -5
View File
@@ -216,8 +216,9 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
Data: state.RawData(),
CID: state.Cid().String(),
},
StateTrieKey: common.HexToHash(stateNode.StateKey),
Leaf: stateNode.Leaf,
StateLeafKey: common.HexToHash(stateNode.StateKey),
Type: ResolveToNodeType(stateNode.NodeType),
Path: stateNode.Path,
}
}
return stateNodes, nil
@@ -246,9 +247,10 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
Data: storage.RawData(),
CID: storage.Cid().String(),
},
StateTrieKey: common.HexToHash(storageNode.StateKey),
StorageTrieKey: common.HexToHash(storageNode.StorageKey),
Leaf: storageNode.Leaf,
StateLeafKey: common.HexToHash(storageNode.StateKey),
StorageLeafKey: common.HexToHash(storageNode.StorageKey),
Type: ResolveToNodeType(storageNode.NodeType),
Path: storageNode.Path,
}
}
return storageNodes, nil
pkg/super_node/eth/ipld_fetcher_test.go
+11 -11
View File
@@ -20,13 +20,13 @@ import (
"bytes"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ipfs/go-block-format"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
)
@@ -71,18 +71,18 @@ var (
},
StateNodes: []eth.StateNodeModel{{
CID: mockStateBlock.Cid().String(),
Leaf: true,
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
}},
StorageNodes: []eth.StorageNodeWithStateKeyModel{{
CID: mockStorageBlock1.Cid().String(),
Leaf: true,
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
StorageKey: "0000000000000000000000000000000000000000000000000000000000000001",
},
{
CID: mockStorageBlock2.Cid().String(),
Leaf: true,
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
StorageKey: "0000000000000000000000000000000000000000000000000000000000000002",
}},
@@ -127,23 +127,23 @@ var _ = Describe("Fetcher", func() {
CID: mockReceiptBlock.Cid().String(),
}))
Expect(len(iplds.StateNodes)).To(Equal(1))
Expect(iplds.StateNodes[0].StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
Expect(iplds.StateNodes[0].Leaf).To(BeTrue())
Expect(iplds.StateNodes[0].StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
Expect(iplds.StateNodes[0].Type).To(Equal(statediff.Leaf))
Expect(iplds.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mockStateBlock.RawData(),
CID: mockStateBlock.Cid().String(),
}))
Expect(len(iplds.StorageNodes)).To(Equal(2))
for _, storage := range iplds.StorageNodes {
Expect(storage.Leaf).To(BeTrue())
Expect(storage.StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
Expect(storage.Type).To(Equal(statediff.Leaf))
Expect(storage.StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock1.RawData(),
CID: mockStorageBlock1.Cid().String(),
}))
}
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock2.RawData(),
CID: mockStorageBlock2.Cid().String(),
pkg/super_node/eth/mocks/test_data.go
+110 -83
View File
@@ -21,12 +21,6 @@ import (
"crypto/elliptic"
"crypto/rand"
"math/big"
rand2 "math/rand"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
@@ -35,9 +29,13 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/statediff/testhelpers"
"github.com/ipfs/go-block-format"
"github.com/multiformats/go-multihash"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
)
@@ -79,11 +77,10 @@ var (
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256)
MockTrxMeta = []eth.TxModel{
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
MockTrxMeta = []eth.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
Src: senderAddr.Hex(),
@@ -161,51 +158,71 @@ var (
}
// statediff data
CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
NonceValue = rand2.Uint64()
anotherNonceValue = rand2.Uint64()
BalanceValue = rand2.Int63()
anotherBalanceValue = rand2.Int63()
ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
StorageValue = common.Hex2Bytes("0x03")
storage = []statediff.StorageDiff{{
Key: StorageKey,
Value: StorageValue,
Path: StoragePath,
Proof: [][]byte{},
Leaf: true,
}}
emptyStorage = make([]statediff.StorageDiff, 0)
ContractLeafKey = crypto.Keccak256Hash(Address.Bytes())
AnotherContractLeafKey = crypto.Keccak256Hash(AnotherAddress.Bytes())
testAccount = state.Account{
Nonce: NonceValue,
Balance: big.NewInt(BalanceValue),
Root: ContractRoot,
CodeHash: CodeHash,
}
anotherTestAccount = state.Account{
Nonce: anotherNonceValue,
Balance: big.NewInt(anotherBalanceValue),
Root: common.HexToHash("0x"),
CodeHash: nil,
}
ValueBytes, _ = rlp.EncodeToBytes(testAccount)
AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount)
CreatedAccountDiffs = []statediff.AccountDiff{
storageLocation = common.HexToHash("0")
StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes()
StorageValue = common.Hex2Bytes("01")
StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
StoragePartialPath,
StorageValue,
})
nonce1 = uint64(1)
contractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
contractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
contractPathHash = crypto.Keccak256Hash([]byte{'\x06'})
ContractAddress = common.HexToAddress("0x703c4b2bD70c169f5717101CaeE543299Fc946C7")
ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress)
ContractAccount, _ = rlp.EncodeToBytes(state.Account{
Nonce: nonce1,
Balance: big.NewInt(0),
CodeHash: contractCodeHash.Bytes(),
Root: common.HexToHash(contractRoot),
})
ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
ContractPartialPath,
ContractAccount,
})
nonce0 = uint64(0)
accountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
accountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
AccountLeafKey = testhelpers.Account2LeafKey
Account, _ = rlp.EncodeToBytes(state.Account{
Nonce: nonce0,
Balance: big.NewInt(1000),
CodeHash: accountCodeHash.Bytes(),
Root: common.HexToHash(accountRoot),
})
AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
AccountPartialPath,
Account,
})
CreatedAccountDiffs = []statediff.AccountDiff{
{
Key: ContractLeafKey.Bytes(),
Value: ValueBytes,
Storage: storage,
Leaf: true,
Path: []byte{'\x06'},
NodeType: statediff.Leaf,
LeafKey: ContractLeafKey,
NodeValue: ContractLeafNode,
Storage: []statediff.StorageDiff{
{
Path: []byte{},
NodeType: statediff.Leaf,
LeafKey: StorageLeafKey,
NodeValue: StorageLeafNode,
},
},
},
{
Key: AnotherContractLeafKey.Bytes(),
Value: AnotherValueBytes,
Storage: emptyStorage,
Leaf: true,
Path: []byte{'\x0c'},
NodeType: statediff.Leaf,
LeafKey: AccountLeafKey,
NodeValue: AccountLeafNode,
Storage: []statediff.StorageDiff{},
},
}
@@ -217,34 +234,39 @@ var (
MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
MockStateNodes = []eth.TrieNode{
{
Key: ContractLeafKey,
Value: ValueBytes,
Leaf: true,
LeafKey: common.BytesToHash(ContractLeafKey),
Path: []byte{'\x06'},
Value: ContractLeafNode,
Type: statediff.Leaf,
},
{
Key: AnotherContractLeafKey,
Value: AnotherValueBytes,
Leaf: true,
LeafKey: common.BytesToHash(AccountLeafKey),
Path: []byte{'\x0c'},
Value: AccountLeafNode,
Type: statediff.Leaf,
},
}
MockStateMetaPostPublish = []eth.StateNodeModel{
{
CID: State1CID.String(),
Leaf: true,
StateKey: ContractLeafKey.String(),
Path: []byte{'\x06'},
NodeType: 2,
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
},
{
CID: State2CID.String(),
Leaf: true,
StateKey: AnotherContractLeafKey.String(),
Path: []byte{'\x0c'},
NodeType: 2,
StateKey: common.BytesToHash(AccountLeafKey).Hex(),
},
}
MockStorageNodes = map[common.Hash][]eth.TrieNode{
ContractLeafKey: {
contractPathHash: {
{
Key: common.BytesToHash(StorageKey),
Value: StorageValue,
Leaf: true,
LeafKey: common.BytesToHash(StorageLeafKey),
Value: StorageLeafNode,
Type: statediff.Leaf,
Path: []byte{},
},
},
}
@@ -284,11 +306,12 @@ var (
},
StateNodeCIDs: MockStateMetaPostPublish,
StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
ContractLeafKey: {
contractPathHash: {
{
CID: StorageCID.String(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
Leaf: true,
Path: []byte{},
StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
NodeType: 2,
},
},
},
@@ -310,10 +333,11 @@ var (
StateNodes: MockStateMetaPostPublish,
StorageNodes: []eth.StorageNodeWithStateKeyModel{
{
Path: []byte{},
CID: StorageCID.String(),
Leaf: true,
StateKey: ContractLeafKey.Hex(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
NodeType: 2,
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
},
},
}
@@ -323,9 +347,9 @@ var (
Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, StorageCID)
State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
MockIPLDs = eth.IPLDs{
BlockNumber: big.NewInt(1),
@@ -355,31 +379,34 @@ var (
},
StateNodes: []eth2.StateNode{
{
StateTrieKey: ContractLeafKey,
Leaf: true,
StateLeafKey: common.BytesToHash(ContractLeafKey),
Type: statediff.Leaf,
IPLD: ipfs.BlockModel{
Data: State1IPLD.RawData(),
CID: State1IPLD.Cid().String(),
},
Path: []byte{'\x06'},
},
{
StateTrieKey: AnotherContractLeafKey,
Leaf: true,
StateLeafKey: common.BytesToHash(AccountLeafKey),
Type: statediff.Leaf,
IPLD: ipfs.BlockModel{
Data: State2IPLD.RawData(),
CID: State2IPLD.Cid().String(),
},
Path: []byte{'\x0c'},
},
},
StorageNodes: []eth2.StorageNode{
{
StateTrieKey: ContractLeafKey,
StorageTrieKey: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"),
Leaf: true,
StateLeafKey: common.BytesToHash(ContractLeafKey),
StorageLeafKey: common.BytesToHash(StorageLeafKey),
Type: statediff.Leaf,
IPLD: ipfs.BlockModel{
Data: StorageIPLD.RawData(),
CID: StorageIPLD.Cid().String(),
},
Path: []byte{},
},
},
}
pkg/super_node/eth/models.go
+6 -3
View File
@@ -67,8 +67,9 @@ type ReceiptModel struct {
type StateNodeModel struct {
ID int64 `db:"id"`
HeaderID int64 `db:"header_id"`
Path []byte `db:"state_path"`
StateKey string `db:"state_key"`
Leaf bool `db:"leaf"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
}
@@ -76,8 +77,9 @@ type StateNodeModel struct {
type StorageNodeModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Path []byte `db:"storage_path"`
StorageKey string `db:"storage_key"`
Leaf bool `db:"leaf"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
}
@@ -85,8 +87,9 @@ type StorageNodeModel struct {
type StorageNodeWithStateKeyModel struct {
ID int64 `db:"id"`
StateID int64 `db:"state_id"`
Path []byte `db:"storage_path"`
StateKey string `db:"state_key"`
StorageKey string `db:"storage_key"`
Leaf bool `db:"leaf"`
NodeType int `db:"node_type"`
CID string `db:"cid"`
}
pkg/super_node/eth/publisher.go
+10 -8
View File
@@ -185,9 +185,10 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
return nil, err
}
stateNodeCids = append(stateNodeCids, StateNodeModel{
StateKey: node.Key.String(),
Path: node.Path,
StateKey: node.LeafKey.String(),
CID: cids[0],
Leaf: node.Leaf,
NodeType: ResolveFromNodeType(node.Type),
})
}
return stateNodeCids, nil
@@ -195,18 +196,19 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
storageLeafCids := make(map[common.Hash][]StorageNodeModel)
for addrKey, storageTrie := range storageNodes {
storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie))
for pathHash, storageTrie := range storageNodes {
storageLeafCids[pathHash] = make([]StorageNodeModel, 0, len(storageTrie))
for _, node := range storageTrie {
cids, err := pub.StoragePutter.DagPut(node.Value)
if err != nil {
return nil, err
}
// Map storage node cids to their state key hashes
storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{
StorageKey: node.Key.Hex(),
// Map storage node cids to their path hashes
storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{
Path: node.Path,
StorageKey: node.LeafKey.Hex(),
CID: cids[0],
Leaf: node.Leaf,
NodeType: ResolveFromNodeType(node.Type),
})
}
}
pkg/super_node/eth/retriever.go
+5 -4
View File
@@ -391,7 +391,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
log.Debug("retrieving state cids for header id ", headerID)
args := make([]interface{}, 0, 2)
pgStr := `SELECT state_cids.id, state_cids.header_id,
state_cids.state_key, state_cids.leaf, state_cids.cid
state_cids.state_key, state_cids.node_type, state_cids.cid, state_cids.state_path
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.id = $1`
args = append(args, headerID)
@@ -405,7 +405,7 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
args = append(args, pq.Array(keys))
}
if !stateFilter.IntermediateNodes {
pgStr += ` AND state_cids.leaf = TRUE`
pgStr += ` AND state_cids.node_type = 2`
}
stateNodeCIDs := make([]StateNodeModel, 0)
return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
@@ -416,7 +416,8 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
log.Debug("retrieving storage cids for header id ", headerID)
args := make([]interface{}, 0, 3)
pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM eth.storage_cids, eth.state_cids, eth.header_cids
storage_cids.node_type, storage_cids.cid, storage_cids.storage_path, state_cids.state_key
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.id = $1`
@@ -437,7 +438,7 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
args = append(args, pq.Array(storageFilter.StorageKeys))
}
if !storageFilter.IntermediateNodes {
pgStr += ` AND storage_cids.leaf = TRUE`
pgStr += ` AND storage_cids.node_type = 2`
}
storageNodeCIDs := make([]StorageNodeWithStateKeyModel, 0)
return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
pkg/super_node/eth/retriever_test.go
+13 -8
View File
@@ -19,6 +19,8 @@ package eth_test
import (
"math/big"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -196,7 +198,7 @@ var (
Off: true,
},
StateFilter: eth.StateFilter{
Addresses: []string{mocks.Address.Hex()},
Addresses: []string{mocks.AccountAddresss.Hex()},
},
StorageFilter: eth.StorageFilter{
Off: true,
@@ -247,12 +249,14 @@ var _ = Describe("Retriever", func() {
Expect(len(cidWrapper.StateNodes)).To(Equal(2))
for _, stateNode := range cidWrapper.StateNodes {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
}
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
}
}
Expect(len(cidWrapper.StorageNodes)).To(Equal(1))
@@ -384,9 +388,10 @@ var _ = Describe("Retriever", func() {
Expect(cidWrapper7.StateNodes[0]).To(Equal(eth.StateNodeModel{
ID: cidWrapper7.StateNodes[0].ID,
HeaderID: cidWrapper7.StateNodes[0].HeaderID,
Leaf: true,
StateKey: mocks.ContractLeafKey.Hex(),
CID: mocks.State1CID.String(),
NodeType: 2,
StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(),
CID: mocks.State2CID.String(),
Path: []byte{'\x0c'},
}))
_, empty, err = retriever.Retrieve(rctTopicsAndContractFilterFail, 1)
pkg/super_node/eth/types.go
+13 -8
View File
@@ -19,6 +19,8 @@ package eth
import (
"math/big"
"github.com/ethereum/go-ethereum/statediff"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
@@ -45,9 +47,10 @@ func (i ConvertedPayload) Height() int64 {
// Trie struct used to flag node as leaf or not
type TrieNode struct {
Key common.Hash
Value []byte
Leaf bool
Path []byte
LeafKey common.Hash
Value []byte
Type statediff.NodeType
}
// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres
@@ -94,14 +97,16 @@ func (i IPLDs) Height() int64 {
}
type StateNode struct {
StateTrieKey common.Hash
Type statediff.NodeType
StateLeafKey common.Hash
Path []byte
IPLD ipfs.BlockModel
Leaf bool
}
type StorageNode struct {
StateTrieKey common.Hash
StorageTrieKey common.Hash
Type statediff.NodeType
StateLeafKey common.Hash
StorageLeafKey common.Hash
Path []byte
IPLD ipfs.BlockModel
Leaf bool
}