cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

pair with new statediffing geth version; travis tests will fail til release is up

Browse Source
This commit is contained in:
Ian Norden
2020-05-21 15:34:43 -05:00
parent 5fb1cc0696
commit 0101c4791a
26 changed files with 169 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/super_node/btc/publishAndIndexer.go → pkg/super_node/btc/publish_and_indexer.go
View File
pkg/super_node/btc/publishAndIndexer_test.go → pkg/super_node/btc/publish_and_indexer_test.go
View File
pkg/super_node/eth/cleaner_test.go
+2 -2
View File
@@ -124,8 +124,8 @@ var (
storageCID = "mockStorageCID1"
storagePath = []byte{'\x01'}
storageKey = crypto.Keccak256Hash(common.Hex2Bytes("0x0000000000000000000000000000000000000000000000000000000000000000"))
storageModels1 = map[common.Hash][]eth2.StorageNodeModel{
crypto.Keccak256Hash(state1Path): {
storageModels1 = map[string][]eth2.StorageNodeModel{
common.Bytes2Hex(state1Path): {
{
CID: storageCID,
StorageKey: storageKey.String(),
pkg/super_node/eth/converter.go
+18 -49
View File
@@ -61,7 +61,7 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
Receipts: make(types.Receipts, 0, trxLen),
ReceiptMetaData: make([]ReceiptModel, 0, trxLen),
StateNodes: make([]TrieNode, 0),
StorageNodes: make(map[common.Hash][]TrieNode),
StorageNodes: make(map[string][]TrieNode),
}
signer := types.MakeSigner(pc.chainConfig, block.Number())
transactions := block.Transactions()
@@ -100,10 +100,12 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
}
mappedContracts[log.Address.String()] = true
}
// These are the contracts seen in the logs
logContracts := make([]string, 0, len(mappedContracts))
for addr := range mappedContracts {
logContracts = append(logContracts, addr)
}
// This is the contract address if this receipt is for a contract creation tx
contract := shared.HandleNullAddr(receipt.ContractAddress)
var contractHash string
if contract != "" {
@@ -124,60 +126,27 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
}
// Unpack state diff rlp to access fields
stateDiff := new(statediff.StateDiff)
if err := rlp.DecodeBytes(stateDiffPayload.StateDiffRlp, stateDiff); err != nil {
stateDiff := new(statediff.StateObject)
if err := rlp.DecodeBytes(stateDiffPayload.StateObjectRlp, stateDiff); err != nil {
return nil, err
}
for _, createdAccount := range stateDiff.CreatedAccounts {
statePathHash := crypto.Keccak256Hash(createdAccount.Path)
for _, stateNode := range stateDiff.Nodes {
statePath := common.Bytes2Hex(stateNode.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Path: createdAccount.Path,
Value: createdAccount.NodeValue,
Type: createdAccount.NodeType,
LeafKey: common.BytesToHash(createdAccount.LeafKey),
Path: stateNode.Path,
Value: stateNode.NodeValue,
Type: stateNode.NodeType,
LeafKey: common.BytesToHash(stateNode.LeafKey),
})
for _, storageDiff := range createdAccount.Storage {
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}
for _, deletedAccount := range stateDiff.DeletedAccounts {
statePathHash := crypto.Keccak256Hash(deletedAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Path: deletedAccount.Path,
Value: deletedAccount.NodeValue,
Type: deletedAccount.NodeType,
LeafKey: common.BytesToHash(deletedAccount.LeafKey),
})
for _, storageDiff := range deletedAccount.Storage {
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
})
}
}
for _, updatedAccount := range stateDiff.UpdatedAccounts {
statePathHash := crypto.Keccak256Hash(updatedAccount.Path)
convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
Path: updatedAccount.Path,
Value: updatedAccount.NodeValue,
Type: updatedAccount.NodeType,
LeafKey: common.BytesToHash(updatedAccount.LeafKey),
})
for _, storageDiff := range updatedAccount.Storage {
convertedPayload.StorageNodes[statePathHash] = append(convertedPayload.StorageNodes[statePathHash], TrieNode{
Path: storageDiff.Path,
Value: storageDiff.NodeValue,
Type: storageDiff.NodeType,
LeafKey: common.BytesToHash(storageDiff.LeafKey),
for _, storageNode := range stateNode.StorageNodes {
convertedPayload.StorageNodes[statePath] = append(convertedPayload.StorageNodes[statePath], TrieNode{
Path: storageNode.Path,
Value: storageNode.NodeValue,
Type: storageNode.NodeType,
LeafKey: common.BytesToHash(storageNode.LeafKey),
})
}
}
return convertedPayload, nil
}
pkg/super_node/eth/filterer.go
+1 -1
View File
@@ -291,7 +291,7 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
}
}
if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
for _, storageNode := range payload.StorageNodes[crypto.Keccak256Hash(stateNode.Path)] {
for _, storageNode := range payload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
if err != nil {
pkg/super_node/eth/helpers.go
+4
View File
@@ -26,6 +26,8 @@ func ResolveFromNodeType(nodeType statediff.NodeType) int {
return 1
case statediff.Leaf:
return 2
case statediff.Removed:
return 3
default:
return -1
}
@@ -39,6 +41,8 @@ func ResolveToNodeType(nodeType int) statediff.NodeType {
return statediff.Extension
case 2:
return statediff.Leaf
case 3:
return statediff.Removed
default:
return statediff.Unknown
}
pkg/super_node/eth/indexer.go
+3 -4
View File
@@ -20,7 +20,6 @@ import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
@@ -159,13 +158,13 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
}
// If we have a state leaf node, index the associated account and storage nodes
if stateCID.NodeType == 2 {
pathKey := crypto.Keccak256Hash(stateCID.Path)
for _, storageCID := range payload.StorageNodeCIDs[pathKey] {
statePath := common.Bytes2Hex(stateCID.Path)
for _, storageCID := range payload.StorageNodeCIDs[statePath] {
if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
return err
}
}
if stateAccount, ok := payload.StateAccounts[pathKey]; ok {
if stateAccount, ok := payload.StateAccounts[statePath]; ok {
if err := in.indexStateAccount(tx, stateAccount, stateID); err != nil {
return err
}
pkg/super_node/eth/mocks/test_data.go
+21 -21
View File
@@ -218,7 +218,7 @@ var (
nonce1 = uint64(1)
ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
contractPathHash = crypto.Keccak256Hash([]byte{'\x06'})
contractPath = common.Bytes2Hex([]byte{'\x06'})
ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress)
ContractAccount, _ = rlp.EncodeToBytes(state.Account{
Nonce: nonce1,
@@ -235,7 +235,7 @@ var (
nonce0 = uint64(0)
AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
accountPathHash = crypto.Keccak256Hash([]byte{'\x0c'})
accountPath = common.Bytes2Hex([]byte{'\x0c'})
AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
AccountLeafKey = testhelpers.Account2LeafKey
Account, _ = rlp.EncodeToBytes(state.Account{
@@ -250,13 +250,13 @@ var (
Account,
})
CreatedAccountDiffs = []statediff.AccountDiff{
StateDiffs = []statediff.StateNode{
{
Path: []byte{'\x06'},
NodeType: statediff.Leaf,
LeafKey: ContractLeafKey,
NodeValue: ContractLeafNode,
Storage: []statediff.StorageDiff{
StorageNodes: []statediff.StorageNode{
{
Path: []byte{},
NodeType: statediff.Leaf,
@@ -266,18 +266,18 @@ var (
},
},
{
Path: []byte{'\x0c'},
NodeType: statediff.Leaf,
LeafKey: AccountLeafKey,
NodeValue: AccountLeafNode,
Storage: []statediff.StorageDiff{},
Path: []byte{'\x0c'},
NodeType: statediff.Leaf,
LeafKey: AccountLeafKey,
NodeValue: AccountLeafNode,
StorageNodes: []statediff.StorageNode{},
},
}
MockStateDiff = statediff.StateDiff{
BlockNumber: BlockNumber,
BlockHash: MockBlock.Hash(),
CreatedAccounts: CreatedAccountDiffs,
MockStateDiff = statediff.StateObject{
BlockNumber: BlockNumber,
BlockHash: MockBlock.Hash(),
Nodes: StateDiffs,
}
MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
MockStateNodes = []eth.TrieNode{
@@ -308,8 +308,8 @@ var (
StateKey: common.BytesToHash(AccountLeafKey).Hex(),
},
}
MockStorageNodes = map[common.Hash][]eth.TrieNode{
contractPathHash: {
MockStorageNodes = map[string][]eth.TrieNode{
contractPath: {
{
LeafKey: common.BytesToHash(StorageLeafKey),
Value: StorageLeafNode,
@@ -322,7 +322,7 @@ var (
// aggregate payloads
MockStateDiffPayload = statediff.Payload{
BlockRlp: MockBlockRlp,
StateDiffRlp: MockStateDiffBytes,
StateObjectRlp: MockStateDiffBytes,
ReceiptsRlp: ReceiptsRlp,
TotalDifficulty: MockBlock.Difficulty(),
}
@@ -360,8 +360,8 @@ var (
MockTransactions[2].Hash(): MockRctMetaPostPublish[2],
},
StateNodeCIDs: MockStateMetaPostPublish,
StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
contractPathHash: {
StorageNodeCIDs: map[string][]eth.StorageNodeModel{
contractPath: {
{
CID: StorageCID.String(),
Path: []byte{},
@@ -370,14 +370,14 @@ var (
},
},
},
StateAccounts: map[common.Hash]eth.StateAccountModel{
contractPathHash: {
StateAccounts: map[string]eth.StateAccountModel{
contractPath: {
Balance: big.NewInt(0).String(),
Nonce: nonce1,
CodeHash: ContractCodeHash.Bytes(),
StorageRoot: common.HexToHash(ContractRoot).String(),
},
accountPathHash: {
accountPath: {
Balance: big.NewInt(1000).String(),
Nonce: nonce0,
CodeHash: AccountCodeHash.Bytes(),
pkg/super_node/eth/payload_fetcher.go
+13 -6
View File
@@ -38,34 +38,41 @@ type PayloadFetcher struct {
// http.Client is thread-safe
client BatchClient
timeout time.Duration
params statediff.Params
}
const method = "statediff_stateDiffAt"
// NewStateDiffFetcher returns a PayloadFetcher
// NewPayloadFetcher returns a PayloadFetcher
func NewPayloadFetcher(bc BatchClient, timeout time.Duration) *PayloadFetcher {
return &PayloadFetcher{
client: bc,
timeout: timeout,
params: statediff.Params{
IncludeReceipts: true,
IncludeTD: true,
IncludeBlock: true,
IntermediateStateNodes: true,
IntermediateStorageNodes: true,
},
}
}
// FetchAt fetches the statediff payloads at the given block heights
// Calls StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
// Calls StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error)
func (fetcher *PayloadFetcher) FetchAt(blockHeights []uint64) ([]shared.RawChainData, error) {
batch := make([]rpc.BatchElem, 0)
for _, height := range blockHeights {
batch = append(batch, rpc.BatchElem{
Method: method,
Args: []interface{}{height},
Args: []interface{}{height, fetcher.params},
Result: new(statediff.Payload),
})
}
ctx, cancel := context.WithTimeout(context.Background(), fetcher.timeout)
defer cancel()
batchErr := fetcher.client.BatchCallContext(ctx, batch)
if batchErr != nil {
return nil, fmt.Errorf("ethereum PayloadFetcher batch err for block range %d-%d: %s", blockHeights[0], blockHeights[len(blockHeights)-1], batchErr.Error())
if err := fetcher.client.BatchCallContext(ctx, batch); err != nil {
return nil, fmt.Errorf("ethereum PayloadFetcher batch err for block range %d-%d: %s", blockHeights[0], blockHeights[len(blockHeights)-1], err.Error())
}
results := make([]shared.RawChainData, 0, len(blockHeights))
for _, batchElem := range batch {
pkg/super_node/eth/payload_fetcher_test.go
+6 -6
View File
@@ -36,10 +36,10 @@ var _ = Describe("StateDiffFetcher", func() {
)
BeforeEach(func() {
mc = new(mocks.BackFillerClient)
setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
Expect(setDiffAtErr1).ToNot(HaveOccurred())
setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
Expect(setDiffAtErr2).ToNot(HaveOccurred())
err := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
Expect(err).ToNot(HaveOccurred())
err = mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
Expect(err).ToNot(HaveOccurred())
stateDiffFetcher = eth.NewPayloadFetcher(mc, time.Second*60)
})
It("Batch calls statediff_stateDiffAt", func() {
@@ -47,8 +47,8 @@ var _ = Describe("StateDiffFetcher", func() {
test_data.BlockNumber.Uint64(),
test_data.BlockNumber2.Uint64(),
}
stateDiffPayloads, fetchErr := stateDiffFetcher.FetchAt(blockHeights)
Expect(fetchErr).ToNot(HaveOccurred())
stateDiffPayloads, err := stateDiffFetcher.FetchAt(blockHeights)
Expect(err).ToNot(HaveOccurred())
Expect(len(stateDiffPayloads)).To(Equal(2))
payload1, ok := stateDiffPayloads[0].(statediff.Payload)
Expect(ok).To(BeTrue())
pkg/super_node/eth/publishAndIndexer.go → pkg/super_node/eth/publish_and_indexer.go
+2 -3
View File
@@ -19,8 +19,8 @@ package eth
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
"github.com/jmoiron/sqlx"
@@ -195,8 +195,7 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
if err := pub.indexer.indexStateAccount(tx, accountModel, stateID); err != nil {
return err
}
statePathHash := crypto.Keccak256Hash(stateNode.Path)
for _, storageNode := range ipldPayload.StorageNodes[statePathHash] {
for _, storageNode := range ipldPayload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
storageCIDStr, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.Value)
if err != nil {
return err
pkg/super_node/eth/publishAndIndexer_test.go → pkg/super_node/eth/publish_and_indexer_test.go
View File
pkg/super_node/eth/publisher.go
+9 -10
View File
@@ -22,7 +22,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
@@ -206,9 +205,9 @@ func (pub *IPLDPublisher) publishReceipts(receipts []*ipld.EthReceipt, receiptTr
return rctCids, nil
}
func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, map[common.Hash]StateAccountModel, error) {
func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, map[string]StateAccountModel, error) {
stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
stateAccounts := make(map[common.Hash]StateAccountModel)
stateAccounts := make(map[string]StateAccountModel)
for _, stateNode := range stateNodes {
node, err := ipld.FromStateTrieRLP(stateNode.Value)
if err != nil {
@@ -238,8 +237,8 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
return nil, nil, err
}
// Map state account to the state path hash
statePathHash := crypto.Keccak256Hash(stateNode.Path)
stateAccounts[statePathHash] = StateAccountModel{
statePath := common.Bytes2Hex(stateNode.Path)
stateAccounts[statePath] = StateAccountModel{
Balance: account.Balance.String(),
Nonce: account.Nonce,
CodeHash: account.CodeHash,
@@ -250,10 +249,10 @@ func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeM
return stateNodeCids, stateAccounts, nil
}
func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
storageLeafCids := make(map[common.Hash][]StorageNodeModel)
for pathHash, storageTrie := range storageNodes {
storageLeafCids[pathHash] = make([]StorageNodeModel, 0, len(storageTrie))
func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[string][]TrieNode) (map[string][]StorageNodeModel, error) {
storageLeafCids := make(map[string][]StorageNodeModel)
for path, storageTrie := range storageNodes {
storageLeafCids[path] = make([]StorageNodeModel, 0, len(storageTrie))
for _, storageNode := range storageTrie {
node, err := ipld.FromStorageTrieRLP(storageNode.Value)
if err != nil {
@@ -264,7 +263,7 @@ func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]Tri
return nil, err
}
// Map storage node cids to the state path hash
storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{
storageLeafCids[path] = append(storageLeafCids[path], StorageNodeModel{
Path: storageNode.Path,
StorageKey: storageNode.LeafKey.Hex(),
CID: cid,
pkg/super_node/eth/streamer.go
+9 -1
View File
@@ -38,12 +38,20 @@ type StreamClient interface {
// PayloadStreamer satisfies the PayloadStreamer interface for ethereum
type PayloadStreamer struct {
Client StreamClient
params statediff.Params
}
// NewPayloadStreamer creates a pointer to a new PayloadStreamer which satisfies the PayloadStreamer interface for ethereum
func NewPayloadStreamer(client StreamClient) *PayloadStreamer {
return &PayloadStreamer{
Client: client,
params: statediff.Params{
IncludeBlock: true,
IncludeTD: true,
IncludeReceipts: true,
IntermediateStorageNodes: true,
IntermediateStateNodes: true,
},
}
}
@@ -60,5 +68,5 @@ func (ps *PayloadStreamer) Stream(payloadChan chan shared.RawChainData) (shared.
}
}
}()
return ps.Client.Subscribe(context.Background(), "statediff", stateDiffChan, "stream")
return ps.Client.Subscribe(context.Background(), "statediff", stateDiffChan, "stream", ps.params)
}
pkg/super_node/eth/types.go
+3 -3
View File
@@ -37,7 +37,7 @@ type ConvertedPayload struct {
Receipts types.Receipts
ReceiptMetaData []ReceiptModel
StateNodes []TrieNode
StorageNodes map[common.Hash][]TrieNode
StorageNodes map[string][]TrieNode
}
// Height satisfies the StreamedIPLDs interface
@@ -62,8 +62,8 @@ type CIDPayload struct {
TransactionCIDs []TxModel
ReceiptCIDs map[common.Hash]ReceiptModel
StateNodeCIDs []StateNodeModel
StateAccounts map[common.Hash]StateAccountModel
StorageNodeCIDs map[common.Hash][]StorageNodeModel
StateAccounts map[string]StateAccountModel
StorageNodeCIDs map[string][]StorageNodeModel
}
// CIDWrapper is used to direct fetching of IPLDs from IPFS
pkg/watcher/eth/converter.go
+2 -2
View File
@@ -56,7 +56,7 @@ func (pc *WatcherConverter) Convert(ethIPLDs eth.IPLDs) (*eth.CIDPayload, error)
cids.TransactionCIDs = make([]eth.TxModel, numTxs)
cids.ReceiptCIDs = make(map[common.Hash]eth.ReceiptModel, numTxs)
cids.StateNodeCIDs = make([]eth.StateNodeModel, len(ethIPLDs.StateNodes))
cids.StorageNodeCIDs = make(map[common.Hash][]eth.StorageNodeModel, len(ethIPLDs.StateNodes))
cids.StorageNodeCIDs = make(map[string][]eth.StorageNodeModel, len(ethIPLDs.StateNodes))
// Unpack header
var header types.Header
@@ -164,7 +164,7 @@ func (pc *WatcherConverter) Convert(ethIPLDs eth.IPLDs) (*eth.CIDPayload, error)
}
// Storage data
for _, storageIPLD := range ethIPLDs.StorageNodes {
cids.StorageNodeCIDs[storageIPLD.StateLeafKey] = append(cids.StorageNodeCIDs[storageIPLD.StateLeafKey], eth.StorageNodeModel{
cids.StorageNodeCIDs[storageIPLD.StateLeafKey.Hex()] = append(cids.StorageNodeCIDs[storageIPLD.StateLeafKey.Hex()], eth.StorageNodeModel{
CID: storageIPLD.IPLD.CID,
NodeType: eth.ResolveFromNodeType(storageIPLD.Type),
StorageKey: storageIPLD.StorageLeafKey.String(),