cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

api test; return full ipld models to subscribers

Browse Source
This commit is contained in:
Ian Norden 2020-02-20 16:13:19 -06:00
parent fb360d8562
commit e3e8700d34
32 changed files with 1105 additions and 774 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/root.go
View File

@ -41,12 +41,9 @@ var (
databaseConfig config.Database
genConfig config.Plugin
ipc string
levelDbPath string
queueRecheckInterval time.Duration
startingBlockNumber int64
storageDiffsPath string
syncAll bool
endingBlockNumber int64
recheckHeadersArg bool
subCommand string
logWithCommand log.Entry
@ -81,7 +78,6 @@ func initFuncs(cmd *cobra.Command, args []string) {
func setViperConfigs() {
ipc = viper.GetString("client.ipcpath")
levelDbPath = viper.GetString("client.leveldbpath")
storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
storageDiffsSource = viper.GetString("storageDiffs.source")
databaseConfig = config.Database{

61
cmd/streamEthSubscribe.go
View File

@ -80,14 +80,13 @@ func streamEthSubscription() {
logWithCommand.Error(payload.Err)
continue
}
data, ok := payload.Data.(eth.StreamResponse)
if !ok {
logWithCommand.Warnf("payload data expected type %T got %T", eth.StreamResponse{}, payload.Data)
continue
var ethData eth.IPLDs
if err := rlp.DecodeBytes(payload.Data, &ethData); err != nil {
logWithCommand.Error(err)
}
for _, headerRlp := range data.HeadersRlp {
for _, headerRlp := range ethData.Headers {
var header types.Header
err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
err = rlp.Decode(bytes.NewBuffer(headerRlp.Data), &header)
if err != nil {
logWithCommand.Error(err)
continue
@ -95,9 +94,9 @@ func streamEthSubscription() {
fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex())
fmt.Printf("header: %v\n", header)
}
for _, trxRlp := range data.TransactionsRlp {
for _, trxRlp := range ethData.Transactions {
var trx types.Transaction
buff := bytes.NewBuffer(trxRlp)
buff := bytes.NewBuffer(trxRlp.Data)
stream := rlp.NewStream(buff, 0)
err := trx.DecodeRLP(stream)
if err != nil {
@ -107,9 +106,9 @@ func streamEthSubscription() {
fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex())
fmt.Printf("trx: %v\n", trx)
}
for _, rctRlp := range data.ReceiptsRlp {
for _, rctRlp := range ethData.Receipts {
var rct types.ReceiptForStorage
buff := bytes.NewBuffer(rctRlp)
buff := bytes.NewBuffer(rctRlp.Data)
stream := rlp.NewStream(buff, 0)
err = rct.DecodeRLP(stream)
if err != nil {
@ -129,40 +128,34 @@ func streamEthSubscription() {
}
}
// This assumes leafs only
for key, stateRlp := range data.StateNodesRlp {
for _, stateNode := range ethData.StateNodes {
var acct state.Account
err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
err = rlp.Decode(bytes.NewBuffer(stateNode.IPLD.Data), &acct)
if err != nil {
logWithCommand.Error(err)
continue
}
fmt.Printf("Account for key %s, and root %s, with balance %d\n",
key.Hex(), acct.Root.Hex(), acct.Balance.Int64())
stateNode.StateTrieKey.Hex(), acct.Root.Hex(), acct.Balance.Int64())
fmt.Printf("state account: %v\n", acct)
}
for stateKey, mappedRlp := range data.StorageNodesRlp {
fmt.Printf("Storage for state key %s ", stateKey.Hex())
for storageKey, storageRlp := range mappedRlp {
fmt.Printf("with storage key %s\n", storageKey.Hex())
var i []interface{}
err := rlp.DecodeBytes(storageRlp, &i)
if err != nil {
logWithCommand.Error(err)
for _, storageNode := range ethData.StorageNodes {
fmt.Printf("Storage for state key %s ", storageNode.StateTrieKey.Hex())
fmt.Printf("with storage key %s\n", storageNode.StorageTrieKey.Hex())
var i []interface{}
err := rlp.DecodeBytes(storageNode.IPLD.Data, &i)
if err != nil {
logWithCommand.Error(err)
continue
}
// if a value node
if len(i) == 1 {
valueBytes, ok := i[0].([]byte)
if !ok {
continue
}
// if a leaf node
if len(i) == 2 {
keyBytes, ok := i[0].([]byte)
if !ok {
continue
}
valueBytes, ok := i[1].([]byte)
if !ok {
continue
}
fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
common.BytesToHash(keyBytes).Hex(), common.BytesToHash(valueBytes).Hex())
}
fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
storageNode.StorageTrieKey.Hex(), common.BytesToHash(valueBytes).Hex())
}
}
case err = <-sub.Err():

4
cmd/superNode.go
View File

@ -68,9 +68,9 @@ func superNode() {
if err != nil {
logWithCommand.Fatal(err)
}
var forwardPayloadChan chan shared.StreamedIPLDs
var forwardPayloadChan chan shared.ConvertedData
if superNodeConfig.Serve {
forwardPayloadChan = make(chan shared.StreamedIPLDs, super_node.PayloadChanBufferSize)
forwardPayloadChan = make(chan shared.ConvertedData, super_node.PayloadChanBufferSize)
superNode.FilterAndServe(wg, forwardPayloadChan)
if err := startServers(superNode, superNodeConfig); err != nil {
logWithCommand.Fatal(err)

1
db/migrations/00012_create_eth_header_cids_table.sql
View File

@ -7,6 +7,7 @@ CREATE TABLE eth.header_cids (
cid TEXT NOT NULL,
td NUMERIC NOT NULL,
node_id INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
reward NUMERIC NOT NULL,
UNIQUE (block_number, block_hash)
);

1
db/migrations/00013_create_eth_uncle_cids_table.sql
View File

@ -5,6 +5,7 @@ CREATE TABLE eth.uncle_cids (
block_hash VARCHAR(66) NOT NULL,
parent_hash VARCHAR(66) NOT NULL,
cid TEXT NOT NULL,
reward NUMERIC NOT NULL,
UNIQUE (header_id, block_hash)
);

2
pkg/eth/contract_watcher/shared/getter/getter_test.go → integration_test/getter_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package getter_test
package integration_test
import (
"github.com/ethereum/go-ethereum/ethclient"

35
pkg/eth/contract_watcher/shared/getter/getter_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package getter_test
import (
"io/ioutil"
"log"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestRepository(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Getter Suite Test")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

40
pkg/eth/converters/common/block_rewards.go
View File

@ -75,3 +75,43 @@ func staticRewardByBlockNumber(blockNumber int64) *big.Int {
}
return staticBlockReward
}
func CalcEthBlockReward(block *types.Block, receipts types.Receipts) *big.Int {
staticBlockReward := staticRewardByBlockNumber(block.Number().Int64())
transactionFees := calcEthTransactionFees(block, receipts)
uncleInclusionRewards := calcEthUncleInclusionRewards(block, block.Uncles())
tmp := transactionFees.Add(transactionFees, uncleInclusionRewards)
return tmp.Add(tmp, staticBlockReward)
}
func CalcUncleMinerReward(blockNumber, uncleBlockNumber int64) *big.Int {
staticBlockReward := staticRewardByBlockNumber(blockNumber)
rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8))
mainBlock := big.NewInt(blockNumber)
uncleBlock := big.NewInt(uncleBlockNumber)
uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8))
uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock)
return rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock)
}
func calcEthTransactionFees(block *types.Block, receipts types.Receipts) *big.Int {
transactionFees := new(big.Int)
for i, transaction := range block.Transactions() {
receipt := receipts[i]
gasPrice := big.NewInt(transaction.GasPrice().Int64())
gasUsed := big.NewInt(int64(receipt.GasUsed))
transactionFee := gasPrice.Mul(gasPrice, gasUsed)
transactionFees = transactionFees.Add(transactionFees, transactionFee)
}
return transactionFees
}
func calcEthUncleInclusionRewards(block *types.Block, uncles []*types.Header) *big.Int {
uncleInclusionRewards := new(big.Int)
for range uncles {
staticBlockReward := staticRewardByBlockNumber(block.Number().Int64())
staticBlockReward.Div(staticBlockReward, big.NewInt(32))
uncleInclusionRewards.Add(uncleInclusionRewards, staticBlockReward)
}
return uncleInclusionRewards
}

2
pkg/ipfs/dag_putters/eth_receipt.go
View File

@ -40,7 +40,7 @@ func (erdp *EthReceiptDagPutter) DagPut(raw interface{}) ([]string, error) {
}
cids := make([]string, len(receipts))
for i, receipt := range receipts {
node, err := ipld.NewReceipt((*types.ReceiptForStorage)(receipt))
node, err := ipld.NewReceipt(receipt)
if err != nil {
return nil, err
}

12
pkg/ipfs/ipld/eth_receipt.go
View File

@ -29,7 +29,7 @@ import (
)
type EthReceipt struct {
*types.ReceiptForStorage
*types.Receipt
rawdata []byte
cid cid.Cid
@ -43,7 +43,7 @@ var _ node.Node = (*EthReceipt)(nil)
*/
// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node
func NewReceipt(receipt *types.ReceiptForStorage) (*EthReceipt, error) {
func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) {
receiptRLP, err := rlp.EncodeToBytes(receipt)
if err != nil {
return nil, err
@ -53,9 +53,9 @@ func NewReceipt(receipt *types.ReceiptForStorage) (*EthReceipt, error) {
return nil, err
}
return &EthReceipt{
ReceiptForStorage: receipt,
cid: c,
rawdata: receiptRLP,
Receipt: receipt,
cid: c,
rawdata: receiptRLP,
}, nil
}
@ -158,7 +158,7 @@ func (r *EthReceipt) Stat() (*node.NodeStat, error) {
// Size will go away. It is here to comply with the interface.
func (r *EthReceipt) Size() (uint64, error) {
return strconv.ParseUint((*types.Receipt)(r.ReceiptForStorage).Size().String(), 10, 64)
return strconv.ParseUint(r.Receipt.Size().String(), 10, 64)
}
/*

440
pkg/ipfs/ipld/trie_node.go
View File

@ -1,440 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipld
import (
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid"
node "github.com/ipfs/go-ipld-format"
)
// TrieNode is the general abstraction for
//ethereum IPLD trie nodes.
type TrieNode struct {
// leaf, extension or branch
nodeKind string
// If leaf or extension: [0] is key, [1] is val.
// If branch: [0] - [16] are children.
elements []interface{}
// IPLD block information
cid cid.Cid
rawdata []byte
}
/*
OUTPUT
*/
type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error)
// decodeTrieNode returns a TrieNode object from an IPLD block's
// cid and rawdata.
func decodeTrieNode(c cid.Cid, b []byte,
leafDecoder trieNodeLeafDecoder) (*TrieNode, error) {
var (
i, decoded, elements []interface{}
nodeKind string
err error
)
err = rlp.DecodeBytes(b, &i)
if err != nil {
return nil, err
}
codec := c.Type()
switch len(i) {
case 2:
nodeKind, decoded, err = decodeCompactKey(i)
if err != nil {
return nil, err
}
if nodeKind == "extension" {
elements, err = parseTrieNodeExtension(decoded, codec)
}
if nodeKind == "leaf" {
elements, err = leafDecoder(decoded)
}
if nodeKind != "extension" && nodeKind != "leaf" {
return nil, fmt.Errorf("unexpected nodeKind returned from decoder")
}
case 17:
nodeKind = "branch"
elements, err = parseTrieNodeBranch(i, codec)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unknown trie node type")
}
return &TrieNode{
nodeKind: nodeKind,
elements: elements,
rawdata: b,
cid: c,
}, nil
}
// decodeCompactKey takes a compact key, and returns its nodeKind and value.
func decodeCompactKey(i []interface{}) (string, []interface{}, error) {
first := i[0].([]byte)
last := i[1].([]byte)
switch first[0] / 16 {
case '\x00':
return "extension", []interface{}{
nibbleToByte(first)[2:],
last,
}, nil
case '\x01':
return "extension", []interface{}{
nibbleToByte(first)[1:],
last,
}, nil
case '\x02':
return "leaf", []interface{}{
nibbleToByte(first)[2:],
last,
}, nil
case '\x03':
return "leaf", []interface{}{
nibbleToByte(first)[1:],
last,
}, nil
default:
return "", nil, fmt.Errorf("unknown hex prefix")
}
}
// parseTrieNodeExtension helper improves readability
func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) {
return []interface{}{
i[0].([]byte),
keccak256ToCid(codec, i[1].([]byte)),
}, nil
}
// parseTrieNodeBranch helper improves readability
func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) {
var out []interface{}
for _, vi := range i {
v := vi.([]byte)
switch len(v) {
case 0:
out = append(out, nil)
case 32:
out = append(out, keccak256ToCid(codec, v))
default:
return nil, fmt.Errorf("unrecognized object: %v", v)
}
}
return out, nil
}
/*
Node INTERFACE
*/
// Resolve resolves a path through this node, stopping at any link boundary
// and returning the object found as well as the remaining path to traverse
func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) {
switch t.nodeKind {
case "extension":
return t.resolveTrieNodeExtension(p)
case "leaf":
return t.resolveTrieNodeLeaf(p)
case "branch":
return t.resolveTrieNodeBranch(p)
default:
return nil, nil, fmt.Errorf("nodeKind case not implemented")
}
}
// Tree lists all paths within the object under 'path', and up to the given depth.
// To list the entire object (similar to `find .`) pass "" and -1
func (t *TrieNode) Tree(p string, depth int) []string {
if p != "" || depth == 0 {
return nil
}
var out []string
switch t.nodeKind {
case "extension":
var val string
for _, e := range t.elements[0].([]byte) {
val += fmt.Sprintf("%x", e)
}
return []string{val}
case "branch":
for i, elem := range t.elements {
if _, ok := elem.(*cid.Cid); ok {
out = append(out, fmt.Sprintf("%x", i))
}
}
return out
default:
return nil
}
}
// ResolveLink is a helper function that calls resolve and asserts the
// output is a link
func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) {
obj, rest, err := t.Resolve(p)
if err != nil {
return nil, nil, err
}
lnk, ok := obj.(*node.Link)
if !ok {
return nil, nil, fmt.Errorf("was not a link")
}
return lnk, rest, nil
}
// Copy will go away. It is here to comply with the interface.
func (t *TrieNode) Copy() node.Node {
panic("dont use this yet")
}
// Links is a helper function that returns all links within this object
func (t *TrieNode) Links() []*node.Link {
var out []*node.Link
for _, i := range t.elements {
c, ok := i.(cid.Cid)
if ok {
out = append(out, &node.Link{Cid: c})
}
}
return out
}
// Stat will go away. It is here to comply with the interface.
func (t *TrieNode) Stat() (*node.NodeStat, error) {
return &node.NodeStat{}, nil
}
// Size will go away. It is here to comply with the interface.
func (t *TrieNode) Size() (uint64, error) {
return 0, nil
}
/*
TrieNode functions
*/
// MarshalJSON processes the transaction trie into readable JSON format.
func (t *TrieNode) MarshalJSON() ([]byte, error) {
var out map[string]interface{}
switch t.nodeKind {
case "extension":
fallthrough
case "leaf":
var hexPrefix string
for _, e := range t.elements[0].([]byte) {
hexPrefix += fmt.Sprintf("%x", e)
}
// if we got a byte we need to do this casting otherwise
// it will be marshaled to a base64 encoded value
if _, ok := t.elements[1].([]byte); ok {
var hexVal string
for _, e := range t.elements[1].([]byte) {
hexVal += fmt.Sprintf("%x", e)
}
t.elements[1] = hexVal
}
out = map[string]interface{}{
"type": t.nodeKind,
hexPrefix: t.elements[1],
}
case "branch":
out = map[string]interface{}{
"type": "branch",
"0": t.elements[0],
"1": t.elements[1],
"2": t.elements[2],
"3": t.elements[3],
"4": t.elements[4],
"5": t.elements[5],
"6": t.elements[6],
"7": t.elements[7],
"8": t.elements[8],
"9": t.elements[9],
"a": t.elements[10],
"b": t.elements[11],
"c": t.elements[12],
"d": t.elements[13],
"e": t.elements[14],
"f": t.elements[15],
}
default:
return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind)
}
return json.Marshal(out)
}
// nibbleToByte expands the nibbles of a byte slice into their own bytes.
func nibbleToByte(k []byte) []byte {
var out []byte
for _, b := range k {
out = append(out, b/16)
out = append(out, b%16)
}
return out
}
// Resolve reading conveniences
func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) {
nibbles := t.elements[0].([]byte)
idx, rest := shiftFromPath(p, len(nibbles))
if len(idx) < len(nibbles) {
return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension")
}
for _, i := range idx {
if getHexIndex(string(i)) == -1 {
return nil, nil, fmt.Errorf("invalid path element")
}
}
for i, n := range nibbles {
if string(idx[i]) != fmt.Sprintf("%x", n) {
return nil, nil, fmt.Errorf("no such link in this extension")
}
}
return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil
}
func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) {
nibbles := t.elements[0].([]byte)
if len(nibbles) != 0 {
idx, rest := shiftFromPath(p, len(nibbles))
if len(idx) < len(nibbles) {
return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf")
}
for _, i := range idx {
if getHexIndex(string(i)) == -1 {
return nil, nil, fmt.Errorf("invalid path element")
}
}
for i, n := range nibbles {
if string(idx[i]) != fmt.Sprintf("%x", n) {
return nil, nil, fmt.Errorf("no such link in this extension")
}
}
p = rest
}
link, ok := t.elements[1].(node.Node)
if !ok {
return nil, nil, fmt.Errorf("leaf children is not an IPLD node")
}
return link.Resolve(p)
}
func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) {
idx, rest := shiftFromPath(p, 1)
hidx := getHexIndex(idx)
if hidx == -1 {
return nil, nil, fmt.Errorf("incorrect path")
}
child := t.elements[hidx]
if child != nil {
return &node.Link{Cid: child.(cid.Cid)}, rest, nil
}
return nil, nil, fmt.Errorf("no such link in this branch")
}
// shiftFromPath extracts from a given path (as a slice of strings)
// the given number of elements as a single string, returning whatever
// it has not taken.
//
// Examples:
// ["0", "a", "something"] and 1 -> "0" and ["a", "something"]
// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"]
func shiftFromPath(p []string, i int) (string, []string) {
var (
out string
rest []string
)
for _, pe := range p {
re := ""
for _, c := range pe {
if len(out) < i {
out += string(c)
} else {
re += string(c)
}
}
if len(out) == i && re != "" {
rest = append(rest, re)
}
}
return out, rest
}
// getHexIndex returns to you the integer 0 - 15 equivalent to your
// string character if applicable, or -1 otherwise.
func getHexIndex(s string) int {
if len(s) != 1 {
return -1
}
c := byte(s[0])
switch {
case '0' <= c && c <= '9':
return int(c - '0')
case 'a' <= c && c <= 'f':
return int(c - 'a' + 10)
}
return -1
}

26
pkg/super_node/btc/filterer.go
View File

@ -21,6 +21,10 @@ import (
"fmt"
"math/big"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@ -63,7 +67,15 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP
if err := payload.Header.Serialize(headerBuffer); err != nil {
return err
}
response.Headers = append(response.Headers, headerBuffer.Bytes())
data := headerBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MBitcoinHeader, data, multihash.DBL_SHA2_256)
if err != nil {
return err
}
response.Headers = append(response.Headers, ipfs.BlockModel{
Data: data,
CID: cid.String(),
})
}
return nil
}
@ -77,14 +89,22 @@ func checkRange(start, end, actual int64) bool {
func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload ConvertedPayload) error {
if !trxFilter.Off {
response.Transactions = make([][]byte, 0, len(payload.TxMetaData))
response.Transactions = make([]ipfs.BlockModel, 0, len(payload.TxMetaData))
for i, txMeta := range payload.TxMetaData {
if checkTransaction(txMeta, trxFilter) {
trxBuffer := new(bytes.Buffer)
if err := payload.Txs[i].MsgTx().Serialize(trxBuffer); err != nil {
return err
}
response.Transactions = append(response.Transactions, trxBuffer.Bytes())
data := trxBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MBitcoinTx, data, multihash.DBL_SHA2_256)
if err != nil {
return err
}
response.Transactions = append(response.Transactions, ipfs.BlockModel{
Data: data,
CID: cid.String(),
})
}
}
}

33
pkg/super_node/btc/ipld_fetcher.go
View File

@ -21,14 +21,13 @@ import (
"errors"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/ipfs/go-block-format"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
var (
@ -74,7 +73,7 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
// FetchHeaders fetches headers
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching header iplds")
headerCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@ -85,20 +84,23 @@ func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
headerCids[i] = dc
}
headers := f.fetchBatch(headerCids)
headersBytes := make([][]byte, len(headers))
headerIPLDs := make([]ipfs.BlockModel, len(headers))
for i, header := range headers {
headersBytes[i] = header.RawData()
headerIPLDs[i] = ipfs.BlockModel{
Data: header.RawData(),
CID: header.Cid().String(),
}
}
if len(headersBytes) != len(headerCids) {
if len(headerIPLDs) != len(headerCids) {
log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(headers), len(headerCids))
return headersBytes, errUnexpectedNumberOfIPLDs
return headerIPLDs, errUnexpectedNumberOfIPLDs
}
return headersBytes, nil
return headerIPLDs, nil
}
// FetchTrxs fetches transactions
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds")
trxCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@ -109,15 +111,18 @@ func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
trxCids[i] = dc
}
trxs := f.fetchBatch(trxCids)
trxBytes := make([][]byte, len(trxs))
trxIPLDs := make([]ipfs.BlockModel, len(trxs))
for i, trx := range trxs {
trxBytes[i] = trx.RawData()
trxIPLDs[i] = ipfs.BlockModel{
Data: trx.RawData(),
CID: trx.Cid().String(),
}
}
if len(trxBytes) != len(trxCids) {
if len(trxIPLDs) != len(trxCids) {
log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
return trxBytes, errUnexpectedNumberOfIPLDs
return trxIPLDs, errUnexpectedNumberOfIPLDs
}
return trxBytes, nil
return trxIPLDs, nil
}
// fetch is used to fetch a single cid

6
pkg/super_node/btc/types.go
View File

@ -19,6 +19,8 @@ package btc
import (
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
)
@ -64,8 +66,8 @@ type CIDWrapper struct {
// Returned by IPLDFetcher and ResponseFilterer
type IPLDs struct {
BlockNumber *big.Int
Headers [][]byte
Transactions [][]byte
Headers []ipfs.BlockModel
Transactions []ipfs.BlockModel
}
// Height satisfies the StreamedIPLDs interface

25
pkg/super_node/eth/api.go
View File

@ -20,6 +20,8 @@ import (
"context"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -63,6 +65,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
topicStrSets := make([][]string, 4)
for i, topicSet := range crit.Topics {
if i > 3 {
// don't allow more than 4 topics
break
}
for _, topic := range topicSet {
@ -173,19 +176,19 @@ func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.H
return nil, err
}
if tx != nil {
return newRPCTransaction(tx, blockHash, blockNumber, index), nil
return NewRPCTransaction(tx, blockHash, blockNumber, index), nil
}
// Transaction unknown, return as such
return nil, nil
}
// extractLogsOfInterest returns logs from the receipt IPLD
func extractLogsOfInterest(rctIPLDs [][]byte, wantedTopics [][]string) ([]*types.Log, error) {
func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
var logs []*types.Log
for _, rctIPLD := range rctIPLDs {
rctRLP := rctIPLD
var rct types.Receipt
if err := rlp.DecodeBytes(rctRLP, &rct); err != nil {
if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
return nil, err
}
for _, log := range rct.Logs {
@ -200,7 +203,7 @@ func extractLogsOfInterest(rctIPLDs [][]byte, wantedTopics [][]string) ([]*types
// returns true if the log matches on the filter
func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
// actualTopics will always have length <= 4
// wantedTopics will always have length == 4
// wantedTopics will always have length 4
matches := 0
for i, actualTopic := range actualTopics {
// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
@ -291,7 +294,7 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]i
}
if fullTx {
formatTx = func(tx *types.Transaction) (interface{}, error) {
return newRPCTransactionFromBlockHash(block, tx.Hash()), nil
return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
}
}
txs := block.Transactions()
@ -314,8 +317,8 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]i
return fields, nil
}
// newRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
func newRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
for idx, tx := range b.Transactions() {
if tx.Hash() == hash {
return newRPCTransactionFromBlockIndex(b, uint64(idx))
@ -330,7 +333,7 @@ func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransacti
if index >= uint64(len(txs)) {
return nil
}
return newRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
}
// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
@ -351,9 +354,9 @@ type RPCTransaction struct {
S *hexutil.Big `json:"s"`
}
// newRPCTransaction returns a transaction that will serialize to the RPC
// NewRPCTransaction returns a transaction that will serialize to the RPC
// representation, with the given location metadata set (if available).
func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
var signer types.Signer = types.FrontierSigner{}
if tx.Protected() {
signer = types.NewEIP155Signer(tx.ChainId())
@ -366,7 +369,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber
Gas: hexutil.Uint64(tx.Gas()),
GasPrice: (*hexutil.Big)(tx.GasPrice()),
Hash: tx.Hash(),
Input: hexutil.Bytes(tx.Data()),
Input: hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
Nonce: hexutil.Uint64(tx.Nonce()),
To: tx.To(),
Value: (*hexutil.Big)(tx.Value()),

588
pkg/super_node/eth/api_test.go Normal file
View File

@ -0,0 +1,588 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth_test
import (
"context"
"strconv"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
mocks3 "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
var (
expectedBlock = map[string]interface{}{
"number": (*hexutil.Big)(mocks.MockBlock.Number()),
"hash": mocks.MockBlock.Hash(),
"parentHash": mocks.MockBlock.ParentHash(),
"nonce": mocks.MockBlock.Header().Nonce,
"mixHash": mocks.MockBlock.MixDigest(),
"sha3Uncles": mocks.MockBlock.UncleHash(),
"logsBloom": mocks.MockBlock.Bloom(),
"stateRoot": mocks.MockBlock.Root(),
"miner": mocks.MockBlock.Coinbase(),
"difficulty": (*hexutil.Big)(mocks.MockBlock.Difficulty()),
"extraData": hexutil.Bytes(mocks.MockBlock.Header().Extra),
"gasLimit": hexutil.Uint64(mocks.MockBlock.GasLimit()),
"gasUsed": hexutil.Uint64(mocks.MockBlock.GasUsed()),
"timestamp": hexutil.Uint64(mocks.MockBlock.Time()),
"transactionsRoot": mocks.MockBlock.TxHash(),
"receiptsRoot": mocks.MockBlock.ReceiptHash(),
"totalDifficulty": (*hexutil.Big)(mocks.MockBlock.Difficulty()),
"size": hexutil.Uint64(mocks.MockBlock.Size()),
}
expectedHeader = map[string]interface{}{
"number": (*hexutil.Big)(mocks.MockBlock.Header().Number),
"hash": mocks.MockBlock.Header().Hash(),
"parentHash": mocks.MockBlock.Header().ParentHash,
"nonce": mocks.MockBlock.Header().Nonce,
"mixHash": mocks.MockBlock.Header().MixDigest,
"sha3Uncles": mocks.MockBlock.Header().UncleHash,
"logsBloom": mocks.MockBlock.Header().Bloom,
"stateRoot": mocks.MockBlock.Header().Root,
"miner": mocks.MockBlock.Header().Coinbase,
"difficulty": (*hexutil.Big)(mocks.MockBlock.Header().Difficulty),
"extraData": hexutil.Bytes(mocks.MockBlock.Header().Extra),
"size": hexutil.Uint64(mocks.MockBlock.Header().Size()),
"gasLimit": hexutil.Uint64(mocks.MockBlock.Header().GasLimit),
"gasUsed": hexutil.Uint64(mocks.MockBlock.Header().GasUsed),
"timestamp": hexutil.Uint64(mocks.MockBlock.Header().Time),
"transactionsRoot": mocks.MockBlock.Header().TxHash,
"receiptsRoot": mocks.MockBlock.Header().ReceiptHash,
"totalDifficulty": (*hexutil.Big)(mocks.MockBlock.Header().Difficulty),
}
expectedTransaction = eth.NewRPCTransaction(mocks.MockTransactions[0], mocks.MockBlock.Hash(), mocks.MockBlock.NumberU64(), 0)
)
var _ = Describe("API", func() {
var (
db *postgres.DB
retriever *eth.CIDRetriever
fetcher *eth.IPLDFetcher
indexer *eth.CIDIndexer
backend *eth.Backend
api *eth.PublicEthAPI
)
BeforeEach(func() {
var err error
db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred())
retriever = eth.NewCIDRetriever(db)
blocksToReturn := map[cid.Cid]blocks.Block{
mocks.HeaderCID: mocks.HeaderIPLD,
mocks.Trx1CID: mocks.Trx1IPLD,
mocks.Trx2CID: mocks.Trx2IPLD,
mocks.Rct1CID: mocks.Rct1IPLD,
mocks.Rct2CID: mocks.Rct2IPLD,
mocks.State1CID: mocks.State1IPLD,
mocks.State2CID: mocks.State2IPLD,
mocks.StorageCID: mocks.StorageIPLD,
}
mockBlockService := &mocks3.MockIPFSBlockService{
Blocks: blocksToReturn,
}
fetcher = &eth.IPLDFetcher{
BlockService: mockBlockService,
}
indexer = eth.NewCIDIndexer(db)
backend = &eth.Backend{
Retriever: retriever,
Fetcher: fetcher,
DB: db,
}
api = eth.NewPublicEthAPI(backend)
err = indexer.Index(mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
uncles := mocks.MockBlock.Uncles()
uncleHashes := make([]common.Hash, len(uncles))
for i, uncle := range uncles {
uncleHashes[i] = uncle.Hash()
}
expectedBlock["uncles"] = uncleHashes
})
AfterEach(func() {
eth.TearDownDB(db)
})
Describe("BlockNumber", func() {
It("Retrieves the head block number", func() {
bn := api.BlockNumber()
ubn := (uint64)(bn)
subn := strconv.FormatUint(ubn, 10)
Expect(subn).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber))
})
})
Describe("GetTransactionByHash", func() {
It("Retrieves the head block number", func() {
hash := mocks.MockTransactions[0].Hash()
tx, err := api.GetTransactionByHash(context.Background(), hash)
Expect(err).ToNot(HaveOccurred())
Expect(tx).To(Equal(expectedTransaction))
})
})
Describe("GetBlockByNumber", func() {
It("Retrieves a block by number", func() {
// without full txs
number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
Expect(err).ToNot(HaveOccurred())
block, err := api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), false)
Expect(err).ToNot(HaveOccurred())
transactionHashes := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactionHashes[i] = trx.Hash()
}
expectedBlock["transactions"] = transactionHashes
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
// with full txs
block, err = api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), true)
Expect(err).ToNot(HaveOccurred())
transactions := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactions[i] = eth.NewRPCTransactionFromBlockHash(mocks.MockBlock, trx.Hash())
}
expectedBlock["transactions"] = transactions
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
})
})
Describe("GetHeaderByNumber", func() {
It("Retrieves a header by number", func() {
number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
Expect(err).ToNot(HaveOccurred())
header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number))
Expect(header).To(Equal(expectedHeader))
})
})
Describe("GetBlockByHash", func() {
It("Retrieves a block by hash", func() {
// without full txs
block, err := api.GetBlockByHash(context.Background(), mocks.MockBlock.Hash(), false)
Expect(err).ToNot(HaveOccurred())
transactionHashes := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactionHashes[i] = trx.Hash()
}
expectedBlock["transactions"] = transactionHashes
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
// with full txs
block, err = api.GetBlockByHash(context.Background(), mocks.MockBlock.Hash(), true)
Expect(err).ToNot(HaveOccurred())
transactions := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactions[i] = eth.NewRPCTransactionFromBlockHash(mocks.MockBlock, trx.Hash())
}
expectedBlock["transactions"] = transactions
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
})
})
Describe("GetLogs", func() {
It("Retrieves receipt logs that match the provided topcis within the provided range", func() {
crit := ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err := api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x06"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(0))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x06"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x05"),
},
{
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x06"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
})
It("Uses the provided blockhash if one is provided", func() {
hash := mocks.MockBlock.Hash()
crit := ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x06"),
},
},
}
logs, err := api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x06"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x05"),
},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(0))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
})
It("Filters on contract address if any are provided", func() {
hash := mocks.MockBlock.Hash()
crit := ethereum.FilterQuery{
BlockHash: &hash,
Addresses: []common.Address{
mocks.Address,
},
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
}
logs, err := api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
hash = mocks.MockBlock.Hash()
crit = ethereum.FilterQuery{
BlockHash: &hash,
Addresses: []common.Address{
mocks.Address,
mocks.AnotherAddress,
},
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
hash = mocks.MockBlock.Hash()
crit = ethereum.FilterQuery{
BlockHash: &hash,
Addresses: []common.Address{
mocks.Address,
mocks.AnotherAddress,
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
})
})
})

69
pkg/super_node/eth/backend.go
View File

@ -93,19 +93,19 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
// Decode the first header at this block height and return it
// We throw an error in FetchHeaders() if the number of headers does not match the number of CIDs and we already
// confirmed the number of CIDs is greater than 0 so there is no need to bound check the slice before accessing
header := new(types.Header)
if err := rlp.DecodeBytes(headerIPLDs[0], header); err != nil {
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
return nil, err
}
return header, nil
return &header, nil
}
// GetTd retrieves and returns the total difficulty at the given block hash
func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
pgStr := `SELECT header_cids.td FROM header_cids
pgStr := `SELECT td FROM eth.header_cids
WHERE header_cids.block_hash = $1`
var tdStr string
err := b.DB.Select(&tdStr, pgStr, blockHash.String())
err := b.DB.Get(&tdStr, pgStr, blockHash.String())
if err != nil {
return nil, err
}
@ -142,7 +142,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log
logs := make([][]*types.Log, len(receiptIPLDs))
for i, rctIPLD := range receiptIPLDs {
var rct types.Receipt
if err := rlp.DecodeBytes(rctIPLD, &rct); err != nil {
if err := rlp.DecodeBytes(rctIPLD.Data, &rct); err != nil {
return nil, err
}
logs[i] = rct.Logs
@ -170,13 +170,14 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
if err != nil {
return nil, err
}
// Fetch and decode the header IPLD
headerIPLDs, err := b.Fetcher.FetchHeaders([]HeaderModel{headerCID})
if err != nil {
return nil, err
}
var header *types.Header
if err := rlp.DecodeBytes(headerIPLDs[0], header); err != nil {
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
return nil, err
}
// Fetch and decode the uncle IPLDs
@ -186,11 +187,11 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
var uncles []*types.Header
for _, uncleIPLD := range uncleIPLDs {
var uncle *types.Header
if err := rlp.DecodeBytes(uncleIPLD, uncle); err != nil {
var uncle types.Header
if err := rlp.DecodeBytes(uncleIPLD.Data, &uncle); err != nil {
return nil, err
}
uncles = append(uncles, uncle)
uncles = append(uncles, &uncle)
}
// Fetch and decode the transaction IPLDs
txIPLDs, err := b.Fetcher.FetchTrxs(txCIDs)
@ -199,11 +200,11 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
var transactions []*types.Transaction
for _, txIPLD := range txIPLDs {
var tx *types.Transaction
if err := rlp.DecodeBytes(txIPLD, tx); err != nil {
var tx types.Transaction
if err := rlp.DecodeBytes(txIPLD.Data, &tx); err != nil {
return nil, err
}
transactions = append(transactions, tx)
transactions = append(transactions, &tx)
}
// Fetch and decode the receipt IPLDs
rctIPLDs, err := b.Fetcher.FetchRcts(rctCIDs)
@ -212,14 +213,14 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs {
var receipt *types.Receipt
if err := rlp.DecodeBytes(rctIPLD, receipt); err != nil {
var receipt types.Receipt
if err := rlp.DecodeBytes(rctIPLD.Data, &receipt); err != nil {
return nil, err
}
receipts = append(receipts, receipt)
receipts = append(receipts, &receipt)
}
// Compose everything together into a complete block
return types.NewBlock(header, transactions, uncles, receipts), nil
return types.NewBlock(&header, transactions, uncles, receipts), nil
}
// BlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
@ -235,8 +236,8 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
if err != nil {
return nil, err
}
var header *types.Header
if err := rlp.DecodeBytes(headerIPLDs[0], header); err != nil {
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
return nil, err
}
// Fetch and decode the uncle IPLDs
@ -246,11 +247,11 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
var uncles []*types.Header
for _, uncleIPLD := range uncleIPLDs {
var uncle *types.Header
if err := rlp.DecodeBytes(uncleIPLD, uncle); err != nil {
var uncle types.Header
if err := rlp.DecodeBytes(uncleIPLD.Data, &uncle); err != nil {
return nil, err
}
uncles = append(uncles, uncle)
uncles = append(uncles, &uncle)
}
// Fetch and decode the transaction IPLDs
txIPLDs, err := b.Fetcher.FetchTrxs(txCIDs)
@ -259,11 +260,11 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
var transactions []*types.Transaction
for _, txIPLD := range txIPLDs {
var tx *types.Transaction
if err := rlp.DecodeBytes(txIPLD, tx); err != nil {
var tx types.Transaction
if err := rlp.DecodeBytes(txIPLD.Data, &tx); err != nil {
return nil, err
}
transactions = append(transactions, tx)
transactions = append(transactions, &tx)
}
// Fetch and decode the receipt IPLDs
rctIPLDs, err := b.Fetcher.FetchRcts(rctCIDs)
@ -272,21 +273,21 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs {
var receipt *types.Receipt
if err := rlp.DecodeBytes(rctIPLD, receipt); err != nil {
var receipt types.Receipt
if err := rlp.DecodeBytes(rctIPLD.Data, &receipt); err != nil {
return nil, err
}
receipts = append(receipts, receipt)
receipts = append(receipts, &receipt)
}
// Compose everything together into a complete block
return types.NewBlock(header, transactions, uncles, receipts), nil
return types.NewBlock(&header, transactions, uncles, receipts), nil
}
// GetTransaction retrieves a tx by hash
// It also returns the blockhash, blocknumber, and tx index associated with the transaction
func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) {
pgStr := `SELECT transaction_cids.cid, transaction_cids.index, header_cids.block_hash, header_cids.block_number
FROM transaction_cids, header_cids
FROM eth.transaction_cids, eth.header_cids
WHERE transaction_cids.header_id = header_cids.id
AND transaction_cids.tx_hash = $1`
var txCIDWithHeaderInfo struct {
@ -302,9 +303,9 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
if err != nil {
return nil, common.Hash{}, 0, 0, err
}
var transaction *types.Transaction
if err := rlp.DecodeBytes(txIPLD[0], transaction); err != nil {
var transaction types.Transaction
if err := rlp.DecodeBytes(txIPLD[0].Data, &transaction); err != nil {
return nil, common.Hash{}, 0, 0, err
}
return transaction, common.HexToHash(txCIDWithHeaderInfo.BlockHash), uint64(txCIDWithHeaderInfo.BlockNumber), uint64(txCIDWithHeaderInfo.Index), nil
return &transaction, common.HexToHash(txCIDWithHeaderInfo.BlockHash), uint64(txCIDWithHeaderInfo.BlockNumber), uint64(txCIDWithHeaderInfo.Index), nil
}

72
pkg/super_node/eth/filterer.go
View File

@ -24,7 +24,10 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@ -80,15 +83,29 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP
if err != nil {
return err
}
response.Headers = append(response.Headers, headerRLP)
cid, err := ipld.RawdataToCid(ipld.MEthHeader, headerRLP, multihash.KECCAK_256)
if err != nil {
return err
}
response.Headers = append(response.Headers, ipfs.BlockModel{
Data: headerRLP,
CID: cid.String(),
})
if headerFilter.Uncles {
response.Uncles = make([][]byte, len(payload.Block.Body().Uncles))
response.Uncles = make([]ipfs.BlockModel, len(payload.Block.Body().Uncles))
for i, uncle := range payload.Block.Body().Uncles {
uncleRlp, err := rlp.EncodeToBytes(uncle)
if err != nil {
return err
}
response.Uncles[i] = uncleRlp
cid, err := ipld.RawdataToCid(ipld.MEthHeader, uncleRlp, multihash.KECCAK_256)
if err != nil {
return err
}
response.Uncles[i] = ipfs.BlockModel{
Data: uncleRlp,
CID: cid.String(),
}
}
}
}
@ -107,14 +124,22 @@ func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLD
if !trxFilter.Off {
trxLen := len(payload.Block.Body().Transactions)
trxHashes = make([]common.Hash, 0, trxLen)
response.Transactions = make([][]byte, 0, trxLen)
response.Transactions = make([]ipfs.BlockModel, 0, trxLen)
for i, trx := range payload.Block.Body().Transactions {
if checkTransactionAddrs(trxFilter.Src, trxFilter.Dst, payload.TxMetaData[i].Src, payload.TxMetaData[i].Dst) {
trxBuffer := new(bytes.Buffer)
if err := trx.EncodeRLP(trxBuffer); err != nil {
return nil, err
}
response.Transactions = append(response.Transactions, trxBuffer.Bytes())
data := trxBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MEthTx, data, multihash.KECCAK_256)
if err != nil {
return nil, err
}
response.Transactions = append(response.Transactions, ipfs.BlockModel{
Data: data,
CID: cid.String(),
})
trxHashes = append(trxHashes, trx.Hash())
}
}
@ -143,17 +168,24 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
if !receiptFilter.Off {
response.Receipts = make([][]byte, 0, len(payload.Receipts))
response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
for i, receipt := range payload.Receipts {
// topics is always length 4
topics := [][]string{payload.ReceiptMetaData[i].Topic0s, payload.ReceiptMetaData[i].Topic1s, payload.ReceiptMetaData[i].Topic2s, payload.ReceiptMetaData[i].Topic3s}
if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.Contracts, payload.ReceiptMetaData[i].Contract, trxHashes) {
receiptForStorage := (*types.ReceiptForStorage)(receipt)
receiptBuffer := new(bytes.Buffer)
if err := receiptForStorage.EncodeRLP(receiptBuffer); err != nil {
if err := receipt.EncodeRLP(receiptBuffer); err != nil {
return err
}
response.Receipts = append(response.Receipts, receiptBuffer.Bytes())
data := receiptBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
if err != nil {
return err
}
response.Receipts = append(response.Receipts, ipfs.BlockModel{
Data: data,
CID: cid.String(),
})
}
}
}
@ -231,10 +263,17 @@ func (s *ResponseFilterer) filterState(stateFilter StateFilter, response *IPLDs,
for _, stateNode := range payload.StateNodes {
if checkNodeKeys(keyFilters, stateNode.Key) {
if stateNode.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StateNodes = append(response.StateNodes, StateNode{
StateTrieKey: stateNode.Key,
IPLD: stateNode.Value,
Leaf: stateNode.Leaf,
IPLD: ipfs.BlockModel{
Data: stateNode.Value,
CID: cid.String(),
},
Leaf: stateNode.Leaf,
})
}
}
@ -271,11 +310,18 @@ func (s *ResponseFilterer) filterStorage(storageFilter StorageFilter, response *
if checkNodeKeys(stateKeyFilters, stateKey) {
for _, storageNode := range storageNodes {
if checkNodeKeys(storageKeyFilters, storageNode.Key) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StorageNodes = append(response.StorageNodes, StorageNode{
StateTrieKey: stateKey,
StorageTrieKey: storageNode.Key,
IPLD: storageNode.Value,
Leaf: storageNode.Leaf,
IPLD: ipfs.BlockModel{
Data: storageNode.Value,
CID: cid.String(),
},
Leaf: storageNode.Leaf,
})
}
}

79
pkg/super_node/eth/filterer_test.go
View File

@ -17,9 +17,8 @@
package eth_test
import (
"bytes"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -29,17 +28,13 @@ import (
)
var (
filterer *eth.ResponseFilterer
expectedRctForStorageRLP1 []byte
expectedRctForStorageRLP2 []byte
filterer *eth.ResponseFilterer
)
var _ = Describe("Filterer", func() {
Describe("FilterResponse", func() {
BeforeEach(func() {
filterer = eth.NewResponseFilterer()
expectedRctForStorageRLP1 = getReceiptForStorageRLP(mocks.MockReceipts, 0)
expectedRctForStorageRLP2 = getReceiptForStorageRLP(mocks.MockReceipts, 1)
})
It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
@ -49,22 +44,28 @@ var _ = Describe("Filterer", func() {
Expect(ok).To(BeTrue())
Expect(iplds.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(iplds.Headers).To(Equal(mocks.MockIPLDs.Headers))
var unclesRlp [][]byte
Expect(iplds.Uncles).To(Equal(unclesRlp))
var expectedEmptyUncles []ipfs.BlockModel
Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
Expect(len(iplds.Transactions)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.ListContainsBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(len(iplds.Receipts)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds.Receipts, expectedRctForStorageRLP1)).To(BeTrue())
Expect(shared.ListContainsBytes(iplds.Receipts, expectedRctForStorageRLP2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
Expect(len(iplds.StateNodes)).To(Equal(2))
for _, stateNode := range iplds.StateNodes {
Expect(stateNode.Leaf).To(BeTrue())
if stateNode.StateTrieKey == mocks.ContractLeafKey {
Expect(stateNode.IPLD).To(Equal(mocks.ValueBytes))
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
}
if stateNode.StateTrieKey == mocks.AnotherContractLeafKey {
Expect(stateNode.IPLD).To(Equal(mocks.AnotherValueBytes))
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State2IPLD.RawData(),
CID: mocks.State2IPLD.Cid().String(),
}))
}
}
Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
@ -82,7 +83,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds1.StorageNodes)).To(Equal(0))
Expect(len(iplds1.StateNodes)).To(Equal(0))
Expect(len(iplds1.Receipts)).To(Equal(1))
Expect(iplds1.Receipts[0]).To(Equal(expectedRctForStorageRLP2))
Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct2IPLD.RawData(),
CID: mocks.Rct2IPLD.Cid().String(),
}))
payload2, err := filterer.Filter(rctTopicsFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -95,7 +99,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds2.StorageNodes)).To(Equal(0))
Expect(len(iplds2.StateNodes)).To(Equal(0))
Expect(len(iplds2.Receipts)).To(Equal(1))
Expect(iplds2.Receipts[0]).To(Equal(expectedRctForStorageRLP1))
Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct1IPLD.RawData(),
CID: mocks.Rct1IPLD.Cid().String(),
}))
payload3, err := filterer.Filter(rctTopicsAndContractFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -108,7 +115,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds3.StorageNodes)).To(Equal(0))
Expect(len(iplds3.StateNodes)).To(Equal(0))
Expect(len(iplds3.Receipts)).To(Equal(1))
Expect(iplds3.Receipts[0]).To(Equal(expectedRctForStorageRLP1))
Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct1IPLD.RawData(),
CID: mocks.Rct1IPLD.Cid().String(),
}))
payload4, err := filterer.Filter(rctContractsAndTopicFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -121,7 +131,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds4.StorageNodes)).To(Equal(0))
Expect(len(iplds4.StateNodes)).To(Equal(0))
Expect(len(iplds4.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(expectedRctForStorageRLP2))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct2IPLD.RawData(),
CID: mocks.Rct2IPLD.Cid().String(),
}))
payload5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -131,13 +144,13 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds5.Headers)).To(Equal(0))
Expect(len(iplds5.Uncles)).To(Equal(0))
Expect(len(iplds5.Transactions)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.ListContainsBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(len(iplds5.StorageNodes)).To(Equal(0))
Expect(len(iplds5.StateNodes)).To(Equal(0))
Expect(len(iplds5.Receipts)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds5.Receipts, expectedRctForStorageRLP1)).To(BeTrue())
Expect(shared.ListContainsBytes(iplds5.Receipts, expectedRctForStorageRLP2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
payload6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -147,11 +160,14 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds6.Headers)).To(Equal(0))
Expect(len(iplds6.Uncles)).To(Equal(0))
Expect(len(iplds6.Transactions)).To(Equal(1))
Expect(shared.ListContainsBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(len(iplds6.StorageNodes)).To(Equal(0))
Expect(len(iplds6.StateNodes)).To(Equal(0))
Expect(len(iplds6.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(expectedRctForStorageRLP2))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct2IPLD.RawData(),
CID: mocks.Rct2IPLD.Cid().String(),
}))
payload7, err := filterer.Filter(stateFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -165,7 +181,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds7.Receipts)).To(Equal(0))
Expect(len(iplds7.StateNodes)).To(Equal(1))
Expect(iplds7.StateNodes[0].StateTrieKey).To(Equal(mocks.ContractLeafKey))
Expect(iplds7.StateNodes[0].IPLD).To(Equal(mocks.ValueBytes))
Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
payload8, err := filterer.Filter(rctTopicsAndContractFilterFail, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -181,11 +200,3 @@ var _ = Describe("Filterer", func() {
})
})
})
func getReceiptForStorageRLP(receipts types.Receipts, i int) []byte {
receiptForStorage := (*types.ReceiptForStorage)(receipts[i])
receiptBuffer := new(bytes.Buffer)
err := receiptForStorage.EncodeRLP(receiptBuffer)
Expect(err).ToNot(HaveOccurred())
return receiptBuffer.Bytes()
}

10
pkg/super_node/eth/indexer.go
View File

@ -82,16 +82,16 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id) = ($3, $4, $5, $6)
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward) = ($3, $4, $5, $6, $7)
RETURNING id`,
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID).Scan(&headerID)
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward).Scan(&headerID)
return headerID, err
}
func (in *CIDIndexer) indexUncleCID(tx *sqlx.Tx, uncle UncleModel, headerID int64) error {
_, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid) VALUES ($1, $2, $3, $4)
ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid) = ($3, $4)`,
_, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward) = ($3, $4, $5)`,
uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID)
return err
}

26
pkg/super_node/eth/indexer_test.go
View File

@ -45,18 +45,20 @@ var _ = Describe("Indexer", func() {
It("Indexes CIDs and related metadata into vulcanizedb", func() {
err = repo.Index(mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
pgStr := `SELECT cid, td FROM eth.header_cids
pgStr := `SELECT cid, td, reward FROM eth.header_cids
WHERE block_number = $1`
// check header was properly indexed
type res struct {
CID string
TD string
CID string
TD string
Reward string
}
headers := new(res)
err = db.QueryRowx(pgStr, 1).StructScan(headers)
Expect(err).ToNot(HaveOccurred())
Expect(headers.CID).To(Equal("mockHeaderCID"))
Expect(headers.TD).To(Equal("1337"))
Expect(headers.CID).To(Equal(mocks.HeaderCID.String()))
Expect(headers.TD).To(Equal(mocks.MockBlock.Difficulty().String()))
Expect(headers.Reward).To(Equal("5000000000000000000"))
// check trxs were properly indexed
trxs := make([]string, 0)
pgStr = `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
@ -64,8 +66,8 @@ var _ = Describe("Indexer", func() {
err = db.Select(&trxs, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(trxs)).To(Equal(2))
Expect(shared.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
Expect(shared.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx2CID.String())).To(BeTrue())
// check receipts were properly indexed
rcts := make([]string, 0)
pgStr = `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
@ -75,8 +77,8 @@ var _ = Describe("Indexer", func() {
err = db.Select(&rcts, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(rcts)).To(Equal(2))
Expect(shared.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
Expect(shared.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
// check that state nodes were properly indexed
stateNodes := make([]eth.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
@ -85,11 +87,11 @@ var _ = Describe("Indexer", func() {
Expect(err).ToNot(HaveOccurred())
Expect(len(stateNodes)).To(Equal(2))
for _, stateNode := range stateNodes {
if stateNode.CID == "mockStateCID1" {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
}
if stateNode.CID == "mockStateCID2" {
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
}
@ -104,7 +106,7 @@ var _ = Describe("Indexer", func() {
Expect(err).ToNot(HaveOccurred())
Expect(len(storageNodes)).To(Equal(1))
Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
CID: "mockStorageCID",
CID: mocks.StorageCID.String(),
Leaf: true,
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
StateKey: mocks.ContractLeafKey.Hex(),

71
pkg/super_node/eth/ipld_fetcher.go
View File

@ -90,7 +90,7 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
// FetchHeaders fetches headers
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching header iplds")
headerCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@ -101,20 +101,23 @@ func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
headerCids[i] = dc
}
headers := f.fetchBatch(headerCids)
headersRLP := make([][]byte, len(headers))
headerIPLDs := make([]ipfs.BlockModel, len(headers))
for i, header := range headers {
headersRLP[i] = header.RawData()
headerIPLDs[i] = ipfs.BlockModel{
Data: header.RawData(),
CID: header.Cid().String(),
}
}
if len(headersRLP) != len(headerCids) {
if len(headerIPLDs) != len(headerCids) {
log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(headers), len(headerCids))
return headersRLP, errUnexpectedNumberOfIPLDs
return headerIPLDs, errUnexpectedNumberOfIPLDs
}
return headersRLP, nil
return headerIPLDs, nil
}
// FetchUncles fetches uncles
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchUncles(cids []UncleModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchUncles(cids []UncleModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching uncle iplds")
uncleCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@ -125,20 +128,23 @@ func (f *IPLDFetcher) FetchUncles(cids []UncleModel) ([][]byte, error) {
uncleCids[i] = dc
}
uncles := f.fetchBatch(uncleCids)
unclesRLP := make([][]byte, len(uncles))
uncleIPLDs := make([]ipfs.BlockModel, len(uncles))
for i, uncle := range uncles {
unclesRLP[i] = uncle.RawData()
uncleIPLDs[i] = ipfs.BlockModel{
Data: uncle.RawData(),
CID: uncle.Cid().String(),
}
}
if len(unclesRLP) != len(uncleCids) {
if len(uncleIPLDs) != len(uncleCids) {
log.Errorf("ipfs fetcher: number of uncle blocks returned (%d) does not match number expected (%d)", len(uncles), len(uncleCids))
return unclesRLP, errUnexpectedNumberOfIPLDs
return uncleIPLDs, errUnexpectedNumberOfIPLDs
}
return unclesRLP, nil
return uncleIPLDs, nil
}
// FetchTrxs fetches transactions
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds")
trxCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@ -149,20 +155,24 @@ func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
trxCids[i] = dc
}
trxs := f.fetchBatch(trxCids)
trxsRLP := make([][]byte, len(trxs))
trxIPLDs := make([]ipfs.BlockModel, len(trxs))
for i, trx := range trxs {
trxsRLP[i] = trx.RawData()
trxIPLDs[i] = ipfs.BlockModel{
Data: trx.RawData(),
CID: trx.Cid().String(),
}
}
if len(trxsRLP) != len(trxCids) {
if len(trxIPLDs) != len(trxCids) {
log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
return trxsRLP, errUnexpectedNumberOfIPLDs
return trxIPLDs, errUnexpectedNumberOfIPLDs
}
return trxsRLP, nil
return trxIPLDs, nil
}
// FetchRcts fetches receipts
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([][]byte, error) {
// batch fetch preserves order?
func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching receipt iplds")
rctCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@ -173,15 +183,18 @@ func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([][]byte, error) {
rctCids[i] = dc
}
rcts := f.fetchBatch(rctCids)
rctsRLP := make([][]byte, len(rcts))
rctIPLDs := make([]ipfs.BlockModel, len(rcts))
for i, rct := range rcts {
rctsRLP[i] = rct.RawData()
rctIPLDs[i] = ipfs.BlockModel{
Data: rct.RawData(),
CID: rct.Cid().String(),
}
}
if len(rctsRLP) != len(rctCids) {
if len(rctIPLDs) != len(rctCids) {
log.Errorf("ipfs fetcher: number of receipt blocks returned (%d) does not match number expected (%d)", len(rcts), len(rctCids))
return rctsRLP, errUnexpectedNumberOfIPLDs
return rctIPLDs, errUnexpectedNumberOfIPLDs
}
return rctsRLP, nil
return rctIPLDs, nil
}
// FetchState fetches state nodes
@ -203,7 +216,10 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
return nil, err
}
stateNodes[i] = StateNode{
IPLD: state.RawData(),
IPLD: ipfs.BlockModel{
Data: state.RawData(),
CID: state.Cid().String(),
},
StateTrieKey: common.HexToHash(stateNode.StateKey),
Leaf: stateNode.Leaf,
}
@ -230,7 +246,10 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
return nil, err
}
storageNodes[i] = StorageNode{
IPLD: storage.RawData(),
IPLD: ipfs.BlockModel{
Data: storage.RawData(),
CID: storage.Cid().String(),
},
StateTrieKey: common.HexToHash(storageNode.StateKey),
StorageTrieKey: common.HexToHash(storageNode.StorageKey),
Leaf: storageNode.Leaf,

37
pkg/super_node/eth/ipld_fetcher_test.go
View File

@ -20,6 +20,8 @@ import (
"bytes"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-block-format"
. "github.com/onsi/ginkgo"
@ -106,26 +108,47 @@ var _ = Describe("Fetcher", func() {
Expect(ok).To(BeTrue())
Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber))
Expect(len(iplds.Headers)).To(Equal(1))
Expect(iplds.Headers[0]).To(Equal(mockHeaderBlock.RawData()))
Expect(iplds.Headers[0]).To(Equal(ipfs.BlockModel{
Data: mockHeaderBlock.RawData(),
CID: mockHeaderBlock.Cid().String(),
}))
Expect(len(iplds.Uncles)).To(Equal(1))
Expect(iplds.Uncles[0]).To(Equal(mockUncleBlock.RawData()))
Expect(iplds.Uncles[0]).To(Equal(ipfs.BlockModel{
Data: mockUncleBlock.RawData(),
CID: mockUncleBlock.Cid().String(),
}))
Expect(len(iplds.Transactions)).To(Equal(1))
Expect(iplds.Transactions[0]).To(Equal(mockTrxBlock.RawData()))
Expect(iplds.Transactions[0]).To(Equal(ipfs.BlockModel{
Data: mockTrxBlock.RawData(),
CID: mockTrxBlock.Cid().String(),
}))
Expect(len(iplds.Receipts)).To(Equal(1))
Expect(iplds.Receipts[0]).To(Equal(mockReceiptBlock.RawData()))
Expect(iplds.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mockReceiptBlock.RawData(),
CID: mockReceiptBlock.Cid().String(),
}))
Expect(len(iplds.StateNodes)).To(Equal(1))
Expect(iplds.StateNodes[0].StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
Expect(iplds.StateNodes[0].Leaf).To(BeTrue())
Expect(iplds.StateNodes[0].IPLD).To(Equal(mockStateBlock.RawData()))
Expect(iplds.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mockStateBlock.RawData(),
CID: mockStateBlock.Cid().String(),
}))
Expect(len(iplds.StorageNodes)).To(Equal(2))
for _, storage := range iplds.StorageNodes {
Expect(storage.Leaf).To(BeTrue())
Expect(storage.StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
Expect(storage.IPLD).To(Equal(mockStorageBlock1.RawData()))
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock1.RawData(),
CID: mockStorageBlock1.Cid().String(),
}))
}
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
Expect(storage.IPLD).To(Equal(mockStorageBlock2.RawData()))
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock2.RawData(),
CID: mockStorageBlock2.Cid().String(),
}))
}
}
})

137
pkg/super_node/eth/mocks/test_data.go
View File

@ -23,6 +23,8 @@ import (
"math/big"
rand2 "math/rand"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
@ -50,6 +52,8 @@ var (
Root: common.HexToHash("0x0"),
TxHash: common.HexToHash("0x0"),
ReceiptHash: common.HexToHash("0x0"),
Difficulty: big.NewInt(5000000),
Extra: []byte{},
}
MockTransactions, MockReceipts, senderAddr = createTransactionsAndReceipts()
ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts)
@ -62,7 +66,24 @@ var (
mockTopic12 = common.HexToHash("0x06")
mockTopic21 = common.HexToHash("0x05")
mockTopic22 = common.HexToHash("0x07")
MockTrxMeta = []eth.TxModel{
MockLog1 = &types.Log{
Topics: []common.Hash{mockTopic11, mockTopic12},
Data: []byte{},
}
MockLog2 = &types.Log{
Topics: []common.Hash{mockTopic21, mockTopic22},
Data: []byte{},
}
HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256)
MockTrxMeta = []eth.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
Src: senderAddr.Hex(),
@ -80,14 +101,14 @@ var (
}
MockTrxMetaPostPublsh = []eth.TxModel{
{
CID: "mockTrxCID1", // This is empty until we go to publish to ipfs
CID: Trx1CID.String(), // This is empty until we go to publish to ipfs
Src: senderAddr.Hex(),
Dst: Address.String(),
Index: 0,
TxHash: MockTransactions[0].Hash().String(),
},
{
CID: "mockTrxCID2",
CID: Trx2CID.String(),
Src: senderAddr.Hex(),
Dst: AnotherAddress.String(),
Index: 1,
@ -118,7 +139,7 @@ var (
}
MockRctMetaPostPublish = []eth.ReceiptModel{
{
CID: "mockRctCID1",
CID: Rct1CID.String(),
Topic0s: []string{
mockTopic11.String(),
},
@ -128,7 +149,7 @@ var (
Contract: Address.String(),
},
{
CID: "mockRctCID2",
CID: Rct2CID.String(),
Topic0s: []string{
mockTopic21.String(),
},
@ -208,12 +229,12 @@ var (
}
MockStateMetaPostPublish = []eth.StateNodeModel{
{
CID: "mockStateCID1",
CID: State1CID.String(),
Leaf: true,
StateKey: ContractLeafKey.String(),
},
{
CID: "mockStateCID2",
CID: State2CID.String(),
Leaf: true,
StateKey: AnotherContractLeafKey.String(),
},
@ -233,11 +254,11 @@ var (
BlockRlp: MockBlockRlp,
StateDiffRlp: MockStateDiffBytes,
ReceiptsRlp: ReceiptsRlp,
TotalDifficulty: big.NewInt(1337),
TotalDifficulty: MockBlock.Difficulty(),
}
MockConvertedPayload = eth.ConvertedPayload{
TotalDifficulty: big.NewInt(1337),
TotalDifficulty: MockBlock.Difficulty(),
Block: MockBlock,
Receipts: MockReceipts,
TxMetaData: MockTrxMeta,
@ -250,9 +271,10 @@ var (
HeaderCID: eth2.HeaderModel{
BlockHash: MockBlock.Hash().String(),
BlockNumber: MockBlock.Number().String(),
CID: "mockHeaderCID",
CID: HeaderCID.String(),
ParentHash: MockBlock.ParentHash().String(),
TotalDifficulty: "1337",
TotalDifficulty: MockBlock.Difficulty().String(),
Reward: "5000000000000000000",
},
UncleCIDs: []eth2.UncleModel{},
TransactionCIDs: MockTrxMetaPostPublsh,
@ -264,7 +286,7 @@ var (
StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
ContractLeafKey: {
{
CID: "mockStorageCID",
CID: StorageCID.String(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
Leaf: true,
},
@ -279,8 +301,9 @@ var (
BlockNumber: "1",
BlockHash: MockBlock.Hash().String(),
ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
CID: "mockHeaderCID",
TotalDifficulty: "1337",
CID: HeaderCID.String(),
TotalDifficulty: MockBlock.Difficulty().String(),
Reward: "5000000000000000000",
},
},
Transactions: MockTrxMetaPostPublsh,
@ -289,54 +312,67 @@ var (
StateNodes: MockStateMetaPostPublish,
StorageNodes: []eth.StorageNodeWithStateKeyModel{
{
CID: "mockStorageCID",
CID: StorageCID.String(),
Leaf: true,
StateKey: ContractLeafKey.Hex(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
},
},
}
headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256)
state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256)
storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256)
HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, headerCID)
Trx1IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), trx1CID)
Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), trx2CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), rct2CID)
State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, state1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, state2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, storageCID)
HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
Trx1IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), Trx1CID)
Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, StorageCID)
MockIPLDs = eth.IPLDs{
BlockNumber: big.NewInt(1),
Headers: [][]byte{
HeaderIPLD.RawData(),
Headers: []ipfs.BlockModel{
{
Data: HeaderIPLD.RawData(),
CID: HeaderIPLD.Cid().String(),
},
},
Transactions: [][]byte{
Trx1IPLD.RawData(),
Trx2IPLD.RawData(),
Transactions: []ipfs.BlockModel{
{
Data: Trx1IPLD.RawData(),
CID: Trx1IPLD.Cid().String(),
},
{
Data: Trx2IPLD.RawData(),
CID: Trx2IPLD.Cid().String(),
},
},
Receipts: [][]byte{
Rct1IPLD.RawData(),
Rct2IPLD.RawData(),
Receipts: []ipfs.BlockModel{
{
Data: Rct1IPLD.RawData(),
CID: Rct1IPLD.Cid().String(),
},
{
Data: Rct2IPLD.RawData(),
CID: Rct2IPLD.Cid().String(),
},
},
StateNodes: []eth2.StateNode{
{
StateTrieKey: ContractLeafKey,
Leaf: true,
IPLD: State1IPLD.RawData(),
IPLD: ipfs.BlockModel{
Data: State1IPLD.RawData(),
CID: State1IPLD.Cid().String(),
},
},
{
StateTrieKey: AnotherContractLeafKey,
Leaf: true,
IPLD: State2IPLD.RawData(),
IPLD: ipfs.BlockModel{
Data: State2IPLD.RawData(),
CID: State2IPLD.Cid().String(),
},
},
},
StorageNodes: []eth2.StorageNode{
@ -344,7 +380,10 @@ var (
StateTrieKey: ContractLeafKey,
StorageTrieKey: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"),
Leaf: true,
IPLD: StorageIPLD.RawData(),
IPLD: ipfs.BlockModel{
Data: StorageIPLD.RawData(),
CID: StorageIPLD.Cid().String(),
},
},
},
}
@ -353,8 +392,8 @@ var (
// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
// make transactions
trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), nil)
trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), nil)
trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
transactionSigner := types.MakeSigner(params.MainnetChainConfig, BlockNumber)
mockCurve := elliptic.P256()
mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
@ -375,16 +414,10 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common
}
// make receipts
mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
mockLog1 := &types.Log{
Topics: []common.Hash{mockTopic11, mockTopic12},
}
mockReceipt1.Logs = []*types.Log{mockLog1}
mockReceipt1.Logs = []*types.Log{MockLog1}
mockReceipt1.TxHash = signedTrx1.Hash()
mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
mockLog2 := &types.Log{
Topics: []common.Hash{mockTopic21, mockTopic22},
}
mockReceipt2.Logs = []*types.Log{mockLog2}
mockReceipt2.Logs = []*types.Log{MockLog2}
mockReceipt2.TxHash = signedTrx2.Hash()
return types.Transactions{signedTrx1, signedTrx2}, types.Receipts{mockReceipt1, mockReceipt2}, senderAddr
}

2
pkg/super_node/eth/models.go
View File

@ -27,6 +27,7 @@ type HeaderModel struct {
CID string `db:"cid"`
TotalDifficulty string `db:"td"`
NodeID int64 `db:"node_id"`
Reward string `db:"reward"`
}
// UncleModel is the db model for eth.uncle_cids
@ -36,6 +37,7 @@ type UncleModel struct {
BlockHash string `db:"block_hash"`
ParentHash string `db:"parent_hash"`
CID string `db:"cid"`
Reward string `db:"reward"`
}
// TxModel is the db model for eth.transaction_cids

9
pkg/super_node/eth/publisher.go
View File

@ -20,12 +20,13 @@ import (
"errors"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/dag_putters"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
// IPLDPublisher satisfies the IPLDPublisher for ethereum
@ -63,12 +64,14 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
if err != nil {
return nil, err
}
reward := common2.CalcEthBlockReward(ipldPayload.Block, ipldPayload.Receipts)
header := HeaderModel{
CID: headerCid,
ParentHash: ipldPayload.Block.ParentHash().String(),
BlockNumber: ipldPayload.Block.Number().String(),
BlockHash: ipldPayload.Block.Hash().String(),
TotalDifficulty: ipldPayload.TotalDifficulty.String(),
Reward: reward.String(),
}
// Process and publish uncles
@ -78,10 +81,12 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
if err != nil {
return nil, err
}
uncleReward := common2.CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncle.Number.Int64())
uncleCids = append(uncleCids, UncleModel{
CID: uncleCid,
ParentHash: uncle.ParentHash.String(),
BlockHash: uncle.Hash().String(),
Reward: uncleReward.String(),
})
}

13
pkg/super_node/eth/publisher_test.go
View File

@ -45,16 +45,16 @@ var _ = Describe("Publisher", func() {
Describe("Publish", func() {
It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() {
mockHeaderDagPutter.CIDsToReturn = []string{"mockHeaderCID"}
mockTrxDagPutter.CIDsToReturn = []string{"mockTrxCID1", "mockTrxCID2"}
mockRctDagPutter.CIDsToReturn = []string{"mockRctCID1", "mockRctCID2"}
mockHeaderDagPutter.CIDsToReturn = []string{mocks.HeaderCID.String()}
mockTrxDagPutter.CIDsToReturn = []string{mocks.Trx1CID.String(), mocks.Trx2CID.String()}
mockRctDagPutter.CIDsToReturn = []string{mocks.Rct1CID.String(), mocks.Rct2CID.String()}
val1 := common.BytesToHash(mocks.MockConvertedPayload.StateNodes[0].Value)
val2 := common.BytesToHash(mocks.MockConvertedPayload.StateNodes[1].Value)
mockStateDagPutter.CIDsToReturn = map[common.Hash][]string{
val1: {"mockStateCID1"},
val2: {"mockStateCID2"},
val1: {mocks.State1CID.String()},
val2: {mocks.State2CID.String()},
}
mockStorageDagPutter.CIDsToReturn = []string{"mockStorageCID"}
mockStorageDagPutter.CIDsToReturn = []string{mocks.StorageCID.String()}
publisher := eth.IPLDPublisher{
HeaderPutter: mockHeaderDagPutter,
TransactionPutter: mockTrxDagPutter,
@ -69,6 +69,7 @@ var _ = Describe("Publisher", func() {
Expect(cidPayload.HeaderCID.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty.String()))
Expect(cidPayload.HeaderCID.BlockNumber).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber))
Expect(cidPayload.HeaderCID.BlockHash).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockHash))
Expect(cidPayload.HeaderCID.Reward).To(Equal(mocks.MockCIDPayload.HeaderCID.Reward))
Expect(cidPayload.UncleCIDs).To(Equal(mocks.MockCIDPayload.UncleCIDs))
Expect(cidPayload.HeaderCID).To(Equal(mocks.MockCIDPayload.HeaderCID))
Expect(len(cidPayload.TransactionCIDs)).To(Equal(2))

27
pkg/super_node/eth/retriever.go
View File

@ -193,6 +193,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, blockNum
pgStr += fmt.Sprintf(` AND transaction_cids.src = ANY($%d::VARCHAR(66)[])`, id)
args = append(args, pq.Array(txFilter.Src))
}
pgStr += ` ORDER BY transaction_cids.index`
return results, tx.Select(&results, pgStr, args...)
}
@ -224,7 +225,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
args = append(args, pq.Array(rctFilter.Contracts))
id++
// Filter on topics if there are any
if len(rctFilter.Topics) > 0 {
if hasTopics(rctFilter.Topics) {
pgStr += " AND ("
first := true
for i, topicSet := range rctFilter.Topics {
@ -250,7 +251,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
pgStr += ")"
} else { // If there are no contract addresses to filter on
// Filter on topics if there are any
if len(rctFilter.Topics) > 0 {
if hasTopics(rctFilter.Topics) {
pgStr += " AND (("
first := true
for i, topicSet := range rctFilter.Topics {
@ -279,10 +280,20 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
args = append(args, pq.Array(trxIds))
}
}
pgStr += ` ORDER BY transaction_cids.index`
receiptCids := make([]ReceiptModel, 0)
return receiptCids, tx.Select(&receiptCids, pgStr, args...)
}
func hasTopics(topics [][]string) bool {
for _, topicSet := range topics {
if len(topicSet) > 0 {
return true
}
}
return false
}
// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided blockheight that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, blockNumber int64) ([]StateNodeModel, error) {
log.Debug("retrieving state cids for block ", blockNumber)
@ -474,7 +485,8 @@ func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.H
func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]TxModel, error) {
log.Debug("retrieving tx cids for block id ", headerID)
pgStr := `SELECT * FROM eth.transaction_cids
WHERE header_id = $1`
WHERE header_id = $1
ORDER BY index`
var txCIDs []TxModel
return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
}
@ -482,8 +494,13 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) (
// RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]ReceiptModel, error) {
log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
pgStr := `SELECT * FROM eth.receipt_cids
WHERE tx_id = ANY($1::INTEGER[])`
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid,
receipt_cids.contract, receipt_cids.topic0s, receipt_cids.topic1s,
receipt_cids.topic2s, receipt_cids.topic3s
FROM eth.receipt_cids, eth.transaction_cids
WHERE tx_id = ANY($1::INTEGER[])
AND receipt_cids.tx_id = transaction_cids.id
ORDER BY transaction_cids.index`
var rctCIDs []ReceiptModel
return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
}

14
pkg/super_node/eth/retriever_test.go
View File

@ -246,11 +246,11 @@ var _ = Describe("Retriever", func() {
Expect(eth.ReceiptModelsContainsCID(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
Expect(len(cidWrapper.StateNodes)).To(Equal(2))
for _, stateNode := range cidWrapper.StateNodes {
if stateNode.CID == "mockStateCID1" {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
}
if stateNode.CID == "mockStateCID2" {
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
}
@ -335,13 +335,13 @@ var _ = Describe("Retriever", func() {
Expect(cidWrapper5.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper5.Headers)).To(Equal(0))
Expect(len(cidWrapper5.Transactions)).To(Equal(2))
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID1")).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID2")).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, mocks.Trx1CID.String())).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, mocks.Trx2CID.String())).To(BeTrue())
Expect(len(cidWrapper5.StateNodes)).To(Equal(0))
Expect(len(cidWrapper5.StorageNodes)).To(Equal(0))
Expect(len(cidWrapper5.Receipts)).To(Equal(2))
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, mocks.Rct1CID.String())).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, mocks.Rct2CID.String())).To(BeTrue())
cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
Expect(err).ToNot(HaveOccurred())
@ -379,7 +379,7 @@ var _ = Describe("Retriever", func() {
HeaderID: cidWrapper7.StateNodes[0].HeaderID,
Leaf: true,
StateKey: mocks.ContractLeafKey.Hex(),
CID: "mockStateCID1",
CID: mocks.State1CID.String(),
}))
_, empty, err = retriever.Retrieve(rctTopicsAndContractFilterFail, 1)

14
pkg/super_node/eth/types.go
View File

@ -19,6 +19,8 @@ package eth
import (
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
@ -77,10 +79,10 @@ type CIDWrapper struct {
// Returned by IPLDFetcher and ResponseFilterer
type IPLDs struct {
BlockNumber *big.Int
Headers [][]byte
Uncles [][]byte
Transactions [][]byte
Receipts [][]byte
Headers []ipfs.BlockModel
Uncles []ipfs.BlockModel
Transactions []ipfs.BlockModel
Receipts []ipfs.BlockModel
StateNodes []StateNode
StorageNodes []StorageNode
}
@ -92,13 +94,13 @@ func (i IPLDs) Height() int64 {
type StateNode struct {
StateTrieKey common.Hash
IPLD []byte
IPLD ipfs.BlockModel
Leaf bool
}
type StorageNode struct {
StateTrieKey common.Hash
StorageTrieKey common.Hash
IPLD []byte
IPLD ipfs.BlockModel
Leaf bool
}

9
pkg/super_node/shared/config.go
View File

@ -23,14 +23,11 @@ import (
"time"
"github.com/btcsuite/btcd/rpcclient"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/eth"
"github.com/vulcanize/vulcanizedb/pkg/eth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/node"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
@ -191,10 +188,6 @@ func getEthNodeAndClient(path string) (core.Node, interface{}, error) {
return core.Node{}, nil, err
}
rpcClient := client.NewRPCClient(rawRPCClient, path)
ethClient := ethclient.NewClient(rawRPCClient)
vdbEthClient := client.NewEthClient(ethClient)
vdbNode := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRPCTransactionConverter(ethClient)
blockChain := eth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
return blockChain.Node(), rpcClient, nil
return vdbNode, rpcClient, nil
}

10
pkg/super_node/shared/functions.go
View File

@ -18,6 +18,8 @@ package shared
import (
"bytes"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
)
// ListContainsString used to check if a list of strings contains a particular string
@ -30,10 +32,10 @@ func ListContainsString(sss []string, s string) bool {
return false
}
// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
func ListContainsBytes(bbb [][]byte, b []byte) bool {
for _, by := range bbb {
if bytes.Equal(by, b) {
// IPLDsContainBytes used to check if a list of strings contains a particular string
func IPLDsContainBytes(iplds []ipfs.BlockModel, b []byte) bool {
for _, ipld := range iplds {
if bytes.Equal(ipld.Data, b) {
return true
}
}