From 4f4ab1dd4fc8bf718ca1f9ac4f65d768cd1d1372 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 26 Oct 2020 08:58:37 -0500 Subject: [PATCH] update eth backend --- pkg/eth/api.go | 6 +- pkg/eth/backend.go | 336 +++++++++++++++----------------------- pkg/eth/backend_utils.go | 185 +++++++++++++++++++++ pkg/eth/interfaces.go | 47 ++++++ pkg/eth/ipld_retriever.go | 64 ++++---- pkg/eth/types.go | 19 +++ pkg/serve/config.go | 1 + 7 files changed, 421 insertions(+), 237 deletions(-) create mode 100644 pkg/eth/backend_utils.go create mode 100644 pkg/eth/interfaces.go diff --git a/pkg/eth/api.go b/pkg/eth/api.go index f61888e5..9a792c0b 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -145,11 +145,7 @@ func (pea *PublicEthAPI) getLogs(ctx context.Context, crit ethereum.FilterQuery) startingBlock := crit.FromBlock endingBlock := crit.ToBlock if startingBlock == nil { - startingBlockInt, err := pea.B.Retriever.RetrieveFirstBlockNumber() - if err != nil { - return nil, err - } - startingBlock = big.NewInt(startingBlockInt) + startingBlock = common.Big0 } if endingBlock == nil { endingBlockInt, err := pea.B.Retriever.RetrieveLastBlockNumber() diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index ba51dd54..4e5ff088 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -21,6 +21,9 @@ import ( "errors" "math/big" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" @@ -36,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb" - "github.com/vulcanize/ipld-eth-indexer/pkg/ipfs" "github.com/vulcanize/ipld-eth-indexer/pkg/postgres" "github.com/vulcanize/ipld-eth-server/pkg/shared" ) @@ -53,6 +55,12 @@ const ( RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE id = (SELECT canonical_header($1))` + RetrieveTD = `SELECT td FROM eth.header_cids + WHERE header_cids.block_hash = $1` + RetrieveRPCTransaction = `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids + WHERE blocks.key = transaction_cids.mh_key + AND transaction_cids.header_id = header_cids.id + AND transaction_cids.tx_hash = $1` ) type Backend struct { @@ -92,6 +100,11 @@ func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) { }, nil } +// ChainDb returns the backend's underlying chain database +func (b *Backend) ChainDb() ethdb.Database { + return b.EthDB +} + // HeaderByNumber gets the canonical header for the provided block number func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Header, error) { var err error @@ -133,12 +146,43 @@ func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.He return header, rlp.DecodeBytes(headerRLP, header) } +// HeaderByNumberOrHash gets the header for the provided block hash or number +func (b *Backend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.HeaderByNumber(ctx, blockNr) + } + if hash, ok := blockNrOrHash.Hash(); ok { + header, err := b.HeaderByHash(ctx, hash) + if err != nil { + return nil, err + } + if header == nil { + return nil, errors.New("header for hash not found") + } + if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash { + return nil, errors.New("hash is not currently canonical") + } + return header, nil + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires +// a `PublicEthAPI`. +func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) { + fields := RPCMarshalHeader(header) + td, err := pea.B.GetTd(header.Hash()) + if err != nil { + return nil, err + } + fields["totalDifficulty"] = (*hexutil.Big)(td) + return fields, nil +} + // GetTd gets the total difficulty at the given block hash func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) { - pgStr := `SELECT td FROM eth.header_cids - WHERE header_cids.block_hash = $1` var tdStr string - err := b.DB.Get(&tdStr, pgStr, blockHash.String()) + err := b.DB.Get(&tdStr, RetrieveTD, blockHash.String()) if err != nil { return nil, err } @@ -149,21 +193,38 @@ func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) { return td, nil } -// GetLogs returns all the logs for the given block hash -func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - _, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash) - if err != nil { - return nil, err +// CurrentBlock returns the current block +func (b *Backend) CurrentBlock() *types.Block { + block, _ := b.BlockByNumber(context.Background(), rpc.LatestBlockNumber) + return block +} + +// BlockByNumberOrHash returns block by number or hash +func (b *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.BlockByNumber(ctx, blockNr) } - logs := make([][]*types.Log, len(receiptBytes)) - for i, rctBytes := range receiptBytes { - var rct types.Receipt - if err := rlp.DecodeBytes(rctBytes, &rct); err != nil { + if hash, ok := blockNrOrHash.Hash(); ok { + header, err := b.HeaderByHash(ctx, hash) + if err != nil { return nil, err } - logs[i] = rct.Logs + if header == nil { + return nil, errors.New("header for hash not found") + } + if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash { + return nil, errors.New("hash is not currently canonical") + } + block, err := b.BlockByHash(ctx, hash) + if err != nil { + return nil, err + } + if block == nil { + return nil, errors.New("header found, but block body is missing") + } + return block, nil } - return logs, nil + return nil, errors.New("invalid arguments; neither block nor hash specified") } // BlockByNumber returns the requested canonical block. @@ -354,11 +415,7 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type BlockNumber uint64 `db:"block_number"` Index uint64 `db:"index"` } - pgStr := `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids - WHERE blocks.key = transaction_cids.mh_key - AND transaction_cids.header_id = header_cids.id - AND transaction_cids.tx_hash = $1` - if err := b.DB.Get(&tempTxStruct, pgStr, txHash.String()); err != nil { + if err := b.DB.Get(&tempTxStruct, RetrieveRPCTransaction, txHash.String()); err != nil { return nil, common.Hash{}, 0, 0, err } var transaction types.Transaction @@ -368,88 +425,38 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type return &transaction, common.HexToHash(tempTxStruct.BlockHash), tempTxStruct.BlockNumber, tempTxStruct.Index, nil } -// extractLogsOfInterest returns logs from the receipt IPLD -func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) { - var logs []*types.Log - for _, rctIPLD := range rctIPLDs { - rctRLP := rctIPLD - var rct types.Receipt - if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil { - return nil, err - } - for _, log := range rct.Logs { - if wanted := wantedLog(wantedTopics, log.Topics); wanted == true { - logs = append(logs, log) - } - } - } - return logs, nil -} - -// returns true if the log matches on the filter -func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool { - // actualTopics will always have length <= 4 - // wantedTopics will always have length 4 - matches := 0 - for i, actualTopic := range actualTopics { - // If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot - if len(wantedTopics[i]) > 0 { - matches += sliceContainsHash(wantedTopics[i], actualTopic) - } else { - // Filter slot is empty, not matching any topics at this slot => counts as a match - matches++ - } - } - if matches == len(actualTopics) { - return true - } - return false -} - -// returns 1 if the slice contains the hash, 0 if it does not -func sliceContainsHash(slice []string, hash common.Hash) int { - for _, str := range slice { - if str == hash.String() { - return 1 - } - } - return 0 -} - -// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires -// a `PublicEthAPI`. -func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) { - fields := RPCMarshalHeader(header) - td, err := pea.B.GetTd(header.Hash()) +// GetReceipts retrieves receipts for provided block hash +func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + _, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash) if err != nil { return nil, err } - fields["totalDifficulty"] = (*hexutil.Big)(td) - return fields, nil + rcts := make(types.Receipts, 0, len(receiptBytes)) + for i, rctBytes := range receiptBytes { + rct := new(types.Receipt) + if err := rlp.DecodeBytes(rctBytes, rct); err != nil { + return nil, err + } + rcts[i] = rct + } + return rcts, nil } -// RPCMarshalHeader converts the given header to the RPC output. -// This function is eth/internal so we have to make our own version here... -func RPCMarshalHeader(head *types.Header) map[string]interface{} { - return map[string]interface{}{ - "number": (*hexutil.Big)(head.Number), - "hash": head.Hash(), - "parentHash": head.ParentHash, - "nonce": head.Nonce, - "mixHash": head.MixDigest, - "sha3Uncles": head.UncleHash, - "logsBloom": head.Bloom, - "stateRoot": head.Root, - "miner": head.Coinbase, - "difficulty": (*hexutil.Big)(head.Difficulty), - "extraData": hexutil.Bytes(head.Extra), - "size": hexutil.Uint64(head.Size()), - "gasLimit": hexutil.Uint64(head.GasLimit), - "gasUsed": hexutil.Uint64(head.GasUsed), - "timestamp": hexutil.Uint64(head.Time), - "transactionsRoot": head.TxHash, - "receiptsRoot": head.ReceiptHash, +// GetLogs returns all the logs for the given block hash +func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + _, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash) + if err != nil { + return nil, err } + logs := make([][]*types.Log, len(receiptBytes)) + for i, rctBytes := range receiptBytes { + var rct types.Receipt + if err := rlp.DecodeBytes(rctBytes, &rct); err != nil { + return nil, err + } + logs[i] = rct.Logs + } + return logs, nil } // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires @@ -467,110 +474,6 @@ func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx boo return fields, err } -// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are -// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain -// transaction hashes. -func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields := RPCMarshalHeader(block.Header()) - fields["size"] = hexutil.Uint64(block.Size()) - - if inclTx { - formatTx := func(tx *types.Transaction) (interface{}, error) { - return tx.Hash(), nil - } - if fullTx { - formatTx = func(tx *types.Transaction) (interface{}, error) { - return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil - } - } - txs := block.Transactions() - transactions := make([]interface{}, len(txs)) - var err error - for i, tx := range txs { - if transactions[i], err = formatTx(tx); err != nil { - return nil, err - } - } - fields["transactions"] = transactions - } - uncles := block.Uncles() - uncleHashes := make([]common.Hash, len(uncles)) - for i, uncle := range uncles { - uncleHashes[i] = uncle.Hash() - } - fields["uncles"] = uncleHashes - - return fields, nil -} - -// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation. -func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction { - for idx, tx := range b.Transactions() { - if tx.Hash() == hash { - return newRPCTransactionFromBlockIndex(b, uint64(idx)) - } - } - return nil -} - -// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation. -func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction { - txs := b.Transactions() - if index >= uint64(len(txs)) { - return nil - } - return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index) -} - -// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction -type RPCTransaction struct { - BlockHash *common.Hash `json:"blockHash"` - BlockNumber *hexutil.Big `json:"blockNumber"` - From common.Address `json:"from"` - Gas hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Hash common.Hash `json:"hash"` - Input hexutil.Bytes `json:"input"` - Nonce hexutil.Uint64 `json:"nonce"` - To *common.Address `json:"to"` - TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` - Value *hexutil.Big `json:"value"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` -} - -// NewRPCTransaction returns a transaction that will serialize to the RPC -// representation, with the given location metadata set (if available). -func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction { - var signer types.Signer = types.FrontierSigner{} - if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) - } - from, _ := types.Sender(signer, tx) - v, r, s := tx.RawSignatureValues() - - result := &RPCTransaction{ - From: from, - Gas: hexutil.Uint64(tx.Gas()), - GasPrice: (*hexutil.Big)(tx.GasPrice()), - Hash: tx.Hash(), - Input: hexutil.Bytes(tx.Data()), // somehow this is ending up `nil` - Nonce: hexutil.Uint64(tx.Nonce()), - To: tx.To(), - Value: (*hexutil.Big)(tx.Value()), - V: (*hexutil.Big)(v), - R: (*hexutil.Big)(r), - S: (*hexutil.Big)(s), - } - if blockHash != (common.Hash{}) { - result.BlockHash = &blockHash - result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) - result.TransactionIndex = (*hexutil.Uint64)(&index) - } - return result -} - // StateAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { if blockNr, ok := blockNrOrHash.Number(); ok { @@ -652,3 +555,36 @@ func (b *Backend) GetHeader(hash common.Hash, height uint64) *types.Header { } return header } + +// RPCGasCap returns the configured gas cap for the rpc server +func (b *Backend) RPCGasCap() *big.Int { + return b.Config.RPCGasCap +} + +func (b *Backend) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { + panic("implement me") +} + +func (b *Backend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { + panic("implement me") +} + +func (b *Backend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { + panic("implement me") +} + +func (b *Backend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { + panic("implement me") +} + +func (b *Backend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + panic("implement me") +} + +func (b *Backend) BloomStatus() (uint64, uint64) { + panic("implement me") +} + +func (b *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { + panic("implement me") +} diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go new file mode 100644 index 00000000..9d7277eb --- /dev/null +++ b/pkg/eth/backend_utils.go @@ -0,0 +1,185 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package eth + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/vulcanize/ipld-eth-indexer/pkg/ipfs" +) + +// RPCMarshalHeader converts the given header to the RPC output. +// This function is eth/internal so we have to make our own version here... +func RPCMarshalHeader(head *types.Header) map[string]interface{} { + return map[string]interface{}{ + "number": (*hexutil.Big)(head.Number), + "hash": head.Hash(), + "parentHash": head.ParentHash, + "nonce": head.Nonce, + "mixHash": head.MixDigest, + "sha3Uncles": head.UncleHash, + "logsBloom": head.Bloom, + "stateRoot": head.Root, + "miner": head.Coinbase, + "difficulty": (*hexutil.Big)(head.Difficulty), + "extraData": hexutil.Bytes(head.Extra), + "size": hexutil.Uint64(head.Size()), + "gasLimit": hexutil.Uint64(head.GasLimit), + "gasUsed": hexutil.Uint64(head.GasUsed), + "timestamp": hexutil.Uint64(head.Time), + "transactionsRoot": head.TxHash, + "receiptsRoot": head.ReceiptHash, + } +} + +// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are +// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain +// transaction hashes. +func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields := RPCMarshalHeader(block.Header()) + fields["size"] = hexutil.Uint64(block.Size()) + + if inclTx { + formatTx := func(tx *types.Transaction) (interface{}, error) { + return tx.Hash(), nil + } + if fullTx { + formatTx = func(tx *types.Transaction) (interface{}, error) { + return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil + } + } + txs := block.Transactions() + transactions := make([]interface{}, len(txs)) + var err error + for i, tx := range txs { + if transactions[i], err = formatTx(tx); err != nil { + return nil, err + } + } + fields["transactions"] = transactions + } + uncles := block.Uncles() + uncleHashes := make([]common.Hash, len(uncles)) + for i, uncle := range uncles { + uncleHashes[i] = uncle.Hash() + } + fields["uncles"] = uncleHashes + + return fields, nil +} + +// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation. +func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction { + for idx, tx := range b.Transactions() { + if tx.Hash() == hash { + return newRPCTransactionFromBlockIndex(b, uint64(idx)) + } + } + return nil +} + +// NewRPCTransaction returns a transaction that will serialize to the RPC +// representation, with the given location metadata set (if available). +func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction { + var signer types.Signer = types.FrontierSigner{} + if tx.Protected() { + signer = types.NewEIP155Signer(tx.ChainId()) + } + from, _ := types.Sender(signer, tx) + v, r, s := tx.RawSignatureValues() + + result := &RPCTransaction{ + From: from, + Gas: hexutil.Uint64(tx.Gas()), + GasPrice: (*hexutil.Big)(tx.GasPrice()), + Hash: tx.Hash(), + Input: hexutil.Bytes(tx.Data()), // somehow this is ending up `nil` + Nonce: hexutil.Uint64(tx.Nonce()), + To: tx.To(), + Value: (*hexutil.Big)(tx.Value()), + V: (*hexutil.Big)(v), + R: (*hexutil.Big)(r), + S: (*hexutil.Big)(s), + } + if blockHash != (common.Hash{}) { + result.BlockHash = &blockHash + result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) + result.TransactionIndex = (*hexutil.Uint64)(&index) + } + return result +} + +// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation. +func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction { + txs := b.Transactions() + if index >= uint64(len(txs)) { + return nil + } + return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index) +} + +// extractLogsOfInterest returns logs from the receipt IPLD +func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) { + var logs []*types.Log + for _, rctIPLD := range rctIPLDs { + rctRLP := rctIPLD + var rct types.Receipt + if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil { + return nil, err + } + for _, log := range rct.Logs { + if wanted := wantedLog(wantedTopics, log.Topics); wanted == true { + logs = append(logs, log) + } + } + } + return logs, nil +} + +// returns true if the log matches on the filter +func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool { + // actualTopics will always have length <= 4 + // wantedTopics will always have length 4 + matches := 0 + for i, actualTopic := range actualTopics { + // If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot + if len(wantedTopics[i]) > 0 { + matches += sliceContainsHash(wantedTopics[i], actualTopic) + } else { + // Filter slot is empty, not matching any topics at this slot => counts as a match + matches++ + } + } + if matches == len(actualTopics) { + return true + } + return false +} + +// returns 1 if the slice contains the hash, 0 if it does not +func sliceContainsHash(slice []string, hash common.Hash) int { + for _, str := range slice { + if str == hash.String() { + return 1 + } + } + return 0 +} diff --git a/pkg/eth/interfaces.go b/pkg/eth/interfaces.go new file mode 100644 index 00000000..9e2dcae3 --- /dev/null +++ b/pkg/eth/interfaces.go @@ -0,0 +1,47 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package eth + +import ( + "context" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/rpc" +) + +// FilterBackend is the geth interface we need to satisfy to use their filters +type FilterBackend interface { + ChainDb() ethdb.Database + HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) + HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) + GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) + + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription + SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription + + BloomStatus() (uint64, uint64) + ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) +} diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index c3e3aa08..b2800300 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -96,8 +96,8 @@ const ( ) type ipldResult struct { - cid string `db:"cid"` - data []byte `db:"data"` + CID string `db:"cid"` + Data []byte `db:"data"` } type IPLDRetriever struct { db *postgres.DB @@ -122,8 +122,8 @@ func (r *IPLDRetriever) RetrieveHeadersByHashes(hashes []common.Hash) ([]string, cids := make([]string, len(headerResults)) headers := make([][]byte, len(headerResults)) for i, res := range headerResults { - cids[i] = res.cid - headers[i] = res.data + cids[i] = res.CID + headers[i] = res.Data } return cids, headers, nil } @@ -138,8 +138,8 @@ func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [ cids := make([]string, len(headerResults)) headers := make([][]byte, len(headerResults)) for i, res := range headerResults { - cids[i] = res.cid - headers[i] = res.data + cids[i] = res.CID + headers[i] = res.Data } return cids, headers, nil } @@ -147,7 +147,7 @@ func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [ // RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash func (r *IPLDRetriever) RetrieveHeaderByHash(hash common.Hash) (string, []byte, error) { headerResult := new(ipldResult) - return headerResult.cid, headerResult.data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex()) + return headerResult.CID, headerResult.Data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex()) } // RetrieveUnclesByHashes returns the cids and rlp bytes for the uncles corresponding to the provided uncle hashes @@ -163,8 +163,8 @@ func (r *IPLDRetriever) RetrieveUnclesByHashes(hashes []common.Hash) ([]string, cids := make([]string, len(uncleResults)) uncles := make([][]byte, len(uncleResults)) for i, res := range uncleResults { - cids[i] = res.cid - uncles[i] = res.data + cids[i] = res.CID + uncles[i] = res.Data } return cids, uncles, nil } @@ -178,8 +178,8 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockHash(hash common.Hash) ([]string, [ cids := make([]string, len(uncleResults)) uncles := make([][]byte, len(uncleResults)) for i, res := range uncleResults { - cids[i] = res.cid - uncles[i] = res.data + cids[i] = res.CID + uncles[i] = res.Data } return cids, uncles, nil } @@ -193,8 +193,8 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, [] cids := make([]string, len(uncleResults)) uncles := make([][]byte, len(uncleResults)) for i, res := range uncleResults { - cids[i] = res.cid - uncles[i] = res.data + cids[i] = res.CID + uncles[i] = res.Data } return cids, uncles, nil } @@ -202,7 +202,7 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, [] // RetrieveUncleByHash returns the cid and rlp bytes for the uncle corresponding to the provided uncle hash func (r *IPLDRetriever) RetrieveUncleByHash(hash common.Hash) (string, []byte, error) { uncleResult := new(ipldResult) - return uncleResult.cid, uncleResult.data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex()) + return uncleResult.CID, uncleResult.Data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex()) } // RetrieveTransactionsByHashes returns the cids and rlp bytes for the transactions corresponding to the provided tx hashes @@ -218,8 +218,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByHashes(hashes []common.Hash) ([]st cids := make([]string, len(txResults)) txs := make([][]byte, len(txResults)) for i, res := range txResults { - cids[i] = res.cid - txs[i] = res.data + cids[i] = res.CID + txs[i] = res.Data } return cids, txs, nil } @@ -233,8 +233,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(hash common.Hash) ([]str cids := make([]string, len(txResults)) txs := make([][]byte, len(txResults)) for i, res := range txResults { - cids[i] = res.cid - txs[i] = res.data + cids[i] = res.CID + txs[i] = res.Data } return cids, txs, nil } @@ -248,8 +248,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]stri cids := make([]string, len(txResults)) txs := make([][]byte, len(txResults)) for i, res := range txResults { - cids[i] = res.cid - txs[i] = res.data + cids[i] = res.CID + txs[i] = res.Data } return cids, txs, nil } @@ -257,7 +257,7 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]stri // RetrieveTransactionByTxHash returns the cid and rlp bytes for the transaction corresponding to the provided tx hash func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, []byte, error) { txResult := new(ipldResult) - return txResult.cid, txResult.data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex()) + return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex()) } // RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes @@ -273,8 +273,8 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri cids := make([]string, len(rctResults)) rcts := make([][]byte, len(rctResults)) for i, res := range rctResults { - cids[i] = res.cid - rcts[i] = res.data + cids[i] = res.CID + rcts[i] = res.Data } return cids, rcts, nil } @@ -288,8 +288,8 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, cids := make([]string, len(rctResults)) rcts := make([][]byte, len(rctResults)) for i, res := range rctResults { - cids[i] = res.cid - rcts[i] = res.data + cids[i] = res.CID + rcts[i] = res.Data } return cids, rcts, nil } @@ -303,8 +303,8 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, cids := make([]string, len(rctResults)) rcts := make([][]byte, len(rctResults)) for i, res := range rctResults { - cids[i] = res.cid - rcts[i] = res.data + cids[i] = res.CID + rcts[i] = res.Data } return cids, rcts, nil } @@ -312,7 +312,7 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, // RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) { rctResult := new(ipldResult) - return rctResult.cid, rctResult.data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()) + return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()) } // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash @@ -323,13 +323,13 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr return "", nil, err } var i []interface{} - if err := rlp.DecodeBytes(accountResult.data, &i); err != nil { + if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) } if len(i) != 2 { return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements") } - return accountResult.cid, i[1].([]byte), nil + return accountResult.CID, i[1].([]byte), nil } // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number @@ -343,9 +343,9 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad cids := make([]string, len(accountResults)) accounts := make([][]byte, len(accountResults)) for i, res := range accountResults { - cids[i] = res.cid + cids[i] = res.CID var iface []interface{} - if err := rlp.DecodeBytes(res.data, &iface); err != nil { + if err := rlp.DecodeBytes(res.Data, &iface); err != nil { return nil, nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) } if len(iface) != 2 { diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 353c15eb..44a2f7e2 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -20,11 +20,30 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/statediff" "github.com/vulcanize/ipld-eth-indexer/pkg/eth" "github.com/vulcanize/ipld-eth-indexer/pkg/ipfs" ) +// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction +type RPCTransaction struct { + BlockHash *common.Hash `json:"blockHash"` + BlockNumber *hexutil.Big `json:"blockNumber"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + Hash common.Hash `json:"hash"` + Input hexutil.Bytes `json:"input"` + Nonce hexutil.Uint64 `json:"nonce"` + To *common.Address `json:"to"` + TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` + Value *hexutil.Big `json:"value"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` +} + // IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server // Returned by IPLDFetcher and ResponseFilterer type IPLDs struct { diff --git a/pkg/serve/config.go b/pkg/serve/config.go index a501f9e7..6d2c5637 100644 --- a/pkg/serve/config.go +++ b/pkg/serve/config.go @@ -68,6 +68,7 @@ type Config struct { func NewConfig() (*Config, error) { c := new(Config) + viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH) viper.BindEnv("server.wsPath", SERVER_WS_PATH) viper.BindEnv("server.ipcPath", SERVER_IPC_PATH) viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)