update eth backend

pkg/eth/api.go

@@ -145,11 +145,7 @@ func (pea *PublicEthAPI) getLogs(ctx context.Context, crit ethereum.FilterQuery)
 	startingBlock := crit.FromBlock
 	endingBlock := crit.ToBlock
 	if startingBlock == nil {
-		startingBlockInt, err := pea.B.Retriever.RetrieveFirstBlockNumber()
+		startingBlock = common.Big0
-		if err != nil {
-			return nil, err
-		}
-		startingBlock = big.NewInt(startingBlockInt)
 	}
 	if endingBlock == nil {
 		endingBlockInt, err := pea.B.Retriever.RetrieveLastBlockNumber()

pkg/eth/backend.go

@@ -21,6 +21,9 @@ import (
 	"errors"
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/core/bloombits"
+	"github.com/ethereum/go-ethereum/event"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
@@ -36,7 +39,6 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
@@ -53,6 +55,12 @@ const (
 	RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids
 									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
 									WHERE id = (SELECT canonical_header($1))`
+	RetrieveTD = `SELECT td FROM eth.header_cids
+			WHERE header_cids.block_hash = $1`
+	RetrieveRPCTransaction = `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids
+			WHERE blocks.key = transaction_cids.mh_key
+			AND transaction_cids.header_id = header_cids.id
+			AND transaction_cids.tx_hash = $1`
 )
 
 type Backend struct {
@@ -92,6 +100,11 @@ func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) {
 	}, nil
 }
 
+// ChainDb returns the backend's underlying chain database
+func (b *Backend) ChainDb() ethdb.Database {
+	return b.EthDB
+}
+
 // HeaderByNumber gets the canonical header for the provided block number
 func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Header, error) {
 	var err error
@@ -133,12 +146,43 @@ func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.He
 	return header, rlp.DecodeBytes(headerRLP, header)
 }
 
+// HeaderByNumberOrHash gets the header for the provided block hash or number
+func (b *Backend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.HeaderByNumber(ctx, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
+		if err != nil {
+			return nil, err
+		}
+		if header == nil {
+			return nil, errors.New("header for hash not found")
+		}
+		if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash {
+			return nil, errors.New("hash is not currently canonical")
+		}
+		return header, nil
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires
+// a `PublicEthAPI`.
+func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(header)
+	td, err := pea.B.GetTd(header.Hash())
+	if err != nil {
+		return nil, err
+	}
+	fields["totalDifficulty"] = (*hexutil.Big)(td)
+	return fields, nil
+}
+
 // GetTd gets the total difficulty at the given block hash
 func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
-	pgStr := `SELECT td FROM eth.header_cids
-			WHERE header_cids.block_hash = $1`
 	var tdStr string
-	err := b.DB.Get(&tdStr, pgStr, blockHash.String())
+	err := b.DB.Get(&tdStr, RetrieveTD, blockHash.String())
 	if err != nil {
 		return nil, err
 	}
@@ -149,21 +193,38 @@ func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
 	return td, nil
 }
 
-// GetLogs returns all the logs for the given block hash
+// CurrentBlock returns the current block
-func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
+func (b *Backend) CurrentBlock() *types.Block {
-	_, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
+	block, _ := b.BlockByNumber(context.Background(), rpc.LatestBlockNumber)
+	return block
+}
+
+// BlockByNumberOrHash returns block by number or hash
+func (b *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.BlockByNumber(ctx, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
 		if err != nil {
 			return nil, err
 		}
-	logs := make([][]*types.Log, len(receiptBytes))
+		if header == nil {
-	for i, rctBytes := range receiptBytes {
+			return nil, errors.New("header for hash not found")
-		var rct types.Receipt
+		}
-		if err := rlp.DecodeBytes(rctBytes, &rct); err != nil {
+		if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash {
+			return nil, errors.New("hash is not currently canonical")
+		}
+		block, err := b.BlockByHash(ctx, hash)
+		if err != nil {
 			return nil, err
 		}
-		logs[i] = rct.Logs
+		if block == nil {
+			return nil, errors.New("header found, but block body is missing")
 		}
-	return logs, nil
+		return block, nil
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
 // BlockByNumber returns the requested canonical block.
@@ -354,11 +415,7 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
 		BlockNumber uint64 `db:"block_number"`
 		Index       uint64 `db:"index"`
 	}
-	pgStr := `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids
+	if err := b.DB.Get(&tempTxStruct, RetrieveRPCTransaction, txHash.String()); err != nil {
-			WHERE blocks.key = transaction_cids.mh_key
-			AND transaction_cids.header_id = header_cids.id
-			AND transaction_cids.tx_hash = $1`
-	if err := b.DB.Get(&tempTxStruct, pgStr, txHash.String()); err != nil {
 		return nil, common.Hash{}, 0, 0, err
 	}
 	var transaction types.Transaction
@@ -368,88 +425,38 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
 	return &transaction, common.HexToHash(tempTxStruct.BlockHash), tempTxStruct.BlockNumber, tempTxStruct.Index, nil
 }
 
-// extractLogsOfInterest returns logs from the receipt IPLD
+// GetReceipts retrieves receipts for provided block hash
-func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
+func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
-	var logs []*types.Log
+	_, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
-	for _, rctIPLD := range rctIPLDs {
-		rctRLP := rctIPLD
-		var rct types.Receipt
-		if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
-			return nil, err
-		}
-		for _, log := range rct.Logs {
-			if wanted := wantedLog(wantedTopics, log.Topics); wanted == true {
-				logs = append(logs, log)
-			}
-		}
-	}
-	return logs, nil
-}
-
-// returns true if the log matches on the filter
-func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
-	// actualTopics will always have length <= 4
-	// wantedTopics will always have length 4
-	matches := 0
-	for i, actualTopic := range actualTopics {
-		// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
-		if len(wantedTopics[i]) > 0 {
-			matches += sliceContainsHash(wantedTopics[i], actualTopic)
-		} else {
-			// Filter slot is empty, not matching any topics at this slot => counts as a match
-			matches++
-		}
-	}
-	if matches == len(actualTopics) {
-		return true
-	}
-	return false
-}
-
-// returns 1 if the slice contains the hash, 0 if it does not
-func sliceContainsHash(slice []string, hash common.Hash) int {
-	for _, str := range slice {
-		if str == hash.String() {
-			return 1
-		}
-	}
-	return 0
-}
-
-// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires
-// a `PublicEthAPI`.
-func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) {
-	fields := RPCMarshalHeader(header)
-	td, err := pea.B.GetTd(header.Hash())
 	if err != nil {
 		return nil, err
 	}
-	fields["totalDifficulty"] = (*hexutil.Big)(td)
+	rcts := make(types.Receipts, 0, len(receiptBytes))
-	return fields, nil
+	for i, rctBytes := range receiptBytes {
+		rct := new(types.Receipt)
+		if err := rlp.DecodeBytes(rctBytes, rct); err != nil {
+			return nil, err
+		}
+		rcts[i] = rct
+	}
+	return rcts, nil
 }
 
-// RPCMarshalHeader converts the given header to the RPC output.
+// GetLogs returns all the logs for the given block hash
-// This function is eth/internal so we have to make our own version here...
+func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
-func RPCMarshalHeader(head *types.Header) map[string]interface{} {
+	_, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
-	return map[string]interface{}{
+	if err != nil {
-		"number":           (*hexutil.Big)(head.Number),
+		return nil, err
-		"hash":             head.Hash(),
-		"parentHash":       head.ParentHash,
-		"nonce":            head.Nonce,
-		"mixHash":          head.MixDigest,
-		"sha3Uncles":       head.UncleHash,
-		"logsBloom":        head.Bloom,
-		"stateRoot":        head.Root,
-		"miner":            head.Coinbase,
-		"difficulty":       (*hexutil.Big)(head.Difficulty),
-		"extraData":        hexutil.Bytes(head.Extra),
-		"size":             hexutil.Uint64(head.Size()),
-		"gasLimit":         hexutil.Uint64(head.GasLimit),
-		"gasUsed":          hexutil.Uint64(head.GasUsed),
-		"timestamp":        hexutil.Uint64(head.Time),
-		"transactionsRoot": head.TxHash,
-		"receiptsRoot":     head.ReceiptHash,
 	}
+	logs := make([][]*types.Log, len(receiptBytes))
+	for i, rctBytes := range receiptBytes {
+		var rct types.Receipt
+		if err := rlp.DecodeBytes(rctBytes, &rct); err != nil {
+			return nil, err
+		}
+		logs[i] = rct.Logs
+	}
+	return logs, nil
 }
 
 // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires
@@ -467,110 +474,6 @@ func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx boo
 	return fields, err
 }
 
-// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
-// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
-// transaction hashes.
-func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
-	fields := RPCMarshalHeader(block.Header())
-	fields["size"] = hexutil.Uint64(block.Size())
-
-	if inclTx {
-		formatTx := func(tx *types.Transaction) (interface{}, error) {
-			return tx.Hash(), nil
-		}
-		if fullTx {
-			formatTx = func(tx *types.Transaction) (interface{}, error) {
-				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
-			}
-		}
-		txs := block.Transactions()
-		transactions := make([]interface{}, len(txs))
-		var err error
-		for i, tx := range txs {
-			if transactions[i], err = formatTx(tx); err != nil {
-				return nil, err
-			}
-		}
-		fields["transactions"] = transactions
-	}
-	uncles := block.Uncles()
-	uncleHashes := make([]common.Hash, len(uncles))
-	for i, uncle := range uncles {
-		uncleHashes[i] = uncle.Hash()
-	}
-	fields["uncles"] = uncleHashes
-
-	return fields, nil
-}
-
-// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
-func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
-	for idx, tx := range b.Transactions() {
-		if tx.Hash() == hash {
-			return newRPCTransactionFromBlockIndex(b, uint64(idx))
-		}
-	}
-	return nil
-}
-
-// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
-func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction {
-	txs := b.Transactions()
-	if index >= uint64(len(txs)) {
-		return nil
-	}
-	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
-}
-
-// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
-type RPCTransaction struct {
-	BlockHash        *common.Hash    `json:"blockHash"`
-	BlockNumber      *hexutil.Big    `json:"blockNumber"`
-	From             common.Address  `json:"from"`
-	Gas              hexutil.Uint64  `json:"gas"`
-	GasPrice         *hexutil.Big    `json:"gasPrice"`
-	Hash             common.Hash     `json:"hash"`
-	Input            hexutil.Bytes   `json:"input"`
-	Nonce            hexutil.Uint64  `json:"nonce"`
-	To               *common.Address `json:"to"`
-	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
-	Value            *hexutil.Big    `json:"value"`
-	V                *hexutil.Big    `json:"v"`
-	R                *hexutil.Big    `json:"r"`
-	S                *hexutil.Big    `json:"s"`
-}
-
-// NewRPCTransaction returns a transaction that will serialize to the RPC
-// representation, with the given location metadata set (if available).
-func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
-	var signer types.Signer = types.FrontierSigner{}
-	if tx.Protected() {
-		signer = types.NewEIP155Signer(tx.ChainId())
-	}
-	from, _ := types.Sender(signer, tx)
-	v, r, s := tx.RawSignatureValues()
-
-	result := &RPCTransaction{
-		From:     from,
-		Gas:      hexutil.Uint64(tx.Gas()),
-		GasPrice: (*hexutil.Big)(tx.GasPrice()),
-		Hash:     tx.Hash(),
-		Input:    hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
-		Nonce:    hexutil.Uint64(tx.Nonce()),
-		To:       tx.To(),
-		Value:    (*hexutil.Big)(tx.Value()),
-		V:        (*hexutil.Big)(v),
-		R:        (*hexutil.Big)(r),
-		S:        (*hexutil.Big)(s),
-	}
-	if blockHash != (common.Hash{}) {
-		result.BlockHash = &blockHash
-		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
-		result.TransactionIndex = (*hexutil.Uint64)(&index)
-	}
-	return result
-}
-
 // StateAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash
 func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) {
 	if blockNr, ok := blockNrOrHash.Number(); ok {
@@ -652,3 +555,36 @@ func (b *Backend) GetHeader(hash common.Hash, height uint64) *types.Header {
 	}
 	return header
 }
+
+// RPCGasCap returns the configured gas cap for the rpc server
+func (b *Backend) RPCGasCap() *big.Int {
+	return b.Config.RPCGasCap
+}
+
+func (b *Backend) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) BloomStatus() (uint64, uint64) {
+	panic("implement me")
+}
+
+func (b *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
+	panic("implement me")
+}

pkg/eth/backend_utils.go

@@ -0,0 +1,185 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+)
+
+// RPCMarshalHeader converts the given header to the RPC output.
+// This function is eth/internal so we have to make our own version here...
+func RPCMarshalHeader(head *types.Header) map[string]interface{} {
+	return map[string]interface{}{
+		"number":           (*hexutil.Big)(head.Number),
+		"hash":             head.Hash(),
+		"parentHash":       head.ParentHash,
+		"nonce":            head.Nonce,
+		"mixHash":          head.MixDigest,
+		"sha3Uncles":       head.UncleHash,
+		"logsBloom":        head.Bloom,
+		"stateRoot":        head.Root,
+		"miner":            head.Coinbase,
+		"difficulty":       (*hexutil.Big)(head.Difficulty),
+		"extraData":        hexutil.Bytes(head.Extra),
+		"size":             hexutil.Uint64(head.Size()),
+		"gasLimit":         hexutil.Uint64(head.GasLimit),
+		"gasUsed":          hexutil.Uint64(head.GasUsed),
+		"timestamp":        hexutil.Uint64(head.Time),
+		"transactionsRoot": head.TxHash,
+		"receiptsRoot":     head.ReceiptHash,
+	}
+}
+
+// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
+// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
+// transaction hashes.
+func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(block.Header())
+	fields["size"] = hexutil.Uint64(block.Size())
+
+	if inclTx {
+		formatTx := func(tx *types.Transaction) (interface{}, error) {
+			return tx.Hash(), nil
+		}
+		if fullTx {
+			formatTx = func(tx *types.Transaction) (interface{}, error) {
+				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
+			}
+		}
+		txs := block.Transactions()
+		transactions := make([]interface{}, len(txs))
+		var err error
+		for i, tx := range txs {
+			if transactions[i], err = formatTx(tx); err != nil {
+				return nil, err
+			}
+		}
+		fields["transactions"] = transactions
+	}
+	uncles := block.Uncles()
+	uncleHashes := make([]common.Hash, len(uncles))
+	for i, uncle := range uncles {
+		uncleHashes[i] = uncle.Hash()
+	}
+	fields["uncles"] = uncleHashes
+
+	return fields, nil
+}
+
+// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
+func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
+	for idx, tx := range b.Transactions() {
+		if tx.Hash() == hash {
+			return newRPCTransactionFromBlockIndex(b, uint64(idx))
+		}
+	}
+	return nil
+}
+
+// NewRPCTransaction returns a transaction that will serialize to the RPC
+// representation, with the given location metadata set (if available).
+func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
+	var signer types.Signer = types.FrontierSigner{}
+	if tx.Protected() {
+		signer = types.NewEIP155Signer(tx.ChainId())
+	}
+	from, _ := types.Sender(signer, tx)
+	v, r, s := tx.RawSignatureValues()
+
+	result := &RPCTransaction{
+		From:     from,
+		Gas:      hexutil.Uint64(tx.Gas()),
+		GasPrice: (*hexutil.Big)(tx.GasPrice()),
+		Hash:     tx.Hash(),
+		Input:    hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
+		Nonce:    hexutil.Uint64(tx.Nonce()),
+		To:       tx.To(),
+		Value:    (*hexutil.Big)(tx.Value()),
+		V:        (*hexutil.Big)(v),
+		R:        (*hexutil.Big)(r),
+		S:        (*hexutil.Big)(s),
+	}
+	if blockHash != (common.Hash{}) {
+		result.BlockHash = &blockHash
+		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
+		result.TransactionIndex = (*hexutil.Uint64)(&index)
+	}
+	return result
+}
+
+// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
+func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction {
+	txs := b.Transactions()
+	if index >= uint64(len(txs)) {
+		return nil
+	}
+	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
+}
+
+// extractLogsOfInterest returns logs from the receipt IPLD
+func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
+	var logs []*types.Log
+	for _, rctIPLD := range rctIPLDs {
+		rctRLP := rctIPLD
+		var rct types.Receipt
+		if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
+			return nil, err
+		}
+		for _, log := range rct.Logs {
+			if wanted := wantedLog(wantedTopics, log.Topics); wanted == true {
+				logs = append(logs, log)
+			}
+		}
+	}
+	return logs, nil
+}
+
+// returns true if the log matches on the filter
+func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
+	// actualTopics will always have length <= 4
+	// wantedTopics will always have length 4
+	matches := 0
+	for i, actualTopic := range actualTopics {
+		// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
+		if len(wantedTopics[i]) > 0 {
+			matches += sliceContainsHash(wantedTopics[i], actualTopic)
+		} else {
+			// Filter slot is empty, not matching any topics at this slot => counts as a match
+			matches++
+		}
+	}
+	if matches == len(actualTopics) {
+		return true
+	}
+	return false
+}
+
+// returns 1 if the slice contains the hash, 0 if it does not
+func sliceContainsHash(slice []string, hash common.Hash) int {
+	for _, str := range slice {
+		if str == hash.String() {
+			return 1
+		}
+	}
+	return 0
+}

pkg/eth/interfaces.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// FilterBackend is the geth interface we need to satisfy to use their filters
+type FilterBackend interface {
+	ChainDb() ethdb.Database
+	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
+	HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error)
+	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
+	GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error)
+
+	SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
+	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
+	SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
+	SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
+	SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
+
+	BloomStatus() (uint64, uint64)
+	ServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
+}

pkg/eth/ipld_retriever.go

@@ -96,8 +96,8 @@ const (
 )
 
 type ipldResult struct {
-	cid  string `db:"cid"`
+	CID  string `db:"cid"`
-	data []byte `db:"data"`
+	Data []byte `db:"data"`
 }
 type IPLDRetriever struct {
 	db *postgres.DB
@@ -122,8 +122,8 @@ func (r *IPLDRetriever) RetrieveHeadersByHashes(hashes []common.Hash) ([]string,
 	cids := make([]string, len(headerResults))
 	headers := make([][]byte, len(headerResults))
 	for i, res := range headerResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		headers[i] = res.data
+		headers[i] = res.Data
 	}
 	return cids, headers, nil
 }
@@ -138,8 +138,8 @@ func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [
 	cids := make([]string, len(headerResults))
 	headers := make([][]byte, len(headerResults))
 	for i, res := range headerResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		headers[i] = res.data
+		headers[i] = res.Data
 	}
 	return cids, headers, nil
 }
@@ -147,7 +147,7 @@ func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [
 // RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
 func (r *IPLDRetriever) RetrieveHeaderByHash(hash common.Hash) (string, []byte, error) {
 	headerResult := new(ipldResult)
-	return headerResult.cid, headerResult.data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+	return headerResult.CID, headerResult.Data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
 }
 
 // RetrieveUnclesByHashes returns the cids and rlp bytes for the uncles corresponding to the provided uncle hashes
@@ -163,8 +163,8 @@ func (r *IPLDRetriever) RetrieveUnclesByHashes(hashes []common.Hash) ([]string,
 	cids := make([]string, len(uncleResults))
 	uncles := make([][]byte, len(uncleResults))
 	for i, res := range uncleResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		uncles[i] = res.data
+		uncles[i] = res.Data
 	}
 	return cids, uncles, nil
 }
@@ -178,8 +178,8 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockHash(hash common.Hash) ([]string, [
 	cids := make([]string, len(uncleResults))
 	uncles := make([][]byte, len(uncleResults))
 	for i, res := range uncleResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		uncles[i] = res.data
+		uncles[i] = res.Data
 	}
 	return cids, uncles, nil
 }
@@ -193,8 +193,8 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, []
 	cids := make([]string, len(uncleResults))
 	uncles := make([][]byte, len(uncleResults))
 	for i, res := range uncleResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		uncles[i] = res.data
+		uncles[i] = res.Data
 	}
 	return cids, uncles, nil
 }
@@ -202,7 +202,7 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, []
 // RetrieveUncleByHash returns the cid and rlp bytes for the uncle corresponding to the provided uncle hash
 func (r *IPLDRetriever) RetrieveUncleByHash(hash common.Hash) (string, []byte, error) {
 	uncleResult := new(ipldResult)
-	return uncleResult.cid, uncleResult.data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex())
+	return uncleResult.CID, uncleResult.Data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex())
 }
 
 // RetrieveTransactionsByHashes returns the cids and rlp bytes for the transactions corresponding to the provided tx hashes
@@ -218,8 +218,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByHashes(hashes []common.Hash) ([]st
 	cids := make([]string, len(txResults))
 	txs := make([][]byte, len(txResults))
 	for i, res := range txResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		txs[i] = res.data
+		txs[i] = res.Data
 	}
 	return cids, txs, nil
 }
@@ -233,8 +233,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(hash common.Hash) ([]str
 	cids := make([]string, len(txResults))
 	txs := make([][]byte, len(txResults))
 	for i, res := range txResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		txs[i] = res.data
+		txs[i] = res.Data
 	}
 	return cids, txs, nil
 }
@@ -248,8 +248,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]stri
 	cids := make([]string, len(txResults))
 	txs := make([][]byte, len(txResults))
 	for i, res := range txResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		txs[i] = res.data
+		txs[i] = res.Data
 	}
 	return cids, txs, nil
 }
@@ -257,7 +257,7 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]stri
 // RetrieveTransactionByTxHash returns the cid and rlp bytes for the transaction corresponding to the provided tx hash
 func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, []byte, error) {
 	txResult := new(ipldResult)
-	return txResult.cid, txResult.data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
+	return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
 }
 
 // RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
@@ -273,8 +273,8 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		rcts[i] = res.data
+		rcts[i] = res.Data
 	}
 	return cids, rcts, nil
 }
@@ -288,8 +288,8 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string,
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		rcts[i] = res.data
+		rcts[i] = res.Data
 	}
 	return cids, rcts, nil
 }
@@ -303,8 +303,8 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string,
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		rcts[i] = res.data
+		rcts[i] = res.Data
 	}
 	return cids, rcts, nil
 }
@@ -312,7 +312,7 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string,
 // RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash
 func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
 	rctResult := new(ipldResult)
-	return rctResult.cid, rctResult.data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
+	return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
 }
 
 // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
@@ -323,13 +323,13 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr
 		return "", nil, err
 	}
 	var i []interface{}
-	if err := rlp.DecodeBytes(accountResult.data, &i); err != nil {
+	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
 		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
 	}
 	if len(i) != 2 {
 		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
 	}
-	return accountResult.cid, i[1].([]byte), nil
+	return accountResult.CID, i[1].([]byte), nil
 }
 
 // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number
@@ -343,9 +343,9 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad
 	cids := make([]string, len(accountResults))
 	accounts := make([][]byte, len(accountResults))
 	for i, res := range accountResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
 		var iface []interface{}
-		if err := rlp.DecodeBytes(res.data, &iface); err != nil {
+		if err := rlp.DecodeBytes(res.Data, &iface); err != nil {
 			return nil, nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
 		}
 		if len(iface) != 2 {

pkg/eth/types.go

@@ -20,11 +20,30 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
 )
 
+// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
+type RPCTransaction struct {
+	BlockHash        *common.Hash    `json:"blockHash"`
+	BlockNumber      *hexutil.Big    `json:"blockNumber"`
+	From             common.Address  `json:"from"`
+	Gas              hexutil.Uint64  `json:"gas"`
+	GasPrice         *hexutil.Big    `json:"gasPrice"`
+	Hash             common.Hash     `json:"hash"`
+	Input            hexutil.Bytes   `json:"input"`
+	Nonce            hexutil.Uint64  `json:"nonce"`
+	To               *common.Address `json:"to"`
+	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
+	Value            *hexutil.Big    `json:"value"`
+	V                *hexutil.Big    `json:"v"`
+	R                *hexutil.Big    `json:"r"`
+	S                *hexutil.Big    `json:"s"`
+}
+
 // IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
 // Returned by IPLDFetcher and ResponseFilterer
 type IPLDs struct {

pkg/serve/config.go

@@ -68,6 +68,7 @@ type Config struct {
 func NewConfig() (*Config, error) {
 	c := new(Config)
 
+	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
 	viper.BindEnv("server.wsPath", SERVER_WS_PATH)
 	viper.BindEnv("server.ipcPath", SERVER_IPC_PATH)
 	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)