Add cold import script

pkg/geth/blockchain.go

@@ -10,13 +10,17 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"golang.org/x/net/context"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
+	vulcRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
 type Blockchain struct {
 	client              *ethclient.Client
+	blockConverter      vulcCommon.BlockConverter
 	readGethHeaders     chan *types.Header
 	outputBlocks        chan core.Block
 	newHeadSubscription ethereum.Subscription
@@ -33,6 +37,8 @@ func NewBlockchain(ipcPath string) *Blockchain {
 	clientWrapper := node.ClientWrapper{ContextCaller: rpcClient, IPCPath: ipcPath}
 	blockchain.node = node.MakeNode(clientWrapper)
 	blockchain.client = client
+	transactionConverter := vulcRpc.NewRpcTransactionConverter(client)
+	blockchain.blockConverter = vulcCommon.NewBlockConverter(transactionConverter)
 	return &blockchain
 }
 
@@ -50,7 +56,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe
 	if err != nil {
 		return []core.Log{}, err
 	}
-	logs := ToCoreLogs(gethLogs)
+	logs := vulcCommon.ToCoreLogs(gethLogs)
 	return logs, nil
 }
 
@@ -63,7 +69,7 @@ func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, e
 	if err != nil {
 		return core.Block{}, err
 	}
-	block, err := ToCoreBlock(gethBlock, blockchain.client)
+	block, err := blockchain.blockConverter.ToCoreBlock(gethBlock)
 	if err != nil {
 		return core.Block{}, err
 	}

pkg/geth/cold_importer.go

@@ -0,0 +1,58 @@
+package geth
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/ethereum"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
+)
+
+type ColdImporter struct {
+	blockRepository   datastore.BlockRepository
+	converter         common.BlockConverter
+	ethDB             ethereum.Database
+	receiptRepository datastore.ReceiptRepository
+}
+
+func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.ReceiptRepository, converter common.BlockConverter) *ColdImporter {
+	return &ColdImporter{
+		blockRepository:   blockRepository,
+		converter:         converter,
+		ethDB:             ethDB,
+		receiptRepository: receiptRepository,
+	}
+}
+
+func (ci *ColdImporter) Execute(startingBlockNumber int64, endingBlockNumber int64) error {
+	for i := startingBlockNumber; i <= endingBlockNumber; i++ {
+		hash := ci.ethDB.GetBlockHash(i)
+
+		blockId, err := ci.createBlocksAndTransactions(hash, i)
+		if err != nil {
+			if err == repositories.ErrBlockExists {
+				continue
+			}
+			return err
+		}
+		err = ci.createReceiptsAndLogs(hash, i, blockId)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ci *ColdImporter) createBlocksAndTransactions(hash []byte, i int64) (int64, error) {
+	block := ci.ethDB.GetBlock(hash, i)
+	coreBlock, err := ci.converter.ToCoreBlock(block)
+	if err != nil {
+		return 0, err
+	}
+	return ci.blockRepository.CreateOrUpdateBlock(coreBlock)
+}
+
+func (ci *ColdImporter) createReceiptsAndLogs(hash []byte, number int64, blockId int64) error {
+	receipts := ci.ethDB.GetBlockReceipts(hash, number)
+	coreReceipts := common.ToCoreReceipts(receipts)
+	return ci.receiptRepository.CreateReceiptsAndLogs(blockId, coreReceipts)
+}

pkg/geth/cold_importer_test.go

@@ -0,0 +1,85 @@
+package geth_test
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
+)
+
+var _ = Describe("Geth cold importer", func() {
+	var fakeGethBlock *types.Block
+
+	BeforeEach(func() {
+		header := &types.Header{}
+		transactions := []*types.Transaction{}
+		uncles := []*types.Header{}
+		receipts := []*types.Receipt{}
+		fakeGethBlock = types.NewBlock(header, transactions, uncles, receipts)
+	})
+
+	It("fetches blocks from level db and persists them to pg", func() {
+		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
+		mockBlockRepository := fakes.NewMockBlockRepository()
+		mockReceiptRepository := fakes.NewMockReceiptRepository()
+		mockTransactionConverter := fakes.NewMockTransactionConverter()
+		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
+
+		blockNumber := int64(123)
+		fakeHash := []byte{1, 2, 3, 4, 5}
+		mockEthereumDatabase.SetReturnHash(fakeHash)
+		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
+		importer := geth.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
+
+		importer.Execute(blockNumber, blockNumber)
+
+		mockEthereumDatabase.AssertGetBlockHashCalledWith(blockNumber)
+		mockEthereumDatabase.AssertGetBlockCalledWith(fakeHash, blockNumber)
+		mockTransactionConverter.AssertConvertTransactionsToCoreCalledWith(fakeGethBlock)
+		convertedBlock, err := blockConverter.ToCoreBlock(fakeGethBlock)
+		Expect(err).NotTo(HaveOccurred())
+		mockBlockRepository.AssertCreateOrUpdateBlockCalledWith(convertedBlock)
+	})
+
+	It("fetches receipts from level db and persists them to pg", func() {
+		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
+		mockBlockRepository := fakes.NewMockBlockRepository()
+		mockReceiptRepository := fakes.NewMockReceiptRepository()
+		mockTransactionConverter := fakes.NewMockTransactionConverter()
+		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
+
+		blockNumber := int64(123)
+		blockId := int64(999)
+		mockBlockRepository.SetCreateOrUpdateBlockReturnVals(blockId, nil)
+		fakeReceipts := types.Receipts{{}}
+		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
+		mockEthereumDatabase.SetReturnReceipts(fakeReceipts)
+		importer := geth.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
+
+		importer.Execute(blockNumber, blockNumber)
+
+		expectedReceipts := vulcCommon.ToCoreReceipts(fakeReceipts)
+		mockReceiptRepository.AssertCreateReceiptsAndLogsCalledWith(blockId, expectedReceipts)
+	})
+
+	It("does not fetch receipts if block already exists", func() {
+		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
+		mockBlockRepository := fakes.NewMockBlockRepository()
+		mockReceiptRepository := fakes.NewMockReceiptRepository()
+		mockTransactionConverter := fakes.NewMockTransactionConverter()
+		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
+
+		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
+		mockBlockRepository.SetCreateOrUpdateBlockReturnVals(0, repositories.ErrBlockExists)
+		importer := geth.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
+
+		err := importer.Execute(1, 1)
+
+		Expect(err).NotTo(HaveOccurred())
+		mockReceiptRepository.AssertCreateReceiptsAndLogsNotCalled()
+	})
+})

pkg/geth/converters/cold_db/transaction_converter.go

@@ -0,0 +1,69 @@
+package cold_db
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"golang.org/x/sync/errgroup"
+	"strings"
+)
+
+type ColdDbTransactionConverter struct{}
+
+func NewColdDbTransactionConverter() *ColdDbTransactionConverter {
+	return &ColdDbTransactionConverter{}
+}
+
+func (cdtc *ColdDbTransactionConverter) ConvertTransactionsToCore(gethBlock *types.Block) ([]core.Transaction, error) {
+	var g errgroup.Group
+	coreTransactions := make([]core.Transaction, len(gethBlock.Transactions()))
+
+	for gethTransactionIndex, gethTransaction := range gethBlock.Transactions() {
+		transaction := gethTransaction
+		transactionIndex := uint(gethTransactionIndex)
+		g.Go(func() error {
+			signer := getSigner(transaction)
+			sender, err := signer.Sender(transaction)
+			if err != nil {
+				return err
+			}
+			coreTransaction := transToCoreTrans(transaction, &sender)
+			coreTransactions[transactionIndex] = coreTransaction
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return coreTransactions, err
+	}
+	return coreTransactions, nil
+}
+
+func getSigner(tx *types.Transaction) types.Signer {
+	v, _, _ := tx.RawSignatureValues()
+	if v.Sign() != 0 && tx.Protected() {
+		return types.NewEIP155Signer(tx.ChainId())
+	}
+	return types.HomesteadSigner{}
+}
+
+func transToCoreTrans(transaction *types.Transaction, from *common.Address) core.Transaction {
+	data := hexutil.Encode(transaction.Data())
+	return core.Transaction{
+		Hash:     transaction.Hash().Hex(),
+		Nonce:    transaction.Nonce(),
+		To:       strings.ToLower(addressToHex(transaction.To())),
+		From:     strings.ToLower(addressToHex(from)),
+		GasLimit: transaction.Gas(),
+		GasPrice: transaction.GasPrice().Int64(),
+		Value:    transaction.Value().String(),
+		Data:     data,
+	}
+}
+
+func addressToHex(to *common.Address) string {
+	if to == nil {
+		return ""
+	}
+	return to.Hex()
+}

pkg/geth/converters/common/block_converter.go

@@ -0,0 +1,41 @@
+package common
+
+import (
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"strings"
+)
+
+type BlockConverter struct {
+	transactionConverter TransactionConverter
+}
+
+func NewBlockConverter(transactionConverter TransactionConverter) BlockConverter {
+	return BlockConverter{transactionConverter: transactionConverter}
+}
+
+func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error) {
+	transactions, err := bc.transactionConverter.ConvertTransactionsToCore(gethBlock)
+	if err != nil {
+		return core.Block{}, err
+	}
+	coreBlock := core.Block{
+		Difficulty:   gethBlock.Difficulty().Int64(),
+		ExtraData:    hexutil.Encode(gethBlock.Extra()),
+		GasLimit:     gethBlock.GasLimit(),
+		GasUsed:      gethBlock.GasUsed(),
+		Hash:         gethBlock.Hash().Hex(),
+		Miner:        strings.ToLower(gethBlock.Coinbase().Hex()),
+		Nonce:        hexutil.Encode(gethBlock.Header().Nonce[:]),
+		Number:       gethBlock.Number().Int64(),
+		ParentHash:   gethBlock.ParentHash().Hex(),
+		Size:         gethBlock.Size().String(),
+		Time:         gethBlock.Time().Int64(),
+		Transactions: transactions,
+		UncleHash:    gethBlock.UncleHash().Hex(),
+	}
+	coreBlock.Reward = CalcBlockReward(coreBlock, gethBlock.Uncles())
+	coreBlock.UnclesReward = CalcUnclesReward(coreBlock, gethBlock.Uncles())
+	return coreBlock, nil
+}

pkg/geth/converters/common/block_converter_test.go

@@ -1,15 +1,11 @@
-package geth_test
+package common_test
 
 import (
-	"math/big"
-
 	"context"
-
 	"fmt"
-
 	"io/ioutil"
 	"log"
-
+	"math/big"
 	"os"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -17,7 +13,9 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
 )
 
 type FakeGethClient struct {
@@ -93,27 +91,29 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 		}
 		block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
 		client := &FakeGethClient{}
-		gethBlock, err := geth.ToCoreBlock(block, client)
+		transactionConverter := rpc.NewRpcTransactionConverter(client)
+		blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
+
+		coreBlock, err := blockConverter.ToCoreBlock(block)
 
 		Expect(err).ToNot(HaveOccurred())
-		Expect(gethBlock.Difficulty).To(Equal(difficulty.Int64()))
-		Expect(gethBlock.GasLimit).To(Equal(gasLimit))
-		Expect(gethBlock.Miner).To(Equal(miner.Hex()))
-		Expect(gethBlock.GasUsed).To(Equal(gasUsed))
-		Expect(gethBlock.Hash).To(Equal(block.Hash().Hex()))
-		Expect(gethBlock.Nonce).To(Equal(hexutil.Encode(header.Nonce[:])))
-		Expect(gethBlock.Number).To(Equal(number))
-		Expect(gethBlock.ParentHash).To(Equal(block.ParentHash().Hex()))
-		Expect(gethBlock.ExtraData).To(Equal(hexutil.Encode(block.Extra())))
-		Expect(gethBlock.Size).To(Equal(block.Size().String()))
-		Expect(gethBlock.Time).To(Equal(time))
-		Expect(gethBlock.UncleHash).To(Equal(block.UncleHash().Hex()))
-		Expect(gethBlock.IsFinal).To(BeFalse())
+		Expect(coreBlock.Difficulty).To(Equal(difficulty.Int64()))
+		Expect(coreBlock.GasLimit).To(Equal(gasLimit))
+		Expect(coreBlock.Miner).To(Equal(miner.Hex()))
+		Expect(coreBlock.GasUsed).To(Equal(gasUsed))
+		Expect(coreBlock.Hash).To(Equal(block.Hash().Hex()))
+		Expect(coreBlock.Nonce).To(Equal(hexutil.Encode(header.Nonce[:])))
+		Expect(coreBlock.Number).To(Equal(number))
+		Expect(coreBlock.ParentHash).To(Equal(block.ParentHash().Hex()))
+		Expect(coreBlock.ExtraData).To(Equal(hexutil.Encode(block.Extra())))
+		Expect(coreBlock.Size).To(Equal(block.Size().String()))
+		Expect(coreBlock.Time).To(Equal(time))
+		Expect(coreBlock.UncleHash).To(Equal(block.UncleHash().Hex()))
+		Expect(coreBlock.IsFinal).To(BeFalse())
 	})
 
 	Describe("The block and uncle rewards calculations", func() {
 		It("calculates block rewards for a block", func() {
-
 			transaction := types.NewTransaction(
 				uint64(226823),
 				common.HexToAddress("0x108fedb097c1dcfed441480170144d8e19bb217f"),
@@ -141,10 +141,13 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			}
 			uncles := []*types.Header{{Number: big.NewInt(1071817)}, {Number: big.NewInt(1071818)}}
 			block := types.NewBlock(&header, transactions, uncles, []*types.Receipt{&receipt})
-			coreBlock, err := geth.ToCoreBlock(block, client)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
+
+			coreBlock, err := blockConverter.ToCoreBlock(block)
 
 			Expect(err).ToNot(HaveOccurred())
-			Expect(geth.CalcBlockReward(coreBlock, block.Uncles())).To(Equal(5.31355))
+			Expect(vulcCommon.CalcBlockReward(coreBlock, block.Uncles())).To(Equal(5.31355))
 		})
 
 		It("calculates the uncles reward for a block", func() {
@@ -175,11 +178,13 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 
 			client := NewFakeClient(nil)
 			client.AddReceipts(receipts)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
-			coreBlock, err := geth.ToCoreBlock(block, client)
+			coreBlock, err := blockConverter.ToCoreBlock(block)
 
 			Expect(err).ToNot(HaveOccurred())
-			Expect(geth.CalcUnclesReward(coreBlock, block.Uncles())).To(Equal(6.875))
+			Expect(vulcCommon.CalcUnclesReward(coreBlock, block.Uncles())).To(Equal(6.875))
 		})
 
 		It("decreases the static block reward from 5 to 3 for blocks after block 4,269,999", func() {
@@ -222,10 +227,13 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 
 			client := NewFakeClient(nil)
 			client.AddReceipts(receipts)
-			coreBlock, err := geth.ToCoreBlock(block, client)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
+
+			coreBlock, err := blockConverter.ToCoreBlock(block)
 
 			Expect(err).ToNot(HaveOccurred())
-			Expect(geth.CalcBlockReward(coreBlock, block.Uncles())).To(Equal(3.024990672))
+			Expect(vulcCommon.CalcBlockReward(coreBlock, block.Uncles())).To(Equal(3.024990672))
 		})
 	})
 
@@ -234,7 +242,10 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			header := types.Header{}
 			block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
 			client := &FakeGethClient{}
-			coreBlock, err := geth.ToCoreBlock(block, client)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
+
+			coreBlock, err := blockConverter.ToCoreBlock(block)
 
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(coreBlock.Transactions)).To(Equal(0))
@@ -263,13 +274,16 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			client.AddReceipts([]*types.Receipt{gethReceipt})
 
 			header := types.Header{}
-			gethBlock := types.NewBlock(
+			block := types.NewBlock(
 				&header,
 				[]*types.Transaction{gethTransaction},
 				[]*types.Header{},
 				[]*types.Receipt{gethReceipt},
 			)
-			coreBlock, err := geth.ToCoreBlock(gethBlock, client)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
+
+			coreBlock, err := blockConverter.ToCoreBlock(block)
 
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(coreBlock.Transactions)).To(Equal(1))
@@ -283,9 +297,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			Expect(coreTransaction.Nonce).To(Equal(gethTransaction.Nonce()))
 
 			coreReceipt := coreTransaction.Receipt
-			expectedReceipt := geth.ReceiptToCoreReceipt(gethReceipt)
+			expectedReceipt := vulcCommon.ToCoreReceipt(gethReceipt)
 			Expect(coreReceipt).To(Equal(expectedReceipt))
-
 		})
 
 		It("has an empty 'To' field when transaction creates a new contract", func() {
@@ -307,21 +320,23 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			client := NewFakeClient(nil)
 			client.AddReceipts([]*types.Receipt{gethReceipt})
 
-			gethBlock := types.NewBlock(
+			block := types.NewBlock(
 				&types.Header{},
 				[]*types.Transaction{gethTransaction},
 				[]*types.Header{},
 				[]*types.Receipt{gethReceipt},
 			)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
-			coreBlock, err := geth.ToCoreBlock(gethBlock, client)
+			coreBlock, err := blockConverter.ToCoreBlock(block)
 
 			Expect(err).ToNot(HaveOccurred())
 			coreTransaction := coreBlock.Transactions[0]
 			Expect(coreTransaction.To).To(Equal(""))
 
 			coreReceipt := coreTransaction.Receipt
-			expectedReceipt := geth.ReceiptToCoreReceipt(gethReceipt)
+			expectedReceipt := vulcCommon.ToCoreReceipt(gethReceipt)
 			Expect(coreReceipt).To(Equal(expectedReceipt))
 		})
 	})
@@ -330,7 +345,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 		var gethTransaction *types.Transaction
 		var gethReceipt *types.Receipt
 		var header *types.Header
-		var gethBlock *types.Block
+		var block *types.Block
 
 		BeforeEach(func() {
 			log.SetOutput(ioutil.Discard)
@@ -344,7 +359,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			)
 			gethReceipt = &types.Receipt{}
 			header = &types.Header{}
-			gethBlock = types.NewBlock(
+			block = types.NewBlock(
 				header,
 				[]*types.Transaction{gethTransaction},
 				[]*types.Header{},
@@ -360,14 +375,22 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 		It("returns an error when transaction sender call fails", func() {
 			client := NewFakeClient(TransactionSenderError{})
 			client.AddReceipts([]*types.Receipt{})
-			_, err := geth.ToCoreBlock(gethBlock, client)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
+
+			_, err := blockConverter.ToCoreBlock(block)
+
 			Expect(err).To(Equal(TransactionSenderError{}))
 		})
 
 		It("returns an error when transaction receipt call fails", func() {
 			client := NewFakeClient(TransActionReceiptError{})
 			client.AddReceipts([]*types.Receipt{})
-			_, err := geth.ToCoreBlock(gethBlock, client)
+			transactionConverter := rpc.NewRpcTransactionConverter(client)
+			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
+
+			_, err := blockConverter.ToCoreBlock(block)
+
 			Expect(err).To(Equal(TransActionReceiptError{}))
 		})
 	})

pkg/geth/converters/common/block_rewards.go

@@ -1,4 +1,4 @@
-package geth
+package common
 
 import (
 	"github.com/ethereum/go-ethereum/core/types"

pkg/geth/converters/common/common_suite_test.go

@@ -0,0 +1,13 @@
+package common_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestCommon(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Common Suite")
+}

pkg/geth/converters/common/log_converter.go

@@ -1,4 +1,4 @@
-package geth
+package common
 
 import (
 	"strings"
@@ -6,6 +6,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
@@ -30,8 +31,7 @@ func ToCoreLog(gethLog types.Log) core.Log {
 	topics := gethLog.Topics
 	hexTopics := makeTopics(topics)
 	return core.Log{
-		Address: strings.ToLower(gethLog.Address.Hex()),
-
+		Address:     strings.ToLower(gethLog.Address.Hex()),
 		BlockNumber: int64(gethLog.BlockNumber),
 		Topics:      hexTopics,
 		TxHash:      gethLog.TxHash.Hex(),

pkg/geth/converters/common/log_converter_test.go

@@ -1,4 +1,4 @@
-package geth_test
+package common_test
 
 import (
 	"strings"
@@ -8,8 +8,9 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 )
 
 var _ = Describe("Conversion of GethLog to core.Log", func() {
@@ -41,7 +42,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
 			},
 		}
 
-		coreLog := geth.ToCoreLog(gethLog)
+		coreLog := vulcCommon.ToCoreLog(gethLog)
 
 		Expect(coreLog.Address).To(Equal(expected.Address))
 		Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber))
@@ -81,15 +82,13 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
 			},
 		}
 
-		expectedOne := geth.ToCoreLog(gethLogOne)
-		expectedTwo := geth.ToCoreLog(gethLogTwo)
+		expectedOne := vulcCommon.ToCoreLog(gethLogOne)
+		expectedTwo := vulcCommon.ToCoreLog(gethLogTwo)
 
-		coreLogs := geth.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo})
+		coreLogs := vulcCommon.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo})
 
 		Expect(len(coreLogs)).To(Equal(2))
 		Expect(coreLogs[0]).To(Equal(expectedOne))
 		Expect(coreLogs[1]).To(Equal(expectedTwo))
-
 	})
-
 })

pkg/geth/converters/common/receipt_converter.go

@@ -1,24 +1,25 @@
-package geth
+package common
 
 import (
-	"math/big"
-
 	"bytes"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
-func BigTo64(n *big.Int) int64 {
-	if n != nil {
-		return n.Int64()
+func ToCoreReceipts(gethReceipts types.Receipts) []core.Receipt {
+	var coreReceipts []core.Receipt
+	for _, receipt := range gethReceipts {
+		coreReceipt := ToCoreReceipt(receipt)
+		coreReceipts = append(coreReceipts, coreReceipt)
 	}
-	return 0
+	return coreReceipts
 }
 
-func ReceiptToCoreReceipt(gethReceipt *types.Receipt) core.Receipt {
+func ToCoreReceipt(gethReceipt *types.Receipt) core.Receipt {
 	bloom := hexutil.Encode(gethReceipt.Bloom.Bytes())
 	var postState string
 	var status int

pkg/geth/converters/common/receipt_converter_test.go

@@ -1,4 +1,4 @@
-package geth_test
+package common_test
 
 import (
 	"github.com/ethereum/go-ethereum/common"
@@ -6,8 +6,9 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 )
 
 var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
@@ -34,7 +35,7 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
 			TxHash:            receipt.TxHash.Hex(),
 		}
 
-		coreReceipt := geth.ReceiptToCoreReceipt(&receipt)
+		coreReceipt := vulcCommon.ToCoreReceipt(&receipt)
 		Expect(coreReceipt.Bloom).To(Equal(expected.Bloom))
 		Expect(coreReceipt.ContractAddress).To(Equal(expected.ContractAddress))
 		Expect(coreReceipt.CumulativeGasUsed).To(Equal(expected.CumulativeGasUsed))
@@ -68,7 +69,7 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
 			TxHash:            receipt.TxHash.Hex(),
 		}
 
-		coreReceipt := geth.ReceiptToCoreReceipt(&receipt)
+		coreReceipt := vulcCommon.ToCoreReceipt(&receipt)
 		Expect(coreReceipt.Bloom).To(Equal(expected.Bloom))
 		Expect(coreReceipt.ContractAddress).To(Equal(""))
 		Expect(coreReceipt.CumulativeGasUsed).To(Equal(expected.CumulativeGasUsed))
@@ -77,7 +78,5 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
 		Expect(coreReceipt.StateRoot).To(Equal(expected.StateRoot))
 		Expect(coreReceipt.Status).To(Equal(expected.Status))
 		Expect(coreReceipt.TxHash).To(Equal(expected.TxHash))
-
 	})
-
 })

pkg/geth/converters/common/transaction_converter.go

@@ -0,0 +1,10 @@
+package common
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type TransactionConverter interface {
+	ConvertTransactionsToCore(gethBlock *types.Block) ([]core.Transaction, error)
+}

pkg/geth/converters/rpc/transaction_converter.go

@@ -1,16 +1,17 @@
-package geth
+package rpc
 
 import (
-	"strings"
-
+	"context"
 	"log"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"golang.org/x/net/context"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 )
 
 type Client interface {
@@ -18,32 +19,15 @@ type Client interface {
 	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
 }
 
-func ToCoreBlock(gethBlock *types.Block, client Client) (core.Block, error) {
-	transactions, err := convertTransactionsToCore(gethBlock, client)
-	if err != nil {
-		return core.Block{}, err
-	}
-	coreBlock := core.Block{
-		Difficulty:   gethBlock.Difficulty().Int64(),
-		ExtraData:    hexutil.Encode(gethBlock.Extra()),
-		GasLimit:     gethBlock.GasLimit(),
-		GasUsed:      gethBlock.GasUsed(),
-		Hash:         gethBlock.Hash().Hex(),
-		Miner:        strings.ToLower(gethBlock.Coinbase().Hex()),
-		Nonce:        hexutil.Encode(gethBlock.Header().Nonce[:]),
-		Number:       gethBlock.Number().Int64(),
-		ParentHash:   gethBlock.ParentHash().Hex(),
-		Size:         gethBlock.Size().String(),
-		Time:         gethBlock.Time().Int64(),
-		Transactions: transactions,
-		UncleHash:    gethBlock.UncleHash().Hex(),
-	}
-	coreBlock.Reward = CalcBlockReward(coreBlock, gethBlock.Uncles())
-	coreBlock.UnclesReward = CalcUnclesReward(coreBlock, gethBlock.Uncles())
-	return coreBlock, nil
+type RpcTransactionConverter struct {
+	client Client
 }
 
-func convertTransactionsToCore(gethBlock *types.Block, client Client) ([]core.Transaction, error) {
+func NewRpcTransactionConverter(client Client) *RpcTransactionConverter {
+	return &RpcTransactionConverter{client: client}
+}
+
+func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.Block) ([]core.Transaction, error) {
 	var g errgroup.Group
 	coreTransactions := make([]core.Transaction, len(gethBlock.Transactions()))
 
@@ -52,13 +36,13 @@ func convertTransactionsToCore(gethBlock *types.Block, client Client) ([]core.Tr
 		transaction := gethTransaction
 		transactionIndex := uint(gethTransactionIndex)
 		g.Go(func() error {
-			from, err := client.TransactionSender(context.Background(), transaction, gethBlock.Hash(), transactionIndex)
+			from, err := rtc.client.TransactionSender(context.Background(), transaction, gethBlock.Hash(), transactionIndex)
 			if err != nil {
 				log.Println("transaction sender: ", err)
 				return err
 			}
 			coreTransaction := transToCoreTrans(transaction, &from)
-			coreTransaction, err = appendReceiptToTransaction(client, coreTransaction)
+			coreTransaction, err = appendReceiptToTransaction(rtc.client, coreTransaction)
 			if err != nil {
 				log.Println("receipt: ", err)
 				return err
@@ -79,7 +63,7 @@ func appendReceiptToTransaction(client Client, transaction core.Transaction) (co
 	if err != nil {
 		return transaction, err
 	}
-	receipt := ReceiptToCoreReceipt(gethReceipt)
+	receipt := vulcCommon.ToCoreReceipt(gethReceipt)
 	transaction.Receipt = receipt
 	return transaction, nil
 }