Merge pull request #29 from 8thlight/light-sync

Add light sync command

README.md

@@ -54,9 +54,10 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli
 - See `environments/infura.toml` to configure commands to run against infura, if a local node is unavailable
 
 ## Start syncing with postgres
-Syncs VulcanizeDB with the configured Ethereum node.
+Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs.
-1. Start node (**if fast syncing wait for initial sync to finish**)
+This command is useful when you want to maintain a broad cache of what's happening on the blockchain.
-1. In a separate terminal start vulcanize_db
+1. Start Ethereum node (**if fast syncing your Ethereum node, wait for initial sync to finish**)
+1. In a separate terminal start VulcanizeDB:
     - `./vulcanizedb sync --config <config.toml> --starting-block-number <block-number>`
 
 ## Alternatively, sync from Geth's underlying LevelDB
@@ -69,6 +70,13 @@ Sync VulcanizeDB from the LevelDB underlying a Geth node.
     - `--ending-block-number`/`-e`: block number to sync to
     - `--all`/`-a`: sync all missing blocks
 
+## Alternatively, sync in "light" mode
+Syncs VulcanizeDB with the configured Ethereum node, populating only block headers.
+This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values).
+1. Start Ethereum node
+2. In a separate terminal start VulcanizeDB:
+    - `./vulcanizedb lightSync --config <config.toml> --starting-block-number <block-number>`
+
 ## Start full environment in docker by single command
 
 ### Geth Rinkeby

cmd/coldImport.go

@@ -1,4 +1,4 @@
-// Copyright © 2018 Rob Mulholand <rmulholand@8thlight.com>
+// Copyright © 2018 Vulcanize
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -44,8 +44,8 @@ Geth must be synced over all of the desired blocks and must not be running in or
 
 func init() {
 	rootCmd.AddCommand(coldImportCmd)
-	coldImportCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Number for first block to cold import.")
+	coldImportCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "BlockNumber for first block to cold import.")
-	coldImportCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "e", 5500000, "Number for last block to cold import.")
+	coldImportCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "e", 5500000, "BlockNumber for last block to cold import.")
 	coldImportCmd.Flags().BoolVarP(&syncAll, "all", "a", false, "Option to sync all missing blocks.")
 }
 
@@ -82,8 +82,8 @@ func coldImport() {
 	// init cold importer deps
 	blockRepository := repositories.NewBlockRepository(&pgDB)
 	receiptRepository := repositories.ReceiptRepository{DB: &pgDB}
-	transactionconverter := cold_db.NewColdDbTransactionConverter()
+	transactionConverter := cold_db.NewColdDbTransactionConverter()
-	blockConverter := vulcCommon.NewBlockConverter(transactionconverter)
+	blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
 	// init and execute cold importer
 	coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)

cmd/erc20.go

@@ -15,13 +15,20 @@
 package cmd
 
 import (
+	"log"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/spf13/cobra"
+
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
 	"github.com/vulcanize/vulcanizedb/libraries/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"log"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	"time"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
 // erc20Cmd represents the erc20 command
@@ -49,14 +56,23 @@ Expects an ethereum node to be running and requires a .toml config file:
 func watchERC20s() {
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
-	blockchain := geth.NewBlockchain(ipc)
+	rawRpcClient, err := rpc.Dial(ipc)
-	db, err := postgres.NewDB(databaseConfig, blockchain.Node())
+	if err != nil {
+		log.Fatal(err)
+	}
+	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
+	ethClient := ethclient.NewClient(rawRpcClient)
+	client := client.NewEthClient(ethClient)
+	node := node.MakeNode(rpcClient)
+	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+	blockChain := geth.NewBlockChain(client, node, transactionConverter)
+	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
 	if err != nil {
 		log.Fatal("Failed to initialize database.")
 	}
 	watcher := shared.Watcher{
 		DB:         *db,
-		Blockchain: blockchain,
+		Blockchain: blockChain,
 	}
 
 	watcher.AddTransformers(every_block.TransformerInitializers())

cmd/lightSync.go

@@ -0,0 +1,107 @@
+// Copyright © 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"log"
+	"os"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/spf13/cobra"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+	"github.com/vulcanize/vulcanizedb/utils"
+)
+
+// lightSyncCmd represents the lightSync command
+var lightSyncCmd = &cobra.Command{
+	Use:   "lightSync",
+	Short: "Syncs VulcanizeDB with local ethereum node's block headers",
+	Long: `Syncs VulcanizeDB with local ethereum node. Populates
+Postgres with block headers.
+
+./vulcanizedb lightSync --starting-block-number 0 --config public.toml
+
+Expects ethereum node to be running and requires a .toml config:
+
+  [database]
+  name = "vulcanize_public"
+  hostname = "localhost"
+  port = 5432
+
+  [client]
+  ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		lightSync()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(lightSyncCmd)
+	lightSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
+}
+
+func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
+	missingBlocksPopulated <- history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
+}
+
+func lightSync() {
+	ticker := time.NewTicker(pollingInterval)
+	defer ticker.Stop()
+	rawRpcClient, err := rpc.Dial(ipc)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
+	ethClient := ethclient.NewClient(rawRpcClient)
+	client := client.NewEthClient(ethClient)
+	node := node.MakeNode(rpcClient)
+	transactionConverter := vRpc.NewRpcTransactionConverter(client)
+	blockChain := geth.NewBlockChain(client, node, transactionConverter)
+
+	lastBlock := blockChain.LastBlock().Int64()
+	if lastBlock == 0 {
+		log.Fatal("geth initial: state sync not finished")
+	}
+	if startingBlockNumber > lastBlock {
+		log.Fatal("starting block number > current block number")
+	}
+
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
+	headerRepository := repositories.NewHeaderRepository(&db)
+	validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow)
+	missingBlocksPopulated := make(chan int)
+	go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
+
+	for {
+		select {
+		case <-ticker.C:
+			window := validator.ValidateHeaders()
+			window.Log(os.Stdout)
+		case <-missingBlocksPopulated:
+			go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber)
+		}
+	}
+}

cmd/root.go

@@ -1,3 +1,17 @@
+// Copyright © 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cmd
 
 import (

cmd/sync.go

@@ -1,17 +1,35 @@
+// Copyright © 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cmd
 
 import (
+	"log"
 	"os"
-
 	"time"
 
-	"log"
+	"github.com/ethereum/go-ethereum/ethclient"
-
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/spf13/cobra"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
@@ -19,9 +37,11 @@ import (
 // syncCmd represents the sync command
 var syncCmd = &cobra.Command{
 	Use:   "sync",
-	Short: "Syncs vulcanizedb with local ethereum node",
+	Short: "Syncs VulcanizeDB with local ethereum node",
-	Long: `Syncs vulcanizedb with local ethereum node. 
+	Long: `Syncs VulcanizeDB with local ethereum node. Populates
-vulcanizedb sync --starting-block-number 0 --config public.toml
+Postgres with blocks, transactions, receipts, and logs.
+
+./vulcanizedb sync --starting-block-number 0 --config public.toml
 
 Expects ethereum node to be running and requires a .toml config:
 
@@ -39,7 +59,8 @@ Expects ethereum node to be running and requires a .toml config:
 }
 
 const (
-	pollingInterval = 7 * time.Second
+	pollingInterval  = 7 * time.Second
+	validationWindow = 15
 )
 
 func init() {
@@ -48,16 +69,25 @@ func init() {
 	syncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
 }
 
-func backFillAllBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
+func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
 	missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
 }
 
 func sync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
-	blockchain := geth.NewBlockchain(ipc)
+	rawRpcClient, err := rpc.Dial(ipc)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
+	ethClient := ethclient.NewClient(rawRpcClient)
+	client := client.NewEthClient(ethClient)
+	node := node.MakeNode(rpcClient)
+	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+	blockChain := geth.NewBlockChain(client, node, transactionConverter)
 
-	lastBlock := blockchain.LastBlock().Int64()
+	lastBlock := blockChain.LastBlock().Int64()
 	if lastBlock == 0 {
 		log.Fatal("geth initial: state sync not finished")
 	}
@@ -65,19 +95,19 @@ func sync() {
 		log.Fatal("starting block number > current block number")
 	}
 
-	db := utils.LoadPostgres(databaseConfig, blockchain.Node())
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
 	blockRepository := repositories.NewBlockRepository(&db)
-	validator := history.NewBlockValidator(blockchain, blockRepository, 15)
+	validator := history.NewBlockValidator(blockChain, blockRepository, validationWindow)
 	missingBlocksPopulated := make(chan int)
-	go backFillAllBlocks(blockchain, blockRepository, missingBlocksPopulated, startingBlockNumber)
+	go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
 
 	for {
 		select {
 		case <-ticker.C:
 			window := validator.ValidateBlocks()
-			validator.Log(os.Stdout, window)
+			window.Log(os.Stdout)
 		case <-missingBlocksPopulated:
-			go backFillAllBlocks(blockchain, blockRepository, missingBlocksPopulated, startingBlockNumber)
+			go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
 		}
 	}
 }

db/migrations/1531758090_create_headers_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE public.headers;

db/migrations/1531758090_create_headers_table.up.sql

@@ -0,0 +1,11 @@
+CREATE TABLE public.headers (
+  id                    SERIAL PRIMARY KEY,
+  hash                  VARCHAR(66),
+  block_number          BIGINT,
+  raw                   bytea,
+  eth_node_id           INTEGER,
+  eth_node_fingerprint  VARCHAR(128),
+  CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id)
+  REFERENCES eth_nodes (id)
+  ON DELETE CASCADE
+);

db/schema.sql

@@ -121,6 +121,40 @@ CREATE TABLE public.eth_nodes (
 );
 
 
+--
+-- Name: headers; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.headers (
+    id integer NOT NULL,
+    hash character varying(66),
+    block_number bigint,
+    raw bytea,
+    eth_node_id integer,
+    eth_node_fingerprint character varying(128)
+);
+
+
+--
+-- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.headers_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id;
+
+
 --
 -- Name: log_filters; Type: TABLE; Schema: public; Owner: -
 --
@@ -385,6 +419,13 @@ ALTER TABLE ONLY public.blocks ALTER COLUMN id SET DEFAULT nextval('public.block
 ALTER TABLE ONLY public.eth_nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
 
 
+--
+-- Name: headers id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass);
+
+
 --
 -- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -459,6 +500,14 @@ ALTER TABLE ONLY public.eth_nodes
     ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id);
 
 
+--
+-- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.headers
+    ADD CONSTRAINT headers_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -574,6 +623,14 @@ ALTER TABLE ONLY public.token_supply
     ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
 
 
+--
+-- Name: headers eth_nodes_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.headers
+    ADD CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE;
+
+
 --
 -- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

examples/erc20_watcher/every_block/fetcher.go

@@ -16,24 +16,25 @@ package every_block
 
 import (
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"log"
 	"math/big"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
 type ERC20FetcherInterface interface {
 	FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error)
-	GetBlockchain() core.Blockchain
+	GetBlockChain() core.BlockChain
 }
 
-func NewFetcher(blockchain core.Blockchain) Fetcher {
+func NewFetcher(blockchain core.BlockChain) Fetcher {
 	return Fetcher{
 		Blockchain: blockchain,
 	}
 }
 
 type Fetcher struct {
-	Blockchain      core.Blockchain
+	Blockchain      core.BlockChain
 	ContractAbi     string
 	ContractAddress string
 }
@@ -65,6 +66,6 @@ func (f Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, block
 	return *result, nil
 }
 
-func (f Fetcher) GetBlockchain() core.Blockchain {
+func (f Fetcher) GetBlockChain() core.BlockChain {
 	return f.Blockchain
 }

examples/erc20_watcher/every_block/fetcher_test.go

@@ -15,46 +15,50 @@
 package every_block_test
 
 import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/examples/constants"
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
-	"github.com/vulcanize/vulcanizedb/examples/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"math/big"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
 var _ = Describe("ERC20 Fetcher", func() {
 	blockNumber := int64(5502914)
 
-	infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
-	realBlockchain := geth.NewBlockchain(infuraIPC)
-	realFetcher := every_block.NewFetcher(realBlockchain)
-
-	fakeBlockchain := &mocks.Blockchain{}
-	testFetcher := every_block.NewFetcher(fakeBlockchain)
-	testAbi := "testAbi"
-	testContractAddress := "testContractAddress"
-
-	errorBlockchain := &mocks.FailureBlockchain{}
-	errorFetcher := every_block.NewFetcher(errorBlockchain)
-
 	Describe("FetchSupplyOf", func() {
 		It("fetches data from the blockchain with the correct arguments", func() {
+			fakeBlockchain := fakes.NewMockBlockChain()
+			testFetcher := every_block.NewFetcher(fakeBlockchain)
+			testAbi := "testAbi"
+			testContractAddress := "testContractAddress"
 			_, err := testFetcher.FetchSupplyOf(testAbi, testContractAddress, blockNumber)
 
 			Expect(err).NotTo(HaveOccurred())
-			Expect(fakeBlockchain.FetchedAbi).To(Equal(testAbi))
-			Expect(fakeBlockchain.FetchedContractAddress).To(Equal(testContractAddress))
-			Expect(fakeBlockchain.FetchedMethod).To(Equal("totalSupply"))
-			Expect(fakeBlockchain.FetchedMethodArg).To(BeNil())
 			expectedResult := big.Int{}
 			expected := &expectedResult
-			Expect(fakeBlockchain.FetchedResult).To(Equal(&expected))
+			fakeBlockchain.AssertFetchContractDataCalledWith(testAbi, testContractAddress, "totalSupply", nil, &expected, blockNumber)
-			Expect(fakeBlockchain.FetchedBlockNumber).To(Equal(blockNumber))
 		})
 
 		It("fetches a token's total supply at the given block height", func() {
+			infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL"
+			rawRpcClient, err := rpc.Dial(infuraIPC)
+			Expect(err).NotTo(HaveOccurred())
+			rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
+			ethClient := ethclient.NewClient(rawRpcClient)
+			blockChainClient := client.NewEthClient(ethClient)
+			node := node.MakeNode(rpcClient)
+			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			realFetcher := every_block.NewFetcher(blockChain)
 			result, err := realFetcher.FetchSupplyOf(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -64,12 +68,15 @@ var _ = Describe("ERC20 Fetcher", func() {
 		})
 
 		It("returns an error if the call to the blockchain fails", func() {
+			blockChain := fakes.NewMockBlockChain()
+			blockChain.SetFetchContractDataErr(fakes.FakeError)
+			errorFetcher := every_block.NewFetcher(blockChain)
 			result, err := errorFetcher.FetchSupplyOf("", "", 0)
 
 			Expect(result.String()).To(Equal("0"))
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("totalSupply"))
-			Expect(err.Error()).To(ContainSubstring(mocks.TestError.Error()))
+			Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		})
 	})
 })

examples/erc20_watcher/every_block/integration_test.go

@@ -20,34 +20,35 @@ import (
 	"github.com/vulcanize/vulcanizedb/examples/constants"
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
-	"github.com/vulcanize/vulcanizedb/examples/mocks"
 	"github.com/vulcanize/vulcanizedb/examples/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"math/big"
 	"strconv"
 )
 
-func setLastBlockOnChain(blockchain *mocks.Blockchain, blockNumber int64) {
+func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) {
 	blockNumberString := strconv.FormatInt(blockNumber, 10)
 	lastBlockOnChain := big.Int{}
 	lastBlockOnChain.SetString(blockNumberString, 10)
-	blockchain.SetLastBlock(&lastBlockOnChain)
+	blockChain.SetLastBlock(&lastBlockOnChain)
 }
 
 var _ = Describe("Everyblock transformers", func() {
 	var db *postgres.DB
-	var blockchain mocks.Blockchain
+	var blockChain *fakes.MockBlockChain
 	var blockNumber int64
 	var blockId int64
 	var err error
 
 	BeforeEach(func() {
+		blockChain = fakes.NewMockBlockChain()
 		blockNumber = erc20_watcher.DaiConfig.FirstBlock
 		lastBlockNumber := blockNumber + 1
 		db = test_helpers.CreateNewDatabase()
-		setLastBlockOnChain(&blockchain, lastBlockNumber)
+		setLastBlockOnChain(blockChain, lastBlockNumber)
 
 		blockRepository := repositories.NewBlockRepository(db)
 
@@ -58,8 +59,8 @@ var _ = Describe("Everyblock transformers", func() {
 	})
 
 	It("creates a token_supply record for each block in the given range", func() {
-		initializer := every_block.TokenSupplyTransformerInitializer{erc20_watcher.DaiConfig}
+		initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig}
-		transformer := initializer.NewTokenSupplyTransformer(db, &blockchain)
+		transformer := initializer.NewTokenSupplyTransformer(db, blockChain)
 		transformer.Execute()
 
 		var tokenSupplyCount int

examples/erc20_watcher/every_block/transformer.go

@@ -38,7 +38,7 @@ type TokenSupplyTransformerInitializer struct {
 	Config erc20_watcher.ContractConfig
 }
 
-func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.Blockchain) shared.Transformer {
+func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
 	fetcher := NewFetcher(blockchain)
 	repository := TokenSupplyRepository{DB: db}
 	transformer := Transformer{
@@ -74,7 +74,7 @@ func newTransformerError(err error, blockNumber int64, msg string) error {
 
 func (t Transformer) Execute() error {
 	var upperBoundBlock int64
-	blockchain := t.Fetcher.GetBlockchain()
+	blockchain := t.Fetcher.GetBlockChain()
 	lastBlock := blockchain.LastBlock().Int64()
 
 	if t.Config.LastBlock == -1 {

examples/erc20_watcher/every_block/transformer_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
 	"github.com/vulcanize/vulcanizedb/examples/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"math/big"
 	"math/rand"
 	"strconv"
@@ -40,7 +41,7 @@ var _ = Describe("Everyblock transformer", func() {
 	var fetcher mocks.Fetcher
 	var repository mocks.TotalSupplyRepository
 	var transformer every_block.Transformer
-	var blockchain mocks.Blockchain
+	var blockChain *fakes.MockBlockChain
 	var initialSupply = "27647235749155415536952630"
 	var initialSupplyPlusOne = "27647235749155415536952631"
 	var initialSupplyPlusTwo = "27647235749155415536952632"
@@ -48,9 +49,9 @@ var _ = Describe("Everyblock transformer", func() {
 	var defaultLastBlock = big.Int{}
 
 	BeforeEach(func() {
-		blockchain = mocks.Blockchain{}
+		blockChain = fakes.NewMockBlockChain()
-		blockchain.SetLastBlock(&defaultLastBlock)
+		blockChain.SetLastBlock(&defaultLastBlock)
-		fetcher = mocks.Fetcher{Blockchain: &blockchain}
+		fetcher = mocks.Fetcher{BlockChain: blockChain}
 		fetcher.SetSupply(initialSupply)
 		repository = mocks.TotalSupplyRepository{}
 		repository.SetMissingBlocks([]int64{config.FirstBlock})
@@ -132,7 +133,7 @@ var _ = Describe("Everyblock transformer", func() {
 		mostRecentBlock := big.Int{}
 		mostRecentBlock.SetString(numberToString, 10)
 
-		blockchain.SetLastBlock(&mostRecentBlock)
+		blockChain.SetLastBlock(&mostRecentBlock)
 
 		err := transformer.Execute()
 		Expect(err).NotTo(HaveOccurred())
@@ -149,13 +150,14 @@ var _ = Describe("Everyblock transformer", func() {
 		}
 		err := transformer.Execute()
 		Expect(err).To(HaveOccurred())
-		Expect(err.Error()).To(ContainSubstring("TestError"))
+		Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		Expect(err.Error()).To(ContainSubstring("fetching missing blocks"))
 	})
 
-	It("returns an error if the call to the blockchain fails", func() {
+	It("returns an error if the call to the blockChain fails", func() {
-		failureBlockchain := mocks.FailureBlockchain{}
+		failureBlockchain := fakes.NewMockBlockChain()
 		failureBlockchain.SetLastBlock(&defaultLastBlock)
+		failureBlockchain.SetFetchContractDataErr(fakes.FakeError)
 		fetcher := every_block.NewFetcher(failureBlockchain)
 		transformer = every_block.Transformer{
 			Fetcher:    &fetcher,
@@ -163,7 +165,7 @@ var _ = Describe("Everyblock transformer", func() {
 		}
 		err := transformer.Execute()
 		Expect(err).To(HaveOccurred())
-		Expect(err.Error()).To(ContainSubstring("TestError"))
+		Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		Expect(err.Error()).To(ContainSubstring("supply"))
 	})
 
@@ -178,7 +180,7 @@ var _ = Describe("Everyblock transformer", func() {
 		}
 		err := transformer.Execute()
 		Expect(err).To(HaveOccurred())
-		Expect(err.Error()).To(ContainSubstring("TestError"))
+		Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		Expect(err.Error()).To(ContainSubstring("supply"))
 	})
 })

examples/erc20_watcher/every_block/transformers.go

@@ -15,8 +15,8 @@
 package every_block
 
 import (
-	"github.com/vulcanize/vulcanizedb/libraries/shared"
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared"
 )
 
 func TransformerInitializers() []shared.TransformerInitializer {

examples/mocks/mocks.go

@@ -15,19 +15,18 @@
 package mocks
 
 import (
-	"errors"
+	"math/big"
+
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"math/big"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 )
 
-var TestError = errors.New("TestError")
-
 type Fetcher struct {
 	ContractAddress string
 	Abi             string
 	FetchedBlocks   []int64
-	Blockchain      core.Blockchain
+	BlockChain      core.BlockChain
 	supply          big.Int
 }
 
@@ -35,8 +34,8 @@ func (f *Fetcher) SetSupply(supply string) {
 	f.supply.SetString(supply, 10)
 }
 
-func (f Fetcher) GetBlockchain() core.Blockchain {
+func (f Fetcher) GetBlockChain() core.BlockChain {
-	return f.Blockchain
+	return f.BlockChain
 }
 
 func (f *Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) {
@@ -80,7 +79,7 @@ type FailureRepository struct {
 
 func (fr *FailureRepository) Create(supply every_block.TokenSupply) error {
 	if fr.createFail {
-		return TestError
+		return fakes.FakeError
 	} else {
 		return nil
 	}
@@ -88,7 +87,7 @@ func (fr *FailureRepository) Create(supply every_block.TokenSupply) error {
 
 func (fr *FailureRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) {
 	if fr.missingBlocksFail {
-		return []int64{}, TestError
+		return []int64{}, fakes.FakeError
 	} else {
 		return fr.missingBlocksNumbers, nil
 	}
@@ -105,71 +104,3 @@ func (fr *FailureRepository) SetMissingBlocksFail(fail bool) {
 func (fr *FailureRepository) SetMissingBlocks(missingBlocks []int64) {
 	fr.missingBlocksNumbers = missingBlocks
 }
-
-type Blockchain struct {
-	FetchedAbi             string
-	FetchedContractAddress string
-	FetchedMethod          string
-	FetchedMethodArg       interface{}
-	FetchedResult          interface{}
-	FetchedBlockNumber     int64
-	lastBlock              *big.Int
-}
-
-func (fb *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
-	fb.FetchedAbi = abiJSON
-	fb.FetchedContractAddress = address
-	fb.FetchedMethod = method
-	fb.FetchedMethodArg = methodArg
-	fb.FetchedResult = result
-	fb.FetchedBlockNumber = blockNumber
-	return nil
-}
-
-func (fb *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
-	panic("implement me")
-}
-
-func (fb *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) {
-	panic("implement me")
-}
-
-func (fb *Blockchain) LastBlock() *big.Int {
-	return fb.lastBlock
-}
-
-func (fb *Blockchain) Node() core.Node {
-	panic("implement me")
-}
-
-func (fb *Blockchain) SetLastBlock(lastBlock *big.Int) {
-	fb.lastBlock = lastBlock
-}
-
-type FailureBlockchain struct {
-	lastBlock *big.Int
-}
-
-func (FailureBlockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
-	return errors.New("TestError")
-}
-
-func (FailureBlockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
-	panic("implement me")
-}
-
-func (FailureBlockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) {
-	panic("implement me")
-}
-
-func (fb FailureBlockchain) LastBlock() *big.Int {
-	return fb.lastBlock
-}
-
-func (FailureBlockchain) Node() core.Node {
-	panic("implement me")
-}
-
-func (fb *FailureBlockchain) SetLastBlock(lastBlock *big.Int) {
-	fb.lastBlock = lastBlock
-}

examples/test_helpers/database.go

@@ -35,21 +35,6 @@ type TransferDBRow struct {
 	VulcanizeLogID int64 `db:"vulcanize_log_id"`
 }
 
-func CreateLogRecord(db *postgres.DB, logRepository repositories.LogRepository, log core.Log) {
-	blockRepository := repositories.NewBlockRepository(db)
-	receiptRepository := repositories.ReceiptRepository{DB: db}
-
-	blockNumber := log.BlockNumber
-	blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
-	Expect(err).NotTo(HaveOccurred())
-
-	receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{})
-	Expect(err).NotTo(HaveOccurred())
-
-	err = logRepository.CreateLogs([]core.Log{log}, receiptId)
-	Expect(err).NotTo(HaveOccurred())
-}
-
 func CreateNewDatabase() *postgres.DB {
 	var node core.Node
 	node = core.Node{

integration_test/block_rewards_test.go

@@ -1,24 +1,44 @@
 package integration
 
 import (
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Rewards calculations", func() {
 
 	It("calculates a block reward for a real block", func() {
-		blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath)
+		rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
-		block, err := blockchain.GetBlockByNumber(1071819)
+		Expect(err).NotTo(HaveOccurred())
+		rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath)
+		ethClient := ethclient.NewClient(rawRpcClient)
+		blockChainClient := client.NewEthClient(ethClient)
+		node := node.MakeNode(rpcClient)
+		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+		blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.Reward).To(Equal(5.31355))
 	})
 
 	It("calculates an uncle reward for a real block", func() {
-		blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath)
+		rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
-		block, err := blockchain.GetBlockByNumber(1071819)
+		Expect(err).NotTo(HaveOccurred())
+		rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath)
+		ethClient := ethclient.NewClient(rawRpcClient)
+		blockChainClient := client.NewEthClient(ethClient)
+		node := node.MakeNode(rpcClient)
+		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+		blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.UnclesReward).To(Equal(6.875))
 	})

integration_test/contract_test.go

@@ -4,10 +4,15 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/testing"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
@@ -27,42 +32,60 @@ var _ = Describe("Reading contracts", func() {
 				},
 				Index: 19,
 				Data:  "0x0000000000000000000000000000000000000000000000000c7d713b49da0000"}
-			blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath)
+			rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
+			Expect(err).NotTo(HaveOccurred())
+			rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath)
+			ethClient := ethclient.NewClient(rawRpcClient)
+			blockChainClient := client.NewEthClient(ethClient)
+			node := node.MakeNode(rpcClient)
+			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
 			contract := testing.SampleContract()
 
-			logs, err := blockchain.GetLogs(contract, big.NewInt(4703824), nil)
+			logs, err := blockChain.GetLogs(contract, big.NewInt(4703824), nil)
 
 			Expect(err).To(BeNil())
 			Expect(len(logs)).To(Equal(3))
 			Expect(logs[0]).To(Equal(expectedLogZero))
-
 		})
 
 		It("returns and empty log array when no events for a given block / contract combo", func() {
-			blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath)
+			rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
+			Expect(err).NotTo(HaveOccurred())
+			rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath)
+			ethClient := ethclient.NewClient(rawRpcClient)
+			blockChainClient := client.NewEthClient(ethClient)
+			node := node.MakeNode(rpcClient)
+			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
 
-			logs, err := blockchain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824), nil)
+			logs, err := blockChain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824), nil)
 
 			Expect(err).To(BeNil())
 			Expect(len(logs)).To(Equal(0))
 
 		})
-
 	})
 
 	Describe("Fetching Contract data", func() {
 		It("returns the correct attribute for a real contract", func() {
-			blockchain := geth.NewBlockchain(test_config.InfuraClient.IPCPath)
+			rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
+			Expect(err).NotTo(HaveOccurred())
+			rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath)
+			ethClient := ethclient.NewClient(rawRpcClient)
+			blockChainClient := client.NewEthClient(ethClient)
+			node := node.MakeNode(rpcClient)
+			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
 
 			contract := testing.SampleContract()
 			var balance = new(big.Int)
 			args := common.HexToHash("0xd26114cd6ee289accf82350c8d8487fedb8a0c07")
-			err := blockchain.FetchContractData(contract.Abi, "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", "balanceOf", args, &balance, 5167471)
+			err = blockChain.FetchContractData(contract.Abi, "0xd26114cd6ee289accf82350c8d8487fedb8a0c07", "balanceOf", args, &balance, 5167471)
 			Expect(err).NotTo(HaveOccurred())
 			expected := new(big.Int)
 			expected.SetString("10897295492887612977137", 10)
 			Expect(balance).To(Equal(expected))
 		})
 	})
-
 })

integration_test/geth_blockchain_test.go

@@ -1,40 +1,49 @@
 package integration_test
 
 import (
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Reading from the Geth blockchain", func() {
-
+	var blockChain *geth.BlockChain
-	var blockchain *geth.Blockchain
-	var inMemory *inmemory.InMemory
 
 	BeforeEach(func() {
-		blockchain = geth.NewBlockchain(test_config.InfuraClient.IPCPath)
+		rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
-		inMemory = inmemory.NewInMemory()
+		Expect(err).NotTo(HaveOccurred())
+		rpcClient := client.NewRpcClient(rawRpcClient, test_config.InfuraClient.IPCPath)
+		ethClient := ethclient.NewClient(rawRpcClient)
+		blockChainClient := client.NewEthClient(ethClient)
+		node := node.MakeNode(rpcClient)
+		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
+		blockChain = geth.NewBlockChain(blockChainClient, node, transactionConverter)
 	})
 
 	It("reads two blocks", func(done Done) {
-		blocks := &inmemory.BlockRepository{InMemory: inMemory}
+		blocks := fakes.NewMockBlockRepository()
-		lastBlock := blockchain.LastBlock()
+		lastBlock := blockChain.LastBlock()
 		queriedBlocks := []int64{lastBlock.Int64() - 5, lastBlock.Int64() - 6}
-		history.RetrieveAndUpdateBlocks(blockchain, blocks, queriedBlocks)
+		history.RetrieveAndUpdateBlocks(blockChain, blocks, queriedBlocks)
-		Expect(blocks.BlockCount()).To(Equal(2))
+		blocks.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(2, []int64{lastBlock.Int64() - 5, lastBlock.Int64() - 6})
 		close(done)
 	}, 30)
 
 	It("retrieves the genesis block and first block", func(done Done) {
-		genesisBlock, err := blockchain.GetBlockByNumber(int64(0))
+		genesisBlock, err := blockChain.GetBlockByNumber(int64(0))
 		Expect(err).ToNot(HaveOccurred())
-		firstBlock, err := blockchain.GetBlockByNumber(int64(1))
+		firstBlock, err := blockChain.GetBlockByNumber(int64(1))
 		Expect(err).ToNot(HaveOccurred())
-		lastBlockNumber := blockchain.LastBlock()
+		lastBlockNumber := blockChain.LastBlock()
 
 		Expect(genesisBlock.Number).To(Equal(int64(0)))
 		Expect(firstBlock.Number).To(Equal(int64(1)))
@@ -43,7 +52,7 @@ var _ = Describe("Reading from the Geth blockchain", func() {
 	}, 15)
 
 	It("retrieves the node info", func(done Done) {
-		node := blockchain.Node()
+		node := blockChain.Node()
 		mainnetID := float64(1)
 
 		Expect(node.GenesisBlock).ToNot(BeNil())
@@ -60,7 +69,7 @@ var _ = Describe("Reading from the Geth blockchain", func() {
 			var blocks []core.Block
 			n := 10
 			for i := 5327459; i > 5327459-n; i-- {
-				block, err := blockchain.GetBlockByNumber(int64(i))
+				block, err := blockChain.GetBlockByNumber(int64(i))
 				Expect(err).ToNot(HaveOccurred())
 				blocks = append(blocks, block)
 			}

libraries/shared/transformer_interface.go

@@ -10,7 +10,7 @@ type Transformer interface {
 	Execute() error
 }
 
-type TransformerInitializer func(db *postgres.DB, blockchain core.Blockchain) Transformer
+type TransformerInitializer func(db *postgres.DB, blockchain core.BlockChain) Transformer
 
 func HexToInt64(byteString string) int64 {
 	value := common.HexToHash(byteString)

libraries/shared/watcher.go

@@ -8,7 +8,7 @@ import (
 type Watcher struct {
 	Transformers []Transformer
 	DB           postgres.DB
-	Blockchain   core.Blockchain
+	Blockchain   core.BlockChain
 }
 
 func (watcher *Watcher) AddTransformers(us []TransformerInitializer) {

libraries/shared/watcher_test.go

@@ -23,7 +23,7 @@ func (mh *MockTransformer) Execute() error {
 	return nil
 }
 
-func fakeTransformerInitializer(db *postgres.DB, blockchain core.Blockchain) shared.Transformer {
+func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
 	return &MockTransformer{}
 }
 

pkg/core/blockchain.go

@@ -2,9 +2,10 @@ package core
 
 import "math/big"
 
-type Blockchain interface {
+type BlockChain interface {
 	ContractDataFetcher
 	GetBlockByNumber(blockNumber int64) (Block, error)
+	GetHeaderByNumber(blockNumber int64) (Header, error)
 	GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error)
 	LastBlock() *big.Int
 	Node() Node

pkg/core/eth_client.go

@@ -0,0 +1,19 @@
+package core
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type EthClient interface {
+	BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error)
+	CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error)
+	FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error)
+	HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error)
+	TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error)
+	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
+}

pkg/core/header.go

@@ -0,0 +1,7 @@
+package core
+
+type Header struct {
+	BlockNumber int64 `db:"block_number"`
+	Hash        string
+	Raw         []byte
+}

pkg/core/rpc_client.go

@@ -0,0 +1,9 @@
+package core
+
+import "context"
+
+type RpcClient interface {
+	CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error
+	IpcPath() string
+	SupportedModules() (map[string]string, error)
+}

pkg/datastore/inmemory/block_repository.go

@@ -1,51 +0,0 @@
-package inmemory
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-)
-
-type BlockRepository struct {
-	*InMemory
-}
-
-func (blockRepository *BlockRepository) CreateOrUpdateBlock(block core.Block) (int64, error) {
-	blockRepository.CreateOrUpdateBlockCallCount++
-	blockRepository.blocks[block.Number] = block
-	for _, transaction := range block.Transactions {
-		blockRepository.receipts[transaction.Hash] = transaction.Receipt
-		blockRepository.logs[transaction.TxHash] = transaction.Logs
-	}
-	return 0, nil
-}
-
-func (blockRepository *BlockRepository) GetBlock(blockNumber int64) (core.Block, error) {
-	if block, ok := blockRepository.blocks[blockNumber]; ok {
-		return block, nil
-	}
-	return core.Block{}, datastore.ErrBlockDoesNotExist(blockNumber)
-}
-
-func (blockRepository *BlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 {
-	missingNumbers := []int64{}
-	for blockNumber := int64(startingBlockNumber); blockNumber <= endingBlockNumber; blockNumber++ {
-		if _, ok := blockRepository.blocks[blockNumber]; !ok {
-			missingNumbers = append(missingNumbers, blockNumber)
-		}
-	}
-	return missingNumbers
-}
-
-func (blockRepository *BlockRepository) SetBlocksStatus(chainHead int64) {
-	for key, block := range blockRepository.blocks {
-		if key < (chainHead - blocksFromHeadBeforeFinal) {
-			tmp := block
-			tmp.IsFinal = true
-			blockRepository.blocks[key] = tmp
-		}
-	}
-}
-
-func (blockRepository *BlockRepository) BlockCount() int {
-	return len(blockRepository.blocks)
-}

pkg/datastore/inmemory/contract_repository.go

@@ -1,35 +0,0 @@
-package inmemory
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-)
-
-type ContractRepostiory struct {
-	*InMemory
-}
-
-func (contractRepository *ContractRepostiory) ContractExists(contractHash string) bool {
-	_, present := contractRepository.contracts[contractHash]
-	return present
-}
-
-func (contractRepository *ContractRepostiory) GetContract(contractHash string) (core.Contract, error) {
-	contract, ok := contractRepository.contracts[contractHash]
-	if !ok {
-		return core.Contract{}, datastore.ErrContractDoesNotExist(contractHash)
-	}
-	for _, block := range contractRepository.blocks {
-		for _, transaction := range block.Transactions {
-			if transaction.To == contractHash {
-				contract.Transactions = append(contract.Transactions, transaction)
-			}
-		}
-	}
-	return contract, nil
-}
-
-func (contractRepository *ContractRepostiory) CreateContract(contract core.Contract) error {
-	contractRepository.contracts[contract.Hash] = contract
-	return nil
-}

pkg/datastore/inmemory/in_memory.go

@@ -1,30 +0,0 @@
-package inmemory
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/filters"
-)
-
-const (
-	blocksFromHeadBeforeFinal = 20
-)
-
-type InMemory struct {
-	blocks                       map[int64]core.Block
-	receipts                     map[string]core.Receipt
-	contracts                    map[string]core.Contract
-	logs                         map[string][]core.Log
-	logFilters                   map[string]filters.LogFilter
-	CreateOrUpdateBlockCallCount int
-}
-
-func NewInMemory() *InMemory {
-	return &InMemory{
-		CreateOrUpdateBlockCallCount: 0,
-		blocks:     make(map[int64]core.Block),
-		receipts:   make(map[string]core.Receipt),
-		contracts:  make(map[string]core.Contract),
-		logs:       make(map[string][]core.Log),
-		logFilters: make(map[string]filters.LogFilter),
-	}
-}

pkg/datastore/postgres/repositories/header_repository.go

@@ -0,0 +1,80 @@
+package repositories
+
+import (
+	"database/sql"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type HeaderRepository struct {
+	database *postgres.DB
+}
+
+func NewHeaderRepository(database *postgres.DB) HeaderRepository {
+	return HeaderRepository{database: database}
+}
+
+func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int64, error) {
+	hash, err := repository.getHeaderHash(header)
+	if err != nil {
+		if headerDoesNotExist(err) {
+			return repository.insertHeader(header)
+		}
+		return 0, err
+	}
+	if headerMustBeReplaced(hash, header) {
+		return repository.replaceHeader(header)
+	}
+	return 0, err
+}
+
+func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
+	var header core.Header
+	err := repository.database.Get(&header, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
+		blockNumber, repository.database.Node.ID)
+	return header, err
+}
+
+func (repository HeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 {
+	numbers := make([]int64, 0)
+	repository.database.Select(&numbers, `SELECT all_block_numbers
+	  FROM (
+		  SELECT generate_series($1::INT, $2::INT) AS all_block_numbers) series
+	  WHERE all_block_numbers NOT IN (
+		  SELECT block_number FROM headers WHERE eth_node_fingerprint = $3
+	  ) `,
+		startingBlockNumber, endingBlockNumber, nodeID)
+	return numbers
+}
+
+func headerMustBeReplaced(hash string, header core.Header) bool {
+	return hash != header.Hash
+}
+
+func headerDoesNotExist(err error) bool {
+	return err == sql.ErrNoRows
+}
+
+func (repository HeaderRepository) getHeaderHash(header core.Header) (string, error) {
+	var hash string
+	err := repository.database.Get(&hash, `SELECT hash FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
+		header.BlockNumber, repository.database.Node.ID)
+	return hash, err
+}
+
+func (repository HeaderRepository) insertHeader(header core.Header) (int64, error) {
+	var headerId int64
+	err := repository.database.QueryRowx(
+		`INSERT INTO public.headers (block_number, hash, raw, eth_node_id, eth_node_fingerprint) VALUES ($1, $2, $3, $4, $5) RETURNING id`,
+		header.BlockNumber, header.Hash, header.Raw, repository.database.NodeID, repository.database.Node.ID).Scan(&headerId)
+	return headerId, err
+}
+
+func (repository HeaderRepository) replaceHeader(header core.Header) (int64, error) {
+	_, err := repository.database.Exec(`DELETE FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
+		header.BlockNumber, repository.database.Node.ID)
+	if err != nil {
+		return 0, err
+	}
+	return repository.insertHeader(header)
+}

pkg/datastore/postgres/repositories/header_repository_test.go

@@ -0,0 +1,248 @@
+package repositories_test
+
+import (
+	"database/sql"
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Block header repository", func() {
+	Describe("creating or updating a header", func() {
+
+		It("adds a header", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+
+			_, err := repo.CreateOrUpdateHeader(header)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeader core.Header
+			err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
+			Expect(dbHeader.Hash).To(Equal(header.Hash))
+			Expect(dbHeader.Raw).To(Equal(header.Raw))
+		})
+
+		It("adds node data to header", func() {
+			node := core.Node{ID: "EthNodeFingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{BlockNumber: 100}
+
+			_, err := repo.CreateOrUpdateHeader(header)
+
+			Expect(err).NotTo(HaveOccurred())
+			var ethNodeId int64
+			err = db.Get(&ethNodeId, `SELECT eth_node_id FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ethNodeId).To(Equal(db.NodeID))
+			var ethNodeFingerprint string
+			err = db.Get(&ethNodeFingerprint, `SELECT eth_node_fingerprint FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ethNodeFingerprint).To(Equal(db.Node.ID))
+		})
+
+		It("does not duplicate headers", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+
+			var dbHeaders []core.Header
+			err = db.Select(&dbHeaders, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(dbHeaders)).To(Equal(1))
+		})
+
+		It("replaces header if hash is different", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+			headerTwo := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
+				Raw:         []byte{5, 4, 3, 2, 1},
+			}
+
+			_, err = repo.CreateOrUpdateHeader(headerTwo)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeader core.Header
+			err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbHeader.Hash).To(Equal(headerTwo.Hash))
+			Expect(dbHeader.Raw).To(Equal(headerTwo.Raw))
+		})
+
+		It("does not replace header if node fingerprint is different", func() {
+			node := core.Node{ID: "Fingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			nodeTwo := core.Node{ID: "FingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+			headerTwo := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
+				Raw:         []byte{5, 4, 3, 2, 1},
+			}
+
+			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeaders []core.Header
+			err = dbTwo.Select(&dbHeaders, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(dbHeaders)).To(Equal(2))
+		})
+
+		It("only replaces header with matching node fingerprint", func() {
+			node := core.Node{ID: "Fingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			nodeTwo := core.Node{ID: "FingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+			headerTwo := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
+				Raw:         []byte{5, 4, 3, 2, 1},
+			}
+			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
+			headerThree := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{1, 1, 1, 1, 1}).Hex(),
+				Raw:         []byte{1, 1, 1, 1, 1},
+			}
+
+			_, err = repoTwo.CreateOrUpdateHeader(headerThree)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeaders []core.Header
+			err = dbTwo.Select(&dbHeaders, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(dbHeaders)).To(Equal(2))
+			Expect(dbHeaders[0].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
+			Expect(dbHeaders[1].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
+			Expect(dbHeaders[0].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+			Expect(dbHeaders[1].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+		})
+	})
+
+	Describe("Getting a header", func() {
+		It("returns header if it exists", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+
+			dbHeader, err := repo.GetHeader(header.BlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbHeader).To(Equal(header))
+		})
+
+		It("does not return header for a different node fingerprint", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+			nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+
+			_, err = repoTwo.GetHeader(header.BlockNumber)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("Getting missing headers", func() {
+		It("returns block numbers for headers not in the database", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+
+			missingBlockNumbers := repo.MissingBlockNumbers(1, 5, node.ID)
+
+			Expect(missingBlockNumbers).To(ConsistOf([]int64{2, 4}))
+		})
+
+		It("does not count headers created by a different node fingerprint", func() {
+			node := core.Node{ID: "NodeFingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+			nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+
+			missingBlockNumbers := repoTwo.MissingBlockNumbers(1, 5, nodeTwo.ID)
+
+			Expect(missingBlockNumbers).To(ConsistOf([]int64{1, 2, 3, 4, 5}))
+		})
+	})
+})

pkg/datastore/repository.go

@@ -14,7 +14,7 @@ var ErrBlockDoesNotExist = func(blockNumber int64) error {
 type BlockRepository interface {
 	CreateOrUpdateBlock(block core.Block) (int64, error)
 	GetBlock(blockNumber int64) (core.Block, error)
-	MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64
+	MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64
 	SetBlocksStatus(chainHead int64)
 }
 
@@ -37,6 +37,12 @@ type FilterRepository interface {
 	GetFilter(name string) (filters.LogFilter, error)
 }
 
+type HeaderRepository interface {
+	CreateOrUpdateHeader(header core.Header) (int64, error)
+	GetHeader(blockNumber int64) (core.Header, error)
+	MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64
+}
+
 type LogRepository interface {
 	CreateLogs(logs []core.Log, receiptId int64) error
 	GetLogs(address string, blockNumber int64) []core.Log

pkg/fakes/blockchain.go

@@ -1,76 +0,0 @@
-package fakes
-
-import (
-	"math/big"
-
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-)
-
-type Blockchain struct {
-	logs                map[string][]core.Log
-	blocks              map[int64]core.Block
-	contractAttributes  map[string]map[string]string
-	blocksChannel       chan core.Block
-	WasToldToStop       bool
-	node                core.Node
-	ContractReturnValue []byte
-	err                 error
-}
-
-func (blockchain *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
-	panic("implement me")
-}
-
-func (blockchain *Blockchain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
-	return blockchain.ContractReturnValue, nil
-}
-
-func (blockchain *Blockchain) LastBlock() *big.Int {
-	var max int64
-	for blockNumber := range blockchain.blocks {
-		if blockNumber > max {
-			max = blockNumber
-		}
-	}
-	return big.NewInt(max)
-}
-
-func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) {
-	return blockchain.logs[contract.Hash], nil
-}
-
-func (blockchain *Blockchain) Node() core.Node {
-	return blockchain.node
-}
-
-func NewBlockchain(err error) *Blockchain {
-	return &Blockchain{
-		blocks:             make(map[int64]core.Block),
-		logs:               make(map[string][]core.Log),
-		contractAttributes: make(map[string]map[string]string),
-		node:               core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "Geth"},
-		err:                err,
-	}
-}
-
-func NewBlockchainWithBlocks(blocks []core.Block) *Blockchain {
-	blockNumberToBlocks := make(map[int64]core.Block)
-	for _, block := range blocks {
-		blockNumberToBlocks[block.Number] = block
-	}
-	return &Blockchain{
-		blocks: blockNumberToBlocks,
-	}
-}
-
-func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
-	if blockchain.err != nil {
-		return core.Block{}, blockchain.err
-	}
-	return blockchain.blocks[blockNumber], nil
-}
-
-func (blockchain *Blockchain) AddBlock(block core.Block) {
-	blockchain.blocks[block.Number] = block
-	blockchain.blocksChannel <- block
-}

pkg/fakes/data.go

@@ -0,0 +1,5 @@
+package fakes
+
+import "errors"
+
+var FakeError = errors.New("failed")

pkg/fakes/mock_block_repository.go

@@ -7,14 +7,16 @@ import (
 )
 
 type MockBlockRepository struct {
+	createOrUpdateBlockCallCount                 int
 	createOrUpdateBlockCalled                    bool
 	createOrUpdateBlockPassedBlock               core.Block
-	createOrUpdateBlockReturnInt                 int64
+	createOrUpdateBlockPassedBlockNumbers        []int64
 	createOrUpdateBlockReturnErr                 error
+	createOrUpdateBlockReturnInt                 int64
 	missingBlockNumbersCalled                    bool
-	missingBlockNumbersPassedStartingBlockNumber int64
 	missingBlockNumbersPassedEndingBlockNumber   int64
 	missingBlockNumbersPassedNodeId              string
+	missingBlockNumbersPassedStartingBlockNumber int64
 	missingBlockNumbersReturnArray               []int64
 	setBlockStatusCalled                         bool
 	setBlockStatusPassedChainHead                int64
@@ -22,65 +24,78 @@ type MockBlockRepository struct {
 
 func NewMockBlockRepository() *MockBlockRepository {
 	return &MockBlockRepository{
+		createOrUpdateBlockCallCount:                 0,
 		createOrUpdateBlockCalled:                    false,
 		createOrUpdateBlockPassedBlock:               core.Block{},
-		createOrUpdateBlockReturnInt:                 0,
+		createOrUpdateBlockPassedBlockNumbers:        nil,
 		createOrUpdateBlockReturnErr:                 nil,
+		createOrUpdateBlockReturnInt:                 0,
 		missingBlockNumbersCalled:                    false,
-		missingBlockNumbersPassedStartingBlockNumber: 0,
 		missingBlockNumbersPassedEndingBlockNumber:   0,
 		missingBlockNumbersPassedNodeId:              "",
+		missingBlockNumbersPassedStartingBlockNumber: 0,
 		missingBlockNumbersReturnArray:               nil,
 		setBlockStatusCalled:                         false,
 		setBlockStatusPassedChainHead:                0,
 	}
 }
 
-func (mbr *MockBlockRepository) SetCreateOrUpdateBlockReturnVals(i int64, err error) {
+func (repository *MockBlockRepository) SetCreateOrUpdateBlockReturnVals(i int64, err error) {
-	mbr.createOrUpdateBlockReturnInt = i
+	repository.createOrUpdateBlockReturnInt = i
-	mbr.createOrUpdateBlockReturnErr = err
+	repository.createOrUpdateBlockReturnErr = err
 }
 
-func (mbr *MockBlockRepository) SetMissingBlockNumbersReturnArray(returnArray []int64) {
+func (repository *MockBlockRepository) SetMissingBlockNumbersReturnArray(returnArray []int64) {
-	mbr.missingBlockNumbersReturnArray = returnArray
+	repository.missingBlockNumbersReturnArray = returnArray
 }
 
-func (mbr *MockBlockRepository) CreateOrUpdateBlock(block core.Block) (int64, error) {
+func (repository *MockBlockRepository) CreateOrUpdateBlock(block core.Block) (int64, error) {
-	mbr.createOrUpdateBlockCalled = true
+	repository.createOrUpdateBlockCallCount++
-	mbr.createOrUpdateBlockPassedBlock = block
+	repository.createOrUpdateBlockCalled = true
-	return mbr.createOrUpdateBlockReturnInt, mbr.createOrUpdateBlockReturnErr
+	repository.createOrUpdateBlockPassedBlock = block
+	repository.createOrUpdateBlockPassedBlockNumbers = append(repository.createOrUpdateBlockPassedBlockNumbers, block.Number)
+	return repository.createOrUpdateBlockReturnInt, repository.createOrUpdateBlockReturnErr
 }
 
-func (mbr *MockBlockRepository) GetBlock(blockNumber int64) (core.Block, error) {
+func (repository *MockBlockRepository) GetBlock(blockNumber int64) (core.Block, error) {
-	panic("implement me")
+	return core.Block{Number: blockNumber}, nil
 }
 
-func (mbr *MockBlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 {
+func (repository *MockBlockRepository) MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64 {
-	mbr.missingBlockNumbersCalled = true
+	repository.missingBlockNumbersCalled = true
-	mbr.missingBlockNumbersPassedStartingBlockNumber = startingBlockNumber
+	repository.missingBlockNumbersPassedStartingBlockNumber = startingBlockNumber
-	mbr.missingBlockNumbersPassedEndingBlockNumber = endingBlockNumber
+	repository.missingBlockNumbersPassedEndingBlockNumber = endingBlockNumber
-	mbr.missingBlockNumbersPassedNodeId = nodeId
+	repository.missingBlockNumbersPassedNodeId = nodeId
-	return mbr.missingBlockNumbersReturnArray
+	return repository.missingBlockNumbersReturnArray
 }
 
-func (mbr *MockBlockRepository) SetBlocksStatus(chainHead int64) {
+func (repository *MockBlockRepository) SetBlocksStatus(chainHead int64) {
-	mbr.setBlockStatusCalled = true
+	repository.setBlockStatusCalled = true
-	mbr.setBlockStatusPassedChainHead = chainHead
+	repository.setBlockStatusPassedChainHead = chainHead
 }
 
-func (mbr *MockBlockRepository) AssertCreateOrUpdateBlockCalledWith(block core.Block) {
+func (repository *MockBlockRepository) AssertCreateOrUpdateBlockCallCountEquals(times int) {
-	Expect(mbr.createOrUpdateBlockCalled).To(BeTrue())
+	Expect(repository.createOrUpdateBlockCallCount).To(Equal(times))
-	Expect(mbr.createOrUpdateBlockPassedBlock).To(Equal(block))
 }
 
-func (mbr *MockBlockRepository) AssertMissingBlockNumbersCalledWith(startingBlockNumber int64, endingBlockNumber int64, nodeId string) {
+func (repository *MockBlockRepository) AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(times int, blockNumbers []int64) {
-	Expect(mbr.missingBlockNumbersCalled).To(BeTrue())
+	Expect(repository.createOrUpdateBlockCallCount).To(Equal(times))
-	Expect(mbr.missingBlockNumbersPassedStartingBlockNumber).To(Equal(startingBlockNumber))
+	Expect(repository.createOrUpdateBlockPassedBlockNumbers).To(Equal(blockNumbers))
-	Expect(mbr.missingBlockNumbersPassedEndingBlockNumber).To(Equal(endingBlockNumber))
-	Expect(mbr.missingBlockNumbersPassedNodeId).To(Equal(nodeId))
 }
 
-func (mbr *MockBlockRepository) AssertSetBlockStatusCalledWith(chainHead int64) {
+func (repository *MockBlockRepository) AssertCreateOrUpdateBlockCalledWith(block core.Block) {
-	Expect(mbr.setBlockStatusCalled).To(BeTrue())
+	Expect(repository.createOrUpdateBlockCalled).To(BeTrue())
-	Expect(mbr.setBlockStatusPassedChainHead).To(Equal(chainHead))
+	Expect(repository.createOrUpdateBlockPassedBlock).To(Equal(block))
+}
+
+func (repository *MockBlockRepository) AssertMissingBlockNumbersCalledWith(startingBlockNumber int64, endingBlockNumber int64, nodeId string) {
+	Expect(repository.missingBlockNumbersCalled).To(BeTrue())
+	Expect(repository.missingBlockNumbersPassedStartingBlockNumber).To(Equal(startingBlockNumber))
+	Expect(repository.missingBlockNumbersPassedEndingBlockNumber).To(Equal(endingBlockNumber))
+	Expect(repository.missingBlockNumbersPassedNodeId).To(Equal(nodeId))
+}
+
+func (repository *MockBlockRepository) AssertSetBlockStatusCalledWith(chainHead int64) {
+	Expect(repository.setBlockStatusCalled).To(BeTrue())
+	Expect(repository.setBlockStatusPassedChainHead).To(Equal(chainHead))
 }

pkg/fakes/mock_blockchain.go

@@ -0,0 +1,83 @@
+package fakes
+
+import (
+	"math/big"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type MockBlockChain struct {
+	fetchContractDataErr               error
+	fetchContractDataPassedAbi         string
+	fetchContractDataPassedAddress     string
+	fetchContractDataPassedMethod      string
+	fetchContractDataPassedMethodArg   interface{}
+	fetchContractDataPassedResult      interface{}
+	fetchContractDataPassedBlockNumber int64
+	getBlockByNumberErr                error
+	lastBlock                          *big.Int
+	node                               core.Node
+}
+
+func NewMockBlockChain() *MockBlockChain {
+	return &MockBlockChain{
+		node: core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "Geth"},
+	}
+}
+
+func (blockChain *MockBlockChain) SetFetchContractDataErr(err error) {
+	blockChain.fetchContractDataErr = err
+}
+
+func (blockChain *MockBlockChain) SetLastBlock(blockNumber *big.Int) {
+	blockChain.lastBlock = blockNumber
+}
+
+func (blockChain *MockBlockChain) SetGetBlockByNumberErr(err error) {
+	blockChain.getBlockByNumberErr = err
+}
+
+func (blockChain *MockBlockChain) GetHeaderByNumber(blockNumber int64) (core.Header, error) {
+	return core.Header{BlockNumber: blockNumber}, nil
+}
+
+func (blockChain *MockBlockChain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
+	blockChain.fetchContractDataPassedAbi = abiJSON
+	blockChain.fetchContractDataPassedAddress = address
+	blockChain.fetchContractDataPassedMethod = method
+	blockChain.fetchContractDataPassedMethodArg = methodArg
+	blockChain.fetchContractDataPassedResult = result
+	blockChain.fetchContractDataPassedBlockNumber = blockNumber
+	return blockChain.fetchContractDataErr
+}
+
+func (blockChain *MockBlockChain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
+	return []byte{}, nil
+}
+
+func (blockChain *MockBlockChain) LastBlock() *big.Int {
+	return blockChain.lastBlock
+}
+
+func (blockChain *MockBlockChain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) {
+	return []core.Log{}, nil
+}
+
+func (blockChain *MockBlockChain) Node() core.Node {
+	return blockChain.node
+}
+
+func (blockChain *MockBlockChain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
+	return core.Block{Number: blockNumber}, blockChain.getBlockByNumberErr
+}
+
+// TODO: handle methodArg being nil (can't match nil to nil in Gomega)
+func (blockChain *MockBlockChain) AssertFetchContractDataCalledWith(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) {
+	Expect(blockChain.fetchContractDataPassedAbi).To(Equal(abiJSON))
+	Expect(blockChain.fetchContractDataPassedAddress).To(Equal(address))
+	Expect(blockChain.fetchContractDataPassedMethod).To(Equal(method))
+	Expect(blockChain.fetchContractDataPassedResult).To(Equal(result))
+	Expect(blockChain.fetchContractDataPassedBlockNumber).To(Equal(blockNumber))
+}

pkg/fakes/mock_eth_client.go

@@ -0,0 +1,162 @@
+package fakes
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/gomega"
+)
+
+type MockEthClient struct {
+	callContractErr             error
+	callContractPassedContext   context.Context
+	callContractPassedMsg       ethereum.CallMsg
+	callContractPassedNumber    *big.Int
+	callContractReturnBytes     []byte
+	blockByNumberErr            error
+	blockByNumberPassedContext  context.Context
+	blockByNumberPassedNumber   *big.Int
+	blockByNumberReturnBlock    *types.Block
+	headerByNumberErr           error
+	headerByNumberPassedContext context.Context
+	headerByNumberPassedNumber  *big.Int
+	headerByNumberReturnHeader  *types.Header
+	filterLogsErr               error
+	filterLogsPassedContext     context.Context
+	filterLogsPassedQuery       ethereum.FilterQuery
+	filterLogsReturnLogs        []types.Log
+	transactionReceipts         map[string]*types.Receipt
+	err                         error
+	transactionSenderErr        error
+	transactionReceiptErr       error
+}
+
+func NewMockEthClient() *MockEthClient {
+	return &MockEthClient{
+		callContractErr:             nil,
+		callContractPassedContext:   nil,
+		callContractPassedMsg:       ethereum.CallMsg{},
+		callContractPassedNumber:    nil,
+		callContractReturnBytes:     nil,
+		blockByNumberErr:            nil,
+		blockByNumberPassedContext:  nil,
+		blockByNumberPassedNumber:   nil,
+		blockByNumberReturnBlock:    nil,
+		headerByNumberErr:           nil,
+		headerByNumberPassedContext: nil,
+		headerByNumberPassedNumber:  nil,
+		headerByNumberReturnHeader:  nil,
+		filterLogsErr:               nil,
+		filterLogsPassedContext:     nil,
+		filterLogsPassedQuery:       ethereum.FilterQuery{},
+		filterLogsReturnLogs:        nil,
+		transactionReceipts:         make(map[string]*types.Receipt),
+		err:                         nil,
+	}
+}
+
+func (client *MockEthClient) SetCallContractErr(err error) {
+	client.callContractErr = err
+}
+
+func (client *MockEthClient) SetCallContractReturnBytes(returnBytes []byte) {
+	client.callContractReturnBytes = returnBytes
+}
+
+func (client *MockEthClient) SetBlockByNumberErr(err error) {
+	client.blockByNumberErr = err
+}
+
+func (client *MockEthClient) SetBlockByNumberReturnBlock(block *types.Block) {
+	client.blockByNumberReturnBlock = block
+}
+
+func (client *MockEthClient) SetHeaderByNumberErr(err error) {
+	client.headerByNumberErr = err
+}
+
+func (client *MockEthClient) SetHeaderByNumberReturnHeader(header *types.Header) {
+	client.headerByNumberReturnHeader = header
+}
+
+func (client *MockEthClient) SetFilterLogsErr(err error) {
+	client.filterLogsErr = err
+}
+
+func (client *MockEthClient) SetFilterLogsReturnLogs(logs []types.Log) {
+	client.filterLogsReturnLogs = logs
+}
+
+func (client *MockEthClient) SetTransactionReceiptErr(err error) {
+	client.transactionReceiptErr = err
+}
+
+func (client *MockEthClient) SetTransactionReceipts(receipts []*types.Receipt) {
+	for _, receipt := range receipts {
+		client.transactionReceipts[receipt.TxHash.Hex()] = receipt
+	}
+}
+
+func (client *MockEthClient) SetTransactionSenderErr(err error) {
+	client.transactionSenderErr = err
+}
+
+func (client *MockEthClient) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
+	client.callContractPassedContext = ctx
+	client.callContractPassedMsg = msg
+	client.callContractPassedNumber = blockNumber
+	return client.callContractReturnBytes, client.callContractErr
+}
+
+func (client *MockEthClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) {
+	client.blockByNumberPassedContext = ctx
+	client.blockByNumberPassedNumber = number
+	return client.blockByNumberReturnBlock, client.blockByNumberErr
+}
+
+func (client *MockEthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) {
+	client.headerByNumberPassedContext = ctx
+	client.headerByNumberPassedNumber = number
+	return client.headerByNumberReturnHeader, client.headerByNumberErr
+}
+
+func (client *MockEthClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
+	client.filterLogsPassedContext = ctx
+	client.filterLogsPassedQuery = q
+	return client.filterLogsReturnLogs, client.filterLogsErr
+}
+
+func (client *MockEthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) {
+	return common.HexToAddress("0x123"), client.transactionSenderErr
+}
+
+func (client *MockEthClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) {
+	if gasUsed, ok := client.transactionReceipts[txHash.Hex()]; ok {
+		return gasUsed, client.transactionReceiptErr
+	}
+	return &types.Receipt{GasUsed: uint64(0)}, client.transactionReceiptErr
+}
+
+func (client *MockEthClient) AssertCallContractCalledWith(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) {
+	Expect(client.callContractPassedContext).To(Equal(ctx))
+	Expect(client.callContractPassedMsg).To(Equal(msg))
+	Expect(client.callContractPassedNumber).To(Equal(blockNumber))
+}
+
+func (client *MockEthClient) AssertBlockByNumberCalledWith(ctx context.Context, number *big.Int) {
+	Expect(client.blockByNumberPassedContext).To(Equal(ctx))
+	Expect(client.blockByNumberPassedNumber).To(Equal(number))
+}
+
+func (client *MockEthClient) AssertHeaderByNumberCalledWith(ctx context.Context, number *big.Int) {
+	Expect(client.headerByNumberPassedContext).To(Equal(ctx))
+	Expect(client.headerByNumberPassedNumber).To(Equal(number))
+}
+
+func (client *MockEthClient) AssertFilterLogsCalledWith(ctx context.Context, q ethereum.FilterQuery) {
+	Expect(client.filterLogsPassedContext).To(Equal(ctx))
+	Expect(client.filterLogsPassedQuery).To(Equal(q))
+}

pkg/fakes/mock_header_repository.go

@@ -0,0 +1,40 @@
+package fakes
+
+import (
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type MockHeaderRepository struct {
+	createOrUpdateBlockNumbersCallCount          int
+	createOrUpdateBlockNumbersPassedBlockNumbers []int64
+	missingBlockNumbers                          []int64
+}
+
+func NewMockHeaderRepository() *MockHeaderRepository {
+	return &MockHeaderRepository{}
+}
+
+func (repository *MockHeaderRepository) SetMissingBlockNumbers(blockNumbers []int64) {
+	repository.missingBlockNumbers = blockNumbers
+}
+
+func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header) (int64, error) {
+	repository.createOrUpdateBlockNumbersCallCount++
+	repository.createOrUpdateBlockNumbersPassedBlockNumbers = append(repository.createOrUpdateBlockNumbersPassedBlockNumbers, header.BlockNumber)
+	return 0, nil
+}
+
+func (*MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
+	return core.Header{BlockNumber: blockNumber}, nil
+}
+
+func (repository *MockHeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 {
+	return repository.missingBlockNumbers
+}
+
+func (repository *MockHeaderRepository) AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(times int, blockNumbers []int64) {
+	Expect(repository.createOrUpdateBlockNumbersCallCount).To(Equal(times))
+	Expect(repository.createOrUpdateBlockNumbersPassedBlockNumbers).To(Equal(blockNumbers))
+}

pkg/fakes/mock_rpc_client.go

@@ -0,0 +1,73 @@
+package fakes
+
+import (
+	"context"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type MockRpcClient struct {
+	ipcPath  string
+	nodeType core.NodeType
+}
+
+func NewMockRpcClient() *MockRpcClient {
+	return &MockRpcClient{}
+}
+
+func (client *MockRpcClient) SetNodeType(nodeType core.NodeType) {
+	client.nodeType = nodeType
+}
+
+func (client *MockRpcClient) SetIpcPath(ipcPath string) {
+	client.ipcPath = ipcPath
+}
+
+func (*MockRpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
+	switch method {
+	case "admin_nodeInfo":
+		if p, ok := result.(*p2p.NodeInfo); ok {
+			p.ID = "enode://GethNode@172.17.0.1:30303"
+			p.Name = "Geth/v1.7"
+		}
+	case "eth_getBlockByNumber":
+		if p, ok := result.(*types.Header); ok {
+			*p = types.Header{}
+		}
+
+	case "parity_versionInfo":
+		if p, ok := result.(*core.ParityNodeInfo); ok {
+			*p = core.ParityNodeInfo{
+				Track: "",
+				ParityVersion: core.ParityVersion{
+					Major: 1,
+					Minor: 2,
+					Patch: 3,
+				},
+				Hash: "",
+			}
+		}
+	case "parity_enode":
+		if p, ok := result.(*string); ok {
+			*p = "enode://ParityNode@172.17.0.1:30303"
+		}
+	case "net_version":
+		if p, ok := result.(*string); ok {
+			*p = "1234"
+		}
+	}
+	return nil
+}
+
+func (client *MockRpcClient) IpcPath() string {
+	return client.ipcPath
+}
+
+func (client *MockRpcClient) SupportedModules() (map[string]string, error) {
+	result := make(map[string]string)
+	if client.nodeType == core.GETH {
+		result["admin"] = "ok"
+	}
+	return result, nil
+}

pkg/geth/blockchain.go

@@ -3,46 +3,47 @@ package geth
 import (
 	"math/big"
 
-	"log"
-
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	"golang.org/x/net/context"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
-	vulcRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
-type Blockchain struct {
+type BlockChain struct {
-	client              *ethclient.Client
+	client          core.EthClient
-	blockConverter      vulcCommon.BlockConverter
+	blockConverter  vulcCommon.BlockConverter
-	readGethHeaders     chan *types.Header
+	headerConverter vulcCommon.HeaderConverter
-	outputBlocks        chan core.Block
+	node            core.Node
-	newHeadSubscription ethereum.Subscription
-	node                core.Node
 }
 
-func NewBlockchain(ipcPath string) *Blockchain {
+func NewBlockChain(client core.EthClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain {
-	blockchain := Blockchain{}
+	return &BlockChain{
-	rpcClient, err := rpc.Dial(ipcPath)
+		client:          client,
-	if err != nil {
+		blockConverter:  vulcCommon.NewBlockConverter(converter),
-		log.Fatal(err)
+		headerConverter: vulcCommon.HeaderConverter{},
+		node:            node,
 	}
-	client := ethclient.NewClient(rpcClient)
-	clientWrapper := node.ClientWrapper{ContextCaller: rpcClient, IPCPath: ipcPath}
-	blockchain.node = node.MakeNode(clientWrapper)
-	blockchain.client = client
-	transactionConverter := vulcRpc.NewRpcTransactionConverter(client)
-	blockchain.blockConverter = vulcCommon.NewBlockConverter(transactionConverter)
-	return &blockchain
 }
 
-func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) {
+func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Block, err error) {
+	gethBlock, err := blockChain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
+	if err != nil {
+		return block, err
+	}
+	return blockChain.blockConverter.ToCoreBlock(gethBlock)
+}
+
+func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) {
+	gethHeader, err := blockChain.client.HeaderByNumber(context.Background(), big.NewInt(blockNumber))
+	if err != nil {
+		return header, err
+	}
+	return blockChain.headerConverter.Convert(gethHeader)
+}
+
+func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) {
 	if endingBlockNumber == nil {
 		endingBlockNumber = startingBlockNumber
 	}
@@ -52,7 +53,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe
 		ToBlock:   endingBlockNumber,
 		Addresses: []common.Address{contractAddress},
 	}
-	gethLogs, err := blockchain.client.FilterLogs(context.Background(), fc)
+	gethLogs, err := blockChain.client.FilterLogs(context.Background(), fc)
 	if err != nil {
 		return []core.Log{}, err
 	}
@@ -60,23 +61,11 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe
 	return logs, nil
 }
 
-func (blockchain *Blockchain) Node() core.Node {
+func (blockChain *BlockChain) LastBlock() *big.Int {
-	return blockchain.node
+	block, _ := blockChain.client.HeaderByNumber(context.Background(), nil)
-}
-
-func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
-	gethBlock, err := blockchain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
-	if err != nil {
-		return core.Block{}, err
-	}
-	block, err := blockchain.blockConverter.ToCoreBlock(gethBlock)
-	if err != nil {
-		return core.Block{}, err
-	}
-	return block, nil
-}
-
-func (blockchain *Blockchain) LastBlock() *big.Int {
-	block, _ := blockchain.client.HeaderByNumber(context.Background(), nil)
 	return block.Number
 }
+
+func (blockChain *BlockChain) Node() core.Node {
+	return blockChain.node
+}

pkg/geth/blockchain_test.go

@@ -0,0 +1,125 @@
+package geth_test
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	vulcCore "github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/converters/cold_db"
+)
+
+var _ = Describe("Geth blockchain", func() {
+	Describe("getting a block", func() {
+		It("fetches block from client", func() {
+			mockClient := fakes.NewMockEthClient()
+			mockClient.SetBlockByNumberReturnBlock(types.NewBlockWithHeader(&types.Header{}))
+			node := vulcCore.Node{}
+			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+			blockNumber := int64(100)
+
+			_, err := blockChain.GetBlockByNumber(blockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			mockClient.AssertBlockByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
+		})
+
+		It("returns err if client returns err", func() {
+			mockClient := fakes.NewMockEthClient()
+			mockClient.SetBlockByNumberErr(fakes.FakeError)
+			node := vulcCore.Node{}
+			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+
+			_, err := blockChain.GetBlockByNumber(100)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
+
+	Describe("getting a header", func() {
+		It("fetches header from client", func() {
+			mockClient := fakes.NewMockEthClient()
+			blockNumber := int64(100)
+			mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
+			node := vulcCore.Node{}
+			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+
+			_, err := blockChain.GetHeaderByNumber(blockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			mockClient.AssertHeaderByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
+		})
+
+		It("returns err if client returns err", func() {
+			mockClient := fakes.NewMockEthClient()
+			mockClient.SetHeaderByNumberErr(fakes.FakeError)
+			node := vulcCore.Node{}
+			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+
+			_, err := blockChain.GetHeaderByNumber(100)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
+
+	Describe("getting logs", func() {
+		It("fetches logs from client", func() {
+			mockClient := fakes.NewMockEthClient()
+			mockClient.SetFilterLogsReturnLogs([]types.Log{{}})
+			node := vulcCore.Node{}
+			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
+			startingBlockNumber := big.NewInt(1)
+			endingBlockNumber := big.NewInt(2)
+
+			_, err := blockChain.GetLogs(contract, startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			expectedQuery := ethereum.FilterQuery{
+				FromBlock: startingBlockNumber,
+				ToBlock:   endingBlockNumber,
+				Addresses: []common.Address{common.HexToAddress(contract.Hash)},
+			}
+			mockClient.AssertFilterLogsCalledWith(context.Background(), expectedQuery)
+		})
+
+		It("returns err if client returns err", func() {
+			mockClient := fakes.NewMockEthClient()
+			mockClient.SetFilterLogsErr(fakes.FakeError)
+			node := vulcCore.Node{}
+			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
+			startingBlockNumber := big.NewInt(1)
+			endingBlockNumber := big.NewInt(2)
+
+			_, err := blockChain.GetLogs(contract, startingBlockNumber, endingBlockNumber)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
+
+	Describe("getting the most recent block number", func() {
+		It("fetches latest header from client", func() {
+			mockClient := fakes.NewMockEthClient()
+			blockNumber := int64(100)
+			mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
+			node := vulcCore.Node{}
+			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+
+			result := blockChain.LastBlock()
+
+			mockClient.AssertHeaderByNumberCalledWith(context.Background(), nil)
+			Expect(result).To(Equal(big.NewInt(blockNumber)))
+		})
+	})
+})

pkg/geth/client/eth_client.go

@@ -0,0 +1,42 @@
+package client
+
+import (
+	"context"
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"math/big"
+)
+
+type EthClient struct {
+	client *ethclient.Client
+}
+
+func NewEthClient(client *ethclient.Client) EthClient {
+	return EthClient{client: client}
+}
+
+func (client EthClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) {
+	return client.client.BlockByNumber(ctx, number)
+}
+
+func (client EthClient) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
+	return client.client.CallContract(ctx, msg, blockNumber)
+}
+
+func (client EthClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
+	return client.client.FilterLogs(ctx, q)
+}
+
+func (client EthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) {
+	return client.client.HeaderByNumber(ctx, number)
+}
+
+func (client EthClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) {
+	return client.client.TransactionSender(ctx, tx, block, index)
+}
+
+func (client EthClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) {
+	return client.client.TransactionReceipt(ctx, txHash)
+}

pkg/geth/client/rpc_client.go

@@ -0,0 +1,30 @@
+package client
+
+import (
+	"context"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+type RpcClient struct {
+	client  *rpc.Client
+	ipcPath string
+}
+
+func NewRpcClient(client *rpc.Client, ipcPath string) RpcClient {
+	return RpcClient{
+		client:  client,
+		ipcPath: ipcPath,
+	}
+}
+
+func (client RpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
+	return client.client.CallContext(ctx, result, method, args)
+}
+
+func (client RpcClient) IpcPath() string {
+	return client.ipcPath
+}
+
+func (client RpcClient) SupportedModules() (map[string]string, error) {
+	return client.client.SupportedModules()
+}

pkg/geth/contract.go

@@ -14,7 +14,7 @@ var (
 	ErrInvalidStateAttribute = errors.New("invalid state attribute")
 )
 
-func (blockchain *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
+func (blockChain *BlockChain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
 	parsed, err := ParseAbi(abiJSON)
 	if err != nil {
 		return err
@@ -28,15 +28,15 @@ func (blockchain *Blockchain) FetchContractData(abiJSON string, address string,
 	if err != nil {
 		return err
 	}
-	output, err := blockchain.callContract(address, input, big.NewInt(blockNumber))
+	output, err := blockChain.callContract(address, input, big.NewInt(blockNumber))
 	if err != nil {
 		return err
 	}
 	return parsed.Unpack(result, method, output)
 }
 
-func (blockchain *Blockchain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
+func (blockChain *BlockChain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
 	to := common.HexToAddress(contractHash)
 	msg := ethereum.CallMsg{To: &to, Data: input}
-	return blockchain.client.CallContract(context.Background(), msg, blockNumber)
+	return blockChain.client.CallContract(context.Background(), msg, blockNumber)
 }

pkg/geth/converters/common/block_converter.go

@@ -20,6 +20,7 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error)
 	if err != nil {
 		return core.Block{}, err
 	}
+
 	coreBlock := core.Block{
 		Difficulty:   gethBlock.Difficulty().Int64(),
 		ExtraData:    hexutil.Encode(gethBlock.Extra()),

pkg/geth/converters/common/block_converter_test.go

@@ -1,8 +1,6 @@
 package common_test
 
 import (
-	"context"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"math/big"
@@ -14,57 +12,11 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
 )
 
-type FakeGethClient struct {
-	receipts map[string]*types.Receipt
-	err      error
-}
-
-type TransActionReceiptError struct{}
-
-func (tarErr TransActionReceiptError) Error() string {
-	return fmt.Sprintf("transaction receipt error")
-}
-
-type TransactionSenderError struct{}
-
-func (tasErr TransactionSenderError) Error() string {
-	return fmt.Sprintf("transaction sender error")
-}
-
-func NewFakeClient(err error) *FakeGethClient {
-	return &FakeGethClient{
-		receipts: make(map[string]*types.Receipt),
-		err:      err,
-	}
-}
-
-func (client *FakeGethClient) AddReceipts(receipts []*types.Receipt) {
-	for _, receipt := range receipts {
-		client.receipts[receipt.TxHash.Hex()] = receipt
-	}
-}
-
-func (client *FakeGethClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) {
-	if err, ok := client.err.(TransActionReceiptError); ok {
-		return &types.Receipt{}, err
-	}
-	if gasUsed, ok := client.receipts[txHash.Hex()]; ok {
-		return gasUsed, nil
-	}
-	return &types.Receipt{GasUsed: uint64(0)}, nil
-}
-
-func (client *FakeGethClient) TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error) {
-	if err, ok := client.err.(TransactionSenderError); ok {
-		return common.Address{}, err
-	}
-	return common.HexToAddress("0x123"), nil
-}
-
 var _ = Describe("Conversion of GethBlock to core.Block", func() {
 
 	It("converts basic Block metadata", func() {
@@ -90,7 +42,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			UncleHash:  common.Hash{128},
 		}
 		block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
-		client := &FakeGethClient{}
+		client := fakes.NewMockEthClient()
 		transactionConverter := rpc.NewRpcTransactionConverter(client)
 		blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
@@ -132,8 +84,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			}
 			receipts := []*types.Receipt{&receipt}
 
-			client := NewFakeClient(nil)
+			client := fakes.NewMockEthClient()
-			client.AddReceipts(receipts)
+			client.SetTransactionReceipts(receipts)
 
 			number := int64(1071819)
 			header := types.Header{
@@ -176,8 +128,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			}
 			block := types.NewBlock(&header, transactions, uncles, receipts)
 
-			client := NewFakeClient(nil)
+			client := fakes.NewMockEthClient()
-			client.AddReceipts(receipts)
+			client.SetTransactionReceipts(receipts)
 			transactionConverter := rpc.NewRpcTransactionConverter(client)
 			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
@@ -225,8 +177,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			var uncles []*types.Header
 			block := types.NewBlock(&header, transactions, uncles, receipts)
 
-			client := NewFakeClient(nil)
+			client := fakes.NewMockEthClient()
-			client.AddReceipts(receipts)
+			client.SetTransactionReceipts(receipts)
 			transactionConverter := rpc.NewRpcTransactionConverter(client)
 			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
@@ -241,7 +193,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 		It("is empty", func() {
 			header := types.Header{}
 			block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
-			client := &FakeGethClient{}
+			client := fakes.NewMockEthClient()
 			transactionConverter := rpc.NewRpcTransactionConverter(client)
 			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
@@ -270,8 +222,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 				TxHash:            gethTransaction.Hash(),
 			}
 
-			client := NewFakeClient(nil)
+			client := fakes.NewMockEthClient()
-			client.AddReceipts([]*types.Receipt{gethReceipt})
+			client.SetTransactionReceipts([]*types.Receipt{gethReceipt})
 
 			header := types.Header{}
 			block := types.NewBlock(
@@ -317,8 +269,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 				ContractAddress:   common.HexToAddress("0x1023342345"),
 			}
 
-			client := NewFakeClient(nil)
+			client := fakes.NewMockEthClient()
-			client.AddReceipts([]*types.Receipt{gethReceipt})
+			client.SetTransactionReceipts([]*types.Receipt{gethReceipt})
 
 			block := types.NewBlock(
 				&types.Header{},
@@ -373,25 +325,25 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 		})
 
 		It("returns an error when transaction sender call fails", func() {
-			client := NewFakeClient(TransactionSenderError{})
+			client := fakes.NewMockEthClient()
-			client.AddReceipts([]*types.Receipt{})
+			client.SetTransactionSenderErr(fakes.FakeError)
 			transactionConverter := rpc.NewRpcTransactionConverter(client)
 			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
 			_, err := blockConverter.ToCoreBlock(block)
 
-			Expect(err).To(Equal(TransactionSenderError{}))
+			Expect(err).To(MatchError(fakes.FakeError))
 		})
 
 		It("returns an error when transaction receipt call fails", func() {
-			client := NewFakeClient(TransActionReceiptError{})
+			client := fakes.NewMockEthClient()
-			client.AddReceipts([]*types.Receipt{})
+			client.SetTransactionReceiptErr(fakes.FakeError)
 			transactionConverter := rpc.NewRpcTransactionConverter(client)
 			blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
 			_, err := blockConverter.ToCoreBlock(block)
 
-			Expect(err).To(Equal(TransActionReceiptError{}))
+			Expect(err).To(MatchError(fakes.FakeError))
 		})
 	})
 

pkg/geth/converters/common/header_converter.go

@@ -0,0 +1,24 @@
+package common
+
+import (
+	"bytes"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type HeaderConverter struct{}
+
+func (converter HeaderConverter) Convert(gethHeader *types.Header) (core.Header, error) {
+	writer := new(bytes.Buffer)
+	err := rlp.Encode(writer, &gethHeader)
+	if err != nil {
+		panic(err)
+	}
+	coreHeader := core.Header{
+		Hash:        gethHeader.Hash().Hex(),
+		BlockNumber: gethHeader.Number.Int64(),
+		Raw:         writer.Bytes(),
+	}
+	return coreHeader, nil
+}

pkg/geth/converters/common/header_converter_test.go

@@ -0,0 +1,47 @@
+package common_test
+
+import (
+	"bytes"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	common2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
+	"math/big"
+)
+
+var _ = Describe("Block header converter", func() {
+	It("converts geth header to core header", func() {
+		gethHeader := &types.Header{
+			Difficulty:  big.NewInt(1),
+			Number:      big.NewInt(2),
+			ParentHash:  common.HexToHash("0xParent"),
+			ReceiptHash: common.HexToHash("0xReceipt"),
+			Root:        common.HexToHash("0xRoot"),
+			Time:        big.NewInt(3),
+			TxHash:      common.HexToHash("0xTransaction"),
+			UncleHash:   common.HexToHash("0xUncle"),
+		}
+		converter := common2.HeaderConverter{}
+
+		coreHeader, err := converter.Convert(gethHeader)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
+		Expect(coreHeader.Hash).To(Equal(gethHeader.Hash().Hex()))
+	})
+
+	It("includes raw bytes for header", func() {
+		headerRLP := []byte{249, 2, 23, 160, 180, 251, 173, 248, 234, 69, 43, 19, 151, 24, 226, 112, 13, 193, 19, 92, 252, 129, 20, 80, 49, 200, 75, 122, 178, 124, 215, 16, 57, 79, 123, 56, 160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71, 148, 42, 101, 172, 164, 213, 252, 91, 92, 133, 144, 144, 166, 195, 77, 22, 65, 53, 57, 130, 38, 160, 14, 6, 111, 60, 34, 151, 165, 203, 48, 5, 147, 5, 38, 23, 209, 188, 165, 148, 111, 12, 170, 6, 53, 253, 177, 184, 90, 199, 229, 35, 111, 52, 160, 101, 186, 136, 127, 203, 8, 38, 246, 22, 208, 31, 115, 108, 29, 45, 103, 123, 202, 189, 226, 247, 252, 37, 170, 145, 207, 188, 11, 59, 173, 92, 179, 160, 32, 227, 83, 69, 64, 202, 241, 99, 120, 230, 232, 106, 43, 241, 35, 109, 159, 135, 109, 50, 24, 251, 192, 57, 88, 230, 219, 28, 99, 75, 35, 51, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 134, 11, 105, 222, 129, 162, 43, 131, 15, 66, 64, 131, 47, 239, 216, 130, 196, 68, 132, 86, 191, 180, 21, 152, 215, 131, 1, 3, 3, 132, 71, 101, 116, 104, 135, 103, 111, 49, 46, 53, 46, 49, 133, 108, 105, 110, 117, 120, 160, 146, 196, 18, 154, 10, 226, 54, 27, 69, 42, 158, 222, 236, 229, 92, 18, 236, 238, 171, 134, 99, 22, 25, 94, 61, 135, 252, 27, 0, 91, 102, 69, 136, 205, 76, 85, 185, 65, 207, 144, 21}
+		var gethHeader types.Header
+		err := rlp.Decode(bytes.NewReader(headerRLP), &gethHeader)
+		Expect(err).NotTo(HaveOccurred())
+		converter := common2.HeaderConverter{}
+
+		coreHeader, err := converter.Convert(&gethHeader)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(coreHeader.Raw).To(Equal(headerRLP))
+	})
+})

pkg/geth/converters/rpc/transaction_converter.go

@@ -14,16 +14,11 @@ import (
 	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 )
 
-type Client interface {
-	TransactionSender(ctx context.Context, tx *types.Transaction, block common.Hash, index uint) (common.Address, error)
-	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
-}
-
 type RpcTransactionConverter struct {
-	client Client
+	client core.EthClient
 }
 
-func NewRpcTransactionConverter(client Client) *RpcTransactionConverter {
+func NewRpcTransactionConverter(client core.EthClient) *RpcTransactionConverter {
 	return &RpcTransactionConverter{client: client}
 }
 
@@ -42,7 +37,7 @@ func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.B
 				return err
 			}
 			coreTransaction := transToCoreTrans(transaction, &from)
-			coreTransaction, err = appendReceiptToTransaction(rtc.client, coreTransaction)
+			coreTransaction, err = rtc.appendReceiptToTransaction(coreTransaction)
 			if err != nil {
 				log.Println("receipt: ", err)
 				return err
@@ -58,8 +53,8 @@ func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.B
 	return coreTransactions, nil
 }
 
-func appendReceiptToTransaction(client Client, transaction core.Transaction) (core.Transaction, error) {
+func (rtc *RpcTransactionConverter) appendReceiptToTransaction(transaction core.Transaction) (core.Transaction, error) {
-	gethReceipt, err := client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash))
+	gethReceipt, err := rtc.client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash))
 	if err != nil {
 		return transaction, err
 	}

pkg/geth/node/node.go

@@ -7,69 +7,38 @@ import (
 
 	"regexp"
 
-	"strings"
-
 	"log"
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"strings"
 )
 
-type PropertiesReader interface {
+type IPropertiesReader interface {
 	NodeInfo() (id string, name string)
 	NetworkId() float64
 	GenesisBlock() string
 }
 
-type ClientWrapper struct {
+type PropertiesReader struct {
-	ContextCaller
+	client core.RpcClient
-	IPCPath string
-}
-
-type ContextCaller interface {
-	CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error
-	SupportedModules() (map[string]string, error)
 }
 
 type ParityClient struct {
-	ClientWrapper
+	PropertiesReader
 }
 
 type GethClient struct {
-	ClientWrapper
+	PropertiesReader
 }
 
 type InfuraClient struct {
-	ClientWrapper
+	PropertiesReader
 }
 
-func (clientWrapper ClientWrapper) NodeType() core.NodeType {
+func MakeNode(rpcClient core.RpcClient) core.Node {
-	if strings.Contains(clientWrapper.IPCPath, "infura") {
+	pr := makePropertiesReader(rpcClient)
-		return core.INFURA
-	}
-	modules, _ := clientWrapper.SupportedModules()
-	if _, ok := modules["admin"]; ok {
-		return core.GETH
-	}
-	return core.PARITY
-}
-
-func makePropertiesReader(wrapper ClientWrapper) PropertiesReader {
-	switch wrapper.NodeType() {
-	case core.GETH:
-		return GethClient{ClientWrapper: wrapper}
-	case core.PARITY:
-		return ParityClient{ClientWrapper: wrapper}
-	case core.INFURA:
-		return InfuraClient{ClientWrapper: wrapper}
-	default:
-		return wrapper
-	}
-}
-
-func MakeNode(wrapper ClientWrapper) core.Node {
-	pr := makePropertiesReader(wrapper)
 	id, name := pr.NodeInfo()
 	return core.Node{
 		GenesisBlock: pr.GenesisBlock(),
@@ -79,9 +48,33 @@ func MakeNode(wrapper ClientWrapper) core.Node {
 	}
 }
 
-func (client ClientWrapper) NetworkId() float64 {
+func makePropertiesReader(client core.RpcClient) IPropertiesReader {
+	switch getNodeType(client) {
+	case core.GETH:
+		return GethClient{PropertiesReader: PropertiesReader{client: client}}
+	case core.PARITY:
+		return ParityClient{PropertiesReader: PropertiesReader{client: client}}
+	case core.INFURA:
+		return InfuraClient{PropertiesReader: PropertiesReader{client: client}}
+	default:
+		return PropertiesReader{client: client}
+	}
+}
+
+func getNodeType(client core.RpcClient) core.NodeType {
+	if strings.Contains(client.IpcPath(), "infura") {
+		return core.INFURA
+	}
+	modules, _ := client.SupportedModules()
+	if _, ok := modules["admin"]; ok {
+		return core.GETH
+	}
+	return core.PARITY
+}
+
+func (reader PropertiesReader) NetworkId() float64 {
 	var version string
-	err := client.CallContext(context.Background(), &version, "net_version")
+	err := reader.client.CallContext(context.Background(), &version, "net_version")
 	if err != nil {
 		log.Println(err)
 	}
@@ -89,17 +82,17 @@ func (client ClientWrapper) NetworkId() float64 {
 	return networkId
 }
 
-func (client ClientWrapper) GenesisBlock() string {
+func (reader PropertiesReader) GenesisBlock() string {
 	var header *types.Header
 	blockZero := "0x0"
 	includeTransactions := false
-	client.CallContext(context.Background(), &header, "eth_getBlockByNumber", blockZero, includeTransactions)
+	reader.client.CallContext(context.Background(), &header, "eth_getBlockByNumber", blockZero, includeTransactions)
 	return header.Hash().Hex()
 }
 
-func (client ClientWrapper) NodeInfo() (string, string) {
+func (reader PropertiesReader) NodeInfo() (string, string) {
 	var info p2p.NodeInfo
-	client.CallContext(context.Background(), &info, "admin_nodeInfo")
+	reader.client.CallContext(context.Background(), &info, "admin_nodeInfo")
 	return info.ID, info.Name
 }
 
@@ -115,14 +108,14 @@ func (client InfuraClient) NodeInfo() (string, string) {
 
 func (client ParityClient) parityNodeInfo() string {
 	var nodeInfo core.ParityNodeInfo
-	client.CallContext(context.Background(), &nodeInfo, "parity_versionInfo")
+	client.client.CallContext(context.Background(), &nodeInfo, "parity_versionInfo")
 	return nodeInfo.String()
 }
 
 func (client ParityClient) parityID() string {
 	var enodeId = regexp.MustCompile(`^enode://(.+)@.+$`)
 	var enodeURL string
-	client.CallContext(context.Background(), &enodeURL, "parity_enode")
+	client.client.CallContext(context.Background(), &enodeURL, "parity_enode")
 	enode := enodeId.FindStringSubmatch(enodeURL)
 	if len(enode) < 2 {
 		return ""

pkg/geth/node/node_test.go

@@ -3,66 +3,15 @@ package node_test
 import (
 	"encoding/json"
 
-	"context"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/p2p"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
-type MockContextCaller struct {
-	nodeType core.NodeType
-}
-
 var EmpytHeaderHash = "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
 
-func (MockContextCaller) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
-	switch method {
-	case "admin_nodeInfo":
-		if p, ok := result.(*p2p.NodeInfo); ok {
-			p.ID = "enode://GethNode@172.17.0.1:30303"
-			p.Name = "Geth/v1.7"
-		}
-	case "eth_getBlockByNumber":
-		if p, ok := result.(*types.Header); ok {
-			*p = types.Header{}
-		}
-
-	case "parity_versionInfo":
-		if p, ok := result.(*core.ParityNodeInfo); ok {
-			*p = core.ParityNodeInfo{
-				Track: "",
-				ParityVersion: core.ParityVersion{
-					Major: 1,
-					Minor: 2,
-					Patch: 3,
-				},
-				Hash: "",
-			}
-		}
-	case "parity_enode":
-		if p, ok := result.(*string); ok {
-			*p = "enode://ParityNode@172.17.0.1:30303"
-		}
-	case "net_version":
-		if p, ok := result.(*string); ok {
-			*p = "1234"
-		}
-	}
-	return nil
-}
-
-func (mcc MockContextCaller) SupportedModules() (map[string]string, error) {
-	result := make(map[string]string)
-	if mcc.nodeType == core.GETH {
-		result["admin"] = "ok"
-	}
-	return result, nil
-}
-
 var _ = Describe("Parity Node Info", func() {
 
 	It("verifies parity_versionInfo can be unmarshalled into ParityNodeInfo", func() {
@@ -99,39 +48,38 @@ var _ = Describe("Parity Node Info", func() {
 	})
 
 	It("returns the genesis block for any client", func() {
-		mcc := MockContextCaller{}
+		client := fakes.NewMockRpcClient()
-		cw := node.ClientWrapper{ContextCaller: mcc}
+		n := node.MakeNode(client)
-		n := node.MakeNode(cw)
 		Expect(n.GenesisBlock).To(Equal(EmpytHeaderHash))
 	})
 
 	It("returns the network id for any client", func() {
-		mcc := MockContextCaller{}
+		client := fakes.NewMockRpcClient()
-		cw := node.ClientWrapper{ContextCaller: mcc}
+		n := node.MakeNode(client)
-		n := node.MakeNode(cw)
 		Expect(n.NetworkID).To(Equal(float64(1234)))
 	})
 
 	It("returns parity ID and client name for parity node", func() {
-		mcc := MockContextCaller{core.PARITY}
+		client := fakes.NewMockRpcClient()
-		cw := node.ClientWrapper{ContextCaller: mcc}
+		client.SetNodeType(core.PARITY)
-		n := node.MakeNode(cw)
+		n := node.MakeNode(client)
 		Expect(n.ID).To(Equal("ParityNode"))
 		Expect(n.ClientName).To(Equal("Parity/v1.2.3/"))
 	})
 
 	It("returns geth ID and client name for geth node", func() {
-		mcc := MockContextCaller{core.GETH}
+		client := fakes.NewMockRpcClient()
-		cw := node.ClientWrapper{ContextCaller: mcc}
+		client.SetNodeType(core.GETH)
-		n := node.MakeNode(cw)
+		n := node.MakeNode(client)
 		Expect(n.ID).To(Equal("enode://GethNode@172.17.0.1:30303"))
 		Expect(n.ClientName).To(Equal("Geth/v1.7"))
 	})
 
 	It("returns infura ID and client name for infura node", func() {
-		mcc := MockContextCaller{core.INFURA}
+		client := fakes.NewMockRpcClient()
-		cw := node.ClientWrapper{ContextCaller: mcc, IPCPath: "https://mainnet.infura.io/123"}
+		client.SetNodeType(core.INFURA)
-		n := node.MakeNode(cw)
+		client.SetIpcPath("infura/path")
+		n := node.MakeNode(client)
 		Expect(n.ID).To(Equal("infura"))
 		Expect(n.ClientName).To(Equal("infura"))
 	})

pkg/history/block_validator.go

@@ -0,0 +1,29 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+type BlockValidator struct {
+	blockchain      core.BlockChain
+	blockRepository datastore.BlockRepository
+	windowSize      int
+}
+
+func NewBlockValidator(blockchain core.BlockChain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
+	return &BlockValidator{
+		blockchain:      blockchain,
+		blockRepository: blockRepository,
+		windowSize:      windowSize,
+	}
+}
+
+func (bv BlockValidator) ValidateBlocks() ValidationWindow {
+	window := MakeValidationWindow(bv.blockchain, bv.windowSize)
+	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
+	RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
+	lastBlock := bv.blockchain.LastBlock().Int64()
+	bv.blockRepository.SetBlocksStatus(lastBlock)
+	return window
+}

pkg/history/block_validator_test.go

@@ -0,0 +1,33 @@
+package history_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+	"math/big"
+)
+
+var _ = Describe("Blocks validator", func() {
+
+	It("calls create or update for all blocks within the window", func() {
+		blockChain := fakes.NewMockBlockChain()
+		blockChain.SetLastBlock(big.NewInt(7))
+		blocksRepository := fakes.NewMockBlockRepository()
+		validator := history.NewBlockValidator(blockChain, blocksRepository, 2)
+
+		window := validator.ValidateBlocks()
+
+		Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
+		blocksRepository.AssertCreateOrUpdateBlockCallCountEquals(3)
+	})
+
+	It("returns the number of largest block", func() {
+		blockChain := fakes.NewMockBlockChain()
+		blockChain.SetLastBlock(big.NewInt(3))
+		maxBlockNumber := blockChain.LastBlock()
+
+		Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
+	})
+})

pkg/history/header_validator.go

@@ -0,0 +1,27 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+type HeaderValidator struct {
+	blockChain       core.BlockChain
+	headerRepository datastore.HeaderRepository
+	windowSize       int
+}
+
+func NewHeaderValidator(blockChain core.BlockChain, repository datastore.HeaderRepository, windowSize int) HeaderValidator {
+	return HeaderValidator{
+		blockChain:       blockChain,
+		headerRepository: repository,
+		windowSize:       windowSize,
+	}
+}
+
+func (validator HeaderValidator) ValidateHeaders() ValidationWindow {
+	window := MakeValidationWindow(validator.blockChain, validator.windowSize)
+	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
+	RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers)
+	return window
+}

pkg/history/header_validator_test.go

@@ -0,0 +1,22 @@
+package history_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+	"math/big"
+)
+
+var _ = Describe("Header validator", func() {
+	It("attempts to create every header in the validation window", func() {
+		headerRepository := fakes.NewMockHeaderRepository()
+		headerRepository.SetMissingBlockNumbers([]int64{})
+		blockChain := fakes.NewMockBlockChain()
+		blockChain.SetLastBlock(big.NewInt(3))
+		validator := history.NewHeaderValidator(blockChain, headerRepository, 2)
+
+		validator.ValidateHeaders()
+
+		headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(3, []int64{1, 2, 3})
+	})
+})

pkg/history/populate_blocks.go

@@ -7,16 +7,16 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 )
 
-func PopulateMissingBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int {
+func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int {
 	lastBlock := blockchain.LastBlock().Int64()
-	blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock-1, blockchain.Node().ID)
+	blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
 	log.SetPrefix("")
 	log.Printf("Backfilling %d blocks\n\n", len(blockRange))
 	RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
 	return len(blockRange)
 }
 
-func RetrieveAndUpdateBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, blockNumbers []int64) int {
+func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, blockNumbers []int64) int {
 	for _, blockNumber := range blockNumbers {
 		block, err := blockchain.GetBlockByNumber(blockNumber)
 		if err != nil {

pkg/history/populate_blocks_test.go

@@ -1,35 +1,26 @@
 package history_test
 
 import (
-	"errors"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
+	"math/big"
 )
 
 var _ = Describe("Populating blocks", func() {
-	var inMemory *inmemory.InMemory
+	var blockRepository *fakes.MockBlockRepository
-	var blockRepository *inmemory.BlockRepository
 
 	BeforeEach(func() {
-		inMemory = inmemory.NewInMemory()
+		blockRepository = fakes.NewMockBlockRepository()
-		blockRepository = &inmemory.BlockRepository{InMemory: inMemory}
 	})
 
-	It("fills in the only missing block (Number 1)", func() {
+	It("fills in the only missing block (BlockNumber 1)", func() {
-		blocks := []core.Block{
+		blockChain := fakes.NewMockBlockChain()
-			{Number: 1},
+		blockChain.SetLastBlock(big.NewInt(2))
-			{Number: 2},
+		blockRepository.SetMissingBlockNumbersReturnArray([]int64{2})
-		}
-		blockchain := fakes.NewBlockchainWithBlocks(blocks)
 
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 2})
+		blocksAdded := history.PopulateMissingBlocks(blockChain, blockRepository, 1)
-
-		blocksAdded := history.PopulateMissingBlocks(blockchain, blockRepository, 1)
 		_, err := blockRepository.GetBlock(1)
 
 		Expect(blocksAdded).To(Equal(1))
@@ -37,77 +28,41 @@ var _ = Describe("Populating blocks", func() {
 	})
 
 	It("fills in the three missing blocks (Numbers: 5,8,10)", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+		blockChain := fakes.NewMockBlockChain()
-			{Number: 4},
+		blockChain.SetLastBlock(big.NewInt(13))
-			{Number: 5},
+		blockRepository.SetMissingBlockNumbersReturnArray([]int64{5, 8, 10})
-			{Number: 6},
-			{Number: 7},
-			{Number: 8},
-			{Number: 9},
-			{Number: 10},
-			{Number: 11},
-			{Number: 12},
-			{Number: 13},
-		})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 1})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 2})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 3})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 7})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 9})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 11})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 12})
 
-		blocksAdded := history.PopulateMissingBlocks(blockchain, blockRepository, 5)
+		blocksAdded := history.PopulateMissingBlocks(blockChain, blockRepository, 5)
 
 		Expect(blocksAdded).To(Equal(3))
-		Expect(blockRepository.BlockCount()).To(Equal(11))
+		blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(3, []int64{5, 8, 10})
-		_, err := blockRepository.GetBlock(4)
-		Expect(err).To(HaveOccurred())
-		_, err = blockRepository.GetBlock(5)
-		Expect(err).ToNot(HaveOccurred())
-		_, err = blockRepository.GetBlock(8)
-		Expect(err).ToNot(HaveOccurred())
-		_, err = blockRepository.GetBlock(10)
-		Expect(err).ToNot(HaveOccurred())
-		_, err = blockRepository.GetBlock(13)
-		Expect(err).To(HaveOccurred())
 	})
 
 	It("returns the number of blocks created", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+		blockChain := fakes.NewMockBlockChain()
-			{Number: 4},
+		blockChain.SetLastBlock(big.NewInt(6))
-			{Number: 5},
+		blockRepository.SetMissingBlockNumbersReturnArray([]int64{4, 5})
-			{Number: 6},
-		})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 3})
-		blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
 
-		numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, blockRepository, 3)
+		numberOfBlocksCreated := history.PopulateMissingBlocks(blockChain, blockRepository, 3)
 
 		Expect(numberOfBlocksCreated).To(Equal(2))
 	})
 
 	It("updates the repository with a range of blocks w/in the range ", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+		blockChain := fakes.NewMockBlockChain()
-			{Number: 1},
-			{Number: 2},
-			{Number: 3},
-			{Number: 4},
-			{Number: 5},
-		})
 
-		history.RetrieveAndUpdateBlocks(blockchain, blockRepository, history.MakeRange(2, 5))
+		history.RetrieveAndUpdateBlocks(blockChain, blockRepository, history.MakeRange(2, 5))
-		Expect(blockRepository.BlockCount()).To(Equal(3))
+
-		Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(3))
+		blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(4, []int64{2, 3, 4, 5})
 	})
 
 	It("does not call repository create block when there is an error", func() {
-		blockchain := fakes.NewBlockchain(errors.New("error getting block"))
+		blockChain := fakes.NewMockBlockChain()
+		blockChain.SetGetBlockByNumberErr(fakes.FakeError)
 		blocks := history.MakeRange(1, 10)
-		history.RetrieveAndUpdateBlocks(blockchain, blockRepository, blocks)
-		Expect(blockRepository.BlockCount()).To(Equal(0))
-		Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(0))
-	})
 
+		history.RetrieveAndUpdateBlocks(blockChain, blockRepository, blocks)
+
+		blockRepository.AssertCreateOrUpdateBlockCallCountEquals(0)
+	})
 })

pkg/history/populate_headers.go

@@ -0,0 +1,29 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"log"
+)
+
+func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) int {
+	lastBlock := blockchain.LastBlock().Int64()
+	blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
+	log.SetPrefix("")
+	log.Printf("Backfilling %d blocks\n\n", len(blockRange))
+	RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
+	return len(blockRange)
+}
+
+func RetrieveAndUpdateHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) int {
+	for _, blockNumber := range blockNumbers {
+		header, err := blockchain.GetHeaderByNumber(blockNumber)
+		if err != nil {
+			log.Printf("failed to retrieve block number: %d\n", blockNumber)
+			return 0
+		}
+		// TODO: handle possible error here
+		headerRepository.CreateOrUpdateHeader(header)
+	}
+	return len(blockNumbers)
+}

pkg/history/populate_headers_test.go

@@ -0,0 +1,43 @@
+package history_test
+
+import (
+	"math/big"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+)
+
+var _ = Describe("Populating headers", func() {
+
+	var headerRepository *fakes.MockHeaderRepository
+
+	BeforeEach(func() {
+		headerRepository = fakes.NewMockHeaderRepository()
+	})
+
+	Describe("When 1 missing header", func() {
+
+		It("returns number of headers added", func() {
+			blockChain := fakes.NewMockBlockChain()
+			blockChain.SetLastBlock(big.NewInt(2))
+			headerRepository.SetMissingBlockNumbers([]int64{2})
+
+			headersAdded := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
+
+			Expect(headersAdded).To(Equal(1))
+		})
+	})
+
+	It("adds missing headers to the db", func() {
+		blockChain := fakes.NewMockBlockChain()
+		blockChain.SetLastBlock(big.NewInt(2))
+		headerRepository.SetMissingBlockNumbers([]int64{2})
+
+		history.PopulateMissingHeaders(blockChain, headerRepository, 1)
+
+		headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2})
+	})
+})

pkg/history/validate_blocks.go

@@ -1,68 +0,0 @@
-package history
-
-import (
-	"io"
-	"text/template"
-
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-)
-
-const WindowTemplate = `Validating Blocks
-|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD)
-
-`
-
-var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate))
-
-type BlockValidator struct {
-	blockchain            core.Blockchain
-	blockRepository       datastore.BlockRepository
-	windowSize            int
-	parsedLoggingTemplate template.Template
-}
-
-func NewBlockValidator(blockchain core.Blockchain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
-	return &BlockValidator{
-		blockchain,
-		blockRepository,
-		windowSize,
-		ParsedWindowTemplate,
-	}
-}
-
-func (bv BlockValidator) ValidateBlocks() ValidationWindow {
-	window := MakeValidationWindow(bv.blockchain, bv.windowSize)
-	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
-	RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
-	lastBlock := bv.blockchain.LastBlock().Int64()
-	bv.blockRepository.SetBlocksStatus(lastBlock)
-	return window
-}
-
-func (bv BlockValidator) Log(out io.Writer, window ValidationWindow) {
-	bv.parsedLoggingTemplate.Execute(out, window)
-}
-
-type ValidationWindow struct {
-	LowerBound int64
-	UpperBound int64
-}
-
-func (window ValidationWindow) Size() int {
-	return int(window.UpperBound - window.LowerBound)
-}
-
-func MakeValidationWindow(blockchain core.Blockchain, windowSize int) ValidationWindow {
-	upperBound := blockchain.LastBlock().Int64()
-	lowerBound := upperBound - int64(windowSize)
-	return ValidationWindow{lowerBound, upperBound}
-}
-
-func MakeRange(min, max int64) []int64 {
-	a := make([]int64, max-min)
-	for i := range a {
-		a[i] = min + int64(i)
-	}
-	return a
-}

pkg/history/validate_blocks_test.go

@@ -1,86 +0,0 @@
-package history_test
-
-import (
-	"bytes"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
-	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/history"
-)
-
-var _ = Describe("Blocks validator", func() {
-
-	It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
-			{Number: 1},
-			{Number: 2},
-			{Number: 3},
-			{Number: 4},
-			{Number: 5},
-		})
-
-		validationWindow := history.MakeValidationWindow(blockchain, 2)
-
-		Expect(validationWindow.LowerBound).To(Equal(int64(3)))
-		Expect(validationWindow.UpperBound).To(Equal(int64(5)))
-	})
-
-	It("returns the window size", func() {
-		window := history.ValidationWindow{LowerBound: 1, UpperBound: 3}
-		Expect(window.Size()).To(Equal(2))
-	})
-
-	It("calls create or update for all blocks within the window", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
-			{Number: 4},
-			{Number: 5},
-			{Number: 6},
-			{Number: 7},
-		})
-		inMemoryDB := inmemory.NewInMemory()
-		blocksRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
-
-		validator := history.NewBlockValidator(blockchain, blocksRepository, 2)
-		window := validator.ValidateBlocks()
-		Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
-		Expect(blocksRepository.BlockCount()).To(Equal(2))
-		Expect(blocksRepository.CreateOrUpdateBlockCallCount).To(Equal(2))
-	})
-
-	It("logs window message", func() {
-		inMemoryDB := inmemory.NewInMemory()
-		blockRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
-
-		expectedMessage := &bytes.Buffer{}
-		window := history.ValidationWindow{LowerBound: 5, UpperBound: 7}
-		history.ParsedWindowTemplate.Execute(expectedMessage, history.ValidationWindow{LowerBound: 5, UpperBound: 7})
-
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{})
-		validator := history.NewBlockValidator(blockchain, blockRepository, 2)
-		actualMessage := &bytes.Buffer{}
-		validator.Log(actualMessage, window)
-		Expect(actualMessage).To(Equal(expectedMessage))
-	})
-
-	It("generates a range of int64s", func() {
-		numberOfBlocksCreated := history.MakeRange(0, 5)
-		expected := []int64{0, 1, 2, 3, 4}
-
-		Expect(numberOfBlocksCreated).To(Equal(expected))
-	})
-
-	It("returns the number of largest block", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
-			{Number: 1},
-			{Number: 2},
-			{Number: 3},
-		})
-		maxBlockNumber := blockchain.LastBlock()
-
-		Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
-	})
-
-})

pkg/history/validation_window.go

@@ -0,0 +1,41 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"io"
+	"text/template"
+)
+
+const WindowTemplate = `Validating Blocks
+|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD)
+
+`
+
+var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate))
+
+type ValidationWindow struct {
+	LowerBound int64
+	UpperBound int64
+}
+
+func (window ValidationWindow) Size() int {
+	return int(window.UpperBound - window.LowerBound)
+}
+
+func MakeValidationWindow(blockchain core.BlockChain, windowSize int) ValidationWindow {
+	upperBound := blockchain.LastBlock().Int64()
+	lowerBound := upperBound - int64(windowSize)
+	return ValidationWindow{lowerBound, upperBound}
+}
+
+func MakeRange(min, max int64) []int64 {
+	a := make([]int64, max-min+1)
+	for i := range a {
+		a[i] = min + int64(i)
+	}
+	return a
+}
+
+func (window ValidationWindow) Log(out io.Writer) {
+	ParsedWindowTemplate.Execute(out, window)
+}

pkg/history/validation_window_test.go

@@ -0,0 +1,48 @@
+package history_test
+
+import (
+	"bytes"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+	"math/big"
+)
+
+var _ = Describe("Validation window", func() {
+	It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
+		blockChain := fakes.NewMockBlockChain()
+		blockChain.SetLastBlock(big.NewInt(5))
+
+		validationWindow := history.MakeValidationWindow(blockChain, 2)
+
+		Expect(validationWindow.LowerBound).To(Equal(int64(3)))
+		Expect(validationWindow.UpperBound).To(Equal(int64(5)))
+	})
+
+	It("returns the window size", func() {
+		window := history.ValidationWindow{LowerBound: 1, UpperBound: 3}
+
+		Expect(window.Size()).To(Equal(2))
+	})
+
+	It("generates a range of int64s", func() {
+		numberOfBlocksCreated := history.MakeRange(0, 5)
+		expected := []int64{0, 1, 2, 3, 4, 5}
+
+		Expect(numberOfBlocksCreated).To(Equal(expected))
+	})
+
+	It("logs window message", func() {
+		expectedMessage := &bytes.Buffer{}
+		window := history.ValidationWindow{LowerBound: 5, UpperBound: 7}
+		history.ParsedWindowTemplate.Execute(expectedMessage, window)
+		actualMessage := &bytes.Buffer{}
+
+		window.Log(actualMessage)
+
+		Expect(actualMessage).To(Equal(expectedMessage))
+	})
+})

test_config/test_config.go

@@ -62,12 +62,13 @@ func setABIPath() {
 
 func NewTestDB(node core.Node) *postgres.DB {
 	db, _ := postgres.NewDB(DBConfig, node)
-	db.MustExec("DELETE FROM watched_contracts")
-	db.MustExec("DELETE FROM transactions")
 	db.MustExec("DELETE FROM blocks")
+	db.MustExec("DELETE FROM headers")
+	db.MustExec("DELETE FROM log_filters")
 	db.MustExec("DELETE FROM logs")
 	db.MustExec("DELETE FROM receipts")
-	db.MustExec("DELETE FROM log_filters")
+	db.MustExec("DELETE FROM transactions")
+	db.MustExec("DELETE FROM watched_contracts")
 	return db
 }