port over lightSync updates from maker repo

cmd/erc20.go

@@ -20,20 +20,14 @@ import (
 	"log"
 	"time"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/spf13/cobra"
 
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/event_triggered/dai"
 	"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
 	"github.com/vulcanize/vulcanizedb/examples/generic"
 	"github.com/vulcanize/vulcanizedb/libraries/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/constants"
+	"github.com/vulcanize/vulcanizedb/utils"
 )
 
 // erc20Cmd represents the erc20 command
@@ -62,25 +56,14 @@ Expects an ethereum node to be running and requires a .toml config file:
 func watchERC20s() {
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
-	rawRpcClient, err := rpc.Dial(ipc)
+
-	if err != nil {
+	blockChain := getBlockChain()
-		log.Fatal(err)
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-	}
-	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
-	ethClient := ethclient.NewClient(rawRpcClient)
-	client := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(client, node, transactionConverter)
-	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
-	if err != nil {
-		log.Fatal("Failed to initialize database.")
-	}
 
 	con := generic.DaiConfig
 	con.Filters = constants.DaiERC20Filters
 	watcher := shared.Watcher{
-		DB:         *db,
+		DB:         db,
 		Blockchain: blockChain,
 	}
 
@@ -89,7 +72,7 @@ func watchERC20s() {
 	// collect balances and allowances at every block
 	transformers := append(dai.DaiEventTriggeredTransformerInitializer(), every_block.ERC20EveryBlockTransformerInitializer()...)
 
-	err = watcher.AddTransformers(transformers, con)
+	err := watcher.AddTransformers(transformers, con)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/lightSync.go

@@ -14,6 +14,20 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+// Copyright © 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cmd
 
 import (
@@ -21,17 +35,12 @@ import (
 	"os"
 	"time"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/spf13/cobra"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
@@ -42,16 +51,12 @@ var lightSyncCmd = &cobra.Command{
 	Short: "Syncs VulcanizeDB with local ethereum node's block headers",
 	Long: `Syncs VulcanizeDB with local ethereum node. Populates
 Postgres with block headers.
-
 ./vulcanizedb lightSync --starting-block-number 0 --config public.toml
-
 Expects ethereum node to be running and requires a .toml config:
-
   [database]
   name = "vulcanize_public"
   hostname = "localhost"
   port = 5432
-
   [client]
   ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
@@ -66,32 +71,20 @@ func init() {
 }
 
 func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
-	missingBlocksPopulated <- history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
+	populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber)
+	if err != nil {
+		log.Fatal("Error populating headers: ", err)
+	}
+	missingBlocksPopulated <- populated
 }
 
 func lightSync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
-	rawRpcClient, err := rpc.Dial(ipc)
+	blockChain := getBlockChain()
-	if err != nil {
+	validateArgs(blockChain)
-		log.Fatal(err)
-	}
-	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
-	ethClient := ethclient.NewClient(rawRpcClient)
-	client := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRpcTransactionConverter(client)
-	blockChain := geth.NewBlockChain(client, node, transactionConverter)
-
-	lastBlock := blockChain.LastBlock().Int64()
-	if lastBlock == 0 {
-		log.Fatal("geth initial: state sync not finished")
-	}
-	if startingBlockNumber > lastBlock {
-		log.Fatal("starting block number > current block number")
-	}
-
 	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
+
 	headerRepository := repositories.NewHeaderRepository(&db)
 	validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow)
 	missingBlocksPopulated := make(chan int)
@@ -107,3 +100,13 @@ func lightSync() {
 		}
 	}
 }
+
+func validateArgs(blockChain *geth.BlockChain) {
+	lastBlock := blockChain.LastBlock().Int64()
+	if lastBlock == 0 {
+		log.Fatal("geth initial: state sync not finished")
+	}
+	if startingBlockNumber > lastBlock {
+		log.Fatal("starting block number > current block number")
+	}
+}

cmd/omniWatcher.go

@@ -19,23 +19,16 @@ package cmd
 import (
 	"bufio"
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"log"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/spf13/cobra"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/transformer"
+	"github.com/vulcanize/vulcanizedb/utils"
 )
 
 // omniWatcherCmd represents the omniWatcher command
@@ -92,9 +85,9 @@ func omniWatcher() {
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 
-	blockChain, db := setupBCandDB()
+	blockChain := getBlockChain()
-
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-	t := transformer.NewTransformer(network, blockChain, db)
+	t := transformer.NewTransformer(network, blockChain, &db)
 
 	contractAddresses = append(contractAddresses, contractAddress)
 	for _, addr := range contractAddresses {
@@ -131,23 +124,3 @@ func init() {
 	omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
 	omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
 }
-
-func setupBCandDB() (core.BlockChain, *postgres.DB) {
-	rawRpcClient, err := rpc.Dial(ipc)
-	if err != nil {
-		log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err))
-	}
-
-	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
-	ethClient := ethclient.NewClient(rawRpcClient)
-	cli := client.NewEthClient(ethClient)
-	n := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(cli, n, transactionConverter)
-	db, err := postgres.NewDB(databaseConfig, blockChain.Node())
-	if err != nil {
-		log.Fatal(fmt.Sprintf("Failed to initialize database\r\nerr: %v\r\n", err))
-	}
-
-	return blockChain, db
-}

cmd/root.go

@@ -18,12 +18,20 @@ package cmd
 
 import (
 	"fmt"
+	"log"
 	"os"
 
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
 	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
 var (
@@ -109,3 +117,17 @@ func initConfig() {
 		fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
 	}
 }
+
+func getBlockChain() *geth.BlockChain {
+	rawRpcClient, err := rpc.Dial(ipc)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
+	ethClient := ethclient.NewClient(rawRpcClient)
+	vdbEthClient := client.NewEthClient(ethClient)
+	vdbNode := node.MakeNode(rpcClient)
+	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+	return geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+}

cmd/sync.go

@@ -21,17 +21,11 @@ import (
 	"os"
 	"time"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/spf13/cobra"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
@@ -78,16 +72,8 @@ func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.Blo
 func sync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
-	rawRpcClient, err := rpc.Dial(ipc)
+
-	if err != nil {
+	blockChain := getBlockChain()
-		log.Fatal(err)
-	}
-	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
-	ethClient := ethclient.NewClient(rawRpcClient)
-	client := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(client, node, transactionConverter)
 
 	lastBlock := blockChain.LastBlock().Int64()
 	if lastBlock == 0 {

db/migrations/1531758090_create_headers_table.up.sql

@@ -2,7 +2,8 @@ CREATE TABLE public.headers (
   id                    SERIAL PRIMARY KEY,
   hash                  VARCHAR(66),
   block_number          BIGINT,
-  raw                   bytea,
+  raw                   JSONB,
+  block_timestamp       NUMERIC,
   eth_node_id           INTEGER,
   eth_node_fingerprint  VARCHAR(128),
   CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id)

db/migrations/1532468317_create_checked_headers_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE public.checked_headers;

db/migrations/1532468317_create_checked_headers_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE public.checked_headers (
+  id                  SERIAL PRIMARY KEY,
+  header_id           INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  price_feeds_checked BOOLEAN NOT NULL DEFAULT FALSE
+);

examples/erc20_watcher/every_block/getter_test.go

@@ -61,7 +61,7 @@ var _ = Describe("ERC20 Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetTotalSupply(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
 
@@ -115,7 +115,7 @@ var _ = Describe("ERC20 Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 
 			testTokenHolderAddress := common.HexToAddress("0x2cccc4b4708b318a6290511aac75d6c3dbe0cf9f")
@@ -179,7 +179,7 @@ var _ = Describe("ERC20 Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 
 			testTokenHolderAddress := common.HexToAddress("0x2cccc4b4708b318a6290511aac75d6c3dbe0cf9f")

examples/generic/every_block/getter_test.go

@@ -61,7 +61,7 @@ var _ = Describe("every_block Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetStoppedStatus(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
 
@@ -106,7 +106,7 @@ var _ = Describe("every_block Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetOwner(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
 
@@ -153,7 +153,7 @@ var _ = Describe("every_block Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetHashName(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
 
@@ -200,7 +200,7 @@ var _ = Describe("every_block Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetHashSymbol(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
 
@@ -247,7 +247,7 @@ var _ = Describe("every_block Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetDecimals(constants.DaiAbiString, constants.DaiContractAddress, blockNumber)
 
@@ -295,7 +295,7 @@ var _ = Describe("every_block Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetStringName(constants.TusdAbiString, constants.TusdContractAddress, blockNumber)
 
@@ -342,7 +342,7 @@ var _ = Describe("every_block Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			realGetter := every_block.NewGetter(blockChain)
 			result, err := realGetter.GetStringName(constants.TusdAbiString, constants.TusdContractAddress, blockNumber)
 

integration_test/block_rewards_test.go

@@ -39,7 +39,7 @@ var _ = Describe("Rewards calculations", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-		blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+		blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.Reward).To(Equal(5.31355))
@@ -53,7 +53,7 @@ var _ = Describe("Rewards calculations", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-		blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+		blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.UnclesReward).To(Equal(6.875))

integration_test/contract_test.go

@@ -56,7 +56,7 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			contract := testing.SampleContract()
 
 			logs, err := blockChain.GetLogs(contract, big.NewInt(4703824), nil)
@@ -74,7 +74,7 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
 			logs, err := blockChain.GetLogs(core.Contract{Hash: "x123"}, big.NewInt(4703824), nil)
 
@@ -93,7 +93,7 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
 			contract := testing.SampleContract()
 			var balance = new(big.Int)

integration_test/geth_blockchain_test.go

@@ -43,7 +43,7 @@ var _ = Describe("Reading from the Geth blockchain", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-		blockChain = geth.NewBlockChain(blockChainClient, node, transactionConverter)
+		blockChain = geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 	})
 
 	It("reads two blocks", func(done Done) {

pkg/core/header.go

@@ -16,8 +16,33 @@
 
 package core
 
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
 type Header struct {
+	Id          int64
 	BlockNumber int64 `db:"block_number"`
 	Hash        string
 	Raw         []byte
+	Timestamp   string `db:"block_timestamp"`
+}
+
+type POAHeader struct {
+	ParentHash  common.Hash    `json:"parentHash"       gencodec:"required"`
+	UncleHash   common.Hash    `json:"sha3Uncles"       gencodec:"required"`
+	Coinbase    common.Address `json:"miner"            gencodec:"required"`
+	Root        common.Hash    `json:"stateRoot"        gencodec:"required"`
+	TxHash      common.Hash    `json:"transactionsRoot" gencodec:"required"`
+	ReceiptHash common.Hash    `json:"receiptsRoot"     gencodec:"required"`
+	Bloom       types.Bloom    `json:"logsBloom"        gencodec:"required"`
+	Difficulty  *hexutil.Big   `json:"difficulty"       gencodec:"required"`
+	Number      *hexutil.Big   `json:"number"           gencodec:"required"`
+	GasLimit    hexutil.Uint64 `json:"gasLimit"         gencodec:"required"`
+	GasUsed     hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
+	Time        *hexutil.Big   `json:"timestamp"        gencodec:"required"`
+	Extra       hexutil.Bytes  `json:"extraData"        gencodec:"required"`
+	Hash        common.Hash    `json:"hash"`
 }

pkg/core/node_info.go

@@ -26,6 +26,11 @@ const (
 	GETH NodeType = iota
 	PARITY
 	INFURA
+	GANACHE
+)
+
+const (
+	KOVAN_NETWORK_ID = 42
 )
 
 type Node struct {

pkg/datastore/postgres/postgres_test.go

@@ -92,6 +92,7 @@ var _ = Describe("Postgres DB", func() {
 		}
 		node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
 		db := test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
 		blocksRepository := repositories.NewBlockRepository(db)
 
 		_, err1 := blocksRepository.CreateOrUpdateBlock(badBlock)

pkg/datastore/postgres/repositories/block_repository_test.go

@@ -42,6 +42,7 @@ var _ = Describe("Saving blocks", func() {
 			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
 		}
 		db = test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
 		blockRepository = repositories.NewBlockRepository(db)
 
 	})
@@ -58,6 +59,7 @@ var _ = Describe("Saving blocks", func() {
 			ClientName:   "Geth",
 		}
 		dbTwo := test_config.NewTestDB(nodeTwo)
+		test_config.CleanTestDB(dbTwo)
 		repositoryTwo := repositories.NewBlockRepository(dbTwo)
 
 		_, err := repositoryTwo.GetBlock(123)
@@ -184,6 +186,7 @@ var _ = Describe("Saving blocks", func() {
 			NetworkID:    1,
 		}
 		dbTwo := test_config.NewTestDB(nodeTwo)
+		test_config.CleanTestDB(dbTwo)
 		repositoryTwo := repositories.NewBlockRepository(dbTwo)
 
 		blockRepository.CreateOrUpdateBlock(blockOne)

pkg/datastore/postgres/repositories/contract_repository_test.go

@@ -41,6 +41,7 @@ var _ = Describe("Creating contracts", func() {
 			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
 		}
 		db = test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
 		contractRepository = repositories.ContractRepository{DB: db}
 	})
 

pkg/datastore/postgres/repositories/header_repository.go

@@ -18,10 +18,14 @@ package repositories
 
 import (
 	"database/sql"
+	"errors"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
+var ErrValidHeaderExists = errors.New("valid header already exists")
+
 type HeaderRepository struct {
 	database *postgres.DB
 }
@@ -41,7 +45,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int
 	if headerMustBeReplaced(hash, header) {
 		return repository.replaceHeader(header)
 	}
-	return 0, err
+	return 0, ErrValidHeaderExists
 }
 
 func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
@@ -81,8 +85,8 @@ func (repository HeaderRepository) getHeaderHash(header core.Header) (string, er
 func (repository HeaderRepository) insertHeader(header core.Header) (int64, error) {
 	var headerId int64
 	err := repository.database.QueryRowx(
-		`INSERT INTO public.headers (block_number, hash, raw, eth_node_id, eth_node_fingerprint) VALUES ($1, $2, $3, $4, $5) RETURNING id`,
+		`INSERT INTO public.headers (block_number, hash, block_timestamp, raw, eth_node_id, eth_node_fingerprint) VALUES ($1, $2, $3::NUMERIC, $4, $5, $6) RETURNING id`,
-		header.BlockNumber, header.Hash, header.Raw, repository.database.NodeID, repository.database.Node.ID).Scan(&headerId)
+		header.BlockNumber, header.Hash, header.Timestamp, header.Raw, repository.database.NodeID, repository.database.Node.ID).Scan(&headerId)
 	return headerId, err
 }
 

pkg/datastore/postgres/repositories/header_repository_test.go

@@ -18,44 +18,66 @@ package repositories_test
 
 import (
 	"database/sql"
+	"encoding/json"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/test_config"
+	"math/big"
 )
 
 var _ = Describe("Block header repository", func() {
-	Describe("creating or updating a header", func() {
+	var (
+		rawHeader []byte
+		err       error
+		timestamp string
+	)
 
+	BeforeEach(func() {
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+		timestamp = big.NewInt(123456789).String()
+	})
+
+	Describe("creating or updating a header", func() {
 		It("adds a header", func() {
 			node := core.Node{}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 
 			_, err := repo.CreateOrUpdateHeader(header)
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbHeader core.Header
-			err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			err = db.Get(&dbHeader, `SELECT block_number, hash, raw, block_timestamp FROM public.headers WHERE block_number = $1`, header.BlockNumber)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
 			Expect(dbHeader.Hash).To(Equal(header.Hash))
-			Expect(dbHeader.Raw).To(Equal(header.Raw))
+			Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
+			Expect(dbHeader.Timestamp).To(Equal(header.Timestamp))
 		})
 
 		It("adds node data to header", func() {
 			node := core.Node{ID: "EthNodeFingerprint"}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
-			header := core.Header{BlockNumber: 100}
+			header := core.Header{
+				BlockNumber: 100,
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
+			}
 
 			_, err := repo.CreateOrUpdateHeader(header)
 
@@ -70,21 +92,24 @@ var _ = Describe("Block header repository", func() {
 			Expect(ethNodeFingerprint).To(Equal(db.Node.ID))
 		})
 
-		It("does not duplicate headers", func() {
+		It("returns valid header exists error if attempting duplicate headers", func() {
 			node := core.Node{}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 
 			_, err := repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = repo.CreateOrUpdateHeader(header)
-			Expect(err).NotTo(HaveOccurred())
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(repositories.ErrValidHeaderExists))
 
 			var dbHeaders []core.Header
 			err = db.Select(&dbHeaders, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
@@ -95,18 +120,21 @@ var _ = Describe("Block header repository", func() {
 		It("replaces header if hash is different", func() {
 			node := core.Node{}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
 			headerTwo := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
-				Raw:         []byte{5, 4, 3, 2, 1},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 
 			_, err = repo.CreateOrUpdateHeader(headerTwo)
@@ -116,17 +144,19 @@ var _ = Describe("Block header repository", func() {
 			err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbHeader.Hash).To(Equal(headerTwo.Hash))
-			Expect(dbHeader.Raw).To(Equal(headerTwo.Raw))
+			Expect(dbHeader.Raw).To(MatchJSON(headerTwo.Raw))
 		})
 
 		It("does not replace header if node fingerprint is different", func() {
 			node := core.Node{ID: "Fingerprint"}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			nodeTwo := core.Node{ID: "FingerprintTwo"}
@@ -136,7 +166,8 @@ var _ = Describe("Block header repository", func() {
 			headerTwo := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
-				Raw:         []byte{5, 4, 3, 2, 1},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 
 			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
@@ -155,7 +186,8 @@ var _ = Describe("Block header repository", func() {
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			nodeTwo := core.Node{ID: "FingerprintTwo"}
@@ -165,13 +197,15 @@ var _ = Describe("Block header repository", func() {
 			headerTwo := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
-				Raw:         []byte{5, 4, 3, 2, 1},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
 			headerThree := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{1, 1, 1, 1, 1}).Hex(),
-				Raw:         []byte{1, 1, 1, 1, 1},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 
 			_, err = repoTwo.CreateOrUpdateHeader(headerThree)
@@ -183,8 +217,8 @@ var _ = Describe("Block header repository", func() {
 			Expect(len(dbHeaders)).To(Equal(2))
 			Expect(dbHeaders[0].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
 			Expect(dbHeaders[1].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
-			Expect(dbHeaders[0].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+			Expect(dbHeaders[0].Raw).To(Or(MatchJSON(header.Raw), MatchJSON(headerThree.Raw)))
-			Expect(dbHeaders[1].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+			Expect(dbHeaders[1].Raw).To(Or(MatchJSON(header.Raw), MatchJSON(headerThree.Raw)))
 		})
 	})
 
@@ -192,11 +226,13 @@ var _ = Describe("Block header repository", func() {
 		It("returns header if it exists", func() {
 			node := core.Node{}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
@@ -204,17 +240,21 @@ var _ = Describe("Block header repository", func() {
 			dbHeader, err := repo.GetHeader(header.BlockNumber)
 
 			Expect(err).NotTo(HaveOccurred())
-			Expect(dbHeader).To(Equal(header))
+			Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
+			Expect(dbHeader.Hash).To(Equal(header.Hash))
+			Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
 		})
 
 		It("does not return header for a different node fingerprint", func() {
 			node := core.Node{}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
 			header := core.Header{
 				BlockNumber: 100,
-				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Hash:        common.BytesToHash(rawHeader).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
@@ -234,10 +274,23 @@ var _ = Describe("Block header repository", func() {
 		It("returns block numbers for headers not in the database", func() {
 			node := core.Node{}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+				BlockNumber: 1,
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 3,
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 5,
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
+			})
 
 			missingBlockNumbers := repo.MissingBlockNumbers(1, 5, node.ID)
 
@@ -247,10 +300,23 @@ var _ = Describe("Block header repository", func() {
 		It("does not count headers created by a different node fingerprint", func() {
 			node := core.Node{ID: "NodeFingerprint"}
 			db := test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+				BlockNumber: 1,
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 3,
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 5,
+				Raw:         rawHeader,
+				Timestamp:   timestamp,
+			})
 			nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
 			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
 			Expect(err).NotTo(HaveOccurred())

pkg/datastore/postgres/repositories/log_filter_repository_test.go

@@ -39,6 +39,7 @@ var _ = Describe("Log Filters Repository", func() {
 			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
 		}
 		db = test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
 		filterRepository = repositories.FilterRepository{DB: db}
 	})
 

pkg/datastore/postgres/repositories/logs_repository_test.go

@@ -44,6 +44,7 @@ var _ = Describe("Logs Repository", func() {
 				ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
 			}
 			db = test_config.NewTestDB(node)
+			test_config.CleanTestDB(db)
 			blockRepository = repositories.NewBlockRepository(db)
 			logsRepository = repositories.LogRepository{DB: db}
 			receiptRepository = repositories.ReceiptRepository{DB: db}

pkg/datastore/postgres/repositories/receipts_repository_test.go

@@ -40,6 +40,7 @@ var _ = Describe("Receipts Repository", func() {
 			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
 		}
 		db = test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
 		blockRepository = repositories.NewBlockRepository(db)
 		logRepository = repositories.LogRepository{DB: db}
 		receiptRepository = repositories.ReceiptRepository{DB: db}

pkg/datastore/postgres/repositories/watched_events_repository_test.go

@@ -37,6 +37,7 @@ var _ = Describe("Watched Events Repository", func() {
 
 	BeforeEach(func() {
 		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
 		blocksRepository = repositories.NewBlockRepository(db)
 		filterRepository = repositories.FilterRepository{DB: db}
 		logRepository = repositories.LogRepository{DB: db}

pkg/fakes/data.go

@@ -16,6 +16,35 @@
 
 package fakes
 
-import "errors"
+import (
+	"encoding/json"
+	"errors"
+	"strconv"
 
-var FakeError = errors.New("failed")
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+var (
+	FakeError     = errors.New("failed")
+	FakeHash      = common.BytesToHash([]byte{1, 2, 3, 4, 5})
+	fakeTimestamp = int64(111111111)
+)
+
+var rawFakeHeader, _ = json.Marshal(types.Header{})
+var FakeHeader = core.Header{
+	Hash:      FakeHash.String(),
+	Raw:       rawFakeHeader,
+	Timestamp: strconv.FormatInt(fakeTimestamp, 10),
+}
+
+func GetFakeHeader(blockNumber int64) core.Header {
+	return core.Header{
+		Hash:        FakeHash.String(),
+		BlockNumber: blockNumber,
+		Raw:         rawFakeHeader,
+		Timestamp:   strconv.FormatInt(fakeTimestamp, 10),
+	}
+}

pkg/fakes/mock_rpc_client.go

@@ -18,29 +18,38 @@ package fakes
 
 import (
 	"context"
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/p2p"
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
 type MockRpcClient struct {
-	ipcPath  string
+	callContextErr   error
-	nodeType core.NodeType
+	ipcPath          string
+	nodeType         core.NodeType
+	passedContext    context.Context
+	passedMethod     string
+	passedResult     interface{}
+	returnPOAHeader  core.POAHeader
+	supportedModules map[string]string
 }
 
 func NewMockRpcClient() *MockRpcClient {
 	return &MockRpcClient{}
 }
 
-func (client *MockRpcClient) SetNodeType(nodeType core.NodeType) {
-	client.nodeType = nodeType
-}
-
 func (client *MockRpcClient) SetIpcPath(ipcPath string) {
 	client.ipcPath = ipcPath
 }
 
-func (*MockRpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
+func (client *MockRpcClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
+	client.passedContext = ctx
+	client.passedResult = result
+	client.passedMethod = method
 	switch method {
 	case "admin_nodeInfo":
 		if p, ok := result.(*p2p.NodeInfo); ok {
@@ -49,9 +58,14 @@ func (*MockRpcClient) CallContext(ctx context.Context, result interface{}, metho
 		}
 	case "eth_getBlockByNumber":
 		if p, ok := result.(*types.Header); ok {
-			*p = types.Header{}
+			*p = types.Header{Number: big.NewInt(123)}
+		}
+		if p, ok := result.(*core.POAHeader); ok {
+			*p = client.returnPOAHeader
+		}
+		if client.callContextErr != nil {
+			return client.callContextErr
 		}
-
 	case "parity_versionInfo":
 		if p, ok := result.(*core.ParityNodeInfo); ok {
 			*p = core.ParityNodeInfo{
@@ -81,9 +95,23 @@ func (client *MockRpcClient) IpcPath() string {
 }
 
 func (client *MockRpcClient) SupportedModules() (map[string]string, error) {
-	result := make(map[string]string)
+	return client.supportedModules, nil
-	if client.nodeType == core.GETH {
+}
-		result["admin"] = "ok"
+
-	}
+func (client *MockRpcClient) SetSupporedModules(supportedModules map[string]string) {
-	return result, nil
+	client.supportedModules = supportedModules
+}
+
+func (client *MockRpcClient) SetCallContextErr(err error) {
+	client.callContextErr = err
+}
+
+func (client *MockRpcClient) SetReturnPOAHeader(header core.POAHeader) {
+	client.returnPOAHeader = header
+}
+
+func (client *MockRpcClient) AssertCallContextCalledWith(ctx context.Context, result interface{}, method string) {
+	Expect(client.passedContext).To(Equal(ctx))
+	Expect(client.passedResult).To(BeAssignableToTypeOf(result))
+	Expect(client.passedMethod).To(Equal(method))
 }

pkg/geth/blockchain.go

@@ -17,34 +17,41 @@
 package geth
 
 import (
+	"errors"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
 	"golang.org/x/net/context"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 )
 
+var ErrEmptyHeader = errors.New("empty header returned over RPC")
+
 type BlockChain struct {
-	client          core.EthClient
 	blockConverter  vulcCommon.BlockConverter
+	ethClient       core.EthClient
 	headerConverter vulcCommon.HeaderConverter
 	node            core.Node
+	rpcClient       core.RpcClient
 }
 
-func NewBlockChain(client core.EthClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain {
+func NewBlockChain(ethClient core.EthClient, rpcClient core.RpcClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain {
 	return &BlockChain{
-		client:          client,
 		blockConverter:  vulcCommon.NewBlockConverter(converter),
+		ethClient:       ethClient,
 		headerConverter: vulcCommon.HeaderConverter{},
 		node:            node,
+		rpcClient:       rpcClient,
 	}
 }
 
 func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Block, err error) {
-	gethBlock, err := blockChain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
+	gethBlock, err := blockChain.ethClient.BlockByNumber(context.Background(), big.NewInt(blockNumber))
 	if err != nil {
 		return block, err
 	}
@@ -52,11 +59,46 @@ func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Bl
 }
 
 func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) {
-	gethHeader, err := blockChain.client.HeaderByNumber(context.Background(), big.NewInt(blockNumber))
+	if blockChain.node.NetworkID == core.KOVAN_NETWORK_ID {
+		return blockChain.getPOAHeader(blockNumber)
+	}
+	return blockChain.getPOWHeader(blockNumber)
+}
+
+func (blockChain *BlockChain) getPOWHeader(blockNumber int64) (header core.Header, err error) {
+	gethHeader, err := blockChain.ethClient.HeaderByNumber(context.Background(), big.NewInt(blockNumber))
 	if err != nil {
 		return header, err
 	}
-	return blockChain.headerConverter.Convert(gethHeader)
+	return blockChain.headerConverter.Convert(gethHeader, gethHeader.Hash().String())
+}
+
+func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Header, err error) {
+	var POAHeader core.POAHeader
+	blockNumberArg := hexutil.EncodeBig(big.NewInt(blockNumber))
+	includeTransactions := false
+	err = blockChain.rpcClient.CallContext(context.Background(), &POAHeader, "eth_getBlockByNumber", blockNumberArg, includeTransactions)
+	if err != nil {
+		return header, err
+	}
+	if POAHeader.Number == nil {
+		return header, ErrEmptyHeader
+	}
+	return blockChain.headerConverter.Convert(&types.Header{
+		ParentHash:  POAHeader.ParentHash,
+		UncleHash:   POAHeader.UncleHash,
+		Coinbase:    POAHeader.Coinbase,
+		Root:        POAHeader.Root,
+		TxHash:      POAHeader.TxHash,
+		ReceiptHash: POAHeader.ReceiptHash,
+		Bloom:       POAHeader.Bloom,
+		Difficulty:  POAHeader.Difficulty.ToInt(),
+		Number:      POAHeader.Number.ToInt(),
+		GasLimit:    uint64(POAHeader.GasLimit),
+		GasUsed:     uint64(POAHeader.GasUsed),
+		Time:        POAHeader.Time.ToInt(),
+		Extra:       POAHeader.Extra,
+	}, POAHeader.Hash.String())
 }
 
 func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) {
@@ -68,8 +110,9 @@ func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumbe
 		FromBlock: startingBlockNumber,
 		ToBlock:   endingBlockNumber,
 		Addresses: []common.Address{contractAddress},
+		Topics:    nil,
 	}
-	gethLogs, err := blockChain.client.FilterLogs(context.Background(), fc)
+	gethLogs, err := blockChain.GetEthLogsWithCustomQuery(fc)
 	if err != nil {
 		return []core.Log{}, err
 	}
@@ -77,8 +120,16 @@ func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumbe
 	return logs, nil
 }
 
+func (blockChain *BlockChain) GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) {
+	gethLogs, err := blockChain.ethClient.FilterLogs(context.Background(), query)
+	if err != nil {
+		return []types.Log{}, err
+	}
+	return gethLogs, nil
+}
+
 func (blockChain *BlockChain) LastBlock() *big.Int {
-	block, _ := blockChain.client.HeaderByNumber(context.Background(), nil)
+	block, _ := blockChain.ethClient.HeaderByNumber(context.Background(), nil)
 	return block.Number
 }
 

pkg/geth/blockchain_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -33,12 +34,21 @@ import (
 )
 
 var _ = Describe("Geth blockchain", func() {
+	var mockClient *fakes.MockEthClient
+	var mockRpcClient *fakes.MockRpcClient
+	var node vulcCore.Node
+	var blockChain *geth.BlockChain
+
+	BeforeEach(func() {
+		mockClient = fakes.NewMockEthClient()
+		mockRpcClient = fakes.NewMockRpcClient()
+		node = vulcCore.Node{}
+		blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
+	})
+
 	Describe("getting a block", func() {
-		It("fetches block from client", func() {
+		It("fetches block from ethClient", func() {
-			mockClient := fakes.NewMockEthClient()
 			mockClient.SetBlockByNumberReturnBlock(types.NewBlockWithHeader(&types.Header{}))
-			node := vulcCore.Node{}
-			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
 			blockNumber := int64(100)
 
 			_, err := blockChain.GetBlockByNumber(blockNumber)
@@ -47,11 +57,8 @@ var _ = Describe("Geth blockchain", func() {
 			mockClient.AssertBlockByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
 		})
 
-		It("returns err if client returns err", func() {
+		It("returns err if ethClient returns err", func() {
-			mockClient := fakes.NewMockEthClient()
 			mockClient.SetBlockByNumberErr(fakes.FakeError)
-			node := vulcCore.Node{}
-			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
 
 			_, err := blockChain.GetBlockByNumber(100)
 
@@ -61,38 +68,66 @@ var _ = Describe("Geth blockchain", func() {
 	})
 
 	Describe("getting a header", func() {
-		It("fetches header from client", func() {
+		Describe("default/mainnet", func() {
-			mockClient := fakes.NewMockEthClient()
+			It("fetches header from ethClient", func() {
-			blockNumber := int64(100)
+				blockNumber := int64(100)
-			mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
+				mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
-			node := vulcCore.Node{}
-			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
 
-			_, err := blockChain.GetHeaderByNumber(blockNumber)
+				_, err := blockChain.GetHeaderByNumber(blockNumber)
 
-			Expect(err).NotTo(HaveOccurred())
+				Expect(err).NotTo(HaveOccurred())
-			mockClient.AssertHeaderByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
+				mockClient.AssertHeaderByNumberCalledWith(context.Background(), big.NewInt(blockNumber))
+			})
+
+			It("returns err if ethClient returns err", func() {
+				mockClient.SetHeaderByNumberErr(fakes.FakeError)
+
+				_, err := blockChain.GetHeaderByNumber(100)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
 		})
 
-		It("returns err if client returns err", func() {
+		Describe("POA/Kovan", func() {
-			mockClient := fakes.NewMockEthClient()
+			It("fetches header from rpcClient", func() {
-			mockClient.SetHeaderByNumberErr(fakes.FakeError)
+				node.NetworkID = vulcCore.KOVAN_NETWORK_ID
-			node := vulcCore.Node{}
+				blockNumber := hexutil.Big(*big.NewInt(123))
-			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
+				mockRpcClient.SetReturnPOAHeader(vulcCore.POAHeader{Number: &blockNumber})
+				blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
 
-			_, err := blockChain.GetHeaderByNumber(100)
+				_, err := blockChain.GetHeaderByNumber(100)
 
-			Expect(err).To(HaveOccurred())
+				Expect(err).NotTo(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
+				mockRpcClient.AssertCallContextCalledWith(context.Background(), &vulcCore.POAHeader{}, "eth_getBlockByNumber")
+			})
+
+			It("returns err if rpcClient returns err", func() {
+				node.NetworkID = vulcCore.KOVAN_NETWORK_ID
+				mockRpcClient.SetCallContextErr(fakes.FakeError)
+				blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
+
+				_, err := blockChain.GetHeaderByNumber(100)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+
+			It("returns error if returned header is empty", func() {
+				node.NetworkID = vulcCore.KOVAN_NETWORK_ID
+				blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter())
+
+				_, err := blockChain.GetHeaderByNumber(100)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(geth.ErrEmptyHeader))
+			})
 		})
 	})
 
-	Describe("getting logs", func() {
+	Describe("getting logs with default FilterQuery", func() {
-		It("fetches logs from client", func() {
+		It("fetches logs from ethClient", func() {
-			mockClient := fakes.NewMockEthClient()
 			mockClient.SetFilterLogsReturnLogs([]types.Log{{}})
-			node := vulcCore.Node{}
-			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
 			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
 			startingBlockNumber := big.NewInt(1)
 			endingBlockNumber := big.NewInt(2)
@@ -108,11 +143,8 @@ var _ = Describe("Geth blockchain", func() {
 			mockClient.AssertFilterLogsCalledWith(context.Background(), expectedQuery)
 		})
 
-		It("returns err if client returns err", func() {
+		It("returns err if ethClient returns err", func() {
-			mockClient := fakes.NewMockEthClient()
 			mockClient.SetFilterLogsErr(fakes.FakeError)
-			node := vulcCore.Node{}
-			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
 			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
 			startingBlockNumber := big.NewInt(1)
 			endingBlockNumber := big.NewInt(2)
@@ -124,13 +156,49 @@ var _ = Describe("Geth blockchain", func() {
 		})
 	})
 
+	Describe("getting logs with a custom FilterQuery", func() {
+		It("fetches logs from ethClient", func() {
+			mockClient.SetFilterLogsReturnLogs([]types.Log{{}})
+			address := common.HexToAddress("0x")
+			startingBlockNumber := big.NewInt(1)
+			endingBlockNumber := big.NewInt(2)
+			topic := common.HexToHash("0x")
+			query := ethereum.FilterQuery{
+				FromBlock: startingBlockNumber,
+				ToBlock:   endingBlockNumber,
+				Addresses: []common.Address{address},
+				Topics:    [][]common.Hash{{topic}},
+			}
+
+			_, err := blockChain.GetEthLogsWithCustomQuery(query)
+
+			Expect(err).NotTo(HaveOccurred())
+			mockClient.AssertFilterLogsCalledWith(context.Background(), query)
+		})
+
+		It("returns err if ethClient returns err", func() {
+			mockClient.SetFilterLogsErr(fakes.FakeError)
+			contract := vulcCore.Contract{Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex()}
+			startingBlockNumber := big.NewInt(1)
+			endingBlockNumber := big.NewInt(2)
+			query := ethereum.FilterQuery{
+				FromBlock: startingBlockNumber,
+				ToBlock:   endingBlockNumber,
+				Addresses: []common.Address{common.HexToAddress(contract.Hash)},
+				Topics:    nil,
+			}
+
+			_, err := blockChain.GetEthLogsWithCustomQuery(query)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
+
 	Describe("getting the most recent block number", func() {
-		It("fetches latest header from client", func() {
+		It("fetches latest header from ethClient", func() {
-			mockClient := fakes.NewMockEthClient()
 			blockNumber := int64(100)
 			mockClient.SetHeaderByNumberReturnHeader(&types.Header{Number: big.NewInt(blockNumber)})
-			node := vulcCore.Node{}
-			blockChain := geth.NewBlockChain(mockClient, node, cold_db.NewColdDbTransactionConverter())
 
 			result := blockChain.LastBlock()
 

pkg/geth/contract.go

@@ -17,9 +17,8 @@
 package geth
 
 import (
-	"errors"
-
 	"context"
+	"errors"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
@@ -54,5 +53,5 @@ func (blockChain *BlockChain) FetchContractData(abiJSON string, address string,
 func (blockChain *BlockChain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
 	to := common.HexToAddress(contractHash)
 	msg := ethereum.CallMsg{To: &to, Data: input}
-	return blockChain.client.CallContract(context.Background(), msg, blockNumber)
+	return blockChain.ethClient.CallContract(context.Background(), msg, blockNumber)
 }

pkg/geth/converters/common/header_converter.go

@@ -17,24 +17,25 @@
 package common
 
 import (
-	"bytes"
+	"encoding/json"
+
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
 type HeaderConverter struct{}
 
-func (converter HeaderConverter) Convert(gethHeader *types.Header) (core.Header, error) {
+func (converter HeaderConverter) Convert(gethHeader *types.Header, blockHash string) (core.Header, error) {
-	writer := new(bytes.Buffer)
+	rawHeader, err := json.Marshal(gethHeader)
-	err := rlp.Encode(writer, &gethHeader)
 	if err != nil {
 		panic(err)
 	}
 	coreHeader := core.Header{
-		Hash:        gethHeader.Hash().Hex(),
+		Hash:        blockHash,
 		BlockNumber: gethHeader.Number.Int64(),
-		Raw:         writer.Bytes(),
+		Raw:         rawHeader,
+		Timestamp:   gethHeader.Time.String(),
 	}
 	return coreHeader, nil
 }

pkg/geth/converters/common/header_converter_test.go

@@ -17,14 +17,16 @@
 package common_test
 
 import (
-	"bytes"
+	"encoding/json"
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	common2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
-	"math/big"
 )
 
 var _ = Describe("Block header converter", func() {
@@ -35,29 +37,30 @@ var _ = Describe("Block header converter", func() {
 			ParentHash:  common.HexToHash("0xParent"),
 			ReceiptHash: common.HexToHash("0xReceipt"),
 			Root:        common.HexToHash("0xRoot"),
-			Time:        big.NewInt(3),
+			Time:        big.NewInt(123456789),
 			TxHash:      common.HexToHash("0xTransaction"),
 			UncleHash:   common.HexToHash("0xUncle"),
 		}
 		converter := common2.HeaderConverter{}
+		hash := fakes.FakeHash.String()
 
-		coreHeader, err := converter.Convert(gethHeader)
+		coreHeader, err := converter.Convert(gethHeader, hash)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
-		Expect(coreHeader.Hash).To(Equal(gethHeader.Hash().Hex()))
+		Expect(coreHeader.Hash).To(Equal(hash))
+		Expect(coreHeader.Timestamp).To(Equal(gethHeader.Time.String()))
 	})
 
-	It("includes raw bytes for header", func() {
+	It("includes raw bytes for header as JSON", func() {
-		headerRLP := []byte{249, 2, 23, 160, 180, 251, 173, 248, 234, 69, 43, 19, 151, 24, 226, 112, 13, 193, 19, 92, 252, 129, 20, 80, 49, 200, 75, 122, 178, 124, 215, 16, 57, 79, 123, 56, 160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71, 148, 42, 101, 172, 164, 213, 252, 91, 92, 133, 144, 144, 166, 195, 77, 22, 65, 53, 57, 130, 38, 160, 14, 6, 111, 60, 34, 151, 165, 203, 48, 5, 147, 5, 38, 23, 209, 188, 165, 148, 111, 12, 170, 6, 53, 253, 177, 184, 90, 199, 229, 35, 111, 52, 160, 101, 186, 136, 127, 203, 8, 38, 246, 22, 208, 31, 115, 108, 29, 45, 103, 123, 202, 189, 226, 247, 252, 37, 170, 145, 207, 188, 11, 59, 173, 92, 179, 160, 32, 227, 83, 69, 64, 202, 241, 99, 120, 230, 232, 106, 43, 241, 35, 109, 159, 135, 109, 50, 24, 251, 192, 57, 88, 230, 219, 28, 99, 75, 35, 51, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 134, 11, 105, 222, 129, 162, 43, 131, 15, 66, 64, 131, 47, 239, 216, 130, 196, 68, 132, 86, 191, 180, 21, 152, 215, 131, 1, 3, 3, 132, 71, 101, 116, 104, 135, 103, 111, 49, 46, 53, 46, 49, 133, 108, 105, 110, 117, 120, 160, 146, 196, 18, 154, 10, 226, 54, 27, 69, 42, 158, 222, 236, 229, 92, 18, 236, 238, 171, 134, 99, 22, 25, 94, 61, 135, 252, 27, 0, 91, 102, 69, 136, 205, 76, 85, 185, 65, 207, 144, 21}
+		gethHeader := types.Header{Number: big.NewInt(123)}
-		var gethHeader types.Header
-		err := rlp.Decode(bytes.NewReader(headerRLP), &gethHeader)
-		Expect(err).NotTo(HaveOccurred())
 		converter := common2.HeaderConverter{}
 
-		coreHeader, err := converter.Convert(&gethHeader)
+		coreHeader, err := converter.Convert(&gethHeader, fakes.FakeHash.String())
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(coreHeader.Raw).To(Equal(headerRLP))
+		expectedJSON, err := json.Marshal(gethHeader)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(coreHeader.Raw).To(Equal(expectedJSON))
 	})
 })

pkg/geth/node/node.go

@@ -53,6 +53,10 @@ type InfuraClient struct {
 	PropertiesReader
 }
 
+type GanacheClient struct {
+	PropertiesReader
+}
+
 func MakeNode(rpcClient core.RpcClient) core.Node {
 	pr := makePropertiesReader(rpcClient)
 	id, name := pr.NodeInfo()
@@ -72,6 +76,8 @@ func makePropertiesReader(client core.RpcClient) IPropertiesReader {
 		return ParityClient{PropertiesReader: PropertiesReader{client: client}}
 	case core.INFURA:
 		return InfuraClient{PropertiesReader: PropertiesReader{client: client}}
+	case core.GANACHE:
+		return GanacheClient{PropertiesReader: PropertiesReader{client: client}}
 	default:
 		return PropertiesReader{client: client}
 	}
@@ -81,6 +87,9 @@ func getNodeType(client core.RpcClient) core.NodeType {
 	if strings.Contains(client.IpcPath(), "infura") {
 		return core.INFURA
 	}
+	if strings.Contains(client.IpcPath(), "127.0.0.1") || strings.Contains(client.IpcPath(), "localhost") {
+		return core.GANACHE
+	}
 	modules, _ := client.SupportedModules()
 	if _, ok := modules["admin"]; ok {
 		return core.GETH
@@ -122,6 +131,10 @@ func (client InfuraClient) NodeInfo() (string, string) {
 	return "infura", "infura"
 }
 
+func (client GanacheClient) NodeInfo() (string, string) {
+	return "ganache", "ganache"
+}
+
 func (client ParityClient) parityNodeInfo() string {
 	var nodeInfo core.ParityNodeInfo
 	client.client.CallContext(context.Background(), &nodeInfo, "parity_versionInfo")

pkg/geth/node/node_test.go

@@ -21,6 +21,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
@@ -28,12 +29,12 @@ import (
 
 var EmpytHeaderHash = "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"
 
-var _ = Describe("Parity Node Info", func() {
+var _ = Describe("Node Info", func() {
-
+	Describe("Parity Node Info", func() {
-	It("verifies parity_versionInfo can be unmarshalled into ParityNodeInfo", func() {
+		It("verifies parity_versionInfo can be unmarshalled into ParityNodeInfo", func() {
-		var parityNodeInfo core.ParityNodeInfo
+			var parityNodeInfo core.ParityNodeInfo
-		nodeInfoJSON := []byte(
+			nodeInfoJSON := []byte(
-			`{
+				`{
         "hash": "0x2ae8b4ca278dd7b896090366615fef81cbbbc0e0",
         "track": "null",
         "version": {
@@ -42,25 +43,34 @@ var _ = Describe("Parity Node Info", func() {
           "patch": 0
              }
         }`)
-		json.Unmarshal(nodeInfoJSON, &parityNodeInfo)
+			json.Unmarshal(nodeInfoJSON, &parityNodeInfo)
-		Expect(parityNodeInfo.Hash).To(Equal("0x2ae8b4ca278dd7b896090366615fef81cbbbc0e0"))
+			Expect(parityNodeInfo.Hash).To(Equal("0x2ae8b4ca278dd7b896090366615fef81cbbbc0e0"))
-		Expect(parityNodeInfo.Track).To(Equal("null"))
+			Expect(parityNodeInfo.Track).To(Equal("null"))
-		Expect(parityNodeInfo.Major).To(Equal(1))
+			Expect(parityNodeInfo.Major).To(Equal(1))
-		Expect(parityNodeInfo.Minor).To(Equal(6))
+			Expect(parityNodeInfo.Minor).To(Equal(6))
-		Expect(parityNodeInfo.Patch).To(Equal(0))
+			Expect(parityNodeInfo.Patch).To(Equal(0))
-	})
+		})
 
-	It("Creates client string", func() {
+		It("Creates client string", func() {
-		parityNodeInfo := core.ParityNodeInfo{
+			parityNodeInfo := core.ParityNodeInfo{
-			Track: "null",
+				Track: "null",
-			ParityVersion: core.ParityVersion{
+				ParityVersion: core.ParityVersion{
-				Major: 1,
+					Major: 1,
-				Minor: 6,
+					Minor: 6,
-				Patch: 0,
+					Patch: 0,
-			},
+				},
-			Hash: "0x1232144j",
+				Hash: "0x1232144j",
-		}
+			}
-		Expect(parityNodeInfo.String()).To(Equal("Parity/v1.6.0/"))
+			Expect(parityNodeInfo.String()).To(Equal("Parity/v1.6.0/"))
+		})
+
+		It("returns parity ID and client name for parity node", func() {
+			client := fakes.NewMockRpcClient()
+
+			n := node.MakeNode(client)
+			Expect(n.ID).To(Equal("ParityNode"))
+			Expect(n.ClientName).To(Equal("Parity/v1.2.3/"))
+		})
 	})
 
 	It("returns the genesis block for any client", func() {
@@ -75,17 +85,12 @@ var _ = Describe("Parity Node Info", func() {
 		Expect(n.NetworkID).To(Equal(float64(1234)))
 	})
 
-	It("returns parity ID and client name for parity node", func() {
-		client := fakes.NewMockRpcClient()
-		client.SetNodeType(core.PARITY)
-		n := node.MakeNode(client)
-		Expect(n.ID).To(Equal("ParityNode"))
-		Expect(n.ClientName).To(Equal("Parity/v1.2.3/"))
-	})
-
 	It("returns geth ID and client name for geth node", func() {
 		client := fakes.NewMockRpcClient()
-		client.SetNodeType(core.GETH)
+		supportedModules := make(map[string]string)
+		supportedModules["admin"] = "ok"
+		client.SetSupporedModules(supportedModules)
+
 		n := node.MakeNode(client)
 		Expect(n.ID).To(Equal("enode://GethNode@172.17.0.1:30303"))
 		Expect(n.ClientName).To(Equal("Geth/v1.7"))
@@ -93,10 +98,22 @@ var _ = Describe("Parity Node Info", func() {
 
 	It("returns infura ID and client name for infura node", func() {
 		client := fakes.NewMockRpcClient()
-		client.SetNodeType(core.INFURA)
 		client.SetIpcPath("infura/path")
 		n := node.MakeNode(client)
 		Expect(n.ID).To(Equal("infura"))
 		Expect(n.ClientName).To(Equal("infura"))
 	})
+
+	It("returns local id and client name for Local node", func() {
+		client := fakes.NewMockRpcClient()
+		client.SetIpcPath("127.0.0.1")
+		n := node.MakeNode(client)
+		Expect(n.ID).To(Equal("ganache"))
+		Expect(n.ClientName).To(Equal("ganache"))
+
+		client.SetIpcPath("localhost")
+		n = node.MakeNode(client)
+		Expect(n.ID).To(Equal("ganache"))
+		Expect(n.ClientName).To(Equal("ganache"))
+	})
 })

pkg/history/populate_headers.go

@@ -17,29 +17,38 @@
 package history
 
 import (
+	"log"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-	"log"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
 
-func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) int {
+func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
 	lastBlock := blockchain.LastBlock().Int64()
 	blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
 	log.SetPrefix("")
 	log.Printf("Backfilling %d blocks\n\n", len(blockRange))
-	RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
+	_, err := RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
-	return len(blockRange)
+	if err != nil {
+		return 0, err
+	}
+	return len(blockRange), nil
 }
 
-func RetrieveAndUpdateHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) int {
+func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
 	for _, blockNumber := range blockNumbers {
-		header, err := blockchain.GetHeaderByNumber(blockNumber)
+		header, err := chain.GetHeaderByNumber(blockNumber)
 		if err != nil {
-			log.Printf("failed to retrieve block number: %d\n", blockNumber)
+			return 0, err
-			return 0
+		}
+		_, err = headerRepository.CreateOrUpdateHeader(header)
+		if err != nil {
+			if err == repositories.ErrValidHeaderExists {
+				continue
+			}
+			return 0, err
 		}
-		// TODO: handle possible error here
-		headerRepository.CreateOrUpdateHeader(header)
 	}
-	return len(blockNumbers)
+	return len(blockNumbers), nil
 }

pkg/history/populate_headers_test.go

@@ -34,17 +34,15 @@ var _ = Describe("Populating headers", func() {
 		headerRepository = fakes.NewMockHeaderRepository()
 	})
 
-	Describe("When 1 missing header", func() {
+	It("returns number of headers added", func() {
+		blockChain := fakes.NewMockBlockChain()
+		blockChain.SetLastBlock(big.NewInt(2))
+		headerRepository.SetMissingBlockNumbers([]int64{2})
 
-		It("returns number of headers added", func() {
+		headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
-			blockChain := fakes.NewMockBlockChain()
-			blockChain.SetLastBlock(big.NewInt(2))
-			headerRepository.SetMissingBlockNumbers([]int64{2})
 
-			headersAdded := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
+		Expect(err).NotTo(HaveOccurred())
-
+		Expect(headersAdded).To(Equal(1))
-			Expect(headersAdded).To(Equal(1))
-		})
 	})
 
 	It("adds missing headers to the db", func() {
@@ -52,8 +50,9 @@ var _ = Describe("Populating headers", func() {
 		blockChain.SetLastBlock(big.NewInt(2))
 		headerRepository.SetMissingBlockNumbers([]int64{2})
 
-		history.PopulateMissingHeaders(blockChain, headerRepository, 1)
+		_, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
 
+		Expect(err).NotTo(HaveOccurred())
 		headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2})
 	})
 })

pkg/omni/converter/converter.go

@@ -74,26 +74,26 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event *types.Event)
 
 	strValues := make(map[string]string, len(values))
 
-	for eventName, input := range values {
+	for fieldName, input := range values {
 		// Postgres cannot handle custom types, resolve to strings
 		switch input.(type) {
 		case *big.Int:
 			var b *big.Int
 			b = input.(*big.Int)
-			strValues[eventName] = b.String()
+			strValues[fieldName] = b.String()
 		case common.Address:
 			var a common.Address
 			a = input.(common.Address)
-			strValues[eventName] = a.String()
+			strValues[fieldName] = a.String()
 			c.contractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses
 		case common.Hash:
 			var h common.Hash
 			h = input.(common.Hash)
-			strValues[eventName] = h.String()
+			strValues[fieldName] = h.String()
 		case string:
-			strValues[eventName] = input.(string)
+			strValues[fieldName] = input.(string)
 		case bool:
-			strValues[eventName] = strconv.FormatBool(input.(bool))
+			strValues[fieldName] = strconv.FormatBool(input.(bool))
 		default:
 			return errors.New("error: unhandled abi type")
 		}

pkg/omni/helpers/test_helpers/database.go

@@ -75,7 +75,7 @@ func SetupBC() core.BlockChain {
 	blockChainClient := client.NewEthClient(ethClient)
 	node := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
 	return blockChain
 }
@@ -89,7 +89,7 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	blockChainClient := client.NewEthClient(ethClient)
 	node := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",

pkg/omni/repository/event_repository.go

@@ -47,6 +47,7 @@ func NewEventDataStore(db *postgres.DB) *eventDatastore {
 // Creates a schema for the contract
 // Creates tables for the watched contract events
 // Persists converted event log data into these custom tables
+// Edit this method to accept a single event
 func (d *eventDatastore) PersistContractEvents(con *contract.Contract) error {
 	_, err := d.CreateContractSchema(con.Address)
 	if err != nil {

pkg/omni/transformer/transformer.go

@@ -209,6 +209,8 @@ func (tr transformer) Execute() error {
 		}
 
 		// After converting all logs for events of interest, persist all of the data
+		// Change this so that event logs are persisted immediately after being created
+		// So as to prevent a huge growth of data in the contract memory
 		err := tr.PersistContractEvents(con)
 		if err != nil {
 			return err

test_config/test_config.go

@@ -20,15 +20,18 @@ import (
 	"log"
 	"os"
 
+	. "github.com/onsi/gomega"
 	"github.com/spf13/viper"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
 
 var TestConfig *viper.Viper
 var DBConfig config.Database
+var TestClient config.Client
 var Infura *viper.Viper
 var InfuraClient config.Client
 var ABIFilePath string
@@ -44,6 +47,7 @@ func setTestConfig() {
 	TestConfig.SetConfigName("private")
 	TestConfig.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
 	err := TestConfig.ReadInConfig()
+	ipc := TestConfig.GetString("client.ipcPath")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -55,6 +59,9 @@ func setTestConfig() {
 		Name:     name,
 		Port:     port,
 	}
+	TestClient = config.Client{
+		IPCPath: ipc,
+	}
 }
 
 func setInfuraConfig() {
@@ -78,14 +85,34 @@ func setABIPath() {
 
 func NewTestDB(node core.Node) *postgres.DB {
 	db, _ := postgres.NewDB(DBConfig, node)
+	return db
+}
+
+func CleanTestDB(db *postgres.DB) {
 	db.MustExec("DELETE FROM blocks")
 	db.MustExec("DELETE FROM headers")
+	db.MustExec("DELETE FROM checked_headers")
 	db.MustExec("DELETE FROM log_filters")
 	db.MustExec("DELETE FROM logs")
 	db.MustExec("DELETE FROM receipts")
 	db.MustExec("DELETE FROM transactions")
 	db.MustExec("DELETE FROM watched_contracts")
-	return db
+}
+
+func NewTestNode() core.Node {
+	return core.Node{
+		GenesisBlock: "GENESIS",
+		NetworkID:    1,
+		ID:           "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
+		ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
+	}
+}
+
+func NewTestBlock(blockNumber int64, repository repositories.BlockRepository) (blockId int64) {
+	blockId, err := repository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
+	Expect(err).NotTo(HaveOccurred())
+
+	return blockId
 }
 
 func NewTestDBWithoutDeletingRecords(node core.Node) *postgres.DB {