Add light sync command

- Only syncs block headers (excludes block bodies, transactions, receipts, and logs)
- Modifies validation window to include the most recent block
- Isolates validation window to the variable defined in the cmd directory (blocks
  have a separate variable defined in the block_repository for determining when
  to set a block as final)

pkg/core/blockchain.go

@@ -5,6 +5,7 @@ import "math/big"
 type Blockchain interface {
 	ContractDataFetcher
 	GetBlockByNumber(blockNumber int64) (Block, error)
+	GetHeaderByNumber(blockNumber int64) (Header, error)
 	GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error)
 	LastBlock() *big.Int
 	Node() Node

pkg/core/header.go

@@ -0,0 +1,7 @@
+package core
+
+type Header struct {
+	BlockNumber int64 `db:"block_number"`
+	Hash        string
+	Raw         []byte
+}

pkg/datastore/inmemory/header_repository.go

@@ -0,0 +1,30 @@
+package inmemory
+
+import "github.com/vulcanize/vulcanizedb/pkg/core"
+
+type HeaderRepository struct {
+	memory *InMemory
+}
+
+func NewHeaderRepository(memory *InMemory) *HeaderRepository {
+	return &HeaderRepository{memory: memory}
+}
+
+func (repository *HeaderRepository) CreateOrUpdateHeader(header core.Header) (int64, error) {
+	repository.memory.headers[header.BlockNumber] = header
+	return 0, nil
+}
+
+func (repository *HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
+	return repository.memory.headers[blockNumber], nil
+}
+
+func (repository *HeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 {
+	missingNumbers := []int64{}
+	for blockNumber := int64(startingBlockNumber); blockNumber <= endingBlockNumber; blockNumber++ {
+		if _, ok := repository.memory.headers[blockNumber]; !ok {
+			missingNumbers = append(missingNumbers, blockNumber)
+		}
+	}
+	return missingNumbers
+}

pkg/datastore/inmemory/in_memory.go

@@ -10,21 +10,23 @@ const (
 )
 
 type InMemory struct {
-	blocks                       map[int64]core.Block
-	receipts                     map[string]core.Receipt
-	contracts                    map[string]core.Contract
-	logs                         map[string][]core.Log
-	logFilters                   map[string]filters.LogFilter
 	CreateOrUpdateBlockCallCount int
+	blocks                       map[int64]core.Block
+	contracts                    map[string]core.Contract
+	headers                      map[int64]core.Header
+	logFilters                   map[string]filters.LogFilter
+	logs                         map[string][]core.Log
+	receipts                     map[string]core.Receipt
 }
 
 func NewInMemory() *InMemory {
 	return &InMemory{
 		CreateOrUpdateBlockCallCount: 0,
 		blocks:     make(map[int64]core.Block),
-		receipts:   make(map[string]core.Receipt),
 		contracts:  make(map[string]core.Contract),
-		logs:       make(map[string][]core.Log),
+		headers:    make(map[int64]core.Header),
 		logFilters: make(map[string]filters.LogFilter),
+		logs:       make(map[string][]core.Log),
+		receipts:   make(map[string]core.Receipt),
 	}
 }

pkg/datastore/postgres/repositories/header_repository.go

@@ -0,0 +1,80 @@
+package repositories
+
+import (
+	"database/sql"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type HeaderRepository struct {
+	database *postgres.DB
+}
+
+func NewHeaderRepository(database *postgres.DB) HeaderRepository {
+	return HeaderRepository{database: database}
+}
+
+func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int64, error) {
+	hash, err := repository.getHeaderHash(header)
+	if err != nil {
+		if headerDoesNotExist(err) {
+			return repository.insertHeader(header)
+		}
+		return 0, err
+	}
+	if headerMustBeReplaced(hash, header) {
+		return repository.replaceHeader(header)
+	}
+	return 0, err
+}
+
+func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
+	var header core.Header
+	err := repository.database.Get(&header, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
+		blockNumber, repository.database.Node.ID)
+	return header, err
+}
+
+func (repository HeaderRepository) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64 {
+	numbers := make([]int64, 0)
+	repository.database.Select(&numbers, `SELECT all_block_numbers
+	  FROM (
+		  SELECT generate_series($1::INT, $2::INT) AS all_block_numbers) series
+	  WHERE all_block_numbers NOT IN (
+		  SELECT block_number FROM headers WHERE eth_node_fingerprint = $3
+	  ) `,
+		startingBlockNumber, endingBlockNumber, nodeID)
+	return numbers
+}
+
+func headerMustBeReplaced(hash string, header core.Header) bool {
+	return hash != header.Hash
+}
+
+func headerDoesNotExist(err error) bool {
+	return err == sql.ErrNoRows
+}
+
+func (repository HeaderRepository) getHeaderHash(header core.Header) (string, error) {
+	var hash string
+	err := repository.database.Get(&hash, `SELECT hash FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
+		header.BlockNumber, repository.database.Node.ID)
+	return hash, err
+}
+
+func (repository HeaderRepository) insertHeader(header core.Header) (int64, error) {
+	var headerId int64
+	err := repository.database.QueryRowx(
+		`INSERT INTO public.headers (block_number, hash, raw, eth_node_id, eth_node_fingerprint) VALUES ($1, $2, $3, $4, $5) RETURNING id`,
+		header.BlockNumber, header.Hash, header.Raw, repository.database.NodeID, repository.database.Node.ID).Scan(&headerId)
+	return headerId, err
+}
+
+func (repository HeaderRepository) replaceHeader(header core.Header) (int64, error) {
+	_, err := repository.database.Exec(`DELETE FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
+		header.BlockNumber, repository.database.Node.ID)
+	if err != nil {
+		return 0, err
+	}
+	return repository.insertHeader(header)
+}

pkg/datastore/postgres/repositories/header_repository_test.go

@@ -0,0 +1,248 @@
+package repositories_test
+
+import (
+	"database/sql"
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Block header repository", func() {
+	Describe("creating or updating a header", func() {
+
+		It("adds a header", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+
+			_, err := repo.CreateOrUpdateHeader(header)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeader core.Header
+			err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
+			Expect(dbHeader.Hash).To(Equal(header.Hash))
+			Expect(dbHeader.Raw).To(Equal(header.Raw))
+		})
+
+		It("adds node data to header", func() {
+			node := core.Node{ID: "EthNodeFingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{BlockNumber: 100}
+
+			_, err := repo.CreateOrUpdateHeader(header)
+
+			Expect(err).NotTo(HaveOccurred())
+			var ethNodeId int64
+			err = db.Get(&ethNodeId, `SELECT eth_node_id FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ethNodeId).To(Equal(db.NodeID))
+			var ethNodeFingerprint string
+			err = db.Get(&ethNodeFingerprint, `SELECT eth_node_fingerprint FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ethNodeFingerprint).To(Equal(db.Node.ID))
+		})
+
+		It("does not duplicate headers", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+
+			var dbHeaders []core.Header
+			err = db.Select(&dbHeaders, `SELECT block_number, hash, raw FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(dbHeaders)).To(Equal(1))
+		})
+
+		It("replaces header if hash is different", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+			headerTwo := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
+				Raw:         []byte{5, 4, 3, 2, 1},
+			}
+
+			_, err = repo.CreateOrUpdateHeader(headerTwo)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeader core.Header
+			err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbHeader.Hash).To(Equal(headerTwo.Hash))
+			Expect(dbHeader.Raw).To(Equal(headerTwo.Raw))
+		})
+
+		It("does not replace header if node fingerprint is different", func() {
+			node := core.Node{ID: "Fingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			nodeTwo := core.Node{ID: "FingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+			headerTwo := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
+				Raw:         []byte{5, 4, 3, 2, 1},
+			}
+
+			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeaders []core.Header
+			err = dbTwo.Select(&dbHeaders, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(dbHeaders)).To(Equal(2))
+		})
+
+		It("only replaces header with matching node fingerprint", func() {
+			node := core.Node{ID: "Fingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			nodeTwo := core.Node{ID: "FingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+			headerTwo := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
+				Raw:         []byte{5, 4, 3, 2, 1},
+			}
+			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
+			headerThree := core.Header{
+				BlockNumber: header.BlockNumber,
+				Hash:        common.BytesToHash([]byte{1, 1, 1, 1, 1}).Hex(),
+				Raw:         []byte{1, 1, 1, 1, 1},
+			}
+
+			_, err = repoTwo.CreateOrUpdateHeader(headerThree)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbHeaders []core.Header
+			err = dbTwo.Select(&dbHeaders, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(dbHeaders)).To(Equal(2))
+			Expect(dbHeaders[0].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
+			Expect(dbHeaders[1].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
+			Expect(dbHeaders[0].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+			Expect(dbHeaders[1].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+		})
+	})
+
+	Describe("Getting a header", func() {
+		It("returns header if it exists", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+
+			dbHeader, err := repo.GetHeader(header.BlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbHeader).To(Equal(header))
+		})
+
+		It("does not return header for a different node fingerprint", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			header := core.Header{
+				BlockNumber: 100,
+				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Raw:         []byte{1, 2, 3, 4, 5},
+			}
+			_, err := repo.CreateOrUpdateHeader(header)
+			Expect(err).NotTo(HaveOccurred())
+			nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+
+			_, err = repoTwo.GetHeader(header.BlockNumber)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("Getting missing headers", func() {
+		It("returns block numbers for headers not in the database", func() {
+			node := core.Node{}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+
+			missingBlockNumbers := repo.MissingBlockNumbers(1, 5, node.ID)
+
+			Expect(missingBlockNumbers).To(ConsistOf([]int64{2, 4}))
+		})
+
+		It("does not count headers created by a different node fingerprint", func() {
+			node := core.Node{ID: "NodeFingerprint"}
+			db := test_config.NewTestDB(node)
+			repo := repositories.NewHeaderRepository(db)
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+			nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
+			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
+			Expect(err).NotTo(HaveOccurred())
+			repoTwo := repositories.NewHeaderRepository(dbTwo)
+
+			missingBlockNumbers := repoTwo.MissingBlockNumbers(1, 5, nodeTwo.ID)
+
+			Expect(missingBlockNumbers).To(ConsistOf([]int64{1, 2, 3, 4, 5}))
+		})
+	})
+})

pkg/datastore/repository.go

@@ -14,7 +14,7 @@ var ErrBlockDoesNotExist = func(blockNumber int64) error {
 type BlockRepository interface {
 	CreateOrUpdateBlock(block core.Block) (int64, error)
 	GetBlock(blockNumber int64) (core.Block, error)
-	MissingBlockNumbers(startingBlockNumber int64, endingBlockNumber int64, nodeId string) []int64
+	MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64
 	SetBlocksStatus(chainHead int64)
 }
 
@@ -37,6 +37,12 @@ type FilterRepository interface {
 	GetFilter(name string) (filters.LogFilter, error)
 }
 
+type HeaderRepository interface {
+	CreateOrUpdateHeader(header core.Header) (int64, error)
+	GetHeader(blockNumber int64) (core.Header, error)
+	MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64
+}
+
 type LogRepository interface {
 	CreateLogs(logs []core.Log, receiptId int64) error
 	GetLogs(address string, blockNumber int64) []core.Log

pkg/fakes/blockchain.go

@@ -6,28 +6,33 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
-type Blockchain struct {
-	logs                map[string][]core.Log
-	blocks              map[int64]core.Block
-	contractAttributes  map[string]map[string]string
-	blocksChannel       chan core.Block
-	WasToldToStop       bool
-	node                core.Node
+type BlockChain struct {
 	ContractReturnValue []byte
+	WasToldToStop       bool
+	blocks              map[int64]core.Block
+	blocksChannel       chan core.Block
+	contractAttributes  map[string]map[string]string
 	err                 error
+	headers             map[int64]core.Header
+	logs                map[string][]core.Log
+	node                core.Node
 }
 
-func (blockchain *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
+func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (core.Header, error) {
+	return blockChain.headers[blockNumber], nil
+}
+
+func (blockChain *BlockChain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
 	panic("implement me")
 }
 
-func (blockchain *Blockchain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
-	return blockchain.ContractReturnValue, nil
+func (blockChain *BlockChain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
+	return blockChain.ContractReturnValue, nil
 }
 
-func (blockchain *Blockchain) LastBlock() *big.Int {
+func (blockChain *BlockChain) LastBlock() *big.Int {
 	var max int64
-	for blockNumber := range blockchain.blocks {
+	for blockNumber := range blockChain.blocks {
 		if blockNumber > max {
 			max = blockNumber
 		}
@@ -35,16 +40,16 @@ func (blockchain *Blockchain) LastBlock() *big.Int {
 	return big.NewInt(max)
 }
 
-func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) {
-	return blockchain.logs[contract.Hash], nil
+func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) {
+	return blockChain.logs[contract.Hash], nil
 }
 
-func (blockchain *Blockchain) Node() core.Node {
-	return blockchain.node
+func (blockChain *BlockChain) Node() core.Node {
+	return blockChain.node
 }
 
-func NewBlockchain(err error) *Blockchain {
-	return &Blockchain{
+func NewBlockchain(err error) *BlockChain {
+	return &BlockChain{
 		blocks:             make(map[int64]core.Block),
 		logs:               make(map[string][]core.Log),
 		contractAttributes: make(map[string]map[string]string),
@@ -53,24 +58,40 @@ func NewBlockchain(err error) *Blockchain {
 	}
 }
 
-func NewBlockchainWithBlocks(blocks []core.Block) *Blockchain {
+func NewBlockchainWithBlocks(blocks []core.Block) *BlockChain {
 	blockNumberToBlocks := make(map[int64]core.Block)
 	for _, block := range blocks {
 		blockNumberToBlocks[block.Number] = block
 	}
-	return &Blockchain{
+	return &BlockChain{
 		blocks: blockNumberToBlocks,
 	}
 }
 
-func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
-	if blockchain.err != nil {
-		return core.Block{}, blockchain.err
+func NewBlockChainWithHeaders(headers []core.Header) *BlockChain {
+	// need to create blocks and headers so that LastBlock() will work in the mock
+	// no reason to implement LastBlock() separately for headers since it checks
+	// the last header in the Node's DB already
+	memoryBlocks := make(map[int64]core.Block)
+	memoryHeaders := make(map[int64]core.Header)
+	for _, header := range headers {
+		memoryBlocks[header.BlockNumber] = core.Block{Number: header.BlockNumber}
+		memoryHeaders[header.BlockNumber] = header
+	}
+	return &BlockChain{
+		blocks:  memoryBlocks,
+		headers: memoryHeaders,
 	}
-	return blockchain.blocks[blockNumber], nil
 }
 
-func (blockchain *Blockchain) AddBlock(block core.Block) {
-	blockchain.blocks[block.Number] = block
-	blockchain.blocksChannel <- block
+func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
+	if blockChain.err != nil {
+		return core.Block{}, blockChain.err
+	}
+	return blockChain.blocks[blockNumber], nil
+}
+
+func (blockChain *BlockChain) AddBlock(block core.Block) {
+	blockChain.blocks[block.Number] = block
+	blockChain.blocksChannel <- block
 }

pkg/geth/blockchain.go

@@ -1,13 +1,11 @@
 package geth
 
 import (
-	"math/big"
-
 	"log"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
 	"golang.org/x/net/context"
@@ -18,31 +16,30 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
-type Blockchain struct {
-	client              *ethclient.Client
-	blockConverter      vulcCommon.BlockConverter
-	readGethHeaders     chan *types.Header
-	outputBlocks        chan core.Block
-	newHeadSubscription ethereum.Subscription
-	node                core.Node
+type BlockChain struct {
+	client          *ethclient.Client
+	blockConverter  vulcCommon.BlockConverter
+	headerConverter vulcCommon.HeaderConverter
+	node            core.Node
 }
 
-func NewBlockchain(ipcPath string) *Blockchain {
-	blockchain := Blockchain{}
+func NewBlockChain(ipcPath string) *BlockChain {
 	rpcClient, err := rpc.Dial(ipcPath)
 	if err != nil {
 		log.Fatal(err)
 	}
 	client := ethclient.NewClient(rpcClient)
 	clientWrapper := node.ClientWrapper{ContextCaller: rpcClient, IPCPath: ipcPath}
-	blockchain.node = node.MakeNode(clientWrapper)
-	blockchain.client = client
 	transactionConverter := vulcRpc.NewRpcTransactionConverter(client)
-	blockchain.blockConverter = vulcCommon.NewBlockConverter(transactionConverter)
-	return &blockchain
+	return &BlockChain{
+		client:          client,
+		blockConverter:  vulcCommon.NewBlockConverter(transactionConverter),
+		headerConverter: vulcCommon.HeaderConverter{},
+		node:            node.MakeNode(clientWrapper),
+	}
 }
 
-func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) {
+func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) {
 	if endingBlockNumber == nil {
 		endingBlockNumber = startingBlockNumber
 	}
@@ -52,7 +49,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe
 		ToBlock:   endingBlockNumber,
 		Addresses: []common.Address{contractAddress},
 	}
-	gethLogs, err := blockchain.client.FilterLogs(context.Background(), fc)
+	gethLogs, err := blockChain.client.FilterLogs(context.Background(), fc)
 	if err != nil {
 		return []core.Log{}, err
 	}
@@ -60,23 +57,27 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe
 	return logs, nil
 }
 
-func (blockchain *Blockchain) Node() core.Node {
-	return blockchain.node
+func (blockChain *BlockChain) Node() core.Node {
+	return blockChain.node
 }
 
-func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) (core.Block, error) {
-	gethBlock, err := blockchain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
+func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Block, err error) {
+	gethBlock, err := blockChain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
 	if err != nil {
-		return core.Block{}, err
+		return block, err
 	}
-	block, err := blockchain.blockConverter.ToCoreBlock(gethBlock)
-	if err != nil {
-		return core.Block{}, err
-	}
-	return block, nil
+	return blockChain.blockConverter.ToCoreBlock(gethBlock)
 }
 
-func (blockchain *Blockchain) LastBlock() *big.Int {
-	block, _ := blockchain.client.HeaderByNumber(context.Background(), nil)
+func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) {
+	gethHeader, err := blockChain.client.HeaderByNumber(context.Background(), big.NewInt(blockNumber))
+	if err != nil {
+		return header, err
+	}
+	return blockChain.headerConverter.Convert(gethHeader)
+}
+
+func (blockChain *BlockChain) LastBlock() *big.Int {
+	block, _ := blockChain.client.HeaderByNumber(context.Background(), nil)
 	return block.Number
 }

pkg/geth/contract.go

@@ -14,7 +14,7 @@ var (
 	ErrInvalidStateAttribute = errors.New("invalid state attribute")
 )
 
-func (blockchain *Blockchain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
+func (blockChain *BlockChain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error {
 	parsed, err := ParseAbi(abiJSON)
 	if err != nil {
 		return err
@@ -28,15 +28,15 @@ func (blockchain *Blockchain) FetchContractData(abiJSON string, address string,
 	if err != nil {
 		return err
 	}
-	output, err := blockchain.callContract(address, input, big.NewInt(blockNumber))
+	output, err := blockChain.callContract(address, input, big.NewInt(blockNumber))
 	if err != nil {
 		return err
 	}
 	return parsed.Unpack(result, method, output)
 }
 
-func (blockchain *Blockchain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
+func (blockChain *BlockChain) callContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) {
 	to := common.HexToAddress(contractHash)
 	msg := ethereum.CallMsg{To: &to, Data: input}
-	return blockchain.client.CallContract(context.Background(), msg, blockNumber)
+	return blockChain.client.CallContract(context.Background(), msg, blockNumber)
 }

pkg/geth/converters/common/block_converter.go

@@ -20,6 +20,7 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error)
 	if err != nil {
 		return core.Block{}, err
 	}
+
 	coreBlock := core.Block{
 		Difficulty:   gethBlock.Difficulty().Int64(),
 		ExtraData:    hexutil.Encode(gethBlock.Extra()),

pkg/geth/converters/common/header_converter.go

@@ -0,0 +1,24 @@
+package common
+
+import (
+	"bytes"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type HeaderConverter struct{}
+
+func (converter HeaderConverter) Convert(gethHeader *types.Header) (core.Header, error) {
+	writer := new(bytes.Buffer)
+	err := rlp.Encode(writer, &gethHeader)
+	if err != nil {
+		panic(err)
+	}
+	coreHeader := core.Header{
+		Hash:        gethHeader.Hash().Hex(),
+		BlockNumber: gethHeader.Number.Int64(),
+		Raw:         writer.Bytes(),
+	}
+	return coreHeader, nil
+}

pkg/geth/converters/common/header_converter_test.go

@@ -0,0 +1,47 @@
+package common_test
+
+import (
+	"bytes"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	common2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
+	"math/big"
+)
+
+var _ = Describe("Block header converter", func() {
+	It("converts geth header to core header", func() {
+		gethHeader := &types.Header{
+			Difficulty:  big.NewInt(1),
+			Number:      big.NewInt(2),
+			ParentHash:  common.HexToHash("0xParent"),
+			ReceiptHash: common.HexToHash("0xReceipt"),
+			Root:        common.HexToHash("0xRoot"),
+			Time:        big.NewInt(3),
+			TxHash:      common.HexToHash("0xTransaction"),
+			UncleHash:   common.HexToHash("0xUncle"),
+		}
+		converter := common2.HeaderConverter{}
+
+		coreHeader, err := converter.Convert(gethHeader)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
+		Expect(coreHeader.Hash).To(Equal(gethHeader.Hash().Hex()))
+	})
+
+	It("includes raw bytes for header", func() {
+		headerRLP := []byte{249, 2, 23, 160, 180, 251, 173, 248, 234, 69, 43, 19, 151, 24, 226, 112, 13, 193, 19, 92, 252, 129, 20, 80, 49, 200, 75, 122, 178, 124, 215, 16, 57, 79, 123, 56, 160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71, 148, 42, 101, 172, 164, 213, 252, 91, 92, 133, 144, 144, 166, 195, 77, 22, 65, 53, 57, 130, 38, 160, 14, 6, 111, 60, 34, 151, 165, 203, 48, 5, 147, 5, 38, 23, 209, 188, 165, 148, 111, 12, 170, 6, 53, 253, 177, 184, 90, 199, 229, 35, 111, 52, 160, 101, 186, 136, 127, 203, 8, 38, 246, 22, 208, 31, 115, 108, 29, 45, 103, 123, 202, 189, 226, 247, 252, 37, 170, 145, 207, 188, 11, 59, 173, 92, 179, 160, 32, 227, 83, 69, 64, 202, 241, 99, 120, 230, 232, 106, 43, 241, 35, 109, 159, 135, 109, 50, 24, 251, 192, 57, 88, 230, 219, 28, 99, 75, 35, 51, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 134, 11, 105, 222, 129, 162, 43, 131, 15, 66, 64, 131, 47, 239, 216, 130, 196, 68, 132, 86, 191, 180, 21, 152, 215, 131, 1, 3, 3, 132, 71, 101, 116, 104, 135, 103, 111, 49, 46, 53, 46, 49, 133, 108, 105, 110, 117, 120, 160, 146, 196, 18, 154, 10, 226, 54, 27, 69, 42, 158, 222, 236, 229, 92, 18, 236, 238, 171, 134, 99, 22, 25, 94, 61, 135, 252, 27, 0, 91, 102, 69, 136, 205, 76, 85, 185, 65, 207, 144, 21}
+		var gethHeader types.Header
+		err := rlp.Decode(bytes.NewReader(headerRLP), &gethHeader)
+		Expect(err).NotTo(HaveOccurred())
+		converter := common2.HeaderConverter{}
+
+		coreHeader, err := converter.Convert(&gethHeader)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(coreHeader.Raw).To(Equal(headerRLP))
+	})
+})

pkg/history/block_validator.go

@@ -0,0 +1,29 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+type BlockValidator struct {
+	blockchain      core.Blockchain
+	blockRepository datastore.BlockRepository
+	windowSize      int
+}
+
+func NewBlockValidator(blockchain core.Blockchain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
+	return &BlockValidator{
+		blockchain:      blockchain,
+		blockRepository: blockRepository,
+		windowSize:      windowSize,
+	}
+}
+
+func (bv BlockValidator) ValidateBlocks() ValidationWindow {
+	window := MakeValidationWindow(bv.blockchain, bv.windowSize)
+	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
+	RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
+	lastBlock := bv.blockchain.LastBlock().Int64()
+	bv.blockRepository.SetBlocksStatus(lastBlock)
+	return window
+}

pkg/history/block_validator_test.go

@@ -0,0 +1,43 @@
+package history_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+)
+
+var _ = Describe("Blocks validator", func() {
+
+	It("calls create or update for all blocks within the window", func() {
+		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+			{Number: 4},
+			{Number: 5},
+			{Number: 6},
+			{Number: 7},
+		})
+		inMemoryDB := inmemory.NewInMemory()
+		blocksRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
+		validator := history.NewBlockValidator(blockchain, blocksRepository, 2)
+
+		window := validator.ValidateBlocks()
+
+		Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
+		Expect(blocksRepository.BlockCount()).To(Equal(3))
+		Expect(blocksRepository.CreateOrUpdateBlockCallCount).To(Equal(3))
+	})
+
+	It("returns the number of largest block", func() {
+		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+			{Number: 1},
+			{Number: 2},
+			{Number: 3},
+		})
+		maxBlockNumber := blockchain.LastBlock()
+
+		Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
+	})
+})

pkg/history/header_validator.go

@@ -0,0 +1,27 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+type HeaderValidator struct {
+	blockChain       core.Blockchain
+	headerRepository datastore.HeaderRepository
+	windowSize       int
+}
+
+func NewHeaderValidator(blockChain core.Blockchain, repository datastore.HeaderRepository, windowSize int) HeaderValidator {
+	return HeaderValidator{
+		blockChain:       blockChain,
+		headerRepository: repository,
+		windowSize:       windowSize,
+	}
+}
+
+func (validator HeaderValidator) ValidateHeaders() ValidationWindow {
+	window := MakeValidationWindow(validator.blockChain, validator.windowSize)
+	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
+	RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers)
+	return window
+}

pkg/history/header_validator_test.go

@@ -0,0 +1,39 @@
+package history_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+)
+
+var _ = Describe("Header validator", func() {
+	It("replaces headers in the validation window that have changed", func() {
+		blockNumber := int64(10)
+		oldHash := common.HexToHash("0x0987654321").Hex()
+		oldHeader := core.Header{
+			BlockNumber: blockNumber,
+			Hash:        oldHash,
+		}
+		inMemory := inmemory.NewInMemory()
+		headerRepository := inmemory.NewHeaderRepository(inMemory)
+		headerRepository.CreateOrUpdateHeader(oldHeader)
+		newHash := common.HexToHash("0x123456789").Hex()
+		newHeader := core.Header{
+			BlockNumber: blockNumber,
+			Hash:        newHash,
+		}
+		headers := []core.Header{newHeader}
+		blockChain := fakes.NewBlockChainWithHeaders(headers)
+		validator := history.NewHeaderValidator(blockChain, headerRepository, 1)
+
+		validator.ValidateHeaders()
+
+		dbHeader, _ := headerRepository.GetHeader(blockNumber)
+		Expect(dbHeader.Hash).NotTo(Equal(oldHash))
+		Expect(dbHeader.Hash).To(Equal(newHash))
+	})
+})

pkg/history/populate_blocks.go

@@ -9,7 +9,7 @@ import (
 
 func PopulateMissingBlocks(blockchain core.Blockchain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int {
 	lastBlock := blockchain.LastBlock().Int64()
-	blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock-1, blockchain.Node().ID)
+	blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
 	log.SetPrefix("")
 	log.Printf("Backfilling %d blocks\n\n", len(blockRange))
 	RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)

pkg/history/populate_blocks_test.go

@@ -20,7 +20,7 @@ var _ = Describe("Populating blocks", func() {
 		blockRepository = &inmemory.BlockRepository{InMemory: inMemory}
 	})
 
-	It("fills in the only missing block (Number 1)", func() {
+	It("fills in the only missing block (BlockNumber 1)", func() {
 		blocks := []core.Block{
 			{Number: 1},
 			{Number: 2},
@@ -57,11 +57,12 @@ var _ = Describe("Populating blocks", func() {
 		blockRepository.CreateOrUpdateBlock(core.Block{Number: 9})
 		blockRepository.CreateOrUpdateBlock(core.Block{Number: 11})
 		blockRepository.CreateOrUpdateBlock(core.Block{Number: 12})
+		blockRepository.CreateOrUpdateBlock(core.Block{Number: 13})
 
 		blocksAdded := history.PopulateMissingBlocks(blockchain, blockRepository, 5)
 
 		Expect(blocksAdded).To(Equal(3))
-		Expect(blockRepository.BlockCount()).To(Equal(11))
+		Expect(blockRepository.BlockCount()).To(Equal(12))
 		_, err := blockRepository.GetBlock(4)
 		Expect(err).To(HaveOccurred())
 		_, err = blockRepository.GetBlock(5)
@@ -70,7 +71,7 @@ var _ = Describe("Populating blocks", func() {
 		Expect(err).ToNot(HaveOccurred())
 		_, err = blockRepository.GetBlock(10)
 		Expect(err).ToNot(HaveOccurred())
-		_, err = blockRepository.GetBlock(13)
+		_, err = blockRepository.GetBlock(14)
 		Expect(err).To(HaveOccurred())
 	})
 
@@ -98,8 +99,8 @@ var _ = Describe("Populating blocks", func() {
 		})
 
 		history.RetrieveAndUpdateBlocks(blockchain, blockRepository, history.MakeRange(2, 5))
-		Expect(blockRepository.BlockCount()).To(Equal(3))
-		Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(3))
+		Expect(blockRepository.BlockCount()).To(Equal(4))
+		Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(4))
 	})
 
 	It("does not call repository create block when there is an error", func() {
@@ -109,5 +110,4 @@ var _ = Describe("Populating blocks", func() {
 		Expect(blockRepository.BlockCount()).To(Equal(0))
 		Expect(blockRepository.CreateOrUpdateBlockCallCount).To(Equal(0))
 	})
-
 })

pkg/history/populate_headers.go

@@ -0,0 +1,29 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"log"
+)
+
+func PopulateMissingHeaders(blockchain core.Blockchain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) int {
+	lastBlock := blockchain.LastBlock().Int64()
+	blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
+	log.SetPrefix("")
+	log.Printf("Backfilling %d blocks\n\n", len(blockRange))
+	RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange)
+	return len(blockRange)
+}
+
+func RetrieveAndUpdateHeaders(blockchain core.Blockchain, headerRepository datastore.HeaderRepository, blockNumbers []int64) int {
+	for _, blockNumber := range blockNumbers {
+		header, err := blockchain.GetHeaderByNumber(blockNumber)
+		if err != nil {
+			log.Printf("failed to retrieve block number: %d\n", blockNumber)
+			return 0
+		}
+		// TODO: handle possible error here
+		headerRepository.CreateOrUpdateHeader(header)
+	}
+	return len(blockNumbers)
+}

pkg/history/populate_headers_test.go

@@ -0,0 +1,52 @@
+package history_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+)
+
+var _ = Describe("Populating headers", func() {
+
+	var inMemory *inmemory.InMemory
+	var headerRepository *inmemory.HeaderRepository
+
+	BeforeEach(func() {
+		inMemory = inmemory.NewInMemory()
+		headerRepository = inmemory.NewHeaderRepository(inMemory)
+	})
+
+	Describe("When 1 missing header", func() {
+
+		It("returns number of headers added", func() {
+			headers := []core.Header{
+				{BlockNumber: 1},
+				{BlockNumber: 2},
+			}
+			blockChain := fakes.NewBlockChainWithHeaders(headers)
+			headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: 2})
+
+			headersAdded := history.PopulateMissingHeaders(blockChain, headerRepository, 1)
+
+			Expect(headersAdded).To(Equal(1))
+		})
+	})
+
+	It("adds missing headers to the db", func() {
+		headers := []core.Header{
+			{BlockNumber: 1},
+			{BlockNumber: 2},
+		}
+		blockChain := fakes.NewBlockChainWithHeaders(headers)
+		dbHeader, _ := headerRepository.GetHeader(1)
+		Expect(dbHeader.BlockNumber).To(BeZero())
+
+		history.PopulateMissingHeaders(blockChain, headerRepository, 1)
+
+		dbHeader, _ = headerRepository.GetHeader(1)
+		Expect(dbHeader.BlockNumber).To(Equal(int64(1)))
+	})
+})

pkg/history/validate_blocks.go

@@ -1,68 +0,0 @@
-package history
-
-import (
-	"io"
-	"text/template"
-
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-)
-
-const WindowTemplate = `Validating Blocks
-|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD)
-
-`
-
-var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate))
-
-type BlockValidator struct {
-	blockchain            core.Blockchain
-	blockRepository       datastore.BlockRepository
-	windowSize            int
-	parsedLoggingTemplate template.Template
-}
-
-func NewBlockValidator(blockchain core.Blockchain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
-	return &BlockValidator{
-		blockchain,
-		blockRepository,
-		windowSize,
-		ParsedWindowTemplate,
-	}
-}
-
-func (bv BlockValidator) ValidateBlocks() ValidationWindow {
-	window := MakeValidationWindow(bv.blockchain, bv.windowSize)
-	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
-	RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
-	lastBlock := bv.blockchain.LastBlock().Int64()
-	bv.blockRepository.SetBlocksStatus(lastBlock)
-	return window
-}
-
-func (bv BlockValidator) Log(out io.Writer, window ValidationWindow) {
-	bv.parsedLoggingTemplate.Execute(out, window)
-}
-
-type ValidationWindow struct {
-	LowerBound int64
-	UpperBound int64
-}
-
-func (window ValidationWindow) Size() int {
-	return int(window.UpperBound - window.LowerBound)
-}
-
-func MakeValidationWindow(blockchain core.Blockchain, windowSize int) ValidationWindow {
-	upperBound := blockchain.LastBlock().Int64()
-	lowerBound := upperBound - int64(windowSize)
-	return ValidationWindow{lowerBound, upperBound}
-}
-
-func MakeRange(min, max int64) []int64 {
-	a := make([]int64, max-min)
-	for i := range a {
-		a[i] = min + int64(i)
-	}
-	return a
-}

pkg/history/validate_blocks_test.go

@@ -1,86 +0,0 @@
-package history_test
-
-import (
-	"bytes"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/inmemory"
-	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/history"
-)
-
-var _ = Describe("Blocks validator", func() {
-
-	It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
-			{Number: 1},
-			{Number: 2},
-			{Number: 3},
-			{Number: 4},
-			{Number: 5},
-		})
-
-		validationWindow := history.MakeValidationWindow(blockchain, 2)
-
-		Expect(validationWindow.LowerBound).To(Equal(int64(3)))
-		Expect(validationWindow.UpperBound).To(Equal(int64(5)))
-	})
-
-	It("returns the window size", func() {
-		window := history.ValidationWindow{LowerBound: 1, UpperBound: 3}
-		Expect(window.Size()).To(Equal(2))
-	})
-
-	It("calls create or update for all blocks within the window", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
-			{Number: 4},
-			{Number: 5},
-			{Number: 6},
-			{Number: 7},
-		})
-		inMemoryDB := inmemory.NewInMemory()
-		blocksRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
-
-		validator := history.NewBlockValidator(blockchain, blocksRepository, 2)
-		window := validator.ValidateBlocks()
-		Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
-		Expect(blocksRepository.BlockCount()).To(Equal(2))
-		Expect(blocksRepository.CreateOrUpdateBlockCallCount).To(Equal(2))
-	})
-
-	It("logs window message", func() {
-		inMemoryDB := inmemory.NewInMemory()
-		blockRepository := &inmemory.BlockRepository{InMemory: inMemoryDB}
-
-		expectedMessage := &bytes.Buffer{}
-		window := history.ValidationWindow{LowerBound: 5, UpperBound: 7}
-		history.ParsedWindowTemplate.Execute(expectedMessage, history.ValidationWindow{LowerBound: 5, UpperBound: 7})
-
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{})
-		validator := history.NewBlockValidator(blockchain, blockRepository, 2)
-		actualMessage := &bytes.Buffer{}
-		validator.Log(actualMessage, window)
-		Expect(actualMessage).To(Equal(expectedMessage))
-	})
-
-	It("generates a range of int64s", func() {
-		numberOfBlocksCreated := history.MakeRange(0, 5)
-		expected := []int64{0, 1, 2, 3, 4}
-
-		Expect(numberOfBlocksCreated).To(Equal(expected))
-	})
-
-	It("returns the number of largest block", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
-			{Number: 1},
-			{Number: 2},
-			{Number: 3},
-		})
-		maxBlockNumber := blockchain.LastBlock()
-
-		Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
-	})
-
-})

pkg/history/validation_window.go

@@ -0,0 +1,41 @@
+package history
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"io"
+	"text/template"
+)
+
+const WindowTemplate = `Validating Blocks
+|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD)
+
+`
+
+var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate))
+
+type ValidationWindow struct {
+	LowerBound int64
+	UpperBound int64
+}
+
+func (window ValidationWindow) Size() int {
+	return int(window.UpperBound - window.LowerBound)
+}
+
+func MakeValidationWindow(blockchain core.Blockchain, windowSize int) ValidationWindow {
+	upperBound := blockchain.LastBlock().Int64()
+	lowerBound := upperBound - int64(windowSize)
+	return ValidationWindow{lowerBound, upperBound}
+}
+
+func MakeRange(min, max int64) []int64 {
+	a := make([]int64, max-min+1)
+	for i := range a {
+		a[i] = min + int64(i)
+	}
+	return a
+}
+
+func (window ValidationWindow) Log(out io.Writer) {
+	ParsedWindowTemplate.Execute(out, window)
+}

pkg/history/validation_window_test.go

@@ -0,0 +1,53 @@
+package history_test
+
+import (
+	"bytes"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/history"
+)
+
+var _ = Describe("", func() {
+	It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
+		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+			{Number: 1},
+			{Number: 2},
+			{Number: 3},
+			{Number: 4},
+			{Number: 5},
+		})
+
+		validationWindow := history.MakeValidationWindow(blockchain, 2)
+
+		Expect(validationWindow.LowerBound).To(Equal(int64(3)))
+		Expect(validationWindow.UpperBound).To(Equal(int64(5)))
+	})
+
+	It("returns the window size", func() {
+		window := history.ValidationWindow{LowerBound: 1, UpperBound: 3}
+
+		Expect(window.Size()).To(Equal(2))
+	})
+
+	It("generates a range of int64s", func() {
+		numberOfBlocksCreated := history.MakeRange(0, 5)
+		expected := []int64{0, 1, 2, 3, 4, 5}
+
+		Expect(numberOfBlocksCreated).To(Equal(expected))
+	})
+
+	It("logs window message", func() {
+		expectedMessage := &bytes.Buffer{}
+		window := history.ValidationWindow{LowerBound: 5, UpperBound: 7}
+		history.ParsedWindowTemplate.Execute(expectedMessage, window)
+		actualMessage := &bytes.Buffer{}
+
+		window.Log(actualMessage)
+
+		Expect(actualMessage).To(Equal(expectedMessage))
+	})
+})