Remove pubsub and replace w/ polling head of chain (#122)

* Rename geth package structs to not be prefaced with package name

* No longer need to dump schema since Travis uses migrate

* Rearrange history package

* Removed double request for receipt from block rewards

* Remove Listener + Observers and Replace w/ Polling Head

* Potential Short term Issue w/ Infura (ignore these tests for now)

pkg/blockchain_listener/blockchain_listener.go

@@ -1,37 +0,0 @@
-package blockchain_listener
-
-import "github.com/8thlight/vulcanizedb/pkg/core"
-
-type BlockchainListener struct {
-	inputBlocks chan core.Block
-	blockchain  core.Blockchain
-	observers   []core.BlockchainObserver
-}
-
-func NewBlockchainListener(blockchain core.Blockchain, observers []core.BlockchainObserver) BlockchainListener {
-	inputBlocks := make(chan core.Block, 10)
-	blockchain.SubscribeToBlocks(inputBlocks)
-	listener := BlockchainListener{
-		inputBlocks: inputBlocks,
-		blockchain:  blockchain,
-		observers:   observers,
-	}
-	return listener
-}
-
-func (listener BlockchainListener) Start() {
-	go listener.blockchain.StartListening()
-	for block := range listener.inputBlocks {
-		listener.notifyObservers(block)
-	}
-}
-
-func (listener BlockchainListener) notifyObservers(block core.Block) {
-	for _, observer := range listener.observers {
-		observer.NotifyBlockAdded(block)
-	}
-}
-
-func (listener BlockchainListener) Stop() {
-	listener.blockchain.StopListening()
-}

pkg/blockchain_listener/blockchain_listener_suite_test.go

@@ -1,13 +0,0 @@
-package blockchain_listener_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"testing"
-)
-
-func TestListener(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Listener Suite")
-}

pkg/blockchain_listener/blockchain_listener_test.go

@@ -1,69 +0,0 @@
-package blockchain_listener_test
-
-import (
-	"github.com/8thlight/vulcanizedb/pkg/blockchain_listener"
-	"github.com/8thlight/vulcanizedb/pkg/core"
-	"github.com/8thlight/vulcanizedb/pkg/fakes"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Blockchain listeners", func() {
-
-	It("starts with no blocks", func(done Done) {
-		observer := fakes.NewFakeBlockchainObserver()
-		blockchain := fakes.NewBlockchain()
-
-		blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
-
-		Expect(len(observer.CurrentBlocks)).To(Equal(0))
-		close(done)
-	}, 1)
-
-	It("sees when one block was added", func(done Done) {
-		observer := fakes.NewFakeBlockchainObserver()
-		blockchain := fakes.NewBlockchain()
-		listener := blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
-		go listener.Start()
-
-		go blockchain.AddBlock(core.Block{Number: 123})
-
-		wasObserverNotified := <-observer.WasNotified
-		Expect(wasObserverNotified).To(BeTrue())
-		Expect(len(observer.CurrentBlocks)).To(Equal(1))
-		addedBlock := observer.CurrentBlocks[0]
-		Expect(addedBlock.Number).To(Equal(int64(123)))
-		close(done)
-	}, 1)
-
-	It("sees a second block", func(done Done) {
-		observer := fakes.NewFakeBlockchainObserver()
-		blockchain := fakes.NewBlockchain()
-		listener := blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
-		go listener.Start()
-
-		go blockchain.AddBlock(core.Block{Number: 123})
-		<-observer.WasNotified
-		go blockchain.AddBlock(core.Block{Number: 456})
-		wasObserverNotified := <-observer.WasNotified
-
-		Expect(wasObserverNotified).To(BeTrue())
-		Expect(len(observer.CurrentBlocks)).To(Equal(2))
-		addedBlock := observer.CurrentBlocks[1]
-		Expect(addedBlock.Number).To(Equal(int64(456)))
-		close(done)
-	}, 1)
-
-	It("stops listening", func(done Done) {
-		observer := fakes.NewFakeBlockchainObserver()
-		blockchain := fakes.NewBlockchain()
-		listener := blockchain_listener.NewBlockchainListener(blockchain, []core.BlockchainObserver{observer})
-		go listener.Start()
-
-		listener.Stop()
-
-		Expect(blockchain.WasToldToStop).To(BeTrue())
-		close(done)
-	}, 1)
-
-})

pkg/core/blockchain.go

@@ -6,9 +6,6 @@ type Blockchain interface {
 	GetBlockByNumber(blockNumber int64) Block
 	LastBlock() *big.Int
 	Node() Node
-	SubscribeToBlocks(blocks chan Block)
-	StartListening()
-	StopListening()
 	GetAttributes(contract Contract) (ContractAttributes, error)
 	GetAttribute(contract Contract, attributeName string, blockNumber *big.Int) (interface{}, error)
 	GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error)

pkg/core/blockchain_observer.go

@@ -1,5 +0,0 @@
-package core
-
-type BlockchainObserver interface {
-	NotifyBlockAdded(Block)
-}

pkg/fakes/blockchain.go

@@ -68,21 +68,11 @@ func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) core.Block {
 	return blockchain.blocks[blockNumber]
 }
 
-func (blockchain *Blockchain) SubscribeToBlocks(outputBlocks chan core.Block) {
-	blockchain.blocksChannel = outputBlocks
-}
-
 func (blockchain *Blockchain) AddBlock(block core.Block) {
 	blockchain.blocks[block.Number] = block
 	blockchain.blocksChannel <- block
 }
 
-func (*Blockchain) StartListening() {}
-
-func (blockchain *Blockchain) StopListening() {
-	blockchain.WasToldToStop = true
-}
-
 func (blockchain *Blockchain) SetContractStateAttribute(contractHash string, blockNumber *big.Int, attributeName string, attributeValue string) {
 	var key string
 	if blockNumber == nil {

pkg/fakes/blockchain_observer.go

@@ -1,23 +0,0 @@
-package fakes
-
-import "github.com/8thlight/vulcanizedb/pkg/core"
-
-type BlockchainObserver struct {
-	CurrentBlocks []core.Block
-	WasNotified   chan bool
-}
-
-func (observer *BlockchainObserver) LastBlock() core.Block {
-	return observer.CurrentBlocks[len(observer.CurrentBlocks)-1]
-}
-
-func NewFakeBlockchainObserver() *BlockchainObserver {
-	return &BlockchainObserver{
-		WasNotified: make(chan bool),
-	}
-}
-
-func (observer *BlockchainObserver) NotifyBlockAdded(block core.Block) {
-	observer.CurrentBlocks = append(observer.CurrentBlocks, block)
-	observer.WasNotified <- true
-}

pkg/geth/block_rewards.go

@@ -0,0 +1,54 @@
+package geth
+
+import (
+	"github.com/8thlight/vulcanizedb/pkg/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+func CalcUnclesReward(block core.Block, uncles []*types.Header) float64 {
+	var unclesReward float64
+	for _, uncle := range uncles {
+		blockNumber := block.Number
+		staticBlockReward := float64(staticRewardByBlockNumber(blockNumber))
+		unclesReward += (1.0 + float64(uncle.Number.Int64()-block.Number)/8.0) * staticBlockReward
+	}
+	return unclesReward
+}
+
+func CalcBlockReward(block core.Block, uncles []*types.Header) float64 {
+	blockNumber := block.Number
+	staticBlockReward := staticRewardByBlockNumber(blockNumber)
+	transactionFees := calcTransactionFees(block)
+	uncleInclusionRewards := calcUncleInclusionRewards(block, uncles)
+	return transactionFees + uncleInclusionRewards + staticBlockReward
+}
+
+func calcTransactionFees(block core.Block) float64 {
+	var transactionFees float64
+	for _, transaction := range block.Transactions {
+		receipt := transaction.Receipt
+		transactionFees += float64(transaction.GasPrice * receipt.GasUsed)
+	}
+	return transactionFees / params.Ether
+}
+
+func calcUncleInclusionRewards(block core.Block, uncles []*types.Header) float64 {
+	var uncleInclusionRewards float64
+	staticBlockReward := staticRewardByBlockNumber(block.Number)
+	for range uncles {
+		uncleInclusionRewards += staticBlockReward * 1 / 32
+	}
+	return uncleInclusionRewards
+}
+
+func staticRewardByBlockNumber(blockNumber int64) float64 {
+	var staticBlockReward float64
+	//https://blog.ethereum.org/2017/10/12/byzantium-hf-announcement/
+	if blockNumber >= 4370000 {
+		staticBlockReward = 3
+	} else {
+		staticBlockReward = 5
+	}
+	return staticBlockReward
+}

pkg/geth/block_to_core_block.go

@@ -17,11 +17,9 @@ type GethClient interface {
 	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
 }
 
-func GethBlockToCoreBlock(gethBlock *types.Block, client GethClient) core.Block {
-	transactions := convertGethTransactionsToCore(gethBlock, client)
-	blockReward := CalcBlockReward(gethBlock, client)
-	uncleReward := CalcUnclesReward(gethBlock)
-	return core.Block{
+func ToCoreBlock(gethBlock *types.Block, client GethClient) core.Block {
+	transactions := convertTransactionsToCore(gethBlock, client)
+	coreBlock := core.Block{
 		Difficulty:   gethBlock.Difficulty().Int64(),
 		ExtraData:    hexutil.Encode(gethBlock.Extra()),
 		GasLimit:     gethBlock.GasLimit().Int64(),
@@ -31,23 +29,24 @@ func GethBlockToCoreBlock(gethBlock *types.Block, client GethClient) core.Block
 		Nonce:        hexutil.Encode(gethBlock.Header().Nonce[:]),
 		Number:       gethBlock.Number().Int64(),
 		ParentHash:   gethBlock.ParentHash().Hex(),
-		Reward:       blockReward,
 		Size:         gethBlock.Size().Int64(),
 		Time:         gethBlock.Time().Int64(),
 		Transactions: transactions,
 		UncleHash:    gethBlock.UncleHash().Hex(),
-		UnclesReward: uncleReward,
 	}
+	coreBlock.Reward = CalcBlockReward(coreBlock, gethBlock.Uncles())
+	coreBlock.UnclesReward = CalcUnclesReward(coreBlock, gethBlock.Uncles())
+	return coreBlock
 }
 
-func convertGethTransactionsToCore(gethBlock *types.Block, client GethClient) []core.Transaction {
+func convertTransactionsToCore(gethBlock *types.Block, client GethClient) []core.Transaction {
 	transactions := make([]core.Transaction, 0)
 	for i, gethTransaction := range gethBlock.Transactions() {
 		from, err := client.TransactionSender(context.Background(), gethTransaction, gethBlock.Hash(), uint(i))
 		if err != nil {
 			log.Println(err)
 		}
-		transaction := gethTransToCoreTrans(gethTransaction, &from)
+		transaction := transToCoreTrans(gethTransaction, &from)
 		transaction, err = appendReceiptToTransaction(client, transaction)
 		if err != nil {
 			log.Println(err)
@@ -59,12 +58,12 @@ func convertGethTransactionsToCore(gethBlock *types.Block, client GethClient) []
 
 func appendReceiptToTransaction(client GethClient, transaction core.Transaction) (core.Transaction, error) {
 	gethReceipt, err := client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash))
-	receipt := GethReceiptToCoreReceipt(gethReceipt)
+	receipt := ReceiptToCoreReceipt(gethReceipt)
 	transaction.Receipt = receipt
 	return transaction, err
 }
 
-func gethTransToCoreTrans(transaction *types.Transaction, from *common.Address) core.Transaction {
+func transToCoreTrans(transaction *types.Transaction, from *common.Address) core.Transaction {
 	data := hexutil.Encode(transaction.Data())
 	return core.Transaction{
 		Hash:     transaction.Hash().Hex(),

pkg/geth/block_to_core_block_test.go

@@ -66,7 +66,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 		}
 		block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
 		client := &FakeGethClient{}
-		gethBlock := geth.GethBlockToCoreBlock(block, client)
+		gethBlock := geth.ToCoreBlock(block, client)
 
 		Expect(gethBlock.Difficulty).To(Equal(difficulty.Int64()))
 		Expect(gethBlock.GasLimit).To(Equal(gasLimit))
@@ -85,6 +85,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 
 	Describe("The block and uncle rewards calculations", func() {
 		It("calculates block rewards for a block", func() {
+
 			transaction := types.NewTransaction(
 				uint64(226823),
 				common.HexToAddress("0x108fedb097c1dcfed441480170144d8e19bb217f"),
@@ -96,20 +97,25 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			transactions := []*types.Transaction{transaction}
 
 			txHash := transaction.Hash()
-			receipt := types.Receipt{TxHash: txHash, GasUsed: big.NewInt(21000)}
+			receipt := types.Receipt{
+				TxHash:            txHash,
+				GasUsed:           big.NewInt(21000),
+				CumulativeGasUsed: big.NewInt(21000),
+			}
 			receipts := []*types.Receipt{&receipt}
 
+			client := NewFakeClient()
+			client.AddReceipts(receipts)
+
 			number := int64(1071819)
 			header := types.Header{
 				Number: big.NewInt(number),
 			}
 			uncles := []*types.Header{{Number: big.NewInt(1071817)}, {Number: big.NewInt(1071818)}}
-			block := types.NewBlock(&header, transactions, uncles, []*types.Receipt{})
+			block := types.NewBlock(&header, transactions, uncles, []*types.Receipt{&receipt})
+			coreBlock := geth.ToCoreBlock(block, client)
 
-			client := NewFakeClient()
-			client.AddReceipts(receipts)
-
-			Expect(geth.CalcBlockReward(block, client)).To(Equal(5.31355))
+			Expect(geth.CalcBlockReward(coreBlock, block.Uncles())).To(Equal(5.31355))
 		})
 
 		It("calculates the uncles reward for a block", func() {
@@ -123,8 +129,9 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			transactions := []*types.Transaction{transaction}
 
 			receipt := types.Receipt{
-				TxHash:  transaction.Hash(),
-				GasUsed: big.NewInt(21000),
+				TxHash:            transaction.Hash(),
+				GasUsed:           big.NewInt(21000),
+				CumulativeGasUsed: big.NewInt(21000),
 			}
 			receipts := []*types.Receipt{&receipt}
 
@@ -135,12 +142,14 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 				{Number: big.NewInt(1071816)},
 				{Number: big.NewInt(1071817)},
 			}
-			block := types.NewBlock(&header, transactions, uncles, []*types.Receipt{})
+			block := types.NewBlock(&header, transactions, uncles, receipts)
 
 			client := NewFakeClient()
 			client.AddReceipts(receipts)
 
-			Expect(geth.CalcUnclesReward(block)).To(Equal(6.875))
+			coreBlock := geth.ToCoreBlock(block, client)
+
+			Expect(geth.CalcUnclesReward(coreBlock, block.Uncles())).To(Equal(6.875))
 		})
 
 		It("decreases the static block reward from 5 to 3 for blocks after block 4,269,999", func() {
@@ -163,12 +172,14 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			transactions := []*types.Transaction{transactionOne, transactionTwo}
 
 			receiptOne := types.Receipt{
-				TxHash:  transactionOne.Hash(),
-				GasUsed: big.NewInt(297508),
+				TxHash:            transactionOne.Hash(),
+				GasUsed:           big.NewInt(297508),
+				CumulativeGasUsed: big.NewInt(0),
 			}
 			receiptTwo := types.Receipt{
-				TxHash:  transactionTwo.Hash(),
-				GasUsed: big.NewInt(297508),
+				TxHash:            transactionTwo.Hash(),
+				GasUsed:           big.NewInt(297508),
+				CumulativeGasUsed: big.NewInt(0),
 			}
 			receipts := []*types.Receipt{&receiptOne, &receiptTwo}
 
@@ -181,8 +192,9 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 
 			client := NewFakeClient()
 			client.AddReceipts(receipts)
+			coreBlock := geth.ToCoreBlock(block, client)
 
-			Expect(geth.CalcBlockReward(block, client)).To(Equal(3.024990672))
+			Expect(geth.CalcBlockReward(coreBlock, block.Uncles())).To(Equal(3.024990672))
 		})
 	})
 
@@ -191,7 +203,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			header := types.Header{}
 			block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
 			client := &FakeGethClient{}
-			coreBlock := geth.GethBlockToCoreBlock(block, client)
+			coreBlock := geth.ToCoreBlock(block, client)
 
 			Expect(len(coreBlock.Transactions)).To(Equal(0))
 		})
@@ -225,7 +237,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 				[]*types.Header{},
 				[]*types.Receipt{gethReceipt},
 			)
-			coreBlock := geth.GethBlockToCoreBlock(gethBlock, client)
+			coreBlock := geth.ToCoreBlock(gethBlock, client)
 
 			Expect(len(coreBlock.Transactions)).To(Equal(1))
 			coreTransaction := coreBlock.Transactions[0]
@@ -238,7 +250,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			Expect(coreTransaction.Nonce).To(Equal(gethTransaction.Nonce()))
 
 			coreReceipt := coreTransaction.Receipt
-			expectedReceipt := geth.GethReceiptToCoreReceipt(gethReceipt)
+			expectedReceipt := geth.ReceiptToCoreReceipt(gethReceipt)
 			Expect(coreReceipt).To(Equal(expectedReceipt))
 
 		})
@@ -269,13 +281,13 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 				[]*types.Receipt{gethReceipt},
 			)
 
-			coreBlock := geth.GethBlockToCoreBlock(gethBlock, client)
+			coreBlock := geth.ToCoreBlock(gethBlock, client)
 
 			coreTransaction := coreBlock.Transactions[0]
 			Expect(coreTransaction.To).To(Equal(""))
 
 			coreReceipt := coreTransaction.Receipt
-			expectedReceipt := geth.GethReceiptToCoreReceipt(gethReceipt)
+			expectedReceipt := geth.ReceiptToCoreReceipt(gethReceipt)
 			Expect(coreReceipt).To(Equal(expectedReceipt))
 		})
 	})

pkg/geth/blockchain.go

@@ -3,8 +3,6 @@ package geth
 import (
 	"math/big"
 
-	"log"
-
 	"github.com/8thlight/vulcanizedb/pkg/core"
 	"github.com/8thlight/vulcanizedb/pkg/geth/node"
 	"github.com/ethereum/go-ethereum"
@@ -15,7 +13,7 @@ import (
 	"golang.org/x/net/context"
 )
 
-type GethBlockchain struct {
+type Blockchain struct {
 	client              *ethclient.Client
 	readGethHeaders     chan *types.Header
 	outputBlocks        chan core.Block
@@ -23,7 +21,16 @@ type GethBlockchain struct {
 	node                core.Node
 }
 
-func (blockchain *GethBlockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) {
+func NewBlockchain(ipcPath string) *Blockchain {
+	blockchain := Blockchain{}
+	rpcClient, _ := rpc.Dial(ipcPath)
+	client := ethclient.NewClient(rpcClient)
+	blockchain.node = node.Retrieve(rpcClient)
+	blockchain.client = client
+	return &blockchain
+}
+
+func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) {
 	if endingBlockNumber == nil {
 		endingBlockNumber = startingBlockNumber
 	}
@@ -41,46 +48,16 @@ func (blockchain *GethBlockchain) GetLogs(contract core.Contract, startingBlockN
 	return logs, nil
 }
 
-func (blockchain *GethBlockchain) Node() core.Node {
+func (blockchain *Blockchain) Node() core.Node {
 	return blockchain.node
 }
 
-func (blockchain *GethBlockchain) GetBlockByNumber(blockNumber int64) core.Block {
+func (blockchain *Blockchain) GetBlockByNumber(blockNumber int64) core.Block {
 	gethBlock, _ := blockchain.client.BlockByNumber(context.Background(), big.NewInt(blockNumber))
-	return GethBlockToCoreBlock(gethBlock, blockchain.client)
+	return ToCoreBlock(gethBlock, blockchain.client)
 }
 
-func NewGethBlockchain(ipcPath string) *GethBlockchain {
-	blockchain := GethBlockchain{}
-	rpcClient, _ := rpc.Dial(ipcPath)
-	client := ethclient.NewClient(rpcClient)
-	blockchain.node = node.Retrieve(rpcClient)
-	blockchain.client = client
-	return &blockchain
-}
-
-func (blockchain *GethBlockchain) SubscribeToBlocks(blocks chan core.Block) {
-	blockchain.outputBlocks = blocks
-	log.Println("SubscribeToBlocks")
-	inputHeaders := make(chan *types.Header, 10)
-	myContext := context.Background()
-	blockchain.readGethHeaders = inputHeaders
-	subscription, _ := blockchain.client.SubscribeNewHead(myContext, inputHeaders)
-	blockchain.newHeadSubscription = subscription
-}
-
-func (blockchain *GethBlockchain) StartListening() {
-	for header := range blockchain.readGethHeaders {
-		block := blockchain.GetBlockByNumber(header.Number.Int64())
-		blockchain.outputBlocks <- block
-	}
-}
-
-func (blockchain *GethBlockchain) StopListening() {
-	blockchain.newHeadSubscription.Unsubscribe()
-}
-
-func (blockchain *GethBlockchain) LastBlock() *big.Int {
+func (blockchain *Blockchain) LastBlock() *big.Int {
 	block, _ := blockchain.client.HeaderByNumber(context.Background(), nil)
 	return block.Number
 }

pkg/geth/contract.go

@@ -17,7 +17,7 @@ var (
 	ErrInvalidStateAttribute = errors.New("invalid state attribute")
 )
 
-func (blockchain *GethBlockchain) GetAttribute(contract core.Contract, attributeName string, blockNumber *big.Int) (interface{}, error) {
+func (blockchain *Blockchain) GetAttribute(contract core.Contract, attributeName string, blockNumber *big.Int) (interface{}, error) {
 	parsed, err := ParseAbi(contract.Abi)
 	var result interface{}
 	if err != nil {
@@ -38,13 +38,13 @@ func (blockchain *GethBlockchain) GetAttribute(contract core.Contract, attribute
 	return result, nil
 }
 
-func callContract(contractHash string, input []byte, blockchain *GethBlockchain, blockNumber *big.Int) ([]byte, error) {
+func callContract(contractHash string, input []byte, blockchain *Blockchain, blockNumber *big.Int) ([]byte, error) {
 	to := common.HexToAddress(contractHash)
 	msg := ethereum.CallMsg{To: &to, Data: input}
 	return blockchain.client.CallContract(context.Background(), msg, blockNumber)
 }
 
-func (blockchain *GethBlockchain) GetAttributes(contract core.Contract) (core.ContractAttributes, error) {
+func (blockchain *Blockchain) GetAttributes(contract core.Contract) (core.ContractAttributes, error) {
 	parsed, _ := ParseAbi(contract.Abi)
 	var contractAttributes core.ContractAttributes
 	for _, abiElement := range parsed.Methods {

pkg/geth/geth_block_rewards.go

@@ -1,58 +0,0 @@
-package geth
-
-import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/params"
-)
-
-func CalcUnclesReward(gethBlock *types.Block) float64 {
-	var unclesReward float64
-	for _, uncle := range gethBlock.Uncles() {
-		blockNumber := gethBlock.Number().Int64()
-		staticBlockReward := float64(staticRewardByBlockNumber(blockNumber))
-		unclesReward += (1.0 + float64(uncle.Number.Int64()-gethBlock.Number().Int64())/8.0) * staticBlockReward
-	}
-	return unclesReward
-}
-
-func CalcBlockReward(gethBlock *types.Block, client GethClient) float64 {
-	blockNumber := gethBlock.Number().Int64()
-	staticBlockReward := staticRewardByBlockNumber(blockNumber)
-	transactionFees := calcTransactionFees(gethBlock, client)
-	uncleInclusionRewards := calcUncleInclusionRewards(gethBlock)
-	return transactionFees + uncleInclusionRewards + staticBlockReward
-}
-
-func calcUncleInclusionRewards(gethBlock *types.Block) float64 {
-	var uncleInclusionRewards float64
-	staticBlockReward := staticRewardByBlockNumber(gethBlock.Number().Int64())
-	for range gethBlock.Uncles() {
-		uncleInclusionRewards += staticBlockReward * 1 / 32
-	}
-	return uncleInclusionRewards
-}
-
-func calcTransactionFees(gethBlock *types.Block, client GethClient) float64 {
-	var transactionFees float64
-	for _, transaction := range gethBlock.Transactions() {
-		receipt, err := client.TransactionReceipt(context.Background(), transaction.Hash())
-		if err != nil {
-			continue
-		}
-		transactionFees += float64(transaction.GasPrice().Int64() * receipt.GasUsed.Int64())
-	}
-	return transactionFees / params.Ether
-}
-
-func staticRewardByBlockNumber(blockNumber int64) float64 {
-	var staticBlockReward float64
-	//https://blog.ethereum.org/2017/10/12/byzantium-hf-announcement/
-	if blockNumber >= 4370000 {
-		staticBlockReward = 3
-	} else {
-		staticBlockReward = 5
-	}
-	return staticBlockReward
-}

pkg/geth/log_to_core_log.go

@@ -6,7 +6,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
-func GethLogToCoreLog(gethLog types.Log) core.Log {
+func LogToCoreLog(gethLog types.Log) core.Log {
 	topics := gethLog.Topics
 	var hexTopics = make(map[int]string)
 	for i, topic := range topics {
@@ -26,7 +26,7 @@ func GethLogToCoreLog(gethLog types.Log) core.Log {
 func GethLogsToCoreLogs(gethLogs []types.Log) []core.Log {
 	var logs []core.Log
 	for _, log := range gethLogs {
-		log := GethLogToCoreLog(log)
+		log := LogToCoreLog(log)
 		logs = append(logs, log)
 	}
 	return logs

pkg/geth/log_to_core_log_test.go

@@ -39,7 +39,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
 			},
 		}
 
-		coreLog := geth.GethLogToCoreLog(gethLog)
+		coreLog := geth.LogToCoreLog(gethLog)
 
 		Expect(coreLog.Address).To(Equal(expected.Address))
 		Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber))
@@ -79,8 +79,8 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
 			},
 		}
 
-		expectedOne := geth.GethLogToCoreLog(gethLogOne)
-		expectedTwo := geth.GethLogToCoreLog(gethLogTwo)
+		expectedOne := geth.LogToCoreLog(gethLogOne)
+		expectedTwo := geth.LogToCoreLog(gethLogTwo)
 
 		coreLogs := geth.GethLogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo})
 

pkg/geth/receipt_to_core_receipt.go

@@ -18,7 +18,7 @@ func BigTo64(n *big.Int) int64 {
 	return 0
 }
 
-func GethReceiptToCoreReceipt(gethReceipt *types.Receipt) core.Receipt {
+func ReceiptToCoreReceipt(gethReceipt *types.Receipt) core.Receipt {
 	bloom := hexutil.Encode(gethReceipt.Bloom.Bytes())
 	var postState string
 	var status int
@@ -49,7 +49,7 @@ func setContractAddress(gethReceipt *types.Receipt) string {
 func dereferenceLogs(gethReceipt *types.Receipt) []core.Log {
 	logs := []core.Log{}
 	for _, log := range gethReceipt.Logs {
-		logs = append(logs, GethLogToCoreLog(*log))
+		logs = append(logs, LogToCoreLog(*log))
 	}
 	return logs
 }

pkg/geth/receipt_to_core_receipt_test.go

@@ -36,7 +36,7 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
 			TxHash:            receipt.TxHash.Hex(),
 		}
 
-		coreReceipt := geth.GethReceiptToCoreReceipt(&receipt)
+		coreReceipt := geth.ReceiptToCoreReceipt(&receipt)
 		Expect(coreReceipt.Bloom).To(Equal(expected.Bloom))
 		Expect(coreReceipt.ContractAddress).To(Equal(expected.ContractAddress))
 		Expect(coreReceipt.CumulativeGasUsed).To(Equal(expected.CumulativeGasUsed))
@@ -70,7 +70,7 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() {
 			TxHash:            receipt.TxHash.Hex(),
 		}
 
-		coreReceipt := geth.GethReceiptToCoreReceipt(&receipt)
+		coreReceipt := geth.ReceiptToCoreReceipt(&receipt)
 		Expect(coreReceipt.Bloom).To(Equal(expected.Bloom))
 		Expect(coreReceipt.ContractAddress).To(Equal(""))
 		Expect(coreReceipt.CumulativeGasUsed).To(Equal(expected.CumulativeGasUsed))

pkg/history/populate_blocks.go

@@ -1,48 +1,25 @@
 package history
 
 import (
+	"log"
+
 	"github.com/8thlight/vulcanizedb/pkg/core"
 	"github.com/8thlight/vulcanizedb/pkg/repositories"
 )
 
-type Window struct {
-	LowerBound     int
-	UpperBound     int
-	MaxBlockNumber int
-}
-
-func (window Window) Size() int {
-	return int(window.UpperBound - window.LowerBound)
-}
-
 func PopulateMissingBlocks(blockchain core.Blockchain, repository repositories.Repository, startingBlockNumber int64) int {
 	lastBlock := blockchain.LastBlock().Int64()
 	blockRange := repository.MissingBlockNumbers(startingBlockNumber, lastBlock-1)
-	updateBlockRange(blockchain, repository, blockRange)
+	log.SetPrefix("")
+	log.Printf("Backfilling %d blocks\n\n", len(blockRange))
+	RetrieveAndUpdateBlocks(blockchain, repository, blockRange)
 	return len(blockRange)
 }
 
-func UpdateBlocksWindow(blockchain core.Blockchain, repository repositories.Repository, windowSize int) Window {
-	maxBlockNumber := blockchain.LastBlock().Int64()
-	upperBound := maxBlockNumber - int64(1)
-	lowerBound := upperBound - int64(windowSize)
-	blockRange := MakeRange(lowerBound, upperBound)
-	updateBlockRange(blockchain, repository, blockRange)
-	return Window{int(lowerBound), int(upperBound), int(maxBlockNumber)}
-}
-
-func updateBlockRange(blockchain core.Blockchain, repository repositories.Repository, blockNumbers []int64) int {
+func RetrieveAndUpdateBlocks(blockchain core.Blockchain, repository repositories.Repository, blockNumbers []int64) int {
 	for _, blockNumber := range blockNumbers {
 		block := blockchain.GetBlockByNumber(blockNumber)
 		repository.CreateOrUpdateBlock(block)
 	}
 	return len(blockNumbers)
 }
-
-func MakeRange(min, max int64) []int64 {
-	a := make([]int64, max-min)
-	for i := range a {
-		a[i] = min + int64(i)
-	}
-	return a
-}

pkg/history/populate_blocks_test.go

@@ -66,30 +66,6 @@ var _ = Describe("Populating blocks", func() {
 		Expect(err).To(HaveOccurred())
 	})
 
-	It("updates the repository with a range of blocks w/in sliding window ", func() {
-		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
-			{Number: 1},
-			{Number: 2},
-			{Number: 3},
-			{Number: 4},
-			{Number: 5},
-		})
-		repository := repositories.NewInMemory()
-		repository.CreateOrUpdateBlock(blockchain.GetBlockByNumber(5))
-
-		history.UpdateBlocksWindow(blockchain, repository, 2)
-
-		Expect(repository.BlockCount()).To(Equal(3))
-		Expect(repository.HandleBlockCallCount).To(Equal(3))
-	})
-
-	It("Generates a range of int64", func() {
-		numberOfBlocksCreated := history.MakeRange(0, 5)
-		expected := []int64{0, 1, 2, 3, 4}
-
-		Expect(numberOfBlocksCreated).To(Equal(expected))
-	})
-
 	It("returns the number of blocks created", func() {
 		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
 			{Number: 4},
@@ -105,19 +81,19 @@ var _ = Describe("Populating blocks", func() {
 		Expect(numberOfBlocksCreated).To(Equal(2))
 	})
 
-	It("returns the window size", func() {
-		window := history.Window{1, 3, 10}
-		Expect(window.Size()).To(Equal(2))
-	})
-
-	It("returns the number of largest block", func() {
+	It("updates the repository with a range of blocks w/in the range ", func() {
 		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
 			{Number: 1},
 			{Number: 2},
 			{Number: 3},
+			{Number: 4},
+			{Number: 5},
 		})
-		maxBlockNumber := blockchain.LastBlock()
+		repository := repositories.NewInMemory()
 
-		Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
+		history.RetrieveAndUpdateBlocks(blockchain, repository, history.MakeRange(2, 5))
+		Expect(repository.BlockCount()).To(Equal(3))
+		Expect(repository.CreateOrUpdateBlockCallCount).To(Equal(3))
 	})
+
 })

pkg/history/validate_blocks.go

@@ -0,0 +1,68 @@
+package history
+
+import (
+	"io"
+	"text/template"
+
+	"github.com/8thlight/vulcanizedb/pkg/core"
+	"github.com/8thlight/vulcanizedb/pkg/repositories"
+)
+
+const WindowTemplate = `Validating Blocks
+|{{.LowerBound}}|-- Validation Window --|{{.UpperBound}}| ({{.UpperBound}}:HEAD)
+
+`
+
+var ParsedWindowTemplate = *template.Must(template.New("window").Parse(WindowTemplate))
+
+type BlockValidator struct {
+	blockchain            core.Blockchain
+	repository            repositories.Repository
+	windowSize            int
+	parsedLoggingTemplate template.Template
+}
+
+func NewBlockValidator(blockchain core.Blockchain, repository repositories.Repository, windowSize int) *BlockValidator {
+	return &BlockValidator{
+		blockchain,
+		repository,
+		windowSize,
+		ParsedWindowTemplate,
+	}
+}
+
+func (bv BlockValidator) ValidateBlocks() ValidationWindow {
+	window := MakeValidationWindow(bv.blockchain, bv.windowSize)
+	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
+	RetrieveAndUpdateBlocks(bv.blockchain, bv.repository, blockNumbers)
+	lastBlock := bv.blockchain.LastBlock().Int64()
+	bv.repository.SetBlocksStatus(lastBlock)
+	return window
+}
+
+func (bv BlockValidator) Log(out io.Writer, window ValidationWindow) {
+	bv.parsedLoggingTemplate.Execute(out, window)
+}
+
+type ValidationWindow struct {
+	LowerBound int64
+	UpperBound int64
+}
+
+func (window ValidationWindow) Size() int {
+	return int(window.UpperBound - window.LowerBound)
+}
+
+func MakeValidationWindow(blockchain core.Blockchain, windowSize int) ValidationWindow {
+	upperBound := blockchain.LastBlock().Int64()
+	lowerBound := upperBound - int64(windowSize)
+	return ValidationWindow{lowerBound, upperBound}
+}
+
+func MakeRange(min, max int64) []int64 {
+	a := make([]int64, max-min)
+	for i := range a {
+		a[i] = min + int64(i)
+	}
+	return a
+}

pkg/history/validate_blocks_test.go

@@ -0,0 +1,90 @@
+package history_test
+
+import (
+	"bytes"
+
+	"io/ioutil"
+	"log"
+
+	"github.com/8thlight/vulcanizedb/pkg/core"
+	"github.com/8thlight/vulcanizedb/pkg/fakes"
+	"github.com/8thlight/vulcanizedb/pkg/history"
+	"github.com/8thlight/vulcanizedb/pkg/repositories"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func init() {
+	log.SetOutput(ioutil.Discard)
+}
+
+var _ = Describe("Blocks validator", func() {
+
+	It("creates a ValidationWindow equal to (HEAD-windowSize, HEAD)", func() {
+		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+			{Number: 1},
+			{Number: 2},
+			{Number: 3},
+			{Number: 4},
+			{Number: 5},
+		})
+
+		validationWindow := history.MakeValidationWindow(blockchain, 2)
+
+		Expect(validationWindow.LowerBound).To(Equal(int64(3)))
+		Expect(validationWindow.UpperBound).To(Equal(int64(5)))
+	})
+
+	It("returns the window size", func() {
+		window := history.ValidationWindow{1, 3}
+		Expect(window.Size()).To(Equal(2))
+	})
+
+	It("calls create or update for all blocks within the window", func() {
+		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+			{Number: 4},
+			{Number: 5},
+			{Number: 6},
+			{Number: 7},
+		})
+		repository := repositories.NewInMemory()
+
+		validator := history.NewBlockValidator(blockchain, repository, 2)
+		window := validator.ValidateBlocks()
+		Expect(window).To(Equal(history.ValidationWindow{5, 7}))
+		Expect(repository.BlockCount()).To(Equal(2))
+		Expect(repository.CreateOrUpdateBlockCallCount).To(Equal(2))
+	})
+
+	It("logs window message", func() {
+		expectedMessage := &bytes.Buffer{}
+		window := history.ValidationWindow{5, 7}
+		history.ParsedWindowTemplate.Execute(expectedMessage, history.ValidationWindow{5, 7})
+
+		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{})
+		repository := repositories.NewInMemory()
+		validator := history.NewBlockValidator(blockchain, repository, 2)
+		actualMessage := &bytes.Buffer{}
+		validator.Log(actualMessage, window)
+		Expect(actualMessage).To(Equal(expectedMessage))
+	})
+
+	It("generates a range of int64s", func() {
+		numberOfBlocksCreated := history.MakeRange(0, 5)
+		expected := []int64{0, 1, 2, 3, 4}
+
+		Expect(numberOfBlocksCreated).To(Equal(expected))
+	})
+
+	It("returns the number of largest block", func() {
+		blockchain := fakes.NewBlockchainWithBlocks([]core.Block{
+			{Number: 1},
+			{Number: 2},
+			{Number: 3},
+		})
+		maxBlockNumber := blockchain.LastBlock()
+
+		Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
+	})
+
+})

pkg/observers/blockchain_db_observer.go

@@ -1,18 +0,0 @@
-package observers
-
-import (
-	"github.com/8thlight/vulcanizedb/pkg/core"
-	"github.com/8thlight/vulcanizedb/pkg/repositories"
-)
-
-type BlockchainDbObserver struct {
-	repository repositories.Repository
-}
-
-func NewBlockchainDbObserver(repository repositories.Repository) BlockchainDbObserver {
-	return BlockchainDbObserver{repository: repository}
-}
-
-func (observer BlockchainDbObserver) NotifyBlockAdded(block core.Block) {
-	observer.repository.CreateOrUpdateBlock(block)
-}

pkg/observers/blockchain_db_observer_test.go

@@ -1,38 +0,0 @@
-package observers_test
-
-import (
-	"github.com/8thlight/vulcanizedb/pkg/core"
-	"github.com/8thlight/vulcanizedb/pkg/observers"
-	"github.com/8thlight/vulcanizedb/pkg/repositories"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Saving blocks to the database", func() {
-
-	var repository *repositories.InMemory
-
-	BeforeEach(func() {
-		repository = repositories.NewInMemory()
-	})
-
-	It("implements the observer interface", func() {
-		var observer core.BlockchainObserver = observers.NewBlockchainDbObserver(repository)
-		Expect(observer).NotTo(BeNil())
-	})
-
-	It("saves a block with one transaction", func() {
-		block := core.Block{
-			Number:       123,
-			Transactions: []core.Transaction{{}},
-		}
-
-		observer := observers.NewBlockchainDbObserver(repository)
-		observer.NotifyBlockAdded(block)
-
-		savedBlock, err := repository.FindBlockByNumber(123)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(len(savedBlock.Transactions)).To(Equal(1))
-	})
-
-})

pkg/observers/blockchain_logging_observer.go

@@ -1,32 +0,0 @@
-package observers
-
-import (
-	"os"
-	"text/template"
-
-	"time"
-
-	"github.com/8thlight/vulcanizedb/pkg/core"
-)
-
-const blockAddedTemplate = `
-    New block was added: {{.Number}}
-    Time: {{.Time | unix_time}}
-    Gas Limit: {{.GasLimit}}
-    Gas Used: {{.GasUsed}}
-    Number of Transactions {{.Transactions | len}}
-
-`
-
-var funcMap = template.FuncMap{
-	"unix_time": func(n int64) time.Time {
-		return time.Unix(n, 0)
-	},
-}
-var tmp = template.Must(template.New("window").Funcs(funcMap).Parse(blockAddedTemplate))
-
-type BlockchainLoggingObserver struct{}
-
-func (blockchainObserver BlockchainLoggingObserver) NotifyBlockAdded(block core.Block) {
-	tmp.Execute(os.Stdout, block)
-}

pkg/observers/observers_suite_test.go

@@ -1,13 +0,0 @@
-package observers_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"testing"
-)
-
-func TestObservers(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Observers Suite")
-}

pkg/repositories/in_memory.go

@@ -7,20 +7,20 @@ import (
 )
 
 type InMemory struct {
-	blocks               map[int64]core.Block
-	receipts             map[string]core.Receipt
-	contracts            map[string]core.Contract
-	logs                 map[string][]core.Log
-	HandleBlockCallCount int
+	blocks                       map[int64]core.Block
+	receipts                     map[string]core.Receipt
+	contracts                    map[string]core.Contract
+	logs                         map[string][]core.Log
+	CreateOrUpdateBlockCallCount int
 }
 
 func NewInMemory() *InMemory {
 	return &InMemory{
-		HandleBlockCallCount: 0,
-		blocks:               make(map[int64]core.Block),
-		receipts:             make(map[string]core.Receipt),
-		contracts:            make(map[string]core.Contract),
-		logs:                 make(map[string][]core.Log),
+		CreateOrUpdateBlockCallCount: 0,
+		blocks:    make(map[int64]core.Block),
+		receipts:  make(map[string]core.Receipt),
+		contracts: make(map[string]core.Contract),
+		logs:      make(map[string][]core.Log),
 	}
 }
 
@@ -98,7 +98,7 @@ func (repository *InMemory) MissingBlockNumbers(startingBlockNumber int64, endin
 }
 
 func (repository *InMemory) CreateOrUpdateBlock(block core.Block) error {
-	repository.HandleBlockCallCount++
+	repository.CreateOrUpdateBlockCallCount++
 	repository.blocks[block.Number] = block
 	for _, transaction := range block.Transactions {
 		repository.receipts[transaction.Hash] = transaction.Receipt

pkg/repositories/postgres.go

@@ -174,7 +174,7 @@ func (repository Postgres) MaxBlockNumber() int64 {
 }
 
 func (repository Postgres) MissingBlockNumbers(startingBlockNumber int64, highestBlockNumber int64) []int64 {
-	numbers := []int64{}
+	numbers := make([]int64, 0)
 	repository.Db.Select(&numbers,
 		`SELECT all_block_numbers
 			FROM (