Vdb 269- fetch logs by hash (#122)

* Upgrade geth from 1.8.15 to 1.8.18

* Update vat_tune to use shared repository methods

* Query blockchain by block hash instead of block number range

* Remove hash validation from repositories

* Fix vow flog integration test

* Update README Travis build sticker

* Update constants formatting per go fmt

* Update EthPublicKeyParser.ParsePublicKey to use discv5.PubkeyID method

* Address PR comments

pkg/transformers/shared/constants/label.go

@@ -1,34 +1,34 @@
 package constants
 
 var (
-	BiteLabel                = "bite"
-	CatFileChopLumpLabel     = "catFileChopLump"
-	CatFileFlipLabel         = "catFileFlip"
-	CatFilePitVowLabel       = "catFilePitVow"
-	DealLabel                = "deal"
-	DentLabel                = "dent"
-	DripDripLabel            = "dripDrip"
-	DripFileIlkLabel         = "dripFileIlk"
-	DripFileRepoLabel        = "dripFileRepo"
-	DripFileVowLabel         = "dripFileVow"
-	FlapKickLabel            = "flapKick"
-	FlipKickLabel            = "flipKick"
-	FlopKickLabel            = "flopKick"
-	FrobLabel                = "frob"
-	PitFileDebtCeilingLabel  = "pitFileDebtCeiling"
-	PitFileIlkLabel          = "pitFileIlk"
-	PriceFeedLabel           = "priceFeed"
-	TendLabel                = "tend"
-	VatFluxLabel             = "vatFlux"
-	VatFoldLabel             = "vatFold"
-	VatGrabLabel             = "vatGrab"
-	VatHealLabel             = "vatHeal"
-	VatInitLabel             = "vatInit"
-	VatMoveLabel             = "vatMove"
-	VatSlipLabel             = "vatSlip"
-	VatTollLabel             = "vatToll"
-	VatTuneLabel             = "vatTune"
-	VowFlogLabel             = "vowFlog"
+	BiteLabel               = "bite"
+	CatFileChopLumpLabel    = "catFileChopLump"
+	CatFileFlipLabel        = "catFileFlip"
+	CatFilePitVowLabel      = "catFilePitVow"
+	DealLabel               = "deal"
+	DentLabel               = "dent"
+	DripDripLabel           = "dripDrip"
+	DripFileIlkLabel        = "dripFileIlk"
+	DripFileRepoLabel       = "dripFileRepo"
+	DripFileVowLabel        = "dripFileVow"
+	FlapKickLabel           = "flapKick"
+	FlipKickLabel           = "flipKick"
+	FlopKickLabel           = "flopKick"
+	FrobLabel               = "frob"
+	PitFileDebtCeilingLabel = "pitFileDebtCeiling"
+	PitFileIlkLabel         = "pitFileIlk"
+	PriceFeedLabel          = "priceFeed"
+	TendLabel               = "tend"
+	VatFluxLabel            = "vatFlux"
+	VatFoldLabel            = "vatFold"
+	VatGrabLabel            = "vatGrab"
+	VatHealLabel            = "vatHeal"
+	VatInitLabel            = "vatInit"
+	VatMoveLabel            = "vatMove"
+	VatSlipLabel            = "vatSlip"
+	VatTollLabel            = "vatToll"
+	VatTuneLabel            = "vatTune"
+	VowFlogLabel            = "vowFlog"
 )
 
 func AllTransformerLabels() []string {

pkg/transformers/shared/log_fetcher.go

@@ -15,8 +15,6 @@
 package shared
 
 import (
-	"math/big"
-
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -25,7 +23,7 @@ import (
 )
 
 type LogFetcher interface {
-	FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error)
+	FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error)
 }
 
 type SettableLogFetcher interface {
@@ -47,12 +45,11 @@ func NewFetcher(blockchain core.BlockChain) Fetcher {
 	}
 }
 
-func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) {
-	block := big.NewInt(blockNumber)
+func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
 	addresses := hexStringsToAddresses(contractAddresses)
+	blockHash := common.HexToHash(header.Hash)
 	query := ethereum.FilterQuery{
-		FromBlock: block,
-		ToBlock:   block,
+		BlockHash: &blockHash,
 		Addresses: addresses,
 		Topics:    topics,
 	}

pkg/transformers/shared/log_fetcher_test.go

@@ -15,13 +15,12 @@
 package shared_test
 
 import (
-	"math/big"
-
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
@@ -31,18 +30,20 @@ var _ = Describe("Fetcher", func() {
 		It("fetches logs based on the given query", func() {
 			blockChain := fakes.NewMockBlockChain()
 			fetcher := shared.NewFetcher(blockChain)
-			blockNumber := int64(123)
+			header := fakes.FakeHeader
+
 			addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"}
 			topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}}
 
-			_, err := fetcher.FetchLogs(addresses, topicZeros, blockNumber)
+			_, err := fetcher.FetchLogs(addresses, topicZeros, header)
 
 			address1 := common.HexToAddress("0xfakeAddress")
 			address2 := common.HexToAddress("0xanotherFakeAddress")
 			Expect(err).NotTo(HaveOccurred())
+
+			blockHash := common.HexToHash(header.Hash)
 			expectedQuery := ethereum.FilterQuery{
-				FromBlock: big.NewInt(blockNumber),
-				ToBlock:   big.NewInt(blockNumber),
+				BlockHash: &blockHash,
 				Addresses: []common.Address{address1, address2},
 				Topics:    topicZeros,
 			}
@@ -54,7 +55,7 @@ var _ = Describe("Fetcher", func() {
 			blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
 			fetcher := shared.NewFetcher(blockChain)
 
-			_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, int64(1))
+			_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(fakes.FakeError))

pkg/transformers/shared/repository.go

@@ -2,41 +2,10 @@ package shared
 
 import (
 	"database/sql"
-	"encoding/json"
-	"errors"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-var ErrHeaderMismatch = errors.New("log's block hash does not match associated header's block hash")
-
-func ValidateHeaderConsistency(headerID int64, rawLog []byte, db *postgres.DB) error {
-	var log types.Log
-	err := json.Unmarshal(rawLog, &log)
-	if err != nil {
-		return err
-	}
-	var hash string
-	err = db.Get(&hash, `SELECT hash FROM public.headers WHERE id = $1`, headerID)
-	if err != nil {
-		return err
-	}
-	if hash != log.BlockHash.String() {
-		err = deleteHeader(headerID, db)
-		if err != nil {
-			return err
-		}
-		return ErrHeaderMismatch
-	}
-	return nil
-}
-
-func deleteHeader(headerID int64, db *postgres.DB) error {
-	_, err := db.Exec(`DELETE FROM public.headers WHERE id = $1`, headerID)
-	return err
-}
-
 func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
 	_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
 		VALUES ($1, $2) 
@@ -59,14 +28,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
 	var err error
 
 	if endingBlockNumber == -1 {
-		query = `SELECT headers.id, headers.block_number FROM headers
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
 				LEFT JOIN checked_headers on headers.id = header_id
 				WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
 				AND headers.block_number >= $1
 				AND headers.eth_node_fingerprint = $2`
 		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
 	} else {
-		query = `SELECT headers.id, headers.block_number FROM headers
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
 				LEFT JOIN checked_headers on headers.id = header_id
 				WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
 				AND headers.block_number >= $1