Convert raw header to JSON

- Enables parsing values off of header via Postgres migration

db/migrations/1531758090_create_headers_table.up.sql

@@ -2,7 +2,7 @@ CREATE TABLE public.headers (
   id                    SERIAL PRIMARY KEY,
   hash                  VARCHAR(66),
   block_number          BIGINT,
-  raw                   bytea,
+  raw                   JSONB,
   eth_node_id           INTEGER,
   eth_node_fingerprint  VARCHAR(128),
   CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id)

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 10.5
+-- Dumped from database version 10.3
--- Dumped by pg_dump version 10.5
+-- Dumped by pg_dump version 10.3
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -1097,7 +1097,7 @@ CREATE TABLE public.headers (
     id integer NOT NULL,
     hash character varying(66),
     block_number bigint,
-    raw bytea,
+    raw jsonb,
     eth_node_id integer,
     eth_node_fingerprint character varying(128)
 );

pkg/datastore/postgres/repositories/header_repository_test.go

@@ -2,7 +2,9 @@ package repositories_test
 
 import (
 	"database/sql"
+	"encoding/json"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -12,6 +14,14 @@ import (
 )
 
 var _ = Describe("Block header repository", func() {
+	var rawHeader []byte
+	var err error
+
+	BeforeEach(func() {
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("creating or updating a header", func() {
 		It("adds a header", func() {
 			node := core.Node{}
@@ -21,7 +31,7 @@ var _ = Describe("Block header repository", func() {
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
 			}
 
 			_, err := repo.CreateOrUpdateHeader(header)
@@ -32,7 +42,7 @@ var _ = Describe("Block header repository", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
 			Expect(dbHeader.Hash).To(Equal(header.Hash))
-			Expect(dbHeader.Raw).To(Equal(header.Raw))
+			Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
 		})
 
 		It("adds node data to header", func() {
@@ -40,7 +50,10 @@ var _ = Describe("Block header repository", func() {
 			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
-			header := core.Header{BlockNumber: 100}
+			header := core.Header{
+				BlockNumber: 100,
+				Raw:         rawHeader,
+			}
 
 			_, err := repo.CreateOrUpdateHeader(header)
 
@@ -63,7 +76,7 @@ var _ = Describe("Block header repository", func() {
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
 			}
 
 			_, err := repo.CreateOrUpdateHeader(header)
@@ -87,14 +100,14 @@ var _ = Describe("Block header repository", func() {
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
 			headerTwo := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
-				Raw:         []byte{5, 4, 3, 2, 1},
+				Raw:         rawHeader,
 			}
 
 			_, err = repo.CreateOrUpdateHeader(headerTwo)
@@ -104,7 +117,7 @@ var _ = Describe("Block header repository", func() {
 			err = db.Get(&dbHeader, `SELECT block_number, hash, raw FROM headers WHERE block_number = $1`, header.BlockNumber)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbHeader.Hash).To(Equal(headerTwo.Hash))
-			Expect(dbHeader.Raw).To(Equal(headerTwo.Raw))
+			Expect(dbHeader.Raw).To(MatchJSON(headerTwo.Raw))
 		})
 
 		It("does not replace header if node fingerprint is different", func() {
@@ -115,7 +128,7 @@ var _ = Describe("Block header repository", func() {
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			nodeTwo := core.Node{ID: "FingerprintTwo"}
@@ -125,7 +138,7 @@ var _ = Describe("Block header repository", func() {
 			headerTwo := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
-				Raw:         []byte{5, 4, 3, 2, 1},
+				Raw:         rawHeader,
 			}
 
 			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
@@ -144,7 +157,7 @@ var _ = Describe("Block header repository", func() {
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			nodeTwo := core.Node{ID: "FingerprintTwo"}
@@ -154,13 +167,13 @@ var _ = Describe("Block header repository", func() {
 			headerTwo := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{5, 4, 3, 2, 1}).Hex(),
-				Raw:         []byte{5, 4, 3, 2, 1},
+				Raw:         rawHeader,
 			}
 			_, err = repoTwo.CreateOrUpdateHeader(headerTwo)
 			headerThree := core.Header{
 				BlockNumber: header.BlockNumber,
 				Hash:        common.BytesToHash([]byte{1, 1, 1, 1, 1}).Hex(),
-				Raw:         []byte{1, 1, 1, 1, 1},
+				Raw:         rawHeader,
 			}
 
 			_, err = repoTwo.CreateOrUpdateHeader(headerThree)
@@ -172,8 +185,8 @@ var _ = Describe("Block header repository", func() {
 			Expect(len(dbHeaders)).To(Equal(2))
 			Expect(dbHeaders[0].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
 			Expect(dbHeaders[1].Hash).To(Or(Equal(header.Hash), Equal(headerThree.Hash)))
-			Expect(dbHeaders[0].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+			Expect(dbHeaders[0].Raw).To(Or(MatchJSON(header.Raw), MatchJSON(headerThree.Raw)))
-			Expect(dbHeaders[1].Raw).To(Or(Equal(header.Raw), Equal(headerThree.Raw)))
+			Expect(dbHeaders[1].Raw).To(Or(MatchJSON(header.Raw), MatchJSON(headerThree.Raw)))
 		})
 	})
 
@@ -186,7 +199,7 @@ var _ = Describe("Block header repository", func() {
 			header := core.Header{
 				BlockNumber: 100,
 				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
@@ -194,7 +207,9 @@ var _ = Describe("Block header repository", func() {
 			dbHeader, err := repo.GetHeader(header.BlockNumber)
 
 			Expect(err).NotTo(HaveOccurred())
-			Expect(dbHeader).To(Equal(header))
+			Expect(dbHeader.BlockNumber).To(Equal(header.BlockNumber))
+			Expect(dbHeader.Hash).To(Equal(header.Hash))
+			Expect(dbHeader.Raw).To(MatchJSON(header.Raw))
 		})
 
 		It("does not return header for a different node fingerprint", func() {
@@ -204,8 +219,8 @@ var _ = Describe("Block header repository", func() {
 			repo := repositories.NewHeaderRepository(db)
 			header := core.Header{
 				BlockNumber: 100,
-				Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
+				Hash:        common.BytesToHash(rawHeader).Hex(),
-				Raw:         []byte{1, 2, 3, 4, 5},
+				Raw:         rawHeader,
 			}
 			_, err := repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
@@ -227,9 +242,18 @@ var _ = Describe("Block header repository", func() {
 			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+				BlockNumber: 1,
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+				Raw:         rawHeader,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 3,
+				Raw:         rawHeader,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 5,
+				Raw:         rawHeader,
+			})
 
 			missingBlockNumbers := repo.MissingBlockNumbers(1, 5, node.ID)
 
@@ -241,9 +265,18 @@ var _ = Describe("Block header repository", func() {
 			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			repo := repositories.NewHeaderRepository(db)
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 1})
+			repo.CreateOrUpdateHeader(core.Header{
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 3})
+				BlockNumber: 1,
-			repo.CreateOrUpdateHeader(core.Header{BlockNumber: 5})
+				Raw:         rawHeader,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 3,
+				Raw:         rawHeader,
+			})
+			repo.CreateOrUpdateHeader(core.Header{
+				BlockNumber: 5,
+				Raw:         rawHeader,
+			})
 			nodeTwo := core.Node{ID: "NodeFingerprintTwo"}
 			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/bite/repository_test.go

@@ -15,11 +15,15 @@
 package bite_test
 
 import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"database/sql"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
@@ -28,27 +32,35 @@ import (
 )
 
 var _ = Describe("Bite repository", func() {
+	var (
+		biteRepository   bite.Repository
+		db               *postgres.DB
+		err              error
+		headerRepository datastore.HeaderRepository
+		rawHeader        []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+		biteRepository = bite.NewBiteRepository(db)
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			biteRepository bite.Repository
-			db             *postgres.DB
-			err            error
-			headerID       int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
 			Expect(err).NotTo(HaveOccurred())
-			biteRepository = bite.NewBiteRepository(db)
 		})
 
 		It("persists a bite record", func() {
-			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbBite bite.BiteModel
 			err = db.Get(&dbBite, `SELECT id, ilk, urn, ink, art, tab, flip, tx_idx, raw_log FROM maker.bite WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -63,9 +75,6 @@ var _ = Describe("Bite repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -73,9 +82,6 @@ var _ = Describe("Bite repository", func() {
 		})
 
 		It("does not duplicate bite events", func() {
-			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			var anotherBiteModel = bite.BiteModel{
 				Id:               "11",
 				Ilk:              test_data.BiteModel.Ilk,
@@ -96,9 +102,6 @@ var _ = Describe("Bite repository", func() {
 		})
 
 		It("removes bite if corresponding header is deleted", func() {
-			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -110,14 +113,14 @@ var _ = Describe("Bite repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		It("creates a row for a new headerID", func() {
+		var headerID int64
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			biteRepository := bite.NewBiteRepository(db)
 
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("creates a row for a new headerID", func() {
 			err = biteRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -128,12 +131,6 @@ var _ = Describe("Bite repository", func() {
 		})
 
 		It("updates row when headerID already exists", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			biteRepository := bite.NewBiteRepository(db)
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
 			err = biteRepository.MarkHeaderChecked(headerID)
@@ -147,31 +144,69 @@ var _ = Describe("Bite repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			node := core.Node{}
+			startingBlock, endingBlock, biteBlock int64
-			db := test_config.NewTestDB(node)
+			blockNumbers, headerIDs               []int64
-			test_config.CleanTestDB(db)
+		)
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			startingBlockNumber := int64(1)
+		BeforeEach(func() {
-			biteBlockNumber := int64(2)
+			startingBlock = GinkgoRandomSeed()
-			endingBlockNumber := int64(3)
+			biteBlock = startingBlock + 1
-			blockNumbers := []int64{startingBlockNumber, biteBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			endingBlock = startingBlock + 2
-			var headerIDs []int64
+
+			blockNumbers = []int64{startingBlock, biteBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			biteRepository := bite.NewBiteRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := biteRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := biteRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := biteRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
+		})
+
+		It("only treats headers as checked if bite logs have been checked", func() {
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := biteRepository.MissingHeaders(startingBlock, endingBlock)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(biteBlock)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(biteBlock)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(biteBlock)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			err := biteRepository.MarkHeaderChecked(headerIDs[0])
+			Expect(err).NotTo(HaveOccurred())
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			for _, n := range blockNumbers {
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			biteRepositoryTwo := bite.NewBiteRepository(dbTwo)
+
+			nodeOneMissingHeaders, err := biteRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
+
+			nodeTwoMissingHeaders, err := biteRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
 		})
 	})
 })

pkg/transformers/cat_file/chop_lump/repository_test.go

@@ -16,11 +16,14 @@ package chop_lump_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/chop_lump"
@@ -29,39 +32,46 @@ import (
 )
 
 var _ = Describe("Cat file chop lump repository", func() {
+	var (
+		catFileRepository chop_lump.CatFileChopLumpRepository
+		db                *postgres.DB
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+		catFileRepository = chop_lump.NewCatFileChopLumpRepository(db)
+	})
+
 	Describe("Create", func() {
-		var catFileRepository chop_lump.CatFileChopLumpRepository
-		var db *postgres.DB
-		var err error
 		var headerID int64
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = catFileRepository.Create(headerID, []chop_lump.CatFileChopLumpModel{test_data.CatFileChopLumpModel})
 			Expect(err).NotTo(HaveOccurred())
-			catFileRepository = chop_lump.NewCatFileChopLumpRepository(db)
 		})
 
 		It("adds a cat file chop lump event", func() {
-			err = catFileRepository.Create(headerID, []chop_lump.CatFileChopLumpModel{test_data.CatFileChopLumpModel})
+			var dbResult chop_lump.CatFileChopLumpModel
-
+			err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile chop_lump.CatFileChopLumpModel
+			Expect(dbResult.Ilk).To(Equal(test_data.CatFileChopLumpModel.Ilk))
-			err = db.Get(&dbPitFile, `SELECT ilk, what, data, tx_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID)
+			Expect(dbResult.What).To(Equal(test_data.CatFileChopLumpModel.What))
-			Expect(err).NotTo(HaveOccurred())
+			Expect(dbResult.Data).To(Equal(test_data.CatFileChopLumpModel.Data))
-			Expect(dbPitFile.Ilk).To(Equal(test_data.CatFileChopLumpModel.Ilk))
+			Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileChopLumpModel.TransactionIndex))
-			Expect(dbPitFile.What).To(Equal(test_data.CatFileChopLumpModel.What))
+			Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileChopLumpModel.Raw))
-			Expect(dbPitFile.Data).To(Equal(test_data.CatFileChopLumpModel.Data))
-			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.CatFileChopLumpModel.TransactionIndex))
-			Expect(dbPitFile.Raw).To(MatchJSON(test_data.CatFileChopLumpModel.Raw))
 		})
 
 		It("marks header as checked for logs", func() {
-			err = catFileRepository.Create(headerID, []chop_lump.CatFileChopLumpModel{test_data.CatFileChopLumpModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT cat_file_chop_lump_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -69,9 +79,6 @@ var _ = Describe("Cat file chop lump repository", func() {
 		})
 
 		It("does not duplicate cat file chop lump events", func() {
-			err = catFileRepository.Create(headerID, []chop_lump.CatFileChopLumpModel{test_data.CatFileChopLumpModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = catFileRepository.Create(headerID, []chop_lump.CatFileChopLumpModel{test_data.CatFileChopLumpModel})
 
 			Expect(err).To(HaveOccurred())
@@ -79,28 +86,25 @@ var _ = Describe("Cat file chop lump repository", func() {
 		})
 
 		It("removes cat file chop lump if corresponding header is deleted", func() {
-			err = catFileRepository.Create(headerID, []chop_lump.CatFileChopLumpModel{test_data.CatFileChopLumpModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile chop_lump.CatFileChopLumpModel
+			var dbResult chop_lump.CatFileChopLumpModel
-			err = db.Get(&dbPitFile, `SELECT ilk, what, data, tx_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID)
+			err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID)
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		It("creates a row for a new headerID", func() {
+		var headerID int64
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			catFileRepository := chop_lump.NewCatFileChopLumpRepository(db)
 
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("creates a row for a new headerID", func() {
 			err = catFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -111,12 +115,6 @@ var _ = Describe("Cat file chop lump repository", func() {
 		})
 
 		It("updates row when headerID already exists", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			catFileRepository := chop_lump.NewCatFileChopLumpRepository(db)
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
 			err = catFileRepository.MarkHeaderChecked(headerID)
@@ -130,78 +128,61 @@ var _ = Describe("Cat file chop lump repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, catFileBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			catFileBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, catFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			catFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, catFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
 				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
 			}
-			catFileRepository := chop_lump.NewCatFileChopLumpRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := catFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := catFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := catFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if cat file chop lump logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			catFiledBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, catFiledBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			catFiledRepository := chop_lump.NewCatFileChopLumpRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := catFiledRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := catFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
-			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-			}
-			catFileRepository := chop_lump.NewCatFileChopLumpRepository(db)
-			catFileRepositoryTwo := chop_lump.NewCatFileChopLumpRepository(dbTwo)
 			err := catFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			for _, n := range blockNumbers {
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			catFileRepositoryTwo := chop_lump.NewCatFileChopLumpRepository(dbTwo)
 
 			nodeOneMissingHeaders, err := catFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/cat_file/flip/repository_test.go

@@ -16,11 +16,14 @@ package flip_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/flip"
@@ -29,39 +32,46 @@ import (
 )
 
 var _ = Describe("Cat file flip repository", func() {
+	var (
+		catFileRepository flip.CatFileFlipRepository
+		db                *postgres.DB
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+		catFileRepository = flip.NewCatFileFlipRepository(db)
+	})
+
 	Describe("Create", func() {
-		var catFileRepository flip.CatFileFlipRepository
-		var db *postgres.DB
-		var err error
 		var headerID int64
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = catFileRepository.Create(headerID, []flip.CatFileFlipModel{test_data.CatFileFlipModel})
 			Expect(err).NotTo(HaveOccurred())
-			catFileRepository = flip.NewCatFileFlipRepository(db)
 		})
 
 		It("adds a cat file flip event", func() {
-			err = catFileRepository.Create(headerID, []flip.CatFileFlipModel{test_data.CatFileFlipModel})
+			var dbResult flip.CatFileFlipModel
-
+			err = db.Get(&dbResult, `SELECT ilk, what, flip, tx_idx, raw_log FROM maker.cat_file_flip WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile flip.CatFileFlipModel
+			Expect(dbResult.Ilk).To(Equal(test_data.CatFileFlipModel.Ilk))
-			err = db.Get(&dbPitFile, `SELECT ilk, what, flip, tx_idx, raw_log FROM maker.cat_file_flip WHERE header_id = $1`, headerID)
+			Expect(dbResult.What).To(Equal(test_data.CatFileFlipModel.What))
-			Expect(err).NotTo(HaveOccurred())
+			Expect(dbResult.Flip).To(Equal(test_data.CatFileFlipModel.Flip))
-			Expect(dbPitFile.Ilk).To(Equal(test_data.CatFileFlipModel.Ilk))
+			Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileFlipModel.TransactionIndex))
-			Expect(dbPitFile.What).To(Equal(test_data.CatFileFlipModel.What))
+			Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileFlipModel.Raw))
-			Expect(dbPitFile.Flip).To(Equal(test_data.CatFileFlipModel.Flip))
-			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.CatFileFlipModel.TransactionIndex))
-			Expect(dbPitFile.Raw).To(MatchJSON(test_data.CatFileFlipModel.Raw))
 		})
 
 		It("marks header as checked for logs", func() {
-			err = catFileRepository.Create(headerID, []flip.CatFileFlipModel{test_data.CatFileFlipModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT cat_file_flip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -69,9 +79,6 @@ var _ = Describe("Cat file flip repository", func() {
 		})
 
 		It("does not duplicate cat file flip events", func() {
-			err = catFileRepository.Create(headerID, []flip.CatFileFlipModel{test_data.CatFileFlipModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = catFileRepository.Create(headerID, []flip.CatFileFlipModel{test_data.CatFileFlipModel})
 
 			Expect(err).To(HaveOccurred())
@@ -79,28 +86,25 @@ var _ = Describe("Cat file flip repository", func() {
 		})
 
 		It("removes cat file flip if corresponding header is deleted", func() {
-			err = catFileRepository.Create(headerID, []flip.CatFileFlipModel{test_data.CatFileFlipModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile flip.CatFileFlipModel
+			var dbResult flip.CatFileFlipModel
-			err = db.Get(&dbPitFile, `SELECT ilk, what, flip, tx_idx, raw_log FROM maker.cat_file_flip WHERE header_id = $1`, headerID)
+			err = db.Get(&dbResult, `SELECT ilk, what, flip, tx_idx, raw_log FROM maker.cat_file_flip WHERE header_id = $1`, headerID)
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		It("creates a row for a new headerID", func() {
+		var headerID int64
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			catFileRepository := flip.NewCatFileFlipRepository(db)
 
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("creates a row for a new headerID", func() {
 			err = catFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -111,12 +115,6 @@ var _ = Describe("Cat file flip repository", func() {
 		})
 
 		It("updates row when headerID already exists", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			catFileRepository := flip.NewCatFileFlipRepository(db)
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
 			err = catFileRepository.MarkHeaderChecked(headerID)
@@ -130,78 +128,61 @@ var _ = Describe("Cat file flip repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, catFileBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			catFileBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, catFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			catFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, catFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			catFileRepository := flip.NewCatFileFlipRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := catFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := catFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := catFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if cat file flip logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			catFiledBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, catFiledBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			catFiledRepository := flip.NewCatFileFlipRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := catFiledRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := catFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
-			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-			}
-			catFileRepository := flip.NewCatFileFlipRepository(db)
-			catFileRepositoryTwo := flip.NewCatFileFlipRepository(dbTwo)
 			err := catFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			for _, n := range blockNumbers {
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			catFileRepositoryTwo := flip.NewCatFileFlipRepository(dbTwo)
 
 			nodeOneMissingHeaders, err := catFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/cat_file/pit_vow/repository_test.go

@@ -16,11 +16,14 @@ package pit_vow_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/pit_vow"
@@ -29,38 +32,45 @@ import (
 )
 
 var _ = Describe("Cat file pit vow repository", func() {
+	var (
+		catFileRepository pit_vow.CatFilePitVowRepository
+		db                *postgres.DB
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+		catFileRepository = pit_vow.NewCatFilePitVowRepository(db)
+	})
+
 	Describe("Create", func() {
-		var catFileRepository pit_vow.CatFilePitVowRepository
-		var db *postgres.DB
-		var err error
 		var headerID int64
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = catFileRepository.Create(headerID, []pit_vow.CatFilePitVowModel{test_data.CatFilePitVowModel})
 			Expect(err).NotTo(HaveOccurred())
-			catFileRepository = pit_vow.NewCatFilePitVowRepository(db)
 		})
 
 		It("adds a cat file pit vow event", func() {
-			err = catFileRepository.Create(headerID, []pit_vow.CatFilePitVowModel{test_data.CatFilePitVowModel})
+			var dbResult pit_vow.CatFilePitVowModel
-
+			err = db.Get(&dbResult, `SELECT what, data, tx_idx, raw_log FROM maker.cat_file_pit_vow WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile pit_vow.CatFilePitVowModel
+			Expect(dbResult.What).To(Equal(test_data.CatFilePitVowModel.What))
-			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.cat_file_pit_vow WHERE header_id = $1`, headerID)
+			Expect(dbResult.Data).To(Equal(test_data.CatFilePitVowModel.Data))
-			Expect(err).NotTo(HaveOccurred())
+			Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFilePitVowModel.TransactionIndex))
-			Expect(dbPitFile.What).To(Equal(test_data.CatFilePitVowModel.What))
+			Expect(dbResult.Raw).To(MatchJSON(test_data.CatFilePitVowModel.Raw))
-			Expect(dbPitFile.Data).To(Equal(test_data.CatFilePitVowModel.Data))
-			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.CatFilePitVowModel.TransactionIndex))
-			Expect(dbPitFile.Raw).To(MatchJSON(test_data.CatFilePitVowModel.Raw))
 		})
 
 		It("marks header as checked for logs", func() {
-			err = catFileRepository.Create(headerID, []pit_vow.CatFilePitVowModel{test_data.CatFilePitVowModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT cat_file_pit_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -68,9 +78,6 @@ var _ = Describe("Cat file pit vow repository", func() {
 		})
 
 		It("does not duplicate cat file pit vow events", func() {
-			err = catFileRepository.Create(headerID, []pit_vow.CatFilePitVowModel{test_data.CatFilePitVowModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = catFileRepository.Create(headerID, []pit_vow.CatFilePitVowModel{test_data.CatFilePitVowModel})
 
 			Expect(err).To(HaveOccurred())
@@ -78,28 +85,25 @@ var _ = Describe("Cat file pit vow repository", func() {
 		})
 
 		It("removes cat file pit vow if corresponding header is deleted", func() {
-			err = catFileRepository.Create(headerID, []pit_vow.CatFilePitVowModel{test_data.CatFilePitVowModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile pit_vow.CatFilePitVowModel
+			var dbResult pit_vow.CatFilePitVowModel
-			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.cat_file_pit_vow WHERE header_id = $1`, headerID)
+			err = db.Get(&dbResult, `SELECT what, data, tx_idx, raw_log FROM maker.cat_file_pit_vow WHERE header_id = $1`, headerID)
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		It("creates a row for a new headerID", func() {
+		var headerID int64
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			catFileRepository := pit_vow.NewCatFilePitVowRepository(db)
 
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("creates a row for a new headerID", func() {
 			err = catFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -110,12 +114,6 @@ var _ = Describe("Cat file pit vow repository", func() {
 		})
 
 		It("updates row when headerID already exists", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			catFileRepository := pit_vow.NewCatFilePitVowRepository(db)
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
 			err = catFileRepository.MarkHeaderChecked(headerID)
@@ -129,78 +127,61 @@ var _ = Describe("Cat file pit vow repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, catFileBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			catFileBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, catFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			endingBlock = startingBlock + 2
-			var headerIDs []int64
+			catFileBlock = startingBlock + 1
+
+			blockNumbers = []int64{startingBlock, catFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
 				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
 			}
-			catFileRepository := pit_vow.NewCatFilePitVowRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := catFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := catFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := catFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if cat file pit vow logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			catFiledBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, catFiledBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			catFiledRepository := pit_vow.NewCatFilePitVowRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := catFiledRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := catFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(catFiledBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(catFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
-			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-			}
-			catFileRepository := pit_vow.NewCatFilePitVowRepository(db)
-			catFileRepositoryTwo := pit_vow.NewCatFilePitVowRepository(dbTwo)
 			err := catFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			for _, n := range blockNumbers {
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			catFileRepositoryTwo := pit_vow.NewCatFilePitVowRepository(dbTwo)
 
 			nodeOneMissingHeaders, err := catFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/deal/repository_test.go

@@ -15,8 +15,9 @@
 package deal_test
 
 import (
-	"math/rand"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,24 +30,28 @@ import (
 )
 
 var _ = Describe("Deal Repository", func() {
-	var node core.Node
+	var (
-	var db *postgres.DB
+		db               *postgres.DB
-	var dealRepository deal.DealRepository
+		dealRepository   deal.DealRepository
-	var headerRepository repositories.HeaderRepository
+		headerRepository repositories.HeaderRepository
-	var headerId int64
+		err              error
-	var err error
+		rawHeader        []byte
+	)
 
 	BeforeEach(func() {
-		node = test_config.NewTestNode()
+		db = test_config.NewTestDB(test_config.NewTestNode())
-		db = test_config.NewTestDB(node)
 		test_config.CleanTestDB(db)
 		dealRepository = deal.NewDealRepository(db)
 		headerRepository = repositories.NewHeaderRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Describe("Create", func() {
+		var headerId int64
+
 		BeforeEach(func() {
-			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 			err := dealRepository.Create(headerId, []deal.DealModel{test_data.DealModel})
 			Expect(err).NotTo(HaveOccurred())
@@ -92,8 +97,10 @@ var _ = Describe("Deal Repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
+		var headerId int64
+
 		BeforeEach(func() {
-			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
@@ -121,79 +128,78 @@ var _ = Describe("Deal Repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		var dealBlockNumber int64
+		var (
-		var startingBlockNumber int64
+			dealBlock, startingBlock, endingBlock int64
-		var endingBlockNumber int64
+			blockNumbers, headerIds               []int64
-		var blockNumbers []int64
+		)
 
 		BeforeEach(func() {
-			dealBlockNumber = rand.Int63()
+			dealBlock = GinkgoRandomSeed()
-			startingBlockNumber = dealBlockNumber - 1
+			startingBlock = dealBlock - 1
-			endingBlockNumber = dealBlockNumber + 1
+			endingBlock = dealBlock + 1
-			outOfRangeBlockNumber := dealBlockNumber + 2
+			outOfRangeBlockNumber := dealBlock + 2
-			blockNumbers = []int64{startingBlockNumber, dealBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
+
-			var headerIds []int64
+			blockNumbers = []int64{startingBlock, dealBlock, endingBlock, outOfRangeBlockNumber}
+
+			headerIds = []int64{}
 			for _, number := range blockNumbers {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 				headerIds = append(headerIds, headerId)
 			}
-			dealRepository.MarkHeaderChecked(headerIds[1])
 		})
 
 		It("returns header records that don't have a corresponding deals", func() {
-			missingHeaders, err := dealRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			err = dealRepository.MarkHeaderChecked(headerIds[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			missingHeaders, err := dealRepository.MissingHeaders(startingBlock, endingBlock)
+
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlock))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlock))
 		})
 
 		It("only treats headers as checked if deal have been checked", func() {
-			startingBlockNumber := int64(1)
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIds[1])
-			dealBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, dealBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, price_feeds_checked) VALUES ($1, $2)`, headerIDs[1], true)
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dealRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dealRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dealBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dealBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dealBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dealBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dealBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dealBlock)))
 		})
 
 		It("only returns missing headers for the given node", func() {
+			err = dealRepository.MarkHeaderChecked(headerIds[1])
+			Expect(err).NotTo(HaveOccurred())
 			node2 := core.Node{}
 			db2 := test_config.NewTestDB(node2)
 			dealRepository2 := deal.NewDealRepository(db2)
 			headerRepository2 := repositories.NewHeaderRepository(db2)
 			var node2HeaderIds []int64
 			for _, number := range blockNumbers {
-				id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
 				node2HeaderIds = append(node2HeaderIds, id)
 				Expect(err).NotTo(HaveOccurred())
 			}
-			missingHeadersNode1, err := dealRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			missingHeadersNode1, err := dealRepository.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(missingHeadersNode1)).To(Equal(2))
-			Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlock))
-			Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlockNumber))
+			Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlock))
-			missingHeadersNode2, err := dealRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			missingHeadersNode2, err := dealRepository2.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(missingHeadersNode2)).To(Equal(3))
-			Expect(missingHeadersNode2[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode2[0].BlockNumber).To(Or(Equal(startingBlock), Equal(dealBlock), Equal(endingBlock)))
-			Expect(missingHeadersNode2[1].BlockNumber).To(Equal(dealBlockNumber))
+			Expect(missingHeadersNode2[1].BlockNumber).To(Or(Equal(startingBlock), Equal(dealBlock), Equal(endingBlock)))
-			Expect(missingHeadersNode2[2].BlockNumber).To(Equal(endingBlockNumber))
+			Expect(missingHeadersNode2[2].BlockNumber).To(Or(Equal(startingBlock), Equal(dealBlock), Equal(endingBlock)))
 		})
 	})
 })

pkg/transformers/dent/repository_test.go

@@ -15,8 +15,9 @@
 package dent_test
 
 import (
-	"math/rand"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,24 +30,28 @@ import (
 )
 
 var _ = Describe("Dent Repository", func() {
-	var node core.Node
+	var (
-	var db *postgres.DB
+		db               *postgres.DB
-	var dentRepository dent.DentRepository
+		dentRepository   dent.DentRepository
-	var headerRepository repositories.HeaderRepository
+		headerRepository repositories.HeaderRepository
-	var headerId int64
+		err              error
-	var err error
+		rawHeader        []byte
+	)
 
 	BeforeEach(func() {
-		node = test_config.NewTestNode()
+		db = test_config.NewTestDB(test_config.NewTestNode())
-		db = test_config.NewTestDB(node)
 		test_config.CleanTestDB(db)
 		dentRepository = dent.NewDentRepository(db)
 		headerRepository = repositories.NewHeaderRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Describe("Create", func() {
+		var headerId int64
+
 		BeforeEach(func() {
-			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 
 			err := dentRepository.Create(headerId, []dent.DentModel{test_data.DentModel})
@@ -99,8 +104,10 @@ var _ = Describe("Dent Repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
+		var headerId int64
+
 		BeforeEach(func() {
-			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
@@ -129,85 +136,75 @@ var _ = Describe("Dent Repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		var dentBlockNumber int64
+		var (
-		var startingBlockNumber int64
+			dentBlock, startingBlock, endingBlock int64
-		var endingBlockNumber int64
+			blockNumbers, headerIds               []int64
-		var blockNumbers []int64
+		)
 
 		BeforeEach(func() {
-			dentBlockNumber = rand.Int63()
+			dentBlock = GinkgoRandomSeed()
-			startingBlockNumber = dentBlockNumber - 1
+			startingBlock = dentBlock - 1
-			endingBlockNumber = dentBlockNumber + 1
+			endingBlock = dentBlock + 1
-			outOfRangeBlockNumber := dentBlockNumber + 2
+			outOfRangeBlockNumber := dentBlock + 2
 
-			blockNumbers = []int64{startingBlockNumber, dentBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
+			blockNumbers = []int64{startingBlock, dentBlock, endingBlock, outOfRangeBlockNumber}
 
-			var headerIds []int64
+			headerIds = []int64{}
 			for _, number := range blockNumbers {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 				headerIds = append(headerIds, headerId)
 			}
-
-			dentRepository.MarkHeaderChecked(headerIds[1])
 		})
 
 		It("returns header records that haven't been checked", func() {
-			missingHeaders, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			dentRepository.MarkHeaderChecked(headerIds[1])
+			missingHeaders, err := dentRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlock))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlock))
 		})
 
-		It("only treats headers as checked if deal have been checked", func() {
+		It("only treats headers as checked if dent has been checked", func() {
-			startingBlockNumber := int64(1)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIds[1])
-			dentBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, dentBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, price_feeds_checked) VALUES ($1, $2)`, headerIDs[1], true)
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dentRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dentBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dentBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dentBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dentBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dentBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dentBlock)))
 		})
 
 		It("only returns missing headers for the given node", func() {
+			dentRepository.MarkHeaderChecked(headerIds[1])
 			node2 := core.Node{}
 			db2 := test_config.NewTestDB(node2)
 			dentRepository2 := dent.NewDentRepository(db2)
 			headerRepository2 := repositories.NewHeaderRepository(db2)
 			var node2HeaderIds []int64
 			for _, number := range blockNumbers {
-				id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
 				node2HeaderIds = append(node2HeaderIds, id)
 				Expect(err).NotTo(HaveOccurred())
 			}
 
-			missingHeadersNode1, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			missingHeadersNode1, err := dentRepository.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(missingHeadersNode1)).To(Equal(2))
-			Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlock))
-			Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlockNumber))
+			Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlock))
 
-			missingHeadersNode2, err := dentRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			missingHeadersNode2, err := dentRepository2.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(missingHeadersNode2)).To(Equal(3))
-			Expect(missingHeadersNode2[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode2[0].BlockNumber).To(Or(Equal(startingBlock), Equal(dentBlock), Equal(endingBlock)))
-			Expect(missingHeadersNode2[1].BlockNumber).To(Equal(dentBlockNumber))
+			Expect(missingHeadersNode2[1].BlockNumber).To(Or(Equal(startingBlock), Equal(dentBlock), Equal(endingBlock)))
-			Expect(missingHeadersNode2[2].BlockNumber).To(Equal(endingBlockNumber))
+			Expect(missingHeadersNode2[2].BlockNumber).To(Or(Equal(startingBlock), Equal(dentBlock), Equal(endingBlock)))
 		})
 	})
 })

pkg/transformers/drip_drip/repository_test.go

@@ -16,11 +16,14 @@ package drip_drip_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
@@ -29,39 +32,44 @@ import (
 )
 
 var _ = Describe("Drip drip repository", func() {
+	var (
+		db                 *postgres.DB
+		dripDripRepository drip_drip.Repository
+		err                error
+		headerRepository   datastore.HeaderRepository
+		rawHeader          []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		dripDripRepository = drip_drip.NewDripDripRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                 *postgres.DB
-			dripDripRepository drip_drip.Repository
-			err                error
-			headerID           int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
 			Expect(err).NotTo(HaveOccurred())
-			dripDripRepository = drip_drip.NewDripDripRepository(db)
 		})
 
 		It("adds a drip drip event", func() {
-			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
+			var dbDripDrip drip_drip.DripDripModel
-
+			err = db.Get(&dbDripDrip, `SELECT ilk, tx_idx, raw_log FROM maker.drip_drip WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile drip_drip.DripDripModel
+			Expect(dbDripDrip.Ilk).To(Equal(test_data.DripDripModel.Ilk))
-			err = db.Get(&dbPitFile, `SELECT ilk, tx_idx, raw_log FROM maker.drip_drip WHERE header_id = $1`, headerID)
+			Expect(dbDripDrip.TransactionIndex).To(Equal(test_data.DripDripModel.TransactionIndex))
-			Expect(err).NotTo(HaveOccurred())
+			Expect(dbDripDrip.Raw).To(MatchJSON(test_data.DripDripModel.Raw))
-			Expect(dbPitFile.Ilk).To(Equal(test_data.DripDripModel.Ilk))
-			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.DripDripModel.TransactionIndex))
-			Expect(dbPitFile.Raw).To(MatchJSON(test_data.DripDripModel.Raw))
 		})
 
 		It("marks header as checked for logs", func() {
-			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -69,9 +77,6 @@ var _ = Describe("Drip drip repository", func() {
 		})
 
 		It("does not duplicate drip drip events", func() {
-			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
 
 			Expect(err).To(HaveOccurred())
@@ -79,34 +84,22 @@ var _ = Describe("Drip drip repository", func() {
 		})
 
 		It("removes drip drip if corresponding header is deleted", func() {
-			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
-			var dbPitFile drip_drip.DripDripModel
+			var dbDripDrip drip_drip.DripDripModel
-			err = db.Get(&dbPitFile, `SELECT ilk, tx_idx, raw_log FROM maker.drip_drip WHERE header_id = $1`, headerID)
+			err = db.Get(&dbDripDrip, `SELECT ilk, tx_idx, raw_log FROM maker.drip_drip WHERE header_id = $1`, headerID)
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                 *postgres.DB
-			dripDripRepository drip_drip.Repository
-			err                error
-			headerID           int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripDripRepository = drip_drip.NewDripDripRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -133,78 +126,62 @@ var _ = Describe("Drip drip repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, dripDripBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                   []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			dripDripBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, dripDripBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			dripDripBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, dripDripBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			dripDripRepository := drip_drip.NewDripDripRepository(db)
+
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := dripDripRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripDripRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if drip drip logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			dripDripBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, dripDripBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			dripDripRepository := drip_drip.NewDripDripRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripDripRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripDripBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripDripBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripDripBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
-			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-			}
-			dripDripRepository := drip_drip.NewDripDripRepository(db)
-			dripDripRepositoryTwo := drip_drip.NewDripDripRepository(dbTwo)
 			err := dripDripRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			for _, n := range blockNumbers {
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			dripDripRepositoryTwo := drip_drip.NewDripDripRepository(dbTwo)
 
 			nodeOneMissingHeaders, err := dripDripRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/drip_file/ilk/repository_test.go

@@ -16,11 +16,14 @@ package ilk_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk"
@@ -29,27 +32,35 @@ import (
 )
 
 var _ = Describe("Drip file ilk repository", func() {
+	var (
+		db                    *postgres.DB
+		dripFileIlkRepository ilk.Repository
+		err                   error
+		headerRepository      datastore.HeaderRepository
+		rawHeader             []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		dripFileIlkRepository = ilk.NewDripFileIlkRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                    *postgres.DB
-			dripFileIlkRepository ilk.Repository
-			err                   error
-			headerID              int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileIlkRepository = ilk.NewDripFileIlkRepository(db)
 		})
 
 		It("adds a drip file ilk event", func() {
-			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbDripFileIlk ilk.DripFileIlkModel
 			err = db.Get(&dbDripFileIlk, `SELECT ilk, vow, tax, tx_idx, raw_log FROM maker.drip_file_ilk WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -61,8 +72,6 @@ var _ = Describe("Drip file ilk repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
-
 			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
@@ -71,9 +80,6 @@ var _ = Describe("Drip file ilk repository", func() {
 		})
 
 		It("does not duplicate drip file events", func() {
-			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
 
 			Expect(err).To(HaveOccurred())
@@ -81,9 +87,6 @@ var _ = Describe("Drip file ilk repository", func() {
 		})
 
 		It("removes drip file if corresponding header is deleted", func() {
-			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -95,20 +98,11 @@ var _ = Describe("Drip file ilk repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                    *postgres.DB
-			dripFileIlkRepository ilk.Repository
-			err                   error
-			headerID              int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileIlkRepository = ilk.NewDripFileIlkRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -135,75 +129,59 @@ var _ = Describe("Drip file ilk repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated drip file event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, dripFileBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                   []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			dripFileIlkBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, dripFileIlkBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			dripFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, dripFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
+
+		})
+
+		It("returns headers with no associated drip file event", func() {
 			err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripFileIlkRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if drip file ilk logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			dripFileIlkdBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, dripFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripFileIlkRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
 			dripFileIlkRepositoryTwo := ilk.NewDripFileIlkRepository(dbTwo)
 			err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/drip_file/repo/repository_test.go

@@ -16,9 +16,14 @@ package repo_test
 
 import (
 	"database/sql"
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo"
@@ -27,27 +32,35 @@ import (
 )
 
 var _ = Describe("Drip file repo repository", func() {
+	var (
+		db                     *postgres.DB
+		dripFileRepoRepository repo.Repository
+		err                    error
+		headerRepository       datastore.HeaderRepository
+		rawHeader              []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		dripFileRepoRepository = repo.NewDripFileRepoRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                     *postgres.DB
-			dripFileRepoRepository repo.Repository
-			err                    error
-			headerID               int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileRepoRepository = repo.NewDripFileRepoRepository(db)
 		})
 
 		It("adds a drip file repo event", func() {
-			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbDripFileRepo repo.DripFileRepoModel
 			err = db.Get(&dbDripFileRepo, `SELECT what, data, tx_idx, raw_log FROM maker.drip_file_repo WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -58,9 +71,6 @@ var _ = Describe("Drip file repo repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -68,9 +78,6 @@ var _ = Describe("Drip file repo repository", func() {
 		})
 
 		It("does not duplicate drip file events", func() {
-			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
 
 			Expect(err).To(HaveOccurred())
@@ -78,9 +85,6 @@ var _ = Describe("Drip file repo repository", func() {
 		})
 
 		It("removes drip file if corresponding header is deleted", func() {
-			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -92,20 +96,11 @@ var _ = Describe("Drip file repo repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                     *postgres.DB
-			dripFileRepoRepository repo.Repository
-			err                    error
-			headerID               int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileRepoRepository = repo.NewDripFileRepoRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -132,75 +127,58 @@ var _ = Describe("Drip file repo repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated drip file event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, dripFileBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                   []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			dripFileRepoBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, dripFileRepoBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			dripFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, dripFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
 				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
 			}
-			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
+		})
+
+		It("returns headers with no associated drip file event", func() {
 			err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripFileRepoRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if drip file repo logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			dripFileRepodBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, dripFileRepodBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripFileRepoRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
 			dripFileRepoRepositoryTwo := repo.NewDripFileRepoRepository(dbTwo)
 			err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/drip_file/vow/repository_test.go

@@ -16,11 +16,14 @@ package vow_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow"
@@ -29,27 +32,35 @@ import (
 )
 
 var _ = Describe("Drip file vow repository", func() {
+	var (
+		db                    *postgres.DB
+		dripFileVowRepository vow.Repository
+		err                   error
+		headerRepository      datastore.HeaderRepository
+		rawHeader             []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		dripFileVowRepository = vow.NewDripFileVowRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                    *postgres.DB
-			dripFileVowRepository vow.Repository
-			err                   error
-			headerID              int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileVowRepository = vow.NewDripFileVowRepository(db)
 		})
 
 		It("adds a drip file vow event", func() {
-			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbDripFileVow vow.DripFileVowModel
 			err = db.Get(&dbDripFileVow, `SELECT what, data, tx_idx, raw_log FROM maker.drip_file_vow WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -60,9 +71,6 @@ var _ = Describe("Drip file vow repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -70,9 +78,6 @@ var _ = Describe("Drip file vow repository", func() {
 		})
 
 		It("does not duplicate drip file events", func() {
-			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
 
 			Expect(err).To(HaveOccurred())
@@ -80,9 +85,6 @@ var _ = Describe("Drip file vow repository", func() {
 		})
 
 		It("removes drip file if corresponding header is deleted", func() {
-			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -94,20 +96,11 @@ var _ = Describe("Drip file vow repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                    *postgres.DB
-			dripFileVowRepository vow.Repository
-			err                   error
-			headerID              int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileVowRepository = vow.NewDripFileVowRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -134,75 +127,58 @@ var _ = Describe("Drip file vow repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, dripFileBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                   []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			dripFileVowBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, dripFileVowBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			dripFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, dripFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
 				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
 			}
-			dripFileVowRepository := vow.NewDripFileVowRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := dripFileVowRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripFileVowRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if drip file vow logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			dripFileVowdBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, dripFileVowdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			dripFileVowRepository := vow.NewDripFileVowRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := dripFileVowRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(dripFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			dripFileVowRepository := vow.NewDripFileVowRepository(db)
 			dripFileVowRepositoryTwo := vow.NewDripFileVowRepository(dbTwo)
 			err := dripFileVowRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/flip_kick/repository_test.go

@@ -15,12 +15,12 @@
 package flip_kick_test
 
 import (
-	"math/rand"
-
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"encoding/json"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
@@ -37,11 +37,10 @@ var _ = Describe("FlipKick Repository", func() {
 	var flipKick = test_data.FlipKickModel
 
 	BeforeEach(func() {
-		node := test_config.NewTestNode()
+		db = test_config.NewTestDB(test_config.NewTestNode())
-		db = test_config.NewTestDB(node)
 		test_config.CleanTestDB(db)
 		flipKickRepository = flip_kick.FlipKickRepository{DB: db}
-		blockNumber = rand.Int63()
+		blockNumber = GinkgoRandomSeed()
 		headerId = createHeader(db, blockNumber)
 
 		_, err := db.Exec(`DELETE from maker.flip_kick;`)
@@ -192,12 +191,14 @@ func assertDBRecordCount(db *postgres.DB, dbTable string, expectedCount int) {
 
 func createHeader(db *postgres.DB, blockNumber int64) (headerId int64) {
 	headerRepository := repositories.NewHeaderRepository(db)
+	rawHeader, err := json.Marshal(types.Header{})
+	Expect(err).NotTo(HaveOccurred())
 	header := core.Header{
 		BlockNumber: blockNumber,
 		Hash:        common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
-		Raw:         []byte{1, 2, 3, 4, 5},
+		Raw:         rawHeader,
 	}
-	_, err := headerRepository.CreateOrUpdateHeader(header)
+	_, err = headerRepository.CreateOrUpdateHeader(header)
 	Expect(err).NotTo(HaveOccurred())
 
 	var dbHeader core.Header

pkg/transformers/flop_kick/repository_test.go

@@ -15,8 +15,9 @@
 package flop_kick_test
 
 import (
-	"math/rand"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,29 +30,37 @@ import (
 )
 
 var _ = Describe("FlopRepository", func() {
-	var db *postgres.DB
+	var (
-	var repository flop_kick.FlopKickRepository
+		db               *postgres.DB
-	var headerRepository repositories.HeaderRepository
+		repository       flop_kick.FlopKickRepository
-	var headerId int64
+		headerRepository repositories.HeaderRepository
-	var err error
+		err              error
-	var dbResult test_data.FlopKickDBResult
+		dbResult         test_data.FlopKickDBResult
+		rawHeader        []byte
+	)
 
 	BeforeEach(func() {
-		node := test_config.NewTestNode()
+		db = test_config.NewTestDB(test_config.NewTestNode())
-		db = test_config.NewTestDB(node)
 		test_config.CleanTestDB(db)
 		repository = flop_kick.NewFlopKickRepository(db)
 		headerRepository = repositories.NewHeaderRepository(db)
-		headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
-		Expect(err).NotTo(HaveOccurred())
 		dbResult = test_data.FlopKickDBResult{}
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Describe("Create", func() {
-		It("creates FlopKick records", func() {
+		var headerId int64
-			err := repository.Create(headerId, []flop_kick.Model{test_data.FlopKickModel})
+
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 
+			err := repository.Create(headerId, []flop_kick.Model{test_data.FlopKickModel})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("creates FlopKick records", func() {
 			err = db.QueryRowx(`SELECT * FROM maker.flop_kick WHERE header_id = $1`, headerId).StructScan(&dbResult)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbResult.HeaderId).To(Equal(headerId))
@@ -65,9 +74,6 @@ var _ = Describe("FlopRepository", func() {
 		})
 
 		It("marks headerId as checked for flop kick logs", func() {
-			err = repository.Create(headerId, []flop_kick.Model{test_data.FlopKickModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT flop_kick_checked FROM public.checked_headers WHERE header_id = $1`, headerId)
 			Expect(err).NotTo(HaveOccurred())
@@ -75,18 +81,12 @@ var _ = Describe("FlopRepository", func() {
 		})
 
 		It("returns an error if inserting the flop_kick record fails", func() {
-			err = repository.Create(headerId, []flop_kick.Model{test_data.FlopKickModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = repository.Create(headerId, []flop_kick.Model{test_data.FlopKickModel})
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("deletes the flop_kick records if its corresponding header record is deleted", func() {
-			err = repository.Create(headerId, []flop_kick.Model{test_data.FlopKickModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			var flopKickCount int
 			err = db.QueryRow(`SELECT count(*) FROM maker.flop_kick`).Scan(&flopKickCount)
 			Expect(err).NotTo(HaveOccurred())
@@ -102,6 +102,13 @@ var _ = Describe("FlopRepository", func() {
 	})
 
 	Describe("MarkedHeadersChecked", func() {
+		var headerId int64
+
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("creates a row for a new headerId", func() {
 			err := repository.MarkHeaderChecked(headerId)
 
@@ -125,74 +132,69 @@ var _ = Describe("FlopRepository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		var flopKickBlockNumber = rand.Int63()
+		var (
-		var startingBlockNumber = flopKickBlockNumber - 1
+			flopKickBlock, startingBlock, endingBlock, outOfRangeBlock int64
-		var endingBlockNumber = flopKickBlockNumber + 1
+			headerIds                                                  []int64
-		var outOfRangeBlockNumber = flopKickBlockNumber + 2
+		)
 
-		It("returns headers haven't been checked", func() {
+		BeforeEach(func() {
-			var headerIds []int64
+			flopKickBlock = GinkgoRandomSeed()
+			startingBlock = flopKickBlock - 1
+			endingBlock = flopKickBlock + 1
+			outOfRangeBlock = flopKickBlock + 2
 
-			for _, number := range []int64{startingBlockNumber, flopKickBlockNumber, endingBlockNumber, outOfRangeBlockNumber} {
+			headerIds = []int64{}
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+			for _, number := range []int64{startingBlock, flopKickBlock, endingBlock, outOfRangeBlock} {
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 				headerIds = append(headerIds, headerId)
 			}
+		})
 
+		It("returns headers haven't been checked", func() {
 			err = repository.MarkHeaderChecked(headerIds[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := repository.MissingHeaders(startingBlock, endingBlock)
+
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if flop_kicks have been checked", func() {
-			var headerIds []int64
-			for _, number := range []int64{startingBlockNumber, flopKickBlockNumber, endingBlockNumber, outOfRangeBlockNumber} {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
-				Expect(err).NotTo(HaveOccurred())
-				headerIds = append(headerIds, headerId)
-			}
-
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIds[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := repository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(flopKickBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(flopKickBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(flopKickBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(flopKickBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(flopKickBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(flopKickBlock)))
 		})
 
 		It("only returns missing headers for the current node", func() {
-			var headerIds []int64
 			node2 := core.Node{}
 			db2 := test_config.NewTestDB(node2)
 			headerRepository2 := repositories.NewHeaderRepository(db2)
 			flopKickRepository2 := flop_kick.NewFlopKickRepository(db2)
 
-			for _, number := range []int64{startingBlockNumber, flopKickBlockNumber, endingBlockNumber} {
+			for _, number := range []int64{startingBlock, flopKickBlock, endingBlock} {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIds = append(headerIds, headerId)
-
-				headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
 				Expect(err).NotTo(HaveOccurred())
 			}
 
 			err = repository.MarkHeaderChecked(headerIds[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			node1MissingHeaders, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			node1MissingHeaders, err := repository.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(node1MissingHeaders)).To(Equal(2))
 
-			node2MissingHeaders, err := flopKickRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			node2MissingHeaders, err := flopKickRepository2.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(node2MissingHeaders)).To(Equal(3))
 		})

pkg/transformers/frob/repository_test.go

@@ -16,11 +16,14 @@ package frob_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
@@ -29,27 +32,35 @@ import (
 )
 
 var _ = Describe("Frob repository", func() {
+	var (
+		db               *postgres.DB
+		frobRepository   frob.Repository
+		err              error
+		headerRepository datastore.HeaderRepository
+		rawHeader        []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		frobRepository = frob.NewFrobRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db             *postgres.DB
-			frobRepository frob.Repository
-			err            error
-			headerID       int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
 			Expect(err).NotTo(HaveOccurred())
-			frobRepository = frob.NewFrobRepository(db)
 		})
 
 		It("adds a frob", func() {
-			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbFrob frob.FrobModel
 			err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, urn, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -65,9 +76,6 @@ var _ = Describe("Frob repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -75,9 +83,6 @@ var _ = Describe("Frob repository", func() {
 		})
 
 		It("does not duplicate frob events", func() {
-			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
 
 			Expect(err).To(HaveOccurred())
@@ -85,9 +90,6 @@ var _ = Describe("Frob repository", func() {
 		})
 
 		It("removes frob if corresponding header is deleted", func() {
-			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -99,20 +101,11 @@ var _ = Describe("Frob repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db             *postgres.DB
-			frobRepository frob.Repository
-			err            error
-			headerID       int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			frobRepository = frob.NewFrobRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -139,75 +132,58 @@ var _ = Describe("Frob repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, frobBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs               []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			frobBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, frobBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			frobBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, frobBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			frobRepository := frob.NewFrobRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := frobRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := frobRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if frob logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			frobdBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, frobdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			frobRepository := frob.NewFrobRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := frobRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(frobBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(frobBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(frobBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			frobRepository := frob.NewFrobRepository(db)
 			frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
 			err := frobRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/pit_file/debt_ceiling/repository_test.go

@@ -16,11 +16,14 @@ package debt_ceiling_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
@@ -29,27 +32,35 @@ import (
 )
 
 var _ = Describe("Pit file debt ceiling repository", func() {
+	var (
+		db                *postgres.DB
+		pitFileRepository debt_ceiling.Repository
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		pitFileRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                           *postgres.DB
-			pitFileDebtCeilingRepository debt_ceiling.Repository
-			err                          error
-			headerID                     int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = pitFileRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
 		})
 
 		It("adds a pit file debt ceiling event", func() {
-			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile debt_ceiling.PitFileDebtCeilingModel
 			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.pit_file_debt_ceiling WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -60,9 +71,6 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -70,19 +78,13 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 		})
 
 		It("does not duplicate pit file events", func() {
-			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
+			err = pitFileRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
-			Expect(err).NotTo(HaveOccurred())
-
-			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file if corresponding header is deleted", func() {
-			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -94,24 +96,15 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                           *postgres.DB
-			pitFileDebtCeilingRepository debt_ceiling.Repository
-			err                          error
-			headerID                     int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
-			err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
+			err = pitFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
@@ -123,7 +116,7 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 		It("updates row when headerID already exists", func() {
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
-			err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
+			err = pitFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
@@ -134,72 +127,56 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, pitFileBlock, endingBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			pitFileBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			pitFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, pitFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if pit file debt ceiling logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			pitFileDebtCeilingdBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, pitFileDebtCeilingdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			pitFileDebtCeilingRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := pitFileDebtCeilingRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
 			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)

pkg/transformers/pit_file/ilk/repository_test.go

@@ -16,11 +16,14 @@ package ilk_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
@@ -29,27 +32,34 @@ import (
 )
 
 var _ = Describe("Pit file ilk repository", func() {
+	var (
+		db                *postgres.DB
+		pitFileRepository ilk.Repository
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		pitFileRepository = ilk.NewPitFileIlkRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                   *postgres.DB
-			pitFileIlkRepository ilk.Repository
-			err                  error
-			headerID             int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = pitFileRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileIlkRepository = ilk.NewPitFileIlkRepository(db)
 		})
 
 		It("adds a pit file ilk event", func() {
-			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile ilk.PitFileIlkModel
 			err = db.Get(&dbPitFile, `SELECT ilk, what, data, tx_idx, raw_log FROM maker.pit_file_ilk WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -61,9 +71,6 @@ var _ = Describe("Pit file ilk repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -71,19 +78,13 @@ var _ = Describe("Pit file ilk repository", func() {
 		})
 
 		It("does not duplicate pit file ilk events", func() {
-			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
+			err = pitFileRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
-			Expect(err).NotTo(HaveOccurred())
-
-			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file ilk if corresponding header is deleted", func() {
-			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -95,24 +96,15 @@ var _ = Describe("Pit file ilk repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                   *postgres.DB
-			pitFileIlkRepository ilk.Repository
-			err                  error
-			headerID             int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileIlkRepository = ilk.NewPitFileIlkRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
-			err = pitFileIlkRepository.MarkHeaderChecked(headerID)
+			err = pitFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
@@ -124,7 +116,7 @@ var _ = Describe("Pit file ilk repository", func() {
 		It("updates row when headerID already exists", func() {
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
-			err = pitFileIlkRepository.MarkHeaderChecked(headerID)
+			err = pitFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
@@ -135,75 +127,58 @@ var _ = Describe("Pit file ilk repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, pitFileBlock, endingBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			pitFileBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			pitFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, pitFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if pit file ilk logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			pitFileIlkdBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, pitFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			pitFileIlkRepository := ilk.NewPitFileIlkRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := pitFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			pitFileRepository := ilk.NewPitFileIlkRepository(db)
 			pitFileRepositoryTwo := ilk.NewPitFileIlkRepository(dbTwo)
 			err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/pit_file/stability_fee/repository_test.go

@@ -16,11 +16,14 @@ package stability_fee_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
@@ -29,27 +32,35 @@ import (
 )
 
 var _ = Describe("Pit file stability fee repository", func() {
+	var (
+		db                *postgres.DB
+		pitFileRepository stability_fee.Repository
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		pitFileRepository = stability_fee.NewPitFileStabilityFeeRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                            *postgres.DB
-			pitFileStabilityFeeRepository stability_fee.Repository
-			err                           error
-			headerID                      int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = pitFileRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db)
 		})
 
 		It("adds a pit file stability fee event", func() {
-			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile stability_fee.PitFileStabilityFeeModel
 			err = db.Get(&dbPitFile, `SELECT what, data, tx_idx, raw_log FROM maker.pit_file_stability_fee WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -60,9 +71,6 @@ var _ = Describe("Pit file stability fee repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -70,19 +78,13 @@ var _ = Describe("Pit file stability fee repository", func() {
 		})
 
 		It("does not duplicate pit file events", func() {
-			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
+			err = pitFileRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
-			Expect(err).NotTo(HaveOccurred())
-
-			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file if corresponding header is deleted", func() {
-			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -94,24 +96,15 @@ var _ = Describe("Pit file stability fee repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                            *postgres.DB
-			pitFileStabilityFeeRepository stability_fee.Repository
-			err                           error
-			headerID                      int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
-			err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID)
+			err = pitFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
@@ -123,7 +116,7 @@ var _ = Describe("Pit file stability fee repository", func() {
 		It("updates row when headerID already exists", func() {
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
-			err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID)
+			err = pitFileRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
@@ -134,75 +127,58 @@ var _ = Describe("Pit file stability fee repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, pitFileBlock, endingBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			pitFileBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			pitFileBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, pitFileBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if pit file stability fee logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			pitFileStabilityFeedBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, pitFileStabilityFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			pitFileStabilityFeeRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := pitFileStabilityFeeRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(pitFileBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
 			pitFileRepositoryTwo := stability_fee.NewPitFileStabilityFeeRepository(dbTwo)
 			err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/price_feeds/integration_test.go

@@ -5,6 +5,7 @@ import (
 	"math/big"
 	"time"
 
+	"encoding/json"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -129,10 +130,15 @@ func persistHeader(rpcClient client.RpcClient, db *postgres.DB, blockNumber int6
 	if err != nil {
 		return err
 	}
+	rawHeader, err := json.Marshal(poaHeader)
+	if err != nil {
+		return err
+	}
 	headerRepository := repositories.NewHeaderRepository(db)
 	_, err = headerRepository.CreateOrUpdateHeader(core.Header{
 		BlockNumber: poaHeader.Number.ToInt().Int64(),
 		Hash:        poaHeader.Hash.String(),
+		Raw:         rawHeader,
 	})
 	return err
 }

pkg/transformers/price_feeds/repository_test.go

@@ -15,10 +15,16 @@
 package price_feeds_test
 
 import (
+	"database/sql"
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -26,18 +32,35 @@ import (
 )
 
 var _ = Describe("Price feeds repository", func() {
+	var (
+		db                  *postgres.DB
+		err                 error
+		headerRepository    datastore.HeaderRepository
+		priceFeedRepository price_feeds.IPriceFeedRepository
+		rawHeader           []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		priceFeedRepository = price_feeds.NewPriceFeedRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		It("persists a price feed update", func() {
+		var headerID int64
-			db := test_config.NewTestDB(core.Node{})
+
-			test_config.CleanTestDB(db)
+		BeforeEach(func() {
-			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
 
 			err = priceFeedRepository.Create(headerID, []price_feeds.PriceFeedModel{test_data.PriceFeedModel})
-
 			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("persists a price feed update", func() {
 			var dbPriceFeedUpdate price_feeds.PriceFeedModel
 			err = db.Get(&dbPriceFeedUpdate, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -49,16 +72,6 @@ var _ = Describe("Price feeds repository", func() {
 		})
 
 		It("marks headerID as checked for price feed logs", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
-
-			err = priceFeedRepository.Create(headerID, []price_feeds.PriceFeedModel{test_data.PriceFeedModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT price_feeds_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -66,31 +79,32 @@ var _ = Describe("Price feeds repository", func() {
 		})
 
 		It("does not duplicate price feed updates", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			header := core.Header{BlockNumber: int64(uint64(12345))}
-			headerID, err := headerRepository.CreateOrUpdateHeader(header)
-			Expect(err).NotTo(HaveOccurred())
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
-			err = priceFeedRepository.Create(headerID, []price_feeds.PriceFeedModel{test_data.PriceFeedModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = priceFeedRepository.Create(headerID, []price_feeds.PriceFeedModel{test_data.PriceFeedModel})
 
 			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes price feed if corresponding header is deleted", func() {
+			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbResult price_feeds.PriceFeedModel
+			err = db.Get(&dbResult, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		It("creates a row for a new headerID", func() {
+		var headerID int64
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
 
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("creates a row for a new headerID", func() {
 			err = priceFeedRepository.MarkHeaderChecked(headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -101,12 +115,6 @@ var _ = Describe("Price feeds repository", func() {
 		})
 
 		It("updates row when headerID already exists", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
 
 			err = priceFeedRepository.MarkHeaderChecked(headerID)
@@ -120,77 +128,59 @@ var _ = Describe("Price feeds repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, priceFeedBlock, endingBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                    []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			priceFeedBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, priceFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			priceFeedBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, priceFeedBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
+
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := priceFeedRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := priceFeedRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := priceFeedRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if price feeds have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			priceFeedBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, priceFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := priceFeedRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := priceFeedRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(priceFeedBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(priceFeedBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(priceFeedBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(priceFeedBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(priceFeedBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(priceFeedBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			nodeOne := core.Node{}
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			db := test_config.NewTestDB(nodeOne)
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
-			nodeTwo := core.Node{ID: "second"}
-			dbTwo := test_config.NewTestDB(nodeTwo)
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			priceFeedRepository := price_feeds.NewPriceFeedRepository(db)
 			priceFeedRepositoryTwo := price_feeds.NewPriceFeedRepository(dbTwo)
 			err := priceFeedRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/tend/repository_test.go

@@ -15,8 +15,9 @@
 package tend_test
 
 import (
-	"math/rand"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,29 +30,36 @@ import (
 )
 
 var _ = Describe("TendRepository", func() {
-	var db *postgres.DB
+	var (
-	var tendRepository tend.TendRepository
+		db               *postgres.DB
-	var headerRepository repositories.HeaderRepository
+		tendRepository   tend.TendRepository
-	var headerId int64
+		headerRepository repositories.HeaderRepository
-	var err error
+		err              error
+		rawHeader        []byte
+	)
 
 	BeforeEach(func() {
 		node := test_config.NewTestNode()
 		db = test_config.NewTestDB(node)
 		test_config.CleanTestDB(db)
-
 		headerRepository = repositories.NewHeaderRepository(db)
-		headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
-		Expect(err).NotTo(HaveOccurred())
-
 		tendRepository = tend.NewTendRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Describe("Create", func() {
-		It("persists a tend record", func() {
+		var headerId int64
-			err := tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel})
+
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 
+			err := tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("persists a tend record", func() {
 			var count int
 			err = db.QueryRow(`SELECT count(*) from maker.tend`).Scan(&count)
 			Expect(err).NotTo(HaveOccurred())
@@ -71,9 +79,6 @@ var _ = Describe("TendRepository", func() {
 		})
 
 		It("marks header as checked", func() {
-			err := tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT tend_checked FROM public.checked_headers WHERE header_id = $1`, headerId)
 			Expect(err).NotTo(HaveOccurred())
@@ -81,18 +86,13 @@ var _ = Describe("TendRepository", func() {
 		})
 
 		It("returns an error if inserting a tend record fails", func() {
-			err := tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel})
+
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("deletes the tend record if its corresponding header record is deleted", func() {
-			err := tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			var count int
 			err = db.QueryRow(`SELECT count(*) from maker.tend`).Scan(&count)
 			Expect(err).NotTo(HaveOccurred())
@@ -108,6 +108,13 @@ var _ = Describe("TendRepository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
+		var headerId int64
+
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("creates a row for a new headerID", func() {
 			err = tendRepository.MarkHeaderChecked(headerId)
 
@@ -132,88 +139,69 @@ var _ = Describe("TendRepository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		var tendBlockNumber int64
+		var (
-		var startingBlockNumber int64
+			tendBlock, startingBlock, endingBlock, outOfRangeBlock int64
-		var endingBlockNumber int64
+			headerIds                                              []int64
-		var outOfRangeBlockNumber int64
+		)
 
 		BeforeEach(func() {
-			tendBlockNumber = rand.Int63()
+			tendBlock = GinkgoRandomSeed()
-			startingBlockNumber = tendBlockNumber - 1
+			startingBlock = tendBlock - 1
-			endingBlockNumber = tendBlockNumber + 1
+			endingBlock = tendBlock + 1
-			outOfRangeBlockNumber = tendBlockNumber + 2
+			outOfRangeBlock = tendBlock + 2
-		})
 
-		It("returns headers for which there isn't an associated tend record", func() {
+			headerIds = []int64{}
-			var headerIds []int64
+			for _, number := range []int64{startingBlock, tendBlock, endingBlock, outOfRangeBlock} {
-
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
-			for _, number := range []int64{startingBlockNumber, tendBlockNumber, endingBlockNumber, outOfRangeBlockNumber} {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
 				Expect(err).NotTo(HaveOccurred())
 				headerIds = append(headerIds, headerId)
 			}
+		})
 
+		It("returns headers for which there isn't an associated tend record", func() {
 			err = tendRepository.MarkHeaderChecked(headerIds[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := tendRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := tendRepository.MissingHeaders(startingBlock, endingBlock)
+
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if deal have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIds[1])
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			dentBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, dentBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			dentRepository := tend.NewTendRepository(db)
-			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, price_feeds_checked) VALUES ($1, $2)`, headerIDs[1], true)
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := tendRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dentBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(tendBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dentBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(tendBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dentBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(tendBlock)))
 		})
 
 		It("only returns missing headers for the current node", func() {
-			var headerIds []int64
 			node2 := core.Node{}
 			db2 := test_config.NewTestDB(node2)
 			headerRepository2 := repositories.NewHeaderRepository(db2)
 			tendRepository2 := tend.NewTendRepository(db2)
 
-			for _, number := range []int64{startingBlockNumber, tendBlockNumber, endingBlockNumber} {
+			for _, number := range []int64{startingBlock, tendBlock, endingBlock} {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIds = append(headerIds, headerId)
-
-				headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
 				Expect(err).NotTo(HaveOccurred())
 			}
 
 			err = tendRepository.MarkHeaderChecked(headerIds[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			node1MissingHeaders, err := tendRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			node1MissingHeaders, err := tendRepository.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(node1MissingHeaders)).To(Equal(2))
 
-			node2MissingHeaders, err := tendRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			node2MissingHeaders, err := tendRepository2.MissingHeaders(startingBlock, endingBlock)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(node2MissingHeaders)).To(Equal(3))
 		})

pkg/transformers/vat_fold/repository_test.go

@@ -16,7 +16,9 @@ package vat_fold_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -29,12 +31,13 @@ import (
 )
 
 var _ = Describe("Vat.fold repository", func() {
-
+	var (
-	var db *postgres.DB
+		db               *postgres.DB
-	var headerID int64
+		repository       vat_fold.VatFoldRepository
-	var repository vat_fold.VatFoldRepository
+		headerRepository repositories.HeaderRepository
-	var headerRepository repositories.HeaderRepository
+		err              error
-	var err error
+		rawHeader        []byte
+	)
 
 	BeforeEach(func() {
 		node := test_config.NewTestNode()
@@ -42,13 +45,17 @@ var _ = Describe("Vat.fold repository", func() {
 		test_config.CleanTestDB(db)
 		repository = vat_fold.NewVatFoldRepository(db)
 		headerRepository = repositories.NewHeaderRepository(db)
-		headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+		rawHeader, err = json.Marshal(types.Header{})
 		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Describe("Create", func() {
+		var headerID int64
 
 		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+
 			err = repository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel})
 			Expect(err).NotTo(HaveOccurred())
 		})
@@ -65,6 +72,13 @@ var _ = Describe("Vat.fold repository", func() {
 			Expect(dbVatFold.Raw).To(MatchJSON(test_data.VatFoldModel.Raw))
 		})
 
+		It("marks header as checked for logs", func() {
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT vat_fold_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
 		It("does not duplicate vat events", func() {
 			err := repository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel})
 
@@ -84,6 +98,13 @@ var _ = Describe("Vat.fold repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
+		var headerID int64
+
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		type CheckedHeaderResult struct {
 			VatFoldChecked bool `db:"vat_fold_checked"`
 		}
@@ -117,70 +138,57 @@ var _ = Describe("Vat.fold repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
+		var (
+			startingBlock, vatFoldBlock, endingBlock int64
+			blockNumbers, headerIDs                  []int64
+		)
+
+		BeforeEach(func() {
+			startingBlock = GinkgoRandomSeed()
+			vatFoldBlock = startingBlock + 1
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, vatFoldBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
+				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
+			}
+		})
 
 		It("returns headers that haven't been checked", func() {
-			startingBlockNumber := int64(1)
-			vatGrabBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, vatGrabBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-
 			err := repository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := repository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if vat fold logs have been checked", func() {
-			startingBlockNumber := int64(1)
-			vatGrabdBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, vatGrabdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := repository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatFoldBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatFoldBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatFoldBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
-
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
 
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
 

pkg/transformers/vat_grab/repository_test.go

@@ -2,9 +2,14 @@ package vat_grab_test
 
 import (
 	"database/sql"
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -13,27 +18,35 @@ import (
 )
 
 var _ = Describe("Vat grab repository", func() {
+	var (
+		db                *postgres.DB
+		vatGrabRepository vat_grab.Repository
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		vatGrabRepository = vat_grab.NewVatGrabRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                *postgres.DB
-			vatGrabRepository vat_grab.Repository
-			err               error
-			headerID          int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
 			Expect(err).NotTo(HaveOccurred())
-			vatGrabRepository = vat_grab.NewVatGrabRepository(db)
 		})
 
 		It("adds a vat grab event", func() {
-			err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbVatGrab vat_grab.VatGrabModel
 			err = db.Get(&dbVatGrab, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_grab WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -48,9 +61,6 @@ var _ = Describe("Vat grab repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -60,16 +70,11 @@ var _ = Describe("Vat grab repository", func() {
 		It("does not duplicate pit file vat_grab events", func() {
 			err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
 
-			Expect(err).NotTo(HaveOccurred())
-			err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file vat_grab if corresponding header is deleted", func() {
-			err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -81,20 +86,11 @@ var _ = Describe("Vat grab repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                *postgres.DB
-			vatGrabRepository vat_grab.Repository
-			err               error
-			headerID          int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			vatGrabRepository = vat_grab.NewVatGrabRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -120,75 +116,58 @@ var _ = Describe("Vat grab repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, vatGrabBlock, endingBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			vatGrabBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, vatGrabBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			vatGrabBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, vatGrabBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			vatGrabRepository := vat_grab.NewVatGrabRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := vatGrabRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := vatGrabRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := vatGrabRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if vat grab logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			vatGrabdBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, vatGrabdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			vatGrabRepository := vat_grab.NewVatGrabRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := vatGrabRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := vatGrabRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatGrabBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatGrabBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatGrabBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			vatGrabRepository := vat_grab.NewVatGrabRepository(db)
 			vatGrabRepositoryTwo := vat_grab.NewVatGrabRepository(dbTwo)
 			err := vatGrabRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/vat_heal/repository_test.go

@@ -15,8 +15,12 @@
 package vat_heal_test
 
 import (
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
@@ -26,19 +30,20 @@ import (
 )
 
 var _ = Describe("VatHeal Repository", func() {
-	var db *postgres.DB
+	var (
-	var repository vat_heal.VatHealRepository
+		db               *postgres.DB
-	var headerRepository repositories.HeaderRepository
+		repository       vat_heal.VatHealRepository
-	var headerId int64
+		headerRepository repositories.HeaderRepository
-	var err error
+		err              error
+		rawHeader        []byte
+	)
 
 	BeforeEach(func() {
-		node := test_config.NewTestNode()
+		db = test_config.NewTestDB(test_config.NewTestNode())
-		db = test_config.NewTestDB(node)
 		test_config.CleanTestDB(db)
 		repository = vat_heal.VatHealRepository{DB: db}
 		headerRepository = repositories.NewHeaderRepository(db)
-		headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+		rawHeader, err = json.Marshal(types.Header{})
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -53,6 +58,13 @@ var _ = Describe("VatHeal Repository", func() {
 	}
 
 	Describe("Create", func() {
+		var headerId int64
+
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("persists vat heal records", func() {
 			anotherVatHeal := test_data.VatHealModel
 			anotherVatHeal.TransactionIndex = test_data.VatHealModel.TransactionIndex + 1
@@ -112,24 +124,32 @@ var _ = Describe("VatHeal Repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			startingBlock := GinkgoRandomSeed()
+			startingBlock, vatHealBlock, endingBlock, outsideRangeBlock int64
-			vatHealBlock := startingBlock + 1
+			blockNumbers, headerIds                                     []int64
-			endingBlock := startingBlock + 2
+		)
-			outsideRangeBlock := startingBlock + 3
 
-			var headerIds []int64
+		BeforeEach(func() {
-			blockNumbers := []int64{startingBlock, vatHealBlock, endingBlock, outsideRangeBlock}
+			startingBlock = GinkgoRandomSeed()
+			vatHealBlock = startingBlock + 1
+			endingBlock = startingBlock + 2
+			outsideRangeBlock = startingBlock + 3
+
+			headerIds = []int64{}
+			blockNumbers = []int64{startingBlock, vatHealBlock, endingBlock, outsideRangeBlock}
 			for _, n := range blockNumbers {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 				headerIds = append(headerIds, headerId)
 			}
+		})
 
+		It("returns headers that haven't been checked", func() {
 			err = repository.MarkCheckedHeader(headerIds[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := repository.MissingHeaders(startingBlock, endingBlock)
+
 			Expect(err).NotTo(HaveOccurred())
 			Expect(headers[0].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2])))
 			Expect(headers[1].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2])))
@@ -137,24 +157,12 @@ var _ = Describe("VatHeal Repository", func() {
 		})
 
 		It("returns header ids when checked_headers.vat_heal is false", func() {
-			startingBlock := GinkgoRandomSeed()
-			vatHealBlock := startingBlock + 1
-			endingBlock := startingBlock + 2
-			outsideRangeBlock := startingBlock + 3
-
-			var headerIds []int64
-			blockNumbers := []int64{startingBlock, vatHealBlock, endingBlock, outsideRangeBlock}
-			for _, n := range blockNumbers {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				Expect(err).NotTo(HaveOccurred())
-				headerIds = append(headerIds, headerId)
-			}
-
 			err = repository.MarkCheckedHeader(headerIds[0])
 			_, err = repository.DB.Exec(`INSERT INTO checked_headers (header_id) VALUES ($1)`, headerIds[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := repository.MissingHeaders(startingBlock, endingBlock)
+
 			Expect(err).NotTo(HaveOccurred())
 			Expect(headers[0].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2])))
 			Expect(headers[1].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2])))
@@ -162,22 +170,12 @@ var _ = Describe("VatHeal Repository", func() {
 		})
 
 		It("only returns header ids for the current node", func() {
-			startingBlock := GinkgoRandomSeed()
-			vatHealBlock := startingBlock + 1
-			endingBlock := startingBlock + 2
-			outsideRangeBlock := startingBlock + 3
 			db2 := test_config.NewTestDB(core.Node{ID: "second node"})
 			headerRepository2 := repositories.NewHeaderRepository(db2)
 			repository2 := vat_heal.NewVatHealRepository(db2)
 
-			var headerIds []int64
-			blockNumbers := []int64{startingBlock, vatHealBlock, endingBlock, outsideRangeBlock}
 			for _, n := range blockNumbers {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIds = append(headerIds, headerId)
-
-				_, err = headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
 
@@ -195,6 +193,13 @@ var _ = Describe("VatHeal Repository", func() {
 	})
 
 	Describe("MarkCheckedHeader", func() {
+		var headerId int64
+
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		It("creates a new checked_header record", func() {
 			err := repository.MarkCheckedHeader(headerId)
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/vat_init/repository_test.go

@@ -20,6 +20,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"encoding/json"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
@@ -34,7 +36,7 @@ var _ = Describe("Vat init repository", func() {
 		vatInitRepository vat_init.Repository
 		headerRepository  repositories.HeaderRepository
 		err               error
-		headerID          int64
+		rawHeader         []byte
 	)
 
 	BeforeEach(func() {
@@ -42,15 +44,22 @@ var _ = Describe("Vat init repository", func() {
 		test_config.CleanTestDB(db)
 		vatInitRepository = vat_init.NewVatInitRepository(db)
 		headerRepository = repositories.NewHeaderRepository(db)
-		headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+		rawHeader, err = json.Marshal(types.Header{})
 		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Describe("Create", func() {
-		It("adds a vat event", func() {
+		var headerID int64
-			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
 
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
+
+			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("adds a vat event", func() {
 			var dbVatInit vat_init.VatInitModel
 			err = db.Get(&dbVatInit, `SELECT ilk,tx_idx, raw_log FROM maker.vat_init WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -60,9 +69,6 @@ var _ = Describe("Vat init repository", func() {
 		})
 
 		It("does not duplicate vat events", func() {
-			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
 
 			Expect(err).To(HaveOccurred())
@@ -70,9 +76,6 @@ var _ = Describe("Vat init repository", func() {
 		})
 
 		It("removes vat if corresponding header is deleted", func() {
-			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -83,9 +86,6 @@ var _ = Describe("Vat init repository", func() {
 		})
 
 		It("marks the header as checked for vat init logs", func() {
-			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -94,6 +94,14 @@ var _ = Describe("Vat init repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
+		var headerID int64
+
+		BeforeEach(func() {
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
+			Expect(err).NotTo(HaveOccurred())
+
+		})
+
 		It("creates a row for a new headerID", func() {
 			err = vatInitRepository.MarkHeaderChecked(headerID)
 
@@ -118,63 +126,56 @@ var _ = Describe("Vat init repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			startingBlockNumber := int64(1)
+			startingBlock, vatInitBlock, endingBlock int64
-			vatInitBlockNumber := int64(2)
+			blockNumbers, headerIDs                  []int64
-			endingBlockNumber := int64(3)
+		)
-			blockNumbers := []int64{startingBlockNumber, vatInitBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+
-			var headerIDs []int64
+		BeforeEach(func() {
+			startingBlock = GinkgoRandomSeed()
+			vatInitBlock = startingBlock + 1
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, vatInitBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			vatInitRepository := vat_init.NewVatInitRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := vatInitRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := vatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := vatInitRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if drip drip logs have been checked", func() {
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			vatInitBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, vatInitBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := vatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := vatInitRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatInitBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatInitBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatInitBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			blockNumbers := []int64{1, 2, 3}
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
 			vatInitRepositoryTwo := vat_init.NewVatInitRepository(dbTwo)

pkg/transformers/vat_move/repository_test.go

@@ -16,6 +16,8 @@ package vat_move_test
 
 import (
 	"database/sql"
+	"encoding/json"
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -31,12 +33,16 @@ var _ = Describe("Vat Move", func() {
 	var db *postgres.DB
 	var headerRepository repositories.HeaderRepository
 	var vatMoveRepository vat_move.VatMoveRepository
+	var rawHeader []byte
+	var err error
 
 	BeforeEach(func() {
 		db = test_config.NewTestDB(core.Node{})
 		test_config.CleanTestDB(db)
 		headerRepository = repositories.NewHeaderRepository(db)
 		vatMoveRepository = vat_move.NewVatMoveRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Describe("Create", func() {
@@ -44,7 +50,7 @@ var _ = Describe("Vat Move", func() {
 		var err error
 
 		BeforeEach(func() {
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 			err = vatMoveRepository.Create(headerID, []vat_move.VatMoveModel{test_data.VatMoveModel})
 			Expect(err).NotTo(HaveOccurred())
@@ -95,7 +101,7 @@ var _ = Describe("Vat Move", func() {
 			var headerIds []int64
 
 			for _, number := range []int64{startingBlockNumber, eventBlockNumber, endingBlockNumber, outOfRangeBlockNumber} {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 				headerIds = append(headerIds, headerId)
 			}
@@ -113,7 +119,7 @@ var _ = Describe("Vat Move", func() {
 		It("only treats headers as checked if vat_move has been checked", func() {
 			var headerIds []int64
 			for _, number := range []int64{startingBlockNumber, eventBlockNumber, endingBlockNumber, outOfRangeBlockNumber} {
-				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 				headerIds = append(headerIds, headerId)
 			}
@@ -137,10 +143,10 @@ var _ = Describe("Vat Move", func() {
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
 			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
 				Expect(err).NotTo(HaveOccurred())
 			}
 			vatMoveRepositoryTwo := vat_move.NewVatMoveRepository(dbTwo)
@@ -163,7 +169,7 @@ var _ = Describe("Vat Move", func() {
 		var err error
 
 		BeforeEach(func() {
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
 			Expect(err).NotTo(HaveOccurred())
 		})
 

pkg/transformers/vat_toll/repository_test.go

@@ -2,11 +2,14 @@ package vat_toll_test
 
 import (
 	"database/sql"
+	"encoding/json"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -15,27 +18,35 @@ import (
 )
 
 var _ = Describe("Vat toll repository", func() {
+	var (
+		db                *postgres.DB
+		vatTollRepository vat_toll.Repository
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		vatTollRepository = vat_toll.NewVatTollRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                *postgres.DB
-			vatTollRepository vat_toll.Repository
-			err               error
-			headerID          int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel})
 			Expect(err).NotTo(HaveOccurred())
-			vatTollRepository = vat_toll.NewVatTollRepository(db)
 		})
 
 		It("adds a vat toll event", func() {
-			err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbVatToll vat_toll.VatTollModel
 			err = db.Get(&dbVatToll, `SELECT ilk, urn, take, tx_idx, raw_log FROM maker.vat_toll WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -47,9 +58,6 @@ var _ = Describe("Vat toll repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT vat_toll_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -57,9 +65,6 @@ var _ = Describe("Vat toll repository", func() {
 		})
 
 		It("does not duplicate vat toll events", func() {
-			err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel})
 
 			Expect(err).To(HaveOccurred())
@@ -67,9 +72,6 @@ var _ = Describe("Vat toll repository", func() {
 		})
 
 		It("removes vat toll if corresponding header is deleted", func() {
-			err = vatTollRepository.Create(headerID, []vat_toll.VatTollModel{test_data.VatTollModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -81,20 +83,11 @@ var _ = Describe("Vat toll repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                *postgres.DB
-			vatTollRepository vat_toll.Repository
-			err               error
-			headerID          int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			vatTollRepository = vat_toll.NewVatTollRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -121,84 +114,67 @@ var _ = Describe("Vat toll repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlock, endingBlock, vatTollBlock int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                  []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			pitFileBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlock = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			vatTollBlock = startingBlock + 1
-			var headerIDs []int64
+			endingBlock = startingBlock + 2
+
+			blockNumbers = []int64{startingBlock, vatTollBlock, endingBlock, endingBlock + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			pitFileRepository := vat_toll.NewVatTollRepository(db)
+		})
-			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
+
+		It("returns headers that haven't been checked", func() {
+			err := vatTollRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := vatTollRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(2))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock)))
 		})
 
 		It("only treats headers as checked if vat toll logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			vatTolldBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, vatTolldBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			vatTollRepository := vat_toll.NewVatTollRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
-			headers, err := vatTollRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			headers, err := vatTollRepository.MissingHeaders(startingBlock, endingBlock)
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatTollBlock)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatTollBlock)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTolldBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlock), Equal(endingBlock), Equal(vatTollBlock)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
-			pitFileRepository := vat_toll.NewVatTollRepository(db)
+			vatTollRepositoryTwo := vat_toll.NewVatTollRepository(dbTwo)
-			pitFileRepositoryTwo := vat_toll.NewVatTollRepository(dbTwo)
+			err := vatTollRepository.MarkHeaderChecked(headerIDs[0])
-			err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
-			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			nodeOneMissingHeaders, err := vatTollRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
 
-			nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			nodeTwoMissingHeaders, err := vatTollRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
 		})

pkg/transformers/vat_tune/repository_test.go

@@ -6,7 +6,10 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"encoding/json"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -15,27 +18,35 @@ import (
 )
 
 var _ = Describe("Vat tune repository", func() {
+	var (
+		db                *postgres.DB
+		vatTuneRepository vat_tune.Repository
+		err               error
+		headerRepository  datastore.HeaderRepository
+		rawHeader         []byte
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(core.Node{})
+		test_config.CleanTestDB(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+		vatTuneRepository = vat_tune.NewVatTuneRepository(db)
+		rawHeader, err = json.Marshal(types.Header{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	Describe("Create", func() {
-		var (
+		var headerID int64
-			db                *postgres.DB
-			vatTuneRepository vat_tune.Repository
-			err               error
-			headerID          int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
-			headerRepository := repositories.NewHeaderRepository(db)
+
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel})
 			Expect(err).NotTo(HaveOccurred())
-			vatTuneRepository = vat_tune.NewVatTuneRepository(db)
 		})
 
 		It("adds a vat tune event", func() {
-			err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var dbVatTune vat_tune.VatTuneModel
 			err = db.Get(&dbVatTune, `SELECT ilk, urn, v, w, dink, dart, tx_idx, raw_log FROM maker.vat_tune WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -50,9 +61,6 @@ var _ = Describe("Vat tune repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
-			err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel})
-
-			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT vat_tune_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -60,9 +68,6 @@ var _ = Describe("Vat tune repository", func() {
 		})
 
 		It("does not duplicate pit file vat_tune events", func() {
-			err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel})
 
 			Expect(err).To(HaveOccurred())
@@ -70,9 +75,6 @@ var _ = Describe("Vat tune repository", func() {
 		})
 
 		It("removes pit file vat_tune if corresponding header is deleted", func() {
-			err = vatTuneRepository.Create(headerID, []vat_tune.VatTuneModel{test_data.VatTuneModel})
-			Expect(err).NotTo(HaveOccurred())
-
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -84,20 +86,11 @@ var _ = Describe("Vat tune repository", func() {
 	})
 
 	Describe("MarkHeaderChecked", func() {
-		var (
+		var headerID int64
-			db                *postgres.DB
-			vatTuneRepository vat_tune.Repository
-			err               error
-			headerID          int64
-		)
 
 		BeforeEach(func() {
-			db = test_config.NewTestDB(core.Node{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{Raw: rawHeader})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			vatTuneRepository = vat_tune.NewVatTuneRepository(db)
 		})
 
 		It("creates a row for a new headerID", func() {
@@ -124,21 +117,27 @@ var _ = Describe("Vat tune repository", func() {
 	})
 
 	Describe("MissingHeaders", func() {
-		It("returns headers that haven't been checked", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			startingBlockNumber, vatTuneBlockNumber, endingBlockNumber int64
-			test_config.CleanTestDB(db)
+			blockNumbers, headerIDs                                    []int64
-			headerRepository := repositories.NewHeaderRepository(db)
+		)
-			startingBlockNumber := int64(1)
+
-			vatTuneBlockNumber := int64(2)
+		BeforeEach(func() {
-			endingBlockNumber := int64(3)
+			startingBlockNumber = GinkgoRandomSeed()
-			blockNumbers := []int64{startingBlockNumber, vatTuneBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			vatTuneBlockNumber = startingBlockNumber + 1
-			var headerIDs []int64
+			endingBlockNumber = startingBlockNumber + 2
+
+			blockNumbers = []int64{startingBlockNumber, vatTuneBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+
+			headerIDs = []int64{}
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				headerIDs = append(headerIDs, headerID)
 				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
 			}
-			vatTuneRepository := vat_tune.NewVatTuneRepository(db)
+		})
+
+		It("returns headers that haven't been checked", func() {
 			err := vatTuneRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
@@ -151,20 +150,6 @@ var _ = Describe("Vat tune repository", func() {
 		})
 
 		It("only treats headers as checked if vat tune logs have been checked", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			startingBlockNumber := int64(1)
-			vatTunedBlockNumber := int64(2)
-			endingBlockNumber := int64(3)
-			blockNumbers := []int64{startingBlockNumber, vatTunedBlockNumber, endingBlockNumber, endingBlockNumber + 1}
-			var headerIDs []int64
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-			vatTuneRepository := vat_tune.NewVatTuneRepository(db)
 			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
@@ -172,24 +157,16 @@ var _ = Describe("Vat tune repository", func() {
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTunedBlockNumber)))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTuneBlockNumber)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTunedBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTuneBlockNumber)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTunedBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatTuneBlockNumber)))
 		})
 
 		It("only returns headers associated with the current node", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			blockNumbers := []int64{1, 2, 3}
-			headerRepository := repositories.NewHeaderRepository(db)
 			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			var headerIDs []int64
 			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n, Raw: rawHeader})
-				Expect(err).NotTo(HaveOccurred())
-				headerIDs = append(headerIDs, headerID)
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
 				Expect(err).NotTo(HaveOccurred())
 			}
 			vatTuneRepository := vat_tune.NewVatTuneRepository(db)