Merge branch 'staging' into VDB-101-vat-fold-transformer

.travis.yml

@@ -16,14 +16,20 @@ before_install:
   - make installtools
   - bash ./scripts/install-postgres-10.sh
   - npm install -g ganache-cli
+  - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+  - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+  - sudo apt-get update && sudo apt-get install yarn
 
 before_script:
   - go get -u github.com/pressly/sup/cmd/sup
   - sudo -u postgres createdb vulcanize_private
   - make migrate NAME=vulcanize_private
   - bash ./pkg/transformers/start_test_chain.sh
+  - cd postgraphile && yarn
 
 script:
+  - yarn test
+  - cd ../
   - make test
 
 notifications:

db/migrations/1538064962_add_bite_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN bite_checked;

db/migrations/1538064962_add_bite_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN bite_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538067790_add_drip_drip_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_drip_checked;

db/migrations/1538067790_add_drip_drip_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_drip_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538070524_add_drip_file_to_checked_headers.down.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_file_ilk_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_file_repo_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_file_vow_checked;

db/migrations/1538070524_add_drip_file_to_checked_headers.up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_file_repo_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_file_vow_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538075501_add_frob_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN frob_checked;

db/migrations/1538075501_add_frob_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN frob_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538078432_add_pit_file_to_checked_headers.down.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN pit_file_debt_ceiling_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN pit_file_ilk_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN pit_file_stability_fee_checked;

db/migrations/1538078432_add_pit_file_to_checked_headers.up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN pit_file_debt_ceiling_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN pit_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN pit_file_stability_fee_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538082658_add_vat_init_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN vat_init_checked;

db/migrations/1538082658_add_vat_init_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN vat_init_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/schema.sql

@@ -856,6 +856,16 @@ CREATE TABLE public.checked_headers (
     cat_file_chop_lump_checked boolean DEFAULT false NOT NULL,
     cat_file_flip_checked boolean DEFAULT false NOT NULL,
     cat_file_pit_vow_checked boolean DEFAULT false NOT NULL,
+    bite_checked boolean DEFAULT false NOT NULL,
+    drip_drip_checked boolean DEFAULT false NOT NULL,
+    drip_file_ilk_checked boolean DEFAULT false NOT NULL,
+    drip_file_repo_checked boolean DEFAULT false NOT NULL,
+    drip_file_vow_checked boolean DEFAULT false NOT NULL,
+    frob_checked boolean DEFAULT false NOT NULL,
+    pit_file_debt_ceiling_checked boolean DEFAULT false NOT NULL,
+    pit_file_ilk_checked boolean DEFAULT false NOT NULL,
+    pit_file_stability_fee_checked boolean DEFAULT false NOT NULL,
+    vat_init_checked boolean DEFAULT false NOT NULL,
     vat_fold_checked boolean DEFAULT false NOT NULL
 );
 

pkg/transformers/bite/converter.go

@@ -19,66 +19,75 @@ package bite
 import (
 	"encoding/json"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type Converter interface {
-	ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error)
+	ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error)
-	ToModel(flipKick BiteEntity) (BiteModel, error)
+	ToModels(biteEntities []BiteEntity) ([]BiteModel, error)
 }
 
 type BiteConverter struct{}
 
-func (BiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) {
+func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) {
-	entity := BiteEntity{}
+	var entities []BiteEntity
-	address := common.HexToAddress(contractAddress)
+	for _, ethLog := range ethLogs {
-	abi, err := geth.ParseAbi(contractAbi)
+		entity := BiteEntity{}
-	if err != nil {
+		address := ethLog.Address
-		return entity, err
+		abi, err := geth.ParseAbi(contractAbi)
+		if err != nil {
+			return nil, err
+		}
+
+		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+
+		err = contract.UnpackLog(&entity, "Bite", ethLog)
+		if err != nil {
+			return nil, err
+		}
+
+		entity.Raw = ethLog
+		entity.TransactionIndex = ethLog.TxIndex
+
+		entities = append(entities, entity)
 	}
 
-	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+	return entities, nil
-
-	err = contract.UnpackLog(&entity, "Bite", ethLog)
-	if err != nil {
-		return entity, err
-	}
-
-	entity.Raw = ethLog
-	entity.TransactionIndex = ethLog.TxIndex
-
-	return entity, nil
 }
-func (converter BiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
 
-	id := entity.Id
+func (converter BiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) {
-	ilk := entity.Ilk[:]
+	var models []BiteModel
-	urn := entity.Urn[:]
+	for _, entity := range entities {
-	ink := entity.Ink
+		id := entity.Id
-	art := entity.Art
+		ilk := entity.Ilk[:]
-	iArt := entity.IArt
+		urn := entity.Urn[:]
-	tab := entity.Tab
+		ink := entity.Ink
-	flip := entity.Flip
+		art := entity.Art
-	txIdx := entity.TransactionIndex
+		iArt := entity.IArt
-	rawLogJson, err := json.Marshal(entity.Raw)
+		tab := entity.Tab
-	rawLogString := string(rawLogJson)
+		flip := entity.Flip
-	if err != nil {
+		txIdx := entity.TransactionIndex
-		return BiteModel{}, err
+		rawLogJson, err := json.Marshal(entity.Raw)
+		rawLogString := string(rawLogJson)
+		if err != nil {
+			return nil, err
+		}
+
+		model := BiteModel{
+			Id:               shared.BigIntToString(id),
+			Ilk:              ilk,
+			Urn:              urn,
+			Ink:              shared.BigIntToString(ink),
+			Art:              shared.BigIntToString(art),
+			IArt:             shared.BigIntToString(iArt),
+			Tab:              shared.BigIntToString(tab),
+			Flip:             shared.BigIntToString(flip),
+			TransactionIndex: txIdx,
+			Raw:              rawLogString,
+		}
+		models = append(models, model)
 	}
-
+	return models, nil
-	return BiteModel{
-		Id:               shared.BigIntToString(id),
-		Ilk:              ilk,
-		Urn:              urn,
-		Ink:              shared.BigIntToString(ink),
-		Art:              shared.BigIntToString(art),
-		IArt:             shared.BigIntToString(iArt),
-		Tab:              shared.BigIntToString(tab),
-		Flip:             shared.BigIntToString(flip),
-		TransactionIndex: txIdx,
-		Raw:              rawLogString,
-	}, nil
 }

pkg/transformers/bite/converter_test.go

@@ -34,9 +34,11 @@ var _ = Describe("Bite Converter", func() {
 
 	Describe("ToEntity", func() {
 		It("converts an eth log to a bite entity", func() {
-			entity, err := converter.ToEntity(shared.CatContractAddress, shared.CatABI, test_data.EthBiteLog)
+			entities, err := converter.ToEntities(shared.CatABI, []types.Log{test_data.EthBiteLog})
 
 			Expect(err).NotTo(HaveOccurred())
+			Expect(len(entities)).To(Equal(1))
+			entity := entities[0]
 			Expect(entity.Ilk).To(Equal(test_data.BiteEntity.Ilk))
 			Expect(entity.Urn).To(Equal(test_data.BiteEntity.Urn))
 			Expect(entity.Ink).To(Equal(test_data.BiteEntity.Ink))
@@ -49,7 +51,7 @@ var _ = Describe("Bite Converter", func() {
 		})
 
 		It("returns an error if converting log to entity fails", func() {
-			_, err := converter.ToEntity(shared.CatContractAddress, "error abi", test_data.EthBiteLog)
+			_, err := converter.ToEntities("error abi", []types.Log{test_data.EthBiteLog})
 
 			Expect(err).To(HaveOccurred())
 		})
@@ -63,9 +65,11 @@ var _ = Describe("Bite Converter", func() {
 		})
 
 		It("converts an Entity to a Model", func() {
-			model, err := converter.ToModel(test_data.BiteEntity)
+			models, err := converter.ToModels([]bite.BiteEntity{test_data.BiteEntity})
 
 			Expect(err).NotTo(HaveOccurred())
+			Expect(len(models)).To(Equal(1))
+			model := models[0]
 			Expect(model).To(Equal(test_data.BiteModel))
 			Expect(model.TransactionIndex).To(Equal(test_data.BiteModel.TransactionIndex))
 		})
@@ -85,9 +89,11 @@ var _ = Describe("Bite Converter", func() {
 				TransactionIndex: 0,
 				Raw:              string(emptyLog),
 			}
-			model, err := converter.ToModel(emptyEntity)
+			models, err := converter.ToModels([]bite.BiteEntity{emptyEntity})
 
 			Expect(err).NotTo(HaveOccurred())
+			Expect(len(models)).To(Equal(1))
+			model := models[0]
 			Expect(model).To(Equal(expectedModel))
 		})
 	})

pkg/transformers/bite/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model BiteModel) error
+	Create(headerID int64, models []BiteModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,26 +33,48 @@ func NewBiteRepository(db *postgres.DB) Repository {
 	return BiteRepository{db: db}
 }
 
-func (repository BiteRepository) Create(headerID int64, model BiteModel) error {
+func (repository BiteRepository) Create(headerID int64, models []BiteModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
-        VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
-		headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
-	)
-
 	if err != nil {
 		return err
 	}
-
+	for _, model := range models {
-	return nil
+		_, err := tx.Exec(
+			`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
+        VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
+			headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET bite_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
 }
+
+func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET bite_checked = $2`, headerID, true)
+	return err
+}
+
 func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
 	var result []core.Header
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.bite on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR bite_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,

pkg/transformers/bite/repository_test.go

@@ -20,6 +20,7 @@ import (
 
 	"database/sql"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -28,16 +29,24 @@ import (
 
 var _ = Describe("Bite repository", func() {
 	Describe("Create", func() {
-		It("persists a bite record", func() {
+		var (
-			node := core.Node{}
+			biteRepository bite.Repository
-			db := test_config.NewTestDB(node)
+			db             *postgres.DB
+			err            error
+			headerID       int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			biteRepository := bite.NewBiteRepository(db)
+			biteRepository = bite.NewBiteRepository(db)
+		})
 
-			err = biteRepository.Create(headerID, test_data.BiteModel)
+		It("persists a bite record", func() {
+			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbBite bite.BiteModel
@@ -53,15 +62,18 @@ var _ = Describe("Bite repository", func() {
 			Expect(dbBite.Raw).To(MatchJSON(test_data.BiteModel.Raw))
 		})
 
-		It("does not duplicate bite events", func() {
+		It("marks header as checked for logs", func() {
-			node := core.Node{}
+			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
-			db := test_config.NewTestDB(node)
+
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			biteRepository := bite.NewBiteRepository(db)
+			var headerChecked bool
-			err = biteRepository.Create(headerID, test_data.BiteModel)
+			err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate bite events", func() {
+			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
 			Expect(err).NotTo(HaveOccurred())
 
 			var anotherBiteModel = bite.BiteModel{
@@ -77,21 +89,14 @@ var _ = Describe("Bite repository", func() {
 				Raw:              test_data.BiteModel.Raw,
 			}
 
-			err = biteRepository.Create(headerID, anotherBiteModel)
+			err = biteRepository.Create(headerID, []bite.BiteModel{anotherBiteModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes bite if corresponding header is deleted", func() {
-			node := core.Node{}
+			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
-			db := test_config.NewTestDB(node)
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			biteRepository := bite.NewBiteRepository(db)
-			err = biteRepository.Create(headerID, test_data.BiteModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -104,8 +109,45 @@ var _ = Describe("Bite repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		It("creates a row for a new headerID", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			biteRepository := bite.NewBiteRepository(db)
+
+			err = biteRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			biteRepository := bite.NewBiteRepository(db)
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = biteRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated bite event", func() {
+		It("returns headers that haven't been checked", func() {
 			node := core.Node{}
 			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
@@ -121,7 +163,7 @@ var _ = Describe("Bite repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			biteRepository := bite.NewBiteRepository(db)
-			err := biteRepository.Create(headerIDs[1], test_data.BiteModel)
+			err := biteRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := biteRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)

pkg/transformers/bite/transformer.go

@@ -67,20 +67,26 @@ func (b BiteTransformer) Execute() error {
 			log.Println("Error fetching matching logs:", err)
 			return err
 		}
-
+		if len(ethLogs) < 1 {
-		for _, ethLog := range ethLogs {
+			err = b.Repository.MarkHeaderChecked(header.Id)
-			entity, err := b.Converter.ToEntity(ethLog.Address.Hex(), config.ContractAbi, ethLog)
-			model, err := b.Converter.ToModel(entity)
 			if err != nil {
-				log.Println("Error converting logs:", err)
 				return err
 			}
+		}
+		entities, err := b.Converter.ToEntities(config.ContractAbi, ethLogs)
+		if err != nil {
+			return err
+		}
+		model, err := b.Converter.ToModels(entities)
+		if err != nil {
+			log.Println("Error converting logs:", err)
+			return err
+		}
 
-			err = b.Repository.Create(header.Id, model)
+		err = b.Repository.Create(header.Id, model)
-			if err != nil {
+		if err != nil {
-				log.Println("Error persisting bite record:", err)
+			log.Println("Error persisting bite record:", err)
-				return err
+			return err
-			}
 		}
 	}
 

pkg/transformers/bite/transfromer_test.go

@@ -17,10 +17,13 @@
 package bite_test
 
 import (
+	"math/rand"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
@@ -28,7 +31,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
 	bite_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/bite"
-	"math/rand"
 )
 
 var _ = Describe("Bite Transformer", func() {
@@ -89,13 +91,32 @@ var _ = Describe("Bite Transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		headerID := int64(123)
+		repository.SetMissingHeaders([]core.Header{{Id: headerID}})
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		repository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		repository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		repository.SetMarkHeaderCheckedErr(fakes.FakeError)
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts an eth log to an Entity", func() {
 		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ConverterContract).To(Equal(test_data.EthBiteLog.Address.Hex()))
 		Expect(converter.ConverterAbi).To(Equal(bite.BiteConfig.ContractAbi))
 		Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthBiteLog}))
 	})
@@ -121,7 +142,7 @@ var _ = Describe("Bite Transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(headerId))
-		Expect(repository.PassedBiteModel).To(Equal(test_data.BiteModel))
+		Expect(repository.PassedBiteModels).To(Equal([]bite.BiteModel{test_data.BiteModel}))
 	})
 
 	It("returns error if persisting bite record fails", func() {

pkg/transformers/drip_drip/converter.go

@@ -22,23 +22,31 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (DripDripModel, error)
+	ToModels(ethLogs []types.Log) ([]DripDripModel, error)
 }
 
 type DripDripConverter struct{}
 
-func (DripDripConverter) ToModel(ethLog types.Log) (DripDripModel, error) {
+func (DripDripConverter) ToModels(ethLogs []types.Log) ([]DripDripModel, error) {
-	err := verifyLog(ethLog)
+	var models []DripDripModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return DripDripModel{}, err
+		err := verifyLog(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		raw, err := json.Marshal(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		model := DripDripModel{
+			Ilk:              ilk,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
 	}
-	ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+	return models, nil
-	raw, err := json.Marshal(ethLog)
-	return DripDripModel{
-		Ilk:              ilk,
-		TransactionIndex: ethLog.TxIndex,
-		Raw:              raw,
-	}, err
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/drip_drip/converter_test.go

@@ -28,7 +28,7 @@ var _ = Describe("Drip drip converter", func() {
 		converter := drip_drip.DripDripConverter{}
 		badLog := types.Log{}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -36,9 +36,9 @@ var _ = Describe("Drip drip converter", func() {
 	It("converts a log to an model", func() {
 		converter := drip_drip.DripDripConverter{}
 
-		model, err := converter.ToModel(test_data.EthDripDripLog)
+		model, err := converter.ToModels([]types.Log{test_data.EthDripDripLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.DripDripModel))
+		Expect(model).To(Equal([]drip_drip.DripDripModel{test_data.DripDripModel}))
 	})
 })

pkg/transformers/drip_drip/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model DripDripModel) error
+	Create(headerID int64, models []DripDripModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,12 +33,38 @@ func NewDripDripRepository(db *postgres.DB) DripDripRepository {
 	return DripDripRepository{db: db}
 }
 
-func (repository DripDripRepository) Create(headerID int64, model DripDripModel) error {
+func (repository DripDripRepository) Create(headerID int64, models []DripDripModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.drip_drip (header_id, ilk, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.drip_drip (header_id, ilk, tx_idx, raw_log)
         VALUES($1, $2, $3, $4)`,
-		headerID, model.Ilk, model.TransactionIndex, model.Raw,
+			headerID, model.Ilk, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_drip_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_drip_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_drip_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_drip_checked = $2`, headerID, true)
 	return err
 }
 
@@ -46,8 +73,8 @@ func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingB
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.drip_drip on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR drip_drip_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,

pkg/transformers/drip_drip/repository_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
 
 var _ = Describe("Drip drip repository", func() {
 	Describe("Create", func() {
-		It("adds a drip drip event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db                 *postgres.DB
+			dripDripRepository drip_drip.Repository
+			err                error
+			headerID           int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripDripRepository := drip_drip.NewDripDripRepository(db)
+			dripDripRepository = drip_drip.NewDripDripRepository(db)
+		})
 
-			err = dripDripRepository.Create(headerID, test_data.DripDripModel)
+		It("adds a drip drip event", func() {
+			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile drip_drip.DripDripModel
@@ -48,30 +58,28 @@ var _ = Describe("Drip drip repository", func() {
 			Expect(dbPitFile.Raw).To(MatchJSON(test_data.DripDripModel.Raw))
 		})
 
-		It("does not duplicate drip drip events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripDripRepository := drip_drip.NewDripDripRepository(db)
+			var headerChecked bool
-			err = dripDripRepository.Create(headerID, test_data.DripDripModel)
+			err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate drip drip events", func() {
+			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = dripDripRepository.Create(headerID, test_data.DripDripModel)
+			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes drip drip if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			dripDripRepository := drip_drip.NewDripDripRepository(db)
-			err = dripDripRepository.Create(headerID, test_data.DripDripModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -84,8 +92,48 @@ var _ = Describe("Drip drip repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                 *postgres.DB
+			dripDripRepository drip_drip.Repository
+			err                error
+			headerID           int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			dripDripRepository = drip_drip.NewDripDripRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = dripDripRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = dripDripRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated drip drip event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -100,7 +148,7 @@ var _ = Describe("Drip drip repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			dripDripRepository := drip_drip.NewDripDripRepository(db)
-			err := dripDripRepository.Create(headerIDs[1], test_data.DripDripModel)
+			err := dripDripRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -111,6 +159,33 @@ var _ = Describe("Drip drip repository", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if drip drip logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			dripDripBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, dripDripBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			dripDripRepository := drip_drip.NewDripDripRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -128,7 +203,7 @@ var _ = Describe("Drip drip repository", func() {
 			}
 			dripDripRepository := drip_drip.NewDripDripRepository(db)
 			dripDripRepositoryTwo := drip_drip.NewDripDripRepository(dbTwo)
-			err := dripDripRepository.Create(headerIDs[0], test_data.DripDripModel)
+			err := dripDripRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := dripDripRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/drip_drip/transformer.go

@@ -59,16 +59,20 @@ func (transformer DripDripTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		models, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, models)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/drip_drip/transformer_test.go

@@ -97,6 +97,42 @@ var _ = Describe("Drip drip transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &drip_drip_mocks.MockDripDripConverter{}
+		mockRepository := &drip_drip_mocks.MockDripDripRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := drip_drip.DripDripTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &drip_drip_mocks.MockDripDripConverter{}
+		mockRepository := &drip_drip_mocks.MockDripDripRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := drip_drip.DripDripTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &drip_drip_mocks.MockDripDripConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -112,7 +148,7 @@ var _ = Describe("Drip drip transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthDripDripLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripDripLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -151,7 +187,7 @@ var _ = Describe("Drip drip transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.DripDripModel))
+		Expect(repository.PassedModels).To(Equal([]drip_drip.DripDripModel{test_data.DripDripModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/drip_file/ilk/converter.go

@@ -25,28 +25,36 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (DripFileIlkModel, error)
+	ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error)
 }
 
 type DripFileIlkConverter struct{}
 
-func (DripFileIlkConverter) ToModel(ethLog types.Log) (DripFileIlkModel, error) {
+func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error) {
-	err := verifyLog(ethLog)
+	var models []DripFileIlkModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return DripFileIlkModel{}, err
+		err := verifyLog(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
+		taxBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
+		tax := big.NewInt(0).SetBytes(taxBytes).String()
+		raw, err := json.Marshal(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		model := DripFileIlkModel{
+			Ilk:              ilk,
+			Vow:              vow,
+			Tax:              tax,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
 	}
-	ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+	return models, nil
-	vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
-	taxBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
-	tax := big.NewInt(0).SetBytes(taxBytes).String()
-	raw, err := json.Marshal(ethLog)
-	return DripFileIlkModel{
-		Ilk:              ilk,
-		Vow:              vow,
-		Tax:              tax,
-		TransactionIndex: ethLog.TxIndex,
-		Raw:              raw,
-	}, err
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/drip_file/ilk/converter_test.go

@@ -32,7 +32,7 @@ var _ = Describe("Drip file ilk converter", func() {
 			Data:   []byte{1, 1, 1, 1, 1},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -43,7 +43,7 @@ var _ = Describe("Drip file ilk converter", func() {
 			Topics: []common.Hash{{}, {}, {}, {}},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 
@@ -52,9 +52,10 @@ var _ = Describe("Drip file ilk converter", func() {
 	It("converts a log to a model", func() {
 		converter := ilk.DripFileIlkConverter{}
 
-		model, err := converter.ToModel(test_data.EthDripFileIlkLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthDripFileIlkLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.DripFileIlkModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.DripFileIlkModel))
 	})
 })

pkg/transformers/drip_file/ilk/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model DripFileIlkModel) error
+	Create(headerID int64, models []DripFileIlkModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,12 +33,38 @@ func NewDripFileIlkRepository(db *postgres.DB) DripFileIlkRepository {
 	return DripFileIlkRepository{db: db}
 }
 
-func (repository DripFileIlkRepository) Create(headerID int64, model DripFileIlkModel) error {
+func (repository DripFileIlkRepository) Create(headerID int64, models []DripFileIlkModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, tx_idx, raw_log)
         VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
-		headerID, model.Ilk, model.Vow, model.Tax, model.TransactionIndex, model.Raw,
+			headerID, model.Ilk, model.Vow, model.Tax, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_file_ilk_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_file_ilk_checked = $2`, headerID, true)
 	return err
 }
 
@@ -46,8 +73,8 @@ func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endi
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.drip_file_ilk on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR drip_file_ilk_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,

pkg/transformers/drip_file/ilk/repository_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
 
 var _ = Describe("Drip file ilk repository", func() {
 	Describe("Create", func() {
-		It("adds a drip file ilk event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db                    *postgres.DB
+			dripFileIlkRepository ilk.Repository
+			err                   error
+			headerID              int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
+			dripFileIlkRepository = ilk.NewDripFileIlkRepository(db)
+		})
 
-			err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
+		It("adds a drip file ilk event", func() {
+			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbDripFileIlk ilk.DripFileIlkModel
@@ -50,30 +60,28 @@ var _ = Describe("Drip file ilk repository", func() {
 			Expect(dbDripFileIlk.Raw).To(MatchJSON(test_data.DripFileIlkModel.Raw))
 		})
 
-		It("does not duplicate drip file events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
+			var headerChecked bool
-			err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
+			err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate drip file events", func() {
+			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
+			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes drip file if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
-			err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -86,6 +94,46 @@ var _ = Describe("Drip file ilk repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                    *postgres.DB
+			dripFileIlkRepository ilk.Repository
+			err                   error
+			headerID              int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			dripFileIlkRepository = ilk.NewDripFileIlkRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = dripFileIlkRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = dripFileIlkRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
 		It("returns headers with no associated drip file event", func() {
 			db := test_config.NewTestDB(core.Node{})
@@ -102,7 +150,7 @@ var _ = Describe("Drip file ilk repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
-			err := dripFileIlkRepository.Create(headerIDs[1], test_data.DripFileIlkModel)
+			err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -113,6 +161,33 @@ var _ = Describe("Drip file ilk repository", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if drip file ilk logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			dripFileIlkdBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, dripFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -130,7 +205,7 @@ var _ = Describe("Drip file ilk repository", func() {
 			}
 			dripFileIlkRepository := ilk.NewDripFileIlkRepository(db)
 			dripFileIlkRepositoryTwo := ilk.NewDripFileIlkRepository(dbTwo)
-			err := dripFileIlkRepository.Create(headerIDs[0], test_data.DripFileIlkModel)
+			err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := dripFileIlkRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/drip_file/ilk/transformer.go

@@ -60,16 +60,20 @@ func (transformer DripFileIlkTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		model, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, model)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/drip_file/ilk/transformer_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file"
@@ -96,6 +97,42 @@ var _ = Describe("Drip file ilk transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &ilk_mocks.MockDripFileIlkConverter{}
+		mockRepository := &ilk_mocks.MockDripFileIlkRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := ilk.DripFileIlkTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &ilk_mocks.MockDripFileIlkConverter{}
+		mockRepository := &ilk_mocks.MockDripFileIlkRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := ilk.DripFileIlkTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &ilk_mocks.MockDripFileIlkConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -111,7 +148,7 @@ var _ = Describe("Drip file ilk transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthDripFileIlkLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileIlkLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -150,7 +187,7 @@ var _ = Describe("Drip file ilk transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.DripFileIlkModel))
+		Expect(repository.PassedModels).To(Equal([]ilk.DripFileIlkModel{test_data.DripFileIlkModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/drip_file/repo/converter.go

@@ -23,25 +23,33 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (DripFileRepoModel, error)
+	ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error)
 }
 
 type DripFileRepoConverter struct{}
 
-func (DripFileRepoConverter) ToModel(ethLog types.Log) (DripFileRepoModel, error) {
+func (DripFileRepoConverter) ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error) {
-	err := verifyLog(ethLog)
+	var models []DripFileRepoModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return DripFileRepoModel{}, err
+		err := verifyLog(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
+		raw, err := json.Marshal(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		model := DripFileRepoModel{
+			What:             what,
+			Data:             data,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
 	}
-	what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+	return models, nil
-	data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
-	raw, err := json.Marshal(ethLog)
-	return DripFileRepoModel{
-		What:             what,
-		Data:             data,
-		TransactionIndex: ethLog.TxIndex,
-		Raw:              raw,
-	}, err
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/drip_file/repo/converter_test.go

@@ -32,7 +32,7 @@ var _ = Describe("Drip file repo converter", func() {
 			Data:   []byte{1, 1, 1, 1, 1},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -40,9 +40,10 @@ var _ = Describe("Drip file repo converter", func() {
 	It("converts a log to a model", func() {
 		converter := repo.DripFileRepoConverter{}
 
-		model, err := converter.ToModel(test_data.EthDripFileRepoLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthDripFileRepoLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.DripFileRepoModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.DripFileRepoModel))
 	})
 })

pkg/transformers/drip_file/repo/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model DripFileRepoModel) error
+	Create(headerID int64, models []DripFileRepoModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,12 +33,38 @@ func NewDripFileRepoRepository(db *postgres.DB) DripFileRepoRepository {
 	return DripFileRepoRepository{db: db}
 }
 
-func (repository DripFileRepoRepository) Create(headerID int64, model DripFileRepoModel) error {
+func (repository DripFileRepoRepository) Create(headerID int64, models []DripFileRepoModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.drip_file_repo (header_id, what, data, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.drip_file_repo (header_id, what, data, tx_idx, raw_log)
         VALUES($1, $2, $3::NUMERIC, $4, $5)`,
-		headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
+			headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_file_repo_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_file_repo_checked = $2`, headerID, true)
 	return err
 }
 
@@ -46,8 +73,8 @@ func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, end
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.drip_file_repo on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR drip_file_repo_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,

pkg/transformers/drip_file/repo/repository_test.go

@@ -19,23 +19,33 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
-var _ = Describe("", func() {
+var _ = Describe("Drip file repo repository", func() {
 	Describe("Create", func() {
-		It("adds a drip file repo event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db                     *postgres.DB
+			dripFileRepoRepository repo.Repository
+			err                    error
+			headerID               int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
+			dripFileRepoRepository = repo.NewDripFileRepoRepository(db)
+		})
 
-			err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
+		It("adds a drip file repo event", func() {
+			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbDripFileRepo repo.DripFileRepoModel
@@ -47,30 +57,28 @@ var _ = Describe("", func() {
 			Expect(dbDripFileRepo.Raw).To(MatchJSON(test_data.DripFileRepoModel.Raw))
 		})
 
-		It("does not duplicate drip file events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
+			var headerChecked bool
-			err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
+			err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate drip file events", func() {
+			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
+			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes drip file if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
-			err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -83,6 +91,46 @@ var _ = Describe("", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                     *postgres.DB
+			dripFileRepoRepository repo.Repository
+			err                    error
+			headerID               int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			dripFileRepoRepository = repo.NewDripFileRepoRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = dripFileRepoRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = dripFileRepoRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
 		It("returns headers with no associated drip file event", func() {
 			db := test_config.NewTestDB(core.Node{})
@@ -99,7 +147,7 @@ var _ = Describe("", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
-			err := dripFileRepoRepository.Create(headerIDs[1], test_data.DripFileRepoModel)
+			err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -110,6 +158,33 @@ var _ = Describe("", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if drip file repo logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			dripFileRepodBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, dripFileRepodBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -127,7 +202,7 @@ var _ = Describe("", func() {
 			}
 			dripFileRepoRepository := repo.NewDripFileRepoRepository(db)
 			dripFileRepoRepositoryTwo := repo.NewDripFileRepoRepository(dbTwo)
-			err := dripFileRepoRepository.Create(headerIDs[0], test_data.DripFileRepoModel)
+			err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := dripFileRepoRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/drip_file/repo/transformer.go

@@ -60,16 +60,20 @@ func (transformer DripFileRepoTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		model, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, model)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/drip_file/repo/transformer_test.go

@@ -97,6 +97,42 @@ var _ = Describe("Drip file repo transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &repo_mocks.MockDripFileRepoConverter{}
+		mockRepository := &repo_mocks.MockDripFileRepoRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := repo.DripFileRepoTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &repo_mocks.MockDripFileRepoConverter{}
+		mockRepository := &repo_mocks.MockDripFileRepoRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := repo.DripFileRepoTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &repo_mocks.MockDripFileRepoConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -112,7 +148,7 @@ var _ = Describe("Drip file repo transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthDripFileRepoLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileRepoLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -151,7 +187,7 @@ var _ = Describe("Drip file repo transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.DripFileRepoModel))
+		Expect(repository.PassedModels).To(Equal([]repo.DripFileRepoModel{test_data.DripFileRepoModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/drip_file/vow/converter.go

@@ -22,25 +22,33 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (DripFileVowModel, error)
+	ToModels(ethLogs []types.Log) ([]DripFileVowModel, error)
 }
 
 type DripFileVowConverter struct{}
 
-func (DripFileVowConverter) ToModel(ethLog types.Log) (DripFileVowModel, error) {
+func (DripFileVowConverter) ToModels(ethLogs []types.Log) ([]DripFileVowModel, error) {
-	err := verifyLog(ethLog)
+	var models []DripFileVowModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return DripFileVowModel{}, err
+		err := verifyLog(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		data := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
+		raw, err := json.Marshal(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		model := DripFileVowModel{
+			What:             what,
+			Data:             data,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
 	}
-	what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+	return models, nil
-	data := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
-	raw, err := json.Marshal(ethLog)
-	return DripFileVowModel{
-		What:             what,
-		Data:             data,
-		TransactionIndex: ethLog.TxIndex,
-		Raw:              raw,
-	}, err
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/drip_file/vow/converter_test.go

@@ -32,7 +32,7 @@ var _ = Describe("Drip file repo converter", func() {
 			Data:   []byte{1, 1, 1, 1, 1},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -40,9 +40,10 @@ var _ = Describe("Drip file repo converter", func() {
 	It("converts a log to a model", func() {
 		converter := vow.DripFileVowConverter{}
 
-		model, err := converter.ToModel(test_data.EthDripFileVowLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthDripFileVowLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.DripFileVowModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.DripFileVowModel))
 	})
 })

pkg/transformers/drip_file/vow/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model DripFileVowModel) error
+	Create(headerID int64, models []DripFileVowModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,12 +33,38 @@ func NewDripFileVowRepository(db *postgres.DB) DripFileVowRepository {
 	return DripFileVowRepository{db: db}
 }
 
-func (repository DripFileVowRepository) Create(headerID int64, model DripFileVowModel) error {
+func (repository DripFileVowRepository) Create(headerID int64, models []DripFileVowModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.drip_file_vow (header_id, what, data, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.drip_file_vow (header_id, what, data, tx_idx, raw_log)
         VALUES($1, $2, $3, $4, $5)`,
-		headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
+			headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_file_vow_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET drip_file_vow_checked = $2`, headerID, true)
 	return err
 }
 
@@ -46,8 +73,8 @@ func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endi
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.drip_file_vow on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR drip_file_vow_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,

pkg/transformers/drip_file/vow/repository_test.go

@@ -16,26 +16,38 @@ package vow_test
 
 import (
 	"database/sql"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
-var _ = Describe("", func() {
+var _ = Describe("Drip file vow repository", func() {
 	Describe("Create", func() {
-		It("adds a drip file vow event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db                    *postgres.DB
+			dripFileVowRepository vow.Repository
+			err                   error
+			headerID              int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileVowRepository := vow.NewDripFileVowRepository(db)
+			dripFileVowRepository = vow.NewDripFileVowRepository(db)
+		})
 
-			err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
+		It("adds a drip file vow event", func() {
+			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbDripFileVow vow.DripFileVowModel
@@ -47,30 +59,28 @@ var _ = Describe("", func() {
 			Expect(dbDripFileVow.Raw).To(MatchJSON(test_data.DripFileVowModel.Raw))
 		})
 
-		It("does not duplicate drip file events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			dripFileVowRepository := vow.NewDripFileVowRepository(db)
+			var headerChecked bool
-			err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
+			err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate drip file events", func() {
+			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
+			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes drip file if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			dripFileVowRepository := vow.NewDripFileVowRepository(db)
-			err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -83,8 +93,48 @@ var _ = Describe("", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                    *postgres.DB
+			dripFileVowRepository vow.Repository
+			err                   error
+			headerID              int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			dripFileVowRepository = vow.NewDripFileVowRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = dripFileVowRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = dripFileVowRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated drip file event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -99,7 +149,7 @@ var _ = Describe("", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			dripFileVowRepository := vow.NewDripFileVowRepository(db)
-			err := dripFileVowRepository.Create(headerIDs[1], test_data.DripFileVowModel)
+			err := dripFileVowRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -110,6 +160,33 @@ var _ = Describe("", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if drip file vow logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			dripFileVowdBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, dripFileVowdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			dripFileVowRepository := vow.NewDripFileVowRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -127,7 +204,7 @@ var _ = Describe("", func() {
 			}
 			dripFileVowRepository := vow.NewDripFileVowRepository(db)
 			dripFileVowRepositoryTwo := vow.NewDripFileVowRepository(dbTwo)
-			err := dripFileVowRepository.Create(headerIDs[0], test_data.DripFileVowModel)
+			err := dripFileVowRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := dripFileVowRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/drip_file/vow/transformer.go

@@ -60,16 +60,20 @@ func (transformer DripFileVowTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		model, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, model)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/drip_file/vow/transformer_test.go

@@ -97,6 +97,42 @@ var _ = Describe("Drip file vow transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &vow_mocks.MockDripFileVowConverter{}
+		mockRepository := &vow_mocks.MockDripFileVowRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := vow.DripFileVowTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &vow_mocks.MockDripFileVowConverter{}
+		mockRepository := &vow_mocks.MockDripFileVowRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := vow.DripFileVowTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &vow_mocks.MockDripFileVowConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -112,7 +148,7 @@ var _ = Describe("Drip file vow transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthDripFileVowLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileVowLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -151,7 +187,7 @@ var _ = Describe("Drip file vow transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.DripFileVowModel))
+		Expect(repository.PassedModels).To(Equal([]vow.DripFileVowModel{test_data.DripFileVowModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/frob/converter.go

@@ -16,7 +16,6 @@ package frob
 
 import (
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
 	"encoding/json"
@@ -24,40 +23,50 @@ import (
 )
 
 type Converter interface {
-	ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error)
+	ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error)
-	ToModel(flipKick FrobEntity) (FrobModel, error)
+	ToModels(entities []FrobEntity) ([]FrobModel, error)
 }
 
 type FrobConverter struct{}
 
-func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error) {
+func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error) {
-	entity := FrobEntity{}
+	var entities []FrobEntity
-	address := common.HexToAddress(contractAddress)
+	for _, ethLog := range ethLogs {
-	abi, err := geth.ParseAbi(contractAbi)
+		entity := FrobEntity{}
-	if err != nil {
+		address := ethLog.Address
-		return entity, err
+		abi, err := geth.ParseAbi(contractAbi)
+		if err != nil {
+			return nil, err
+		}
+		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+		err = contract.UnpackLog(&entity, "Frob", ethLog)
+		entity.TransactionIndex = ethLog.TxIndex
+		entity.Raw = ethLog
+		entities = append(entities, entity)
 	}
-	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+
-	err = contract.UnpackLog(&entity, "Frob", ethLog)
+	return entities, nil
-	entity.TransactionIndex = ethLog.TxIndex
-	entity.Raw = ethLog
-	return entity, err
 }
 
-func (FrobConverter) ToModel(frob FrobEntity) (FrobModel, error) {
+func (FrobConverter) ToModels(entities []FrobEntity) ([]FrobModel, error) {
-	rawLog, err := json.Marshal(frob.Raw)
+	var models []FrobModel
-	if err != nil {
+	for _, entity := range entities {
-		return FrobModel{}, err
+		rawLog, err := json.Marshal(entity.Raw)
+		if err != nil {
+			return nil, err
+		}
+		model := FrobModel{
+			Ilk:              entity.Ilk[:],
+			Urn:              entity.Urn[:],
+			Ink:              entity.Ink.String(),
+			Art:              entity.Art.String(),
+			Dink:             entity.Dink.String(),
+			Dart:             entity.Dart.String(),
+			IArt:             entity.IArt.String(),
+			TransactionIndex: entity.TransactionIndex,
+			Raw:              rawLog,
+		}
+		models = append(models, model)
 	}
-	return FrobModel{
+	return models, nil
-		Ilk:              frob.Ilk[:],
-		Urn:              frob.Urn[:],
-		Ink:              frob.Ink.String(),
-		Art:              frob.Art.String(),
-		Dink:             frob.Dink.String(),
-		Dart:             frob.Dart.String(),
-		IArt:             frob.IArt.String(),
-		TransactionIndex: frob.TransactionIndex,
-		Raw:              rawLog,
-	}, nil
 }

pkg/transformers/frob/converter_test.go

@@ -18,6 +18,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -27,18 +28,20 @@ var _ = Describe("Frob converter", func() {
 	It("converts a log to an entity", func() {
 		converter := frob.FrobConverter{}
 
-		entity, err := converter.ToEntity(shared.PitContractAddress, shared.PitABI, test_data.EthFrobLog)
+		entities, err := converter.ToEntities(shared.PitABI, []types.Log{test_data.EthFrobLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(entity).To(Equal(test_data.FrobEntity))
+		Expect(len(entities)).To(Equal(1))
+		Expect(entities[0]).To(Equal(test_data.FrobEntity))
 	})
 
 	It("converts an entity to a model", func() {
 		converter := frob.FrobConverter{}
 
-		model, err := converter.ToModel(test_data.FrobEntity)
+		models, err := converter.ToModels([]frob.FrobEntity{test_data.FrobEntity})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.FrobModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.FrobModel))
 	})
 })

pkg/transformers/frob/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model FrobModel) error
+	Create(headerID int64, models []FrobModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,10 +33,36 @@ func NewFrobRepository(db *postgres.DB) FrobRepository {
 	return FrobRepository{db: db}
 }
 
-func (repository FrobRepository) Create(headerID int64, model FrobModel) error {
+func (repository FrobRepository) Create(headerID int64, models []FrobModel) error {
-	_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
+	tx, err := repository.db.Begin()
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
 		VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10)`,
-		headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
+			headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET frob_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository FrobRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET frob_checked = $2`, headerID, true)
 	return err
 }
 
@@ -44,8 +71,8 @@ func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlock
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.frob on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR frob_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,

pkg/transformers/frob/repository_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
 
 var _ = Describe("Frob repository", func() {
 	Describe("Create", func() {
-		It("adds a frob", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db             *postgres.DB
+			frobRepository frob.Repository
+			err            error
+			headerID       int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			frobRepository := frob.NewFrobRepository(db)
+			frobRepository = frob.NewFrobRepository(db)
+		})
 
-			err = frobRepository.Create(headerID, test_data.FrobModel)
+		It("adds a frob", func() {
+			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbFrob frob.FrobModel
@@ -54,30 +64,28 @@ var _ = Describe("Frob repository", func() {
 			Expect(dbFrob.Raw).To(MatchJSON(test_data.FrobModel.Raw))
 		})
 
-		It("does not duplicate frob events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			frobRepository := frob.NewFrobRepository(db)
+			var headerChecked bool
-			err = frobRepository.Create(headerID, test_data.FrobModel)
+			err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate frob events", func() {
+			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = frobRepository.Create(headerID, test_data.FrobModel)
+			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes frob if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			frobRepository := frob.NewFrobRepository(db)
-			err = frobRepository.Create(headerID, test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -90,8 +98,48 @@ var _ = Describe("Frob repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db             *postgres.DB
+			frobRepository frob.Repository
+			err            error
+			headerID       int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			frobRepository = frob.NewFrobRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = frobRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = frobRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated frob event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -106,7 +154,7 @@ var _ = Describe("Frob repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			frobRepository := frob.NewFrobRepository(db)
-			err := frobRepository.Create(headerIDs[1], test_data.FrobModel)
+			err := frobRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -117,6 +165,33 @@ var _ = Describe("Frob repository", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if frob logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			frobdBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, frobdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			frobRepository := frob.NewFrobRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -134,7 +209,7 @@ var _ = Describe("Frob repository", func() {
 			}
 			frobRepository := frob.NewFrobRepository(db)
 			frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
-			err := frobRepository.Create(headerIDs[0], test_data.FrobModel)
+			err := frobRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/frob/transformer.go

@@ -59,20 +59,24 @@ func (transformer FrobTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			entity, err := transformer.Converter.ToEntity(log.Address.Hex(), FrobConfig.ContractAbi, log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			model, err := transformer.Converter.ToModel(entity)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		entities, err := transformer.Converter.ToEntities(FrobConfig.ContractAbi, matchingLogs)
+		if err != nil {
+			return err
+		}
+		models, err := transformer.Converter.ToModels(entities)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, models)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/frob/transformer_test.go

@@ -96,6 +96,42 @@ var _ = Describe("Frob transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &frob_mocks.MockFrobConverter{}
+		mockRepository := &frob_mocks.MockFrobRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := frob.FrobTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &frob_mocks.MockFrobConverter{}
+		mockRepository := &frob_mocks.MockFrobRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := frob.FrobTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs to entity", func() {
 		converter := &frob_mocks.MockFrobConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -111,14 +147,13 @@ var _ = Describe("Frob transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedContractAddress).To(Equal(test_data.EthFrobLog.Address.Hex()))
 		Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
-		Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthFrobLog}))
 	})
 
 	It("returns error if converting to entity returns error", func() {
 		converter := &frob_mocks.MockFrobConverter{}
-		converter.SetToEntityError(fakes.FakeError)
+		converter.SetToEntitiesError(fakes.FakeError)
 		fetcher := &mocks.MockLogFetcher{}
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
 		repository := &frob_mocks.MockFrobRepository{}
@@ -150,12 +185,12 @@ var _ = Describe("Frob transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))
+		Expect(converter.PassedEntities).To(Equal([]frob.FrobEntity{test_data.FrobEntity}))
 	})
 
 	It("returns error if converting to model returns error", func() {
 		converter := &frob_mocks.MockFrobConverter{}
-		converter.SetToModelError(fakes.FakeError)
+		converter.SetToModelsError(fakes.FakeError)
 		fetcher := &mocks.MockLogFetcher{}
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
 		repository := &frob_mocks.MockFrobRepository{}
@@ -189,7 +224,7 @@ var _ = Describe("Frob transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel))
+		Expect(repository.PassedModels).To(Equal([]frob.FrobModel{test_data.FrobModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/pit_file/debt_ceiling/converter.go

@@ -25,26 +25,34 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error)
+	ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error)
 }
 
 type PitFileDebtCeilingConverter struct{}
 
-func (PitFileDebtCeilingConverter) ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error) {
+func (PitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error) {
-	err := verifyLog(ethLog)
+	var models []PitFileDebtCeilingModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return PitFileDebtCeilingModel{}, err
+		err := verifyLog(ethLog)
-	}
+		if err != nil {
-	what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+			return nil, err
-	data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
+		}
+		what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
 
-	raw, err := json.Marshal(ethLog)
+		raw, err := json.Marshal(ethLog)
-	return PitFileDebtCeilingModel{
+		if err != nil {
-		What:             what,
+			return nil, err
-		Data:             data,
+		}
-		TransactionIndex: ethLog.TxIndex,
+		model := PitFileDebtCeilingModel{
-		Raw:              raw,
+			What:             what,
-	}, err
+			Data:             data,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
+	}
+	return models, nil
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/pit_file/debt_ceiling/converter_test.go

@@ -31,7 +31,7 @@ var _ = Describe("Pit file debt ceiling converter", func() {
 			Data: []byte{1, 1, 1, 1, 1},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -42,7 +42,7 @@ var _ = Describe("Pit file debt ceiling converter", func() {
 			Topics: []common.Hash{{}, {}, {}, {}},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -50,9 +50,10 @@ var _ = Describe("Pit file debt ceiling converter", func() {
 	It("converts a log to an model", func() {
 		converter := debt_ceiling.PitFileDebtCeilingConverter{}
 
-		model, err := converter.ToModel(test_data.EthPitFileDebtCeilingLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthPitFileDebtCeilingLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.PitFileDebtCeilingModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.PitFileDebtCeilingModel))
 	})
 })

pkg/transformers/pit_file/debt_ceiling/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model PitFileDebtCeilingModel) error
+	Create(headerID int64, models []PitFileDebtCeilingModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -34,12 +35,38 @@ func NewPitFileDebtCeilingRepository(db *postgres.DB) PitFileDebtCeilingReposito
 	}
 }
 
-func (repository PitFileDebtCeilingRepository) Create(headerID int64, model PitFileDebtCeilingModel) error {
+func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []PitFileDebtCeilingModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log)
         VALUES($1, $2, $3::NUMERIC, $4, $5)`,
-		headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
+			headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true)
 	return err
 }
 
@@ -48,8 +75,8 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.pit_file_debt_ceiling on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR pit_file_debt_ceiling_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,
@@ -57,6 +84,5 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe
 		endingBlockNumber,
 		repository.db.Node.ID,
 	)
-
 	return result, err
 }

pkg/transformers/pit_file/debt_ceiling/repository_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
 
 var _ = Describe("Pit file debt ceiling repository", func() {
 	Describe("Create", func() {
-		It("adds a pit file debt ceiling event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db                           *postgres.DB
+			pitFileDebtCeilingRepository debt_ceiling.Repository
+			err                          error
+			headerID                     int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
+		})
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+		It("adds a pit file debt ceiling event", func() {
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile debt_ceiling.PitFileDebtCeilingModel
@@ -49,30 +59,28 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileDebtCeilingModel.Raw))
 		})
 
-		It("does not duplicate pit file events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			var headerChecked bool
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+			err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate pit file events", func() {
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -85,8 +93,48 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                           *postgres.DB
+			pitFileDebtCeilingRepository debt_ceiling.Repository
+			err                          error
+			headerID                     int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated pit file event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -101,7 +149,7 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
-			err := pitFileRepository.Create(headerIDs[1], test_data.PitFileDebtCeilingModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -112,6 +160,33 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if pit file debt ceiling logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileDebtCeilingdBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileDebtCeilingdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileDebtCeilingRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileDebtCeilingRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -129,7 +204,7 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 			}
 			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
 			pitFileRepositoryTwo := debt_ceiling.NewPitFileDebtCeilingRepository(dbTwo)
-			err := pitFileRepository.Create(headerIDs[0], test_data.PitFileDebtCeilingModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/pit_file/debt_ceiling/transformer.go

@@ -60,16 +60,20 @@ func (transformer PitFileDebtCeilingTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		model, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, model)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/pit_file/debt_ceiling/transformer_test.go

@@ -97,6 +97,42 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -112,7 +148,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileDebtCeilingLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileDebtCeilingLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -151,7 +187,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.PitFileDebtCeilingModel))
+		Expect(repository.PassedModels).To(Equal([]debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/pit_file/ilk/converter.go

@@ -25,29 +25,37 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (PitFileIlkModel, error)
+	ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error)
 }
 
 type PitFileIlkConverter struct{}
 
-func (PitFileIlkConverter) ToModel(ethLog types.Log) (PitFileIlkModel, error) {
+func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error) {
-	err := verifyLog(ethLog)
+	var models []PitFileIlkModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return PitFileIlkModel{}, err
+		err := verifyLog(ethLog)
-	}
+		if err != nil {
-	ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+			return nil, err
-	what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
+		}
-	riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
+		ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
-	risk := big.NewInt(0).SetBytes(riskBytes).String()
+		what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
+		riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:]
+		risk := big.NewInt(0).SetBytes(riskBytes).String()
 
-	raw, err := json.Marshal(ethLog)
+		raw, err := json.Marshal(ethLog)
-	return PitFileIlkModel{
+		if err != nil {
-		Ilk:              ilk,
+			return nil, err
-		What:             what,
+		}
-		Data:             risk,
+		model := PitFileIlkModel{
-		TransactionIndex: ethLog.TxIndex,
+			Ilk:              ilk,
-		Raw:              raw,
+			What:             what,
-	}, err
+			Data:             risk,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
+	}
+	return models, nil
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/pit_file/ilk/converter_test.go

@@ -31,7 +31,7 @@ var _ = Describe("Pit file ilk converter", func() {
 			Data: []byte{1, 1, 1, 1, 1},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -42,7 +42,7 @@ var _ = Describe("Pit file ilk converter", func() {
 			Topics: []common.Hash{{}, {}, {}, {}},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -50,9 +50,10 @@ var _ = Describe("Pit file ilk converter", func() {
 	It("converts a log to an model", func() {
 		converter := ilk.PitFileIlkConverter{}
 
-		model, err := converter.ToModel(test_data.EthPitFileIlkLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.PitFileIlkModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.PitFileIlkModel))
 	})
 })

pkg/transformers/pit_file/ilk/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model PitFileIlkModel) error
+	Create(headerID int64, models []PitFileIlkModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -34,12 +35,38 @@ func NewPitFileIlkRepository(db *postgres.DB) PitFileIlkRepository {
 	}
 }
 
-func (repository PitFileIlkRepository) Create(headerID int64, model PitFileIlkModel) error {
+func (repository PitFileIlkRepository) Create(headerID int64, models []PitFileIlkModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log)
         VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
-		headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw,
+			headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_ilk_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_ilk_checked = $2`, headerID, true)
 	return err
 }
 
@@ -48,8 +75,8 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.pit_file_ilk on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR pit_file_ilk_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,
@@ -57,6 +84,5 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin
 		endingBlockNumber,
 		repository.db.Node.ID,
 	)
-
 	return result, err
 }

pkg/transformers/pit_file/ilk/repository_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
 
 var _ = Describe("Pit file ilk repository", func() {
 	Describe("Create", func() {
-		It("adds a pit file ilk event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db                   *postgres.DB
+			pitFileIlkRepository ilk.Repository
+			err                  error
+			headerID             int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+			pitFileIlkRepository = ilk.NewPitFileIlkRepository(db)
+		})
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
+		It("adds a pit file ilk event", func() {
+			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile ilk.PitFileIlkModel
@@ -50,30 +60,28 @@ var _ = Describe("Pit file ilk repository", func() {
 			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkModel.Raw))
 		})
 
-		It("does not duplicate pit file ilk events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := ilk.NewPitFileIlkRepository(db)
+			var headerChecked bool
-			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
+			err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate pit file ilk events", func() {
+			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
+			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file ilk if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := ilk.NewPitFileIlkRepository(db)
-			err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -86,8 +94,48 @@ var _ = Describe("Pit file ilk repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                   *postgres.DB
+			pitFileIlkRepository ilk.Repository
+			err                  error
+			headerID             int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileIlkRepository = ilk.NewPitFileIlkRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = pitFileIlkRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = pitFileIlkRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated pit file ilk event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -102,7 +150,7 @@ var _ = Describe("Pit file ilk repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			pitFileRepository := ilk.NewPitFileIlkRepository(db)
-			err := pitFileRepository.Create(headerIDs[1], test_data.PitFileIlkModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -113,6 +161,33 @@ var _ = Describe("Pit file ilk repository", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if pit file ilk logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileIlkdBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileIlkRepository := ilk.NewPitFileIlkRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -130,7 +205,7 @@ var _ = Describe("Pit file ilk repository", func() {
 			}
 			pitFileRepository := ilk.NewPitFileIlkRepository(db)
 			pitFileRepositoryTwo := ilk.NewPitFileIlkRepository(dbTwo)
-			err := pitFileRepository.Create(headerIDs[0], test_data.PitFileIlkModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/pit_file/ilk/transformer.go

@@ -60,16 +60,20 @@ func (transformer PitFileIlkTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		models, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, models)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/pit_file/ilk/transformer_test.go

@@ -97,6 +97,42 @@ var _ = Describe("Pit file ilk transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
+		mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := ilk.PitFileIlkTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
+		mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := ilk.PitFileIlkTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -112,7 +148,7 @@ var _ = Describe("Pit file ilk transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileIlkLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileIlkLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -151,7 +187,7 @@ var _ = Describe("Pit file ilk transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.PitFileIlkModel))
+		Expect(repository.PassedModels).To(Equal([]ilk.PitFileIlkModel{test_data.PitFileIlkModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/pit_file/stability_fee/converter.go

@@ -24,26 +24,34 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error)
+	ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error)
 }
 
 type PitFileStabilityFeeConverter struct{}
 
-func (PitFileStabilityFeeConverter) ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error) {
+func (PitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error) {
-	err := verifyLog(ethLog)
+	var models []PitFileStabilityFeeModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return PitFileStabilityFeeModel{}, err
+		err := verifyLog(ethLog)
-	}
+		if err != nil {
-	what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+			return nil, err
-	data := common.HexToAddress(ethLog.Topics[3].String()).Hex()
+		}
+		what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		data := common.HexToAddress(ethLog.Topics[3].String()).Hex()
 
-	raw, err := json.Marshal(ethLog)
+		raw, err := json.Marshal(ethLog)
-	return PitFileStabilityFeeModel{
+		if err != nil {
-		What:             what,
+			return nil, err
-		Data:             data,
+		}
-		TransactionIndex: ethLog.TxIndex,
+		model := PitFileStabilityFeeModel{
-		Raw:              raw,
+			What:             what,
-	}, err
+			Data:             data,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
+	}
+	return models, nil
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/pit_file/stability_fee/converter_test.go

@@ -28,7 +28,7 @@ var _ = Describe("Pit file stability fee converter", func() {
 		converter := stability_fee.PitFileStabilityFeeConverter{}
 		badLog := types.Log{}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -36,9 +36,10 @@ var _ = Describe("Pit file stability fee converter", func() {
 	It("converts a log to an model", func() {
 		converter := stability_fee.PitFileStabilityFeeConverter{}
 
-		model, err := converter.ToModel(test_data.EthPitFileStabilityFeeLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthPitFileStabilityFeeLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.PitFileStabilityFeeModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.PitFileStabilityFeeModel))
 	})
 })

pkg/transformers/pit_file/stability_fee/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model PitFileStabilityFeeModel) error
+	Create(headerID int64, models []PitFileStabilityFeeModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -34,12 +35,38 @@ func NewPitFileStabilityFeeRepository(db *postgres.DB) PitFileStabilityFeeReposi
 	}
 }
 
-func (repository PitFileStabilityFeeRepository) Create(headerID int64, model PitFileStabilityFeeModel) error {
+func (repository PitFileStabilityFeeRepository) Create(headerID int64, models []PitFileStabilityFeeModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log)
         VALUES($1, $2, $3, $4, $5)`,
-		headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
+			headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository PitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true)
 	return err
 }
 
@@ -48,8 +75,8 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.pit_file_stability_fee on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR pit_file_stability_fee_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,
@@ -57,6 +84,5 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb
 		endingBlockNumber,
 		repository.db.Node.ID,
 	)
-
 	return result, err
 }

pkg/transformers/pit_file/stability_fee/repository_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
 
 var _ = Describe("Pit file stability fee repository", func() {
 	Describe("Create", func() {
-		It("adds a pit file stability fee event", func() {
+		var (
-			db := test_config.NewTestDB(core.Node{})
+			db                            *postgres.DB
+			pitFileStabilityFeeRepository stability_fee.Repository
+			err                           error
+			headerID                      int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+			pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db)
+		})
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
+		It("adds a pit file stability fee event", func() {
+			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile stability_fee.PitFileStabilityFeeModel
@@ -49,30 +59,28 @@ var _ = Describe("Pit file stability fee repository", func() {
 			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileStabilityFeeModel.Raw))
 		})
 
-		It("does not duplicate pit file events", func() {
+		It("marks header as checked for logs", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
-			test_config.CleanTestDB(db)
+
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+			var headerChecked bool
-			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
+			err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate pit file events", func() {
+			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
+			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
+			err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
-			err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -85,8 +93,48 @@ var _ = Describe("Pit file stability fee repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                            *postgres.DB
+			pitFileStabilityFeeRepository stability_fee.Repository
+			err                           error
+			headerID                      int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated pit file event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -101,7 +149,7 @@ var _ = Describe("Pit file stability fee repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
-			err := pitFileRepository.Create(headerIDs[1], test_data.PitFileStabilityFeeModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -112,6 +160,33 @@ var _ = Describe("Pit file stability fee repository", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if pit file stability fee logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileStabilityFeedBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileStabilityFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileStabilityFeeRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileStabilityFeeRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -129,7 +204,7 @@ var _ = Describe("Pit file stability fee repository", func() {
 			}
 			pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db)
 			pitFileRepositoryTwo := stability_fee.NewPitFileStabilityFeeRepository(dbTwo)
-			err := pitFileRepository.Create(headerIDs[0], test_data.PitFileStabilityFeeModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/pit_file/stability_fee/transformer.go

@@ -60,16 +60,20 @@ func (transformer PitFileStabilityFeeTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		models, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, models)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/pit_file/stability_fee/transformer_test.go

@@ -97,6 +97,42 @@ var _ = Describe("Pit file stability fee transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
+		mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
+		mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := stability_fee.PitFileStabilityFeeTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -112,7 +148,7 @@ var _ = Describe("Pit file stability fee transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileStabilityFeeLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileStabilityFeeLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -151,7 +187,7 @@ var _ = Describe("Pit file stability fee transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.PitFileStabilityFeeModel))
+		Expect(repository.PassedModels).To(Equal([]stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

pkg/transformers/test_data/mocks/bite/converter.go

@@ -22,24 +22,23 @@ import (
 )
 
 type MockBiteConverter struct {
-	ConverterContract string
 	ConverterAbi      string
 	LogsToConvert     []types.Log
 	EntitiesToConvert []BiteEntity
 	ConverterError    error
 }
 
-func (mbc *MockBiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) {
+func (mbc *MockBiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) {
-	mbc.ConverterContract = contractAddress
 	mbc.ConverterAbi = contractAbi
-	mbc.LogsToConvert = append(mbc.LogsToConvert, ethLog)
+	mbc.LogsToConvert = append(mbc.LogsToConvert, ethLogs...)
-	return test_data.BiteEntity, mbc.ConverterError
+	return []BiteEntity{test_data.BiteEntity}, mbc.ConverterError
 }
 
-func (mbc *MockBiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
+func (mbc *MockBiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) {
-	mbc.EntitiesToConvert = append(mbc.EntitiesToConvert, entity)
+	mbc.EntitiesToConvert = append(mbc.EntitiesToConvert, entities...)
-	return test_data.BiteModel, mbc.ConverterError
+	return []BiteModel{test_data.BiteModel}, mbc.ConverterError
 }
+
 func (mbc *MockBiteConverter) SetConverterError(err error) {
 	mbc.ConverterError = err
 }

pkg/transformers/test_data/mocks/bite/repository.go

@@ -15,19 +15,23 @@
 package bite
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
 )
 
 type MockBiteRepository struct {
-	createError               error
+	createError                     error
-	PassedEndingBlockNumber   int64
+	PassedEndingBlockNumber         int64
-	PassedBiteModel           bite.BiteModel
+	PassedBiteModels                []bite.BiteModel
-	PassedHeaderID            int64
+	PassedHeaderID                  int64
-	PassedStartingBlockNumber int64
+	PassedStartingBlockNumber       int64
-	PassedTransactionIndex    uint
+	PassedTransactionIndex          uint
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
+	missingHeaders                  []core.Header
+	missingHeadersErr               error
 }
 
 func (repository *MockBiteRepository) SetCreateError(err error) {
@@ -42,14 +46,27 @@ func (repository *MockBiteRepository) SetMissingHeaders(headers []core.Header) {
 	repository.missingHeaders = headers
 }
 
-func (repository *MockBiteRepository) Create(headerID int64, model bite.BiteModel) error {
+func (repository *MockBiteRepository) Create(headerID int64, models []bite.BiteModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedBiteModel = model
+	repository.PassedBiteModels = models
 	return repository.createError
 }
 
+func (repository *MockBiteRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockBiteRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
+
+func (repository *MockBiteRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
+func (repository *MockBiteRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/drip_drip/converter.go

@@ -21,13 +21,13 @@ import (
 )
 
 type MockDripDripConverter struct {
-	Err       error
+	Err        error
-	PassedLog types.Log
+	PassedLogs []types.Log
 }
 
-func (converter *MockDripDripConverter) ToModel(ethLog types.Log) (drip_drip.DripDripModel, error) {
+func (converter *MockDripDripConverter) ToModels(ethLogs []types.Log) ([]drip_drip.DripDripModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.DripDripModel, converter.Err
+	return []drip_drip.DripDripModel{test_data.DripDripModel}, converter.Err
 }
 
 func (converter *MockDripDripConverter) SetConverterError(e error) {

pkg/transformers/test_data/mocks/drip_drip/repository.go

@@ -15,32 +15,45 @@
 package drip_drip
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
 )
 
 type MockDripDripRepository struct {
-	createErr                 error
+	createErr                       error
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
-	PassedStartingBlockNumber int64
+	missingHeaders                  []core.Header
-	PassedEndingBlockNumber   int64
+	missingHeadersErr               error
-	PassedHeaderID            int64
+	PassedStartingBlockNumber       int64
-	PassedModel               drip_drip.DripDripModel
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []drip_drip.DripDripModel
 }
 
-func (repository *MockDripDripRepository) Create(headerID int64, model drip_drip.DripDripModel) error {
+func (repository *MockDripDripRepository) Create(headerID int64, models []drip_drip.DripDripModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedModel = model
+	repository.PassedModels = models
 	return repository.createErr
 }
 
+func (repository *MockDripDripRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockDripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
 
+func (repository *MockDripDripRepository) SetMarkHeaderCheckedErr(err error) {
+	repository.markHeaderCheckedErr = err
+}
+
 func (repository *MockDripDripRepository) SetMissingHeadersErr(e error) {
 	repository.missingHeadersErr = e
 }
@@ -52,3 +65,6 @@ func (repository *MockDripDripRepository) SetMissingHeaders(headers []core.Heade
 func (repository *MockDripDripRepository) SetCreateError(e error) {
 	repository.createErr = e
 }
+func (repository *MockDripDripRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/drip_file/ilk/converter.go

@@ -22,12 +22,12 @@ import (
 
 type MockDripFileIlkConverter struct {
 	converterErr error
-	PassedLog    types.Log
+	PassedLogs   []types.Log
 }
 
-func (converter *MockDripFileIlkConverter) ToModel(ethLog types.Log) (ilk.DripFileIlkModel, error) {
+func (converter *MockDripFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.DripFileIlkModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.DripFileIlkModel, converter.converterErr
+	return []ilk.DripFileIlkModel{test_data.DripFileIlkModel}, converter.converterErr
 }
 
 func (converter *MockDripFileIlkConverter) SetConverterError(e error) {

pkg/transformers/test_data/mocks/drip_file/ilk/repository.go

@@ -15,32 +15,45 @@
 package ilk
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk"
 )
 
 type MockDripFileIlkRepository struct {
-	createErr                 error
+	createErr                       error
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
-	PassedStartingBlockNumber int64
+	missingHeaders                  []core.Header
-	PassedEndingBlockNumber   int64
+	missingHeadersErr               error
-	PassedHeaderID            int64
+	PassedStartingBlockNumber       int64
-	PassedModel               ilk.DripFileIlkModel
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []ilk.DripFileIlkModel
 }
 
-func (repository *MockDripFileIlkRepository) Create(headerID int64, model ilk.DripFileIlkModel) error {
+func (repository *MockDripFileIlkRepository) Create(headerID int64, models []ilk.DripFileIlkModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedModel = model
+	repository.PassedModels = models
 	return repository.createErr
 }
 
+func (repository *MockDripFileIlkRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockDripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
 
+func (repository *MockDripFileIlkRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockDripFileIlkRepository) SetMissingHeadersErr(e error) {
 	repository.missingHeadersErr = e
 }
@@ -52,3 +65,6 @@ func (repository *MockDripFileIlkRepository) SetMissingHeaders(headers []core.He
 func (repository *MockDripFileIlkRepository) SetCreateError(e error) {
 	repository.createErr = e
 }
+func (repository *MockDripFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/drip_file/repo/converter.go

@@ -22,12 +22,12 @@ import (
 
 type MockDripFileRepoConverter struct {
 	converterErr error
-	PassedLog    types.Log
+	PassedLogs   []types.Log
 }
 
-func (converter *MockDripFileRepoConverter) ToModel(ethLog types.Log) (repo.DripFileRepoModel, error) {
+func (converter *MockDripFileRepoConverter) ToModels(ethLogs []types.Log) ([]repo.DripFileRepoModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.DripFileRepoModel, converter.converterErr
+	return []repo.DripFileRepoModel{test_data.DripFileRepoModel}, converter.converterErr
 }
 
 func (converter *MockDripFileRepoConverter) SetConverterError(e error) {

pkg/transformers/test_data/mocks/drip_file/repo/repository.go

@@ -15,32 +15,45 @@
 package repo
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo"
 )
 
 type MockDripFileRepoRepository struct {
-	createErr                 error
+	createErr                       error
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
-	PassedStartingBlockNumber int64
+	missingHeaders                  []core.Header
-	PassedEndingBlockNumber   int64
+	missingHeadersErr               error
-	PassedHeaderID            int64
+	PassedStartingBlockNumber       int64
-	PassedModel               repo.DripFileRepoModel
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []repo.DripFileRepoModel
 }
 
-func (repository *MockDripFileRepoRepository) Create(headerID int64, model repo.DripFileRepoModel) error {
+func (repository *MockDripFileRepoRepository) Create(headerID int64, models []repo.DripFileRepoModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedModel = model
+	repository.PassedModels = models
 	return repository.createErr
 }
 
+func (repository *MockDripFileRepoRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockDripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
 
+func (repository *MockDripFileRepoRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockDripFileRepoRepository) SetMissingHeadersErr(e error) {
 	repository.missingHeadersErr = e
 }
@@ -52,3 +65,6 @@ func (repository *MockDripFileRepoRepository) SetMissingHeaders(headers []core.H
 func (repository *MockDripFileRepoRepository) SetCreateError(e error) {
 	repository.createErr = e
 }
+func (repository *MockDripFileRepoRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/drip_file/vow/converter.go

@@ -22,12 +22,12 @@ import (
 
 type MockDripFileVowConverter struct {
 	converterErr error
-	PassedLog    types.Log
+	PassedLogs   []types.Log
 }
 
-func (converter *MockDripFileVowConverter) ToModel(ethLog types.Log) (vow.DripFileVowModel, error) {
+func (converter *MockDripFileVowConverter) ToModels(ethLogs []types.Log) ([]vow.DripFileVowModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.DripFileVowModel, converter.converterErr
+	return []vow.DripFileVowModel{test_data.DripFileVowModel}, converter.converterErr
 }
 
 func (converter *MockDripFileVowConverter) SetConverterError(e error) {

pkg/transformers/test_data/mocks/drip_file/vow/repository.go

@@ -15,32 +15,45 @@
 package vow
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow"
 )
 
 type MockDripFileVowRepository struct {
-	createErr                 error
+	createErr                       error
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
-	PassedStartingBlockNumber int64
+	missingHeaders                  []core.Header
-	PassedEndingBlockNumber   int64
+	missingHeadersErr               error
-	PassedHeaderID            int64
+	PassedStartingBlockNumber       int64
-	PassedModel               vow.DripFileVowModel
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []vow.DripFileVowModel
 }
 
-func (repository *MockDripFileVowRepository) Create(headerID int64, model vow.DripFileVowModel) error {
+func (repository *MockDripFileVowRepository) Create(headerID int64, models []vow.DripFileVowModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedModel = model
+	repository.PassedModels = models
 	return repository.createErr
 }
 
+func (repository *MockDripFileVowRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockDripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
 
+func (repository *MockDripFileVowRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockDripFileVowRepository) SetMissingHeadersErr(e error) {
 	repository.missingHeadersErr = e
 }
@@ -52,3 +65,6 @@ func (repository *MockDripFileVowRepository) SetMissingHeaders(headers []core.He
 func (repository *MockDripFileVowRepository) SetCreateError(e error) {
 	repository.createErr = e
 }
+func (repository *MockDripFileVowRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/frob/converter.go

@@ -22,30 +22,28 @@ import (
 )
 
 type MockFrobConverter struct {
-	PassedContractAddress string
+	PassedContractABI string
-	PassedContractABI     string
+	PassedLogs        []types.Log
-	PassedLog             types.Log
+	PassedEntities    []frob.FrobEntity
-	PassedEntity          frob.FrobEntity
+	toEntityError     error
-	toEntityError         error
+	toModelError      error
-	toModelError          error
 }
 
-func (converter *MockFrobConverter) SetToEntityError(err error) {
+func (converter *MockFrobConverter) SetToEntitiesError(err error) {
 	converter.toEntityError = err
 }
 
-func (converter *MockFrobConverter) SetToModelError(err error) {
+func (converter *MockFrobConverter) SetToModelsError(err error) {
 	converter.toModelError = err
 }
 
-func (converter *MockFrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (frob.FrobEntity, error) {
+func (converter *MockFrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]frob.FrobEntity, error) {
-	converter.PassedContractAddress = contractAddress
 	converter.PassedContractABI = contractAbi
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.FrobEntity, converter.toEntityError
+	return []frob.FrobEntity{test_data.FrobEntity}, converter.toEntityError
 }
 
-func (converter *MockFrobConverter) ToModel(frobEntity frob.FrobEntity) (frob.FrobModel, error) {
+func (converter *MockFrobConverter) ToModels(frobEntities []frob.FrobEntity) ([]frob.FrobModel, error) {
-	converter.PassedEntity = frobEntity
+	converter.PassedEntities = frobEntities
-	return test_data.FrobModel, converter.toModelError
+	return []frob.FrobModel{test_data.FrobModel}, converter.toModelError
 }

pkg/transformers/test_data/mocks/frob/repository.go

@@ -15,24 +15,32 @@
 package frob
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
 )
 
 type MockFrobRepository struct {
-	createError               error
+	createError                     error
-	PassedEndingBlockNumber   int64
+	PassedEndingBlockNumber         int64
-	PassedFrobModel           frob.FrobModel
+	PassedModels                    []frob.FrobModel
-	PassedHeaderID            int64
+	PassedHeaderID                  int64
-	PassedStartingBlockNumber int64
+	PassedStartingBlockNumber       int64
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
+	missingHeaders                  []core.Header
+	missingHeadersErr               error
 }
 
 func (repository *MockFrobRepository) SetCreateError(err error) {
 	repository.createError = err
 }
 
+func (repository *MockFrobRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockFrobRepository) SetMissingHeadersErr(err error) {
 	repository.missingHeadersErr = err
 }
@@ -41,14 +49,22 @@ func (repository *MockFrobRepository) SetMissingHeaders(headers []core.Header) {
 	repository.missingHeaders = headers
 }
 
-func (repository *MockFrobRepository) Create(headerID int64, model frob.FrobModel) error {
+func (repository *MockFrobRepository) Create(headerID int64, models []frob.FrobModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedFrobModel = model
+	repository.PassedModels = models
 	return repository.createError
 }
 
+func (repository *MockFrobRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockFrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
+func (repository *MockFrobRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go

@@ -23,12 +23,12 @@ import (
 
 type MockPitFileDebtCeilingConverter struct {
 	converterErr error
-	PassedLog    types.Log
+	PassedLogs   []types.Log
 }
 
-func (converter *MockPitFileDebtCeilingConverter) ToModel(ethLog types.Log) (debt_ceiling.PitFileDebtCeilingModel, error) {
+func (converter *MockPitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]debt_ceiling.PitFileDebtCeilingModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.PitFileDebtCeilingModel, converter.converterErr
+	return []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}, converter.converterErr
 }
 
 func (converter *MockPitFileDebtCeilingConverter) SetConverterError(e error) {

pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go

@@ -15,23 +15,32 @@
 package debt_ceiling
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
 )
 
 type MockPitFileDebtCeilingRepository struct {
-	createErr                 error
+	createErr                       error
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
-	PassedStartingBlockNumber int64
+	missingHeaders                  []core.Header
-	PassedEndingBlockNumber   int64
+	missingHeadersErr               error
-	PassedHeaderID            int64
+	PassedStartingBlockNumber       int64
-	PassedModel               debt_ceiling.PitFileDebtCeilingModel
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []debt_ceiling.PitFileDebtCeilingModel
 }
 
-func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, model debt_ceiling.PitFileDebtCeilingModel) error {
+func (repository *MockPitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
+func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, models []debt_ceiling.PitFileDebtCeilingModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedModel = model
+	repository.PassedModels = models
 	return repository.createErr
 }
 
@@ -41,6 +50,10 @@ func (repository *MockPitFileDebtCeilingRepository) MissingHeaders(startingBlock
 	return repository.missingHeaders, repository.missingHeadersErr
 }
 
+func (repository *MockPitFileDebtCeilingRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockPitFileDebtCeilingRepository) SetMissingHeadersErr(e error) {
 	repository.missingHeadersErr = e
 }
@@ -52,3 +65,7 @@ func (repository *MockPitFileDebtCeilingRepository) SetMissingHeaders(headers []
 func (repository *MockPitFileDebtCeilingRepository) SetCreateError(e error) {
 	repository.createErr = e
 }
+
+func (repository *MockPitFileDebtCeilingRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/pit_file/ilk/converter.go

@@ -22,7 +22,7 @@ import (
 )
 
 type MockPitFileIlkConverter struct {
-	PassedLog      types.Log
+	PassedLogs     []types.Log
 	converterError error
 }
 
@@ -30,7 +30,7 @@ func (converter *MockPitFileIlkConverter) SetConverterError(err error) {
 	converter.converterError = err
 }
 
-func (converter *MockPitFileIlkConverter) ToModel(ethLog types.Log) (ilk.PitFileIlkModel, error) {
+func (converter *MockPitFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.PitFileIlkModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.PitFileIlkModel, converter.converterError
+	return []ilk.PitFileIlkModel{test_data.PitFileIlkModel}, converter.converterError
 }

pkg/transformers/test_data/mocks/pit_file/ilk/repository.go

@@ -15,24 +15,32 @@
 package ilk
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
 )
 
 type MockPitFileIlkRepository struct {
-	createError               error
+	createError                     error
-	PassedEndingBlockNumber   int64
+	PassedEndingBlockNumber         int64
-	PassedModel               ilk.PitFileIlkModel
+	PassedModels                    []ilk.PitFileIlkModel
-	PassedHeaderID            int64
+	PassedHeaderID                  int64
-	PassedStartingBlockNumber int64
+	PassedStartingBlockNumber       int64
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
+	missingHeaders                  []core.Header
+	missingHeadersErr               error
 }
 
 func (repository *MockPitFileIlkRepository) SetCreateError(err error) {
 	repository.createError = err
 }
 
+func (repository *MockPitFileIlkRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockPitFileIlkRepository) SetMissingHeadersErr(err error) {
 	repository.missingHeadersErr = err
 }
@@ -41,14 +49,23 @@ func (repository *MockPitFileIlkRepository) SetMissingHeaders(headers []core.Hea
 	repository.missingHeaders = headers
 }
 
-func (repository *MockPitFileIlkRepository) Create(headerID int64, model ilk.PitFileIlkModel) error {
+func (repository *MockPitFileIlkRepository) Create(headerID int64, models []ilk.PitFileIlkModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedModel = model
+	repository.PassedModels = models
 	return repository.createError
 }
 
+func (repository *MockPitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockPitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
+
+func (repository *MockPitFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go

@@ -23,13 +23,14 @@ import (
 
 type MockPitFileStabilityFeeConverter struct {
 	converterErr error
-	PassedLog    types.Log
+	PassedLogs   []types.Log
 }
 
-func (converter *MockPitFileStabilityFeeConverter) ToModel(ethLog types.Log) (stability_fee.PitFileStabilityFeeModel, error) {
+func (converter *MockPitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]stability_fee.PitFileStabilityFeeModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.PitFileStabilityFeeModel, converter.converterErr
+	return []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}, converter.converterErr
 }
+
 func (converter *MockPitFileStabilityFeeConverter) SetConverterError(e error) {
 	converter.converterErr = e
 }

pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go

@@ -15,38 +15,57 @@
 package stability_fee
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee"
 )
 
 type MockPitFileStabilityFeeRepository struct {
-	createErr                 error
+	createErr                       error
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
-	PassedStartingBlockNumber int64
+	missingHeaders                  []core.Header
-	PassedEndingBlockNumber   int64
+	missingHeadersErr               error
-	PassedHeaderID            int64
+	PassedStartingBlockNumber       int64
-	PassedModel               stability_fee.PitFileStabilityFeeModel
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []stability_fee.PitFileStabilityFeeModel
 }
 
-func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, model stability_fee.PitFileStabilityFeeModel) error {
+func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, models []stability_fee.PitFileStabilityFeeModel) error {
-	repository.PassedModel = model
+	repository.PassedModels = models
 	repository.PassedHeaderID = headerID
 	return repository.createErr
 }
 
+func (repository *MockPitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockPitFileStabilityFeeRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
 
+func (repository *MockPitFileStabilityFeeRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockPitFileStabilityFeeRepository) SetMissingHeadersErr(e error) {
 	repository.missingHeadersErr = e
 }
+
 func (repository *MockPitFileStabilityFeeRepository) SetMissingHeaders(headers []core.Header) {
 	repository.missingHeaders = headers
 }
+
 func (repository *MockPitFileStabilityFeeRepository) SetCreateError(e error) {
 	repository.createErr = e
 }
+
+func (repository *MockPitFileStabilityFeeRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/test_data/mocks/vat_init/converter.go

@@ -23,12 +23,12 @@ import (
 
 type MockVatInitConverter struct {
 	converterErr error
-	PassedLog    types.Log
+	PassedLogs   []types.Log
 }
 
-func (converter *MockVatInitConverter) ToModel(ethLog types.Log) (vat_init.VatInitModel, error) {
+func (converter *MockVatInitConverter) ToModels(ethLogs []types.Log) ([]vat_init.VatInitModel, error) {
-	converter.PassedLog = ethLog
+	converter.PassedLogs = ethLogs
-	return test_data.VatInitModel, converter.converterErr
+	return []vat_init.VatInitModel{test_data.VatInitModel}, converter.converterErr
 }
 
 func (converter *MockVatInitConverter) SetConverterError(e error) {

pkg/transformers/test_data/mocks/vat_init/repository.go

@@ -15,32 +15,45 @@
 package vat_init
 
 import (
+	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init"
 )
 
 type MockVatInitRepository struct {
-	createErr                 error
+	createErr                       error
-	missingHeaders            []core.Header
+	markHeaderCheckedErr            error
-	missingHeadersErr         error
+	markHeaderCheckedPassedHeaderID int64
-	PassedStartingBlockNumber int64
+	missingHeaders                  []core.Header
-	PassedEndingBlockNumber   int64
+	missingHeadersErr               error
-	PassedHeaderID            int64
+	PassedStartingBlockNumber       int64
-	PassedModel               vat_init.VatInitModel
+	PassedEndingBlockNumber         int64
+	PassedHeaderID                  int64
+	PassedModels                    []vat_init.VatInitModel
 }
 
-func (repository *MockVatInitRepository) Create(headerID int64, model vat_init.VatInitModel) error {
+func (repository *MockVatInitRepository) Create(headerID int64, models []vat_init.VatInitModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedModel = model
+	repository.PassedModels = models
 	return repository.createErr
 }
 
+func (repository *MockVatInitRepository) MarkHeaderChecked(headerID int64) error {
+	repository.markHeaderCheckedPassedHeaderID = headerID
+	return repository.markHeaderCheckedErr
+}
+
 func (repository *MockVatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
 	repository.PassedStartingBlockNumber = startingBlockNumber
 	repository.PassedEndingBlockNumber = endingBlockNumber
 	return repository.missingHeaders, repository.missingHeadersErr
 }
 
+func (repository *MockVatInitRepository) SetMarkHeaderCheckedErr(e error) {
+	repository.markHeaderCheckedErr = e
+}
+
 func (repository *MockVatInitRepository) SetMissingHeadersErr(e error) {
 	repository.missingHeadersErr = e
 }
@@ -52,3 +65,7 @@ func (repository *MockVatInitRepository) SetMissingHeaders(headers []core.Header
 func (repository *MockVatInitRepository) SetCreateError(e error) {
 	repository.createErr = e
 }
+
+func (repository *MockVatInitRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
+	Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
+}

pkg/transformers/vat_init/converter.go

@@ -23,23 +23,31 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (VatInitModel, error)
+	ToModels(ethLogs []types.Log) ([]VatInitModel, error)
 }
 
 type VatInitConverter struct{}
 
-func (VatInitConverter) ToModel(ethLog types.Log) (VatInitModel, error) {
+func (VatInitConverter) ToModels(ethLogs []types.Log) ([]VatInitModel, error) {
-	err := verifyLog(ethLog)
+	var models []VatInitModel
-	if err != nil {
+	for _, ethLog := range ethLogs {
-		return VatInitModel{}, err
+		err := verifyLog(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+		raw, err := json.Marshal(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		model := VatInitModel{
+			Ilk:              ilk,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
 	}
-	ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
+	return models, nil
-	raw, err := json.Marshal(ethLog)
-	return VatInitModel{
-		Ilk:              ilk,
-		TransactionIndex: ethLog.TxIndex,
-		Raw:              raw,
-	}, err
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/vat_init/converter_test.go

@@ -28,7 +28,7 @@ var _ = Describe("Vat init converter", func() {
 		converter := vat_init.VatInitConverter{}
 		badLog := types.Log{}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -36,9 +36,10 @@ var _ = Describe("Vat init converter", func() {
 	It("converts a log to an model", func() {
 		converter := vat_init.VatInitConverter{}
 
-		model, err := converter.ToModel(test_data.EthVatInitLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthVatInitLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.VatInitModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.VatInitModel))
 	})
 })

pkg/transformers/vat_init/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model VatInitModel) error
+	Create(headerID int64, models []VatInitModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -34,12 +35,31 @@ func NewVatInitRepository(db *postgres.DB) VatInitRepository {
 	}
 }
 
-func (repository VatInitRepository) Create(headerID int64, model VatInitModel) error {
+func (repository VatInitRepository) Create(headerID int64, models []VatInitModel) error {
-	_, err := repository.db.Exec(
+	tx, err := repository.db.Begin()
-		`INSERT into maker.vat_init (header_id, ilk, tx_idx, raw_log)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.vat_init (header_id, ilk, tx_idx, raw_log)
         VALUES($1, $2, $3, $4)`,
-		headerID, model.Ilk, model.TransactionIndex, model.Raw,
+			headerID, model.Ilk, model.TransactionIndex, model.Raw,
-	)
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET vat_init_checked = $2`, headerID, true)
 	return err
 }
 
@@ -48,8 +68,8 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.vat_init on headers.id = header_id
+               LEFT JOIN checked_headers on headers.id = header_id
-               WHERE header_id ISNULL
+               WHERE (header_id ISNULL OR vat_init_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,
@@ -57,6 +77,5 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl
 		endingBlockNumber,
 		repository.db.Node.ID,
 	)
-
 	return result, err
 }

pkg/transformers/vat_init/repository_test.go

@@ -21,13 +21,14 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
-var _ = Describe("", func() {
+var _ = Describe("Vat init repository", func() {
 	Describe("Create", func() {
 		It("adds a vat event", func() {
 			db := test_config.NewTestDB(core.Node{})
@@ -37,7 +38,7 @@ var _ = Describe("", func() {
 			Expect(err).NotTo(HaveOccurred())
 			vatInitRepository := vat_init.NewVatInitRepository(db)
 
-			err = vatInitRepository.Create(headerID, test_data.VatInitModel)
+			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbVatInit vat_init.VatInitModel
@@ -55,10 +56,10 @@ var _ = Describe("", func() {
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			vatInitRepository := vat_init.NewVatInitRepository(db)
-			err = vatInitRepository.Create(headerID, test_data.VatInitModel)
+			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = vatInitRepository.Create(headerID, test_data.VatInitModel)
+			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
@@ -71,7 +72,7 @@ var _ = Describe("", func() {
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			vatInitRepository := vat_init.NewVatInitRepository(db)
-			err = vatInitRepository.Create(headerID, test_data.VatInitModel)
+			err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel})
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -84,8 +85,48 @@ var _ = Describe("", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                *postgres.DB
+			vatInitRepository vat_init.Repository
+			err               error
+			headerID          int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			vatInitRepository = vat_init.NewVatInitRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = vatInitRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = vatInitRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated vat event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -100,7 +141,7 @@ var _ = Describe("", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			vatInitRepository := vat_init.NewVatInitRepository(db)
-			err := vatInitRepository.Create(headerIDs[1], test_data.VatInitModel)
+			err := vatInitRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := vatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -111,6 +152,33 @@ var _ = Describe("", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if drip drip logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			vatInitBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, vatInitBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			VatInitRepository := vat_init.NewVatInitRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := VatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -128,7 +196,7 @@ var _ = Describe("", func() {
 			}
 			vatInitRepository := vat_init.NewVatInitRepository(db)
 			vatInitRepositoryTwo := vat_init.NewVatInitRepository(dbTwo)
-			err := vatInitRepository.Create(headerIDs[0], test_data.VatInitModel)
+			err := vatInitRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := vatInitRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/vat_init/transformer.go

@@ -59,16 +59,20 @@ func (transformer VatInitTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
+		if len(matchingLogs) < 1 {
-			model, err := transformer.Converter.ToModel(log)
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}
 		}
+		models, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, models)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/vat_init/transformer_test.go

@@ -96,6 +96,42 @@ var _ = Describe("Vat init transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &vat_init_mocks.MockVatInitConverter{}
+		mockRepository := &vat_init_mocks.MockVatInitRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := vat_init.VatInitTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &vat_init_mocks.MockVatInitConverter{}
+		mockRepository := &vat_init_mocks.MockVatInitRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := vat_init.VatInitTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &vat_init_mocks.MockVatInitConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -111,7 +147,7 @@ var _ = Describe("Vat init transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthVatInitLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatInitLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -150,7 +186,7 @@ var _ = Describe("Vat init transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.VatInitModel))
+		Expect(repository.PassedModels).To(Equal([]vat_init.VatInitModel{test_data.VatInitModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {

postgraphile/spec/server/runtime.spec.ts

@@ -27,6 +27,7 @@ describe('bootServer', () => {
       middleware: jasmine.createSpyObj<PostgraphileMiddleware>(['call']),
       options: { 
         pluginHook: jasmine.createSpy('pluginHook'),
+        watchPg: true,
         simpleSubscriptions: true,
         graphiql: true,
         webSocketMiddlewares: [] },

postgraphile/src/adapters/postgraphile.ts

@@ -11,6 +11,7 @@ export interface PostgraphileMiddleware extends RequestHandler {}
 export interface PostgraphileOptions {
   pluginHook: PluginHookFn,
   simpleSubscriptions: boolean;
+  watchPg: boolean;
   graphiql: boolean;
   // NOTE (jchristie@8thlight.com) Shape of the middlewares is not
   // currently important to this application, but if a need arises,

postgraphile/src/server/config.ts

@@ -27,6 +27,7 @@ export function buildServerConfig(
   const options: PostgraphileOptions = {
     pluginHook: pluginHook,
     simpleSubscriptions: true,
+    watchPg: true,
     graphiql: true,
     webSocketMiddlewares: [
       expressSessionHandler,

scripts/create_migration

@@ -2,7 +2,7 @@
 
 if [ $# -eq 1 ]
 then
-  $GOBIN/migrate create -dir ./db/migrations -ext sql $1
+  $GOPATH/bin/migrate create -dir ./db/migrations -ext sql $1
 else
   echo "**An Error Occurred**"
   echo "Usage: ./scripts/create_migration <migration-name>"

scripts/reset_db

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Provide me with a postgres database name, and I will:
+# - Drop the database
+# - Recreate the database
+# - Run the vulcanizedb migration
+
+if [ "$1" = "" ]; then
+    echo "Provide a database name to reset"
+    exit 1
+fi
+
+db=$1
+dir=$(basename "$(pwd)")
+if [ $dir != "vulcanizedb" ]
+then
+  echo "Run me from the vulcanizedb root dir"
+  exit 1
+fi
+
+user=$(whoami)
+psql -c "DROP DATABASE $db" postgres
+if [ $? -eq 0 ]; then
+    psql -c "CREATE DATABASE $db WITH OWNER $user" postgres
+    make migrate HOST_NAME=localhost NAME=$db PORT=5432
+else
+    echo "Couldnt drop the database. Are you connected? Does it exist?"
+fi