Log checked headers for MCD events (#43)

* Log checked headers for Bite events

* Log checked headers for Drip drip events

* Log checked headers for Drip file events

- mark headers checked for 3 implementations of file on the Drip contract

* Log checked headers for Frob frob events

* Log checked headers for Pit file events

- mark headers checked for 3 implementations of file on the Pit contract

* Log checked headers for Vat init events

pkg/transformers/pit_file/debt_ceiling/converter.go

@@ -25,26 +25,34 @@ import (
 )
 
 type Converter interface {
-	ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error)
+	ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error)
 }
 
 type PitFileDebtCeilingConverter struct{}
 
-func (PitFileDebtCeilingConverter) ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error) {
-	err := verifyLog(ethLog)
-	if err != nil {
-		return PitFileDebtCeilingModel{}, err
-	}
-	what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
-	data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
+func (PitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error) {
+	var models []PitFileDebtCeilingModel
+	for _, ethLog := range ethLogs {
+		err := verifyLog(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+		data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
 
-	raw, err := json.Marshal(ethLog)
-	return PitFileDebtCeilingModel{
-		What:             what,
-		Data:             data,
-		TransactionIndex: ethLog.TxIndex,
-		Raw:              raw,
-	}, err
+		raw, err := json.Marshal(ethLog)
+		if err != nil {
+			return nil, err
+		}
+		model := PitFileDebtCeilingModel{
+			What:             what,
+			Data:             data,
+			TransactionIndex: ethLog.TxIndex,
+			Raw:              raw,
+		}
+		models = append(models, model)
+	}
+	return models, nil
 }
 
 func verifyLog(log types.Log) error {

pkg/transformers/pit_file/debt_ceiling/converter_test.go

@@ -31,7 +31,7 @@ var _ = Describe("Pit file debt ceiling converter", func() {
 			Data: []byte{1, 1, 1, 1, 1},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -42,7 +42,7 @@ var _ = Describe("Pit file debt ceiling converter", func() {
 			Topics: []common.Hash{{}, {}, {}, {}},
 		}
 
-		_, err := converter.ToModel(badLog)
+		_, err := converter.ToModels([]types.Log{badLog})
 
 		Expect(err).To(HaveOccurred())
 	})
@@ -50,9 +50,10 @@ var _ = Describe("Pit file debt ceiling converter", func() {
 	It("converts a log to an model", func() {
 		converter := debt_ceiling.PitFileDebtCeilingConverter{}
 
-		model, err := converter.ToModel(test_data.EthPitFileDebtCeilingLog)
+		models, err := converter.ToModels([]types.Log{test_data.EthPitFileDebtCeilingLog})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model).To(Equal(test_data.PitFileDebtCeilingModel))
+		Expect(len(models)).To(Equal(1))
+		Expect(models[0]).To(Equal(test_data.PitFileDebtCeilingModel))
 	})
 })

pkg/transformers/pit_file/debt_ceiling/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model PitFileDebtCeilingModel) error
+	Create(headerID int64, models []PitFileDebtCeilingModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -34,12 +35,38 @@ func NewPitFileDebtCeilingRepository(db *postgres.DB) PitFileDebtCeilingReposito
 	}
 }
 
-func (repository PitFileDebtCeilingRepository) Create(headerID int64, model PitFileDebtCeilingModel) error {
-	_, err := repository.db.Exec(
-		`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log)
+func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []PitFileDebtCeilingModel) error {
+	tx, err := repository.db.Begin()
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err = tx.Exec(
+			`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log)
         VALUES($1, $2, $3::NUMERIC, $4, $5)`,
-		headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
-	)
+			headerID, model.What, model.Data, model.TransactionIndex, model.Raw,
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true)
 	return err
 }
 
@@ -48,8 +75,8 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.pit_file_debt_ceiling on headers.id = header_id
-               WHERE header_id ISNULL
+               LEFT JOIN checked_headers on headers.id = header_id
+               WHERE (header_id ISNULL OR pit_file_debt_ceiling_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,
@@ -57,6 +84,5 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe
 		endingBlockNumber,
 		repository.db.Node.ID,
 	)
-
 	return result, err
 }

pkg/transformers/pit_file/debt_ceiling/repository_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@@ -29,15 +30,24 @@ import (
 
 var _ = Describe("Pit file debt ceiling repository", func() {
 	Describe("Create", func() {
-		It("adds a pit file debt ceiling event", func() {
-			db := test_config.NewTestDB(core.Node{})
+		var (
+			db                           *postgres.DB
+			pitFileDebtCeilingRepository debt_ceiling.Repository
+			err                          error
+			headerID                     int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
+		})
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+		It("adds a pit file debt ceiling event", func() {
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbPitFile debt_ceiling.PitFileDebtCeilingModel
@@ -49,30 +59,28 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileDebtCeilingModel.Raw))
 		})
 
-		It("does not duplicate pit file events", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+		It("marks header as checked for logs", func() {
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
+
 			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("does not duplicate pit file events", func() {
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 			Expect(err).NotTo(HaveOccurred())
 
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes pit file if corresponding header is deleted", func() {
-			db := test_config.NewTestDB(core.Node{})
-			test_config.CleanTestDB(db)
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
-			Expect(err).NotTo(HaveOccurred())
-			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
-			err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel)
+			err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -85,8 +93,48 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 		})
 	})
 
+	Describe("MarkHeaderChecked", func() {
+		var (
+			db                           *postgres.DB
+			pitFileDebtCeilingRepository debt_ceiling.Repository
+			err                          error
+			headerID                     int64
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db)
+		})
+
+		It("creates a row for a new headerID", func() {
+			err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates row when headerID already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+
+			err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
-		It("returns headers with no associated pit file event", func() {
+		It("returns headers that haven't been checked", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
@@ -101,7 +149,7 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
-			err := pitFileRepository.Create(headerIDs[1], test_data.PitFileDebtCeilingModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -112,6 +160,33 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
 		})
 
+		It("only treats headers as checked if pit file debt ceiling logs have been checked", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileDebtCeilingdBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileDebtCeilingdBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileDebtCeilingRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
+			_, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileDebtCeilingRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber)))
+		})
+
 		It("only returns headers associated with the current node", func() {
 			db := test_config.NewTestDB(core.Node{})
 			test_config.CleanTestDB(db)
@@ -129,7 +204,7 @@ var _ = Describe("Pit file debt ceiling repository", func() {
 			}
 			pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db)
 			pitFileRepositoryTwo := debt_ceiling.NewPitFileDebtCeilingRepository(dbTwo)
-			err := pitFileRepository.Create(headerIDs[0], test_data.PitFileDebtCeilingModel)
+			err := pitFileRepository.MarkHeaderChecked(headerIDs[0])
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/pit_file/debt_ceiling/transformer.go

@@ -60,16 +60,20 @@ func (transformer PitFileDebtCeilingTransformer) Execute() error {
 		if err != nil {
 			return err
 		}
-		for _, log := range matchingLogs {
-			model, err := transformer.Converter.ToModel(log)
-			if err != nil {
-				return err
-			}
-			err = transformer.Repository.Create(header.Id, model)
+		if len(matchingLogs) < 1 {
+			err = transformer.Repository.MarkHeaderChecked(header.Id)
 			if err != nil {
 				return err
 			}
 		}
+		model, err := transformer.Converter.ToModels(matchingLogs)
+		if err != nil {
+			return err
+		}
+		err = transformer.Repository.Create(header.Id, model)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/transformers/pit_file/debt_ceiling/transformer_test.go

@@ -97,6 +97,42 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
+	It("marks header checked if no logs returned", func() {
+		mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		headerID := int64(123)
+		mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
+	})
+
+	It("returns error if marking header checked returns err", func() {
+		mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
+		mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{}
+		mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
+		mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		mockFetcher := &mocks.MockLogFetcher{}
+		transformer := debt_ceiling.PitFileDebtCeilingTransformer{
+			Converter:  mockConverter,
+			Fetcher:    mockFetcher,
+			Repository: mockRepository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
 	It("converts matching logs", func() {
 		converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}
 		fetcher := &mocks.MockLogFetcher{}
@@ -112,7 +148,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileDebtCeilingLog))
+		Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileDebtCeilingLog}))
 	})
 
 	It("returns error if converter returns error", func() {
@@ -151,7 +187,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedModel).To(Equal(test_data.PitFileDebtCeilingModel))
+		Expect(repository.PassedModels).To(Equal([]debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}))
 	})
 
 	It("returns error if repository returns error for create", func() {