Decouple log extraction from transformer delegation

- limit missing headers results set to 100 so that extraction doesn't
  excessively block delegation
- wrap checked headers functions in repository struct
- move storage repository to factory, to correspond with event
  repository path
- remove unused files
- reformat sql
- remove line breaks in imports

pkg/datastore/postgres/postgres_test.go

@@ -126,16 +126,16 @@ var _ = Describe("Postgres DB", func() {
 	It("does not commit log if log is invalid", func() {
 		//badTxHash violates db tx_hash field length
 		badTxHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
-		badLog := core.Log{
+		badLog := core.FullSyncLog{
 			Address:     "x123",
 			BlockNumber: 1,
 			TxHash:      badTxHash,
 		}
 		node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
 		db, _ := postgres.NewDB(test_config.DBConfig, node)
-		logRepository := repositories.LogRepository{DB: db}
+		logRepository := repositories.FullSyncLogRepository{DB: db}
 
-		err := logRepository.CreateLogs([]core.Log{badLog}, 123)
+		err := logRepository.CreateLogs([]core.FullSyncLog{badLog}, 123)
 
 		Expect(err).ToNot(BeNil())
 		savedBlock, err := logRepository.GetLogs("x123", 1)

pkg/datastore/postgres/repositories/block_repository.go

@@ -19,10 +19,8 @@ package repositories
 import (
 	"database/sql"
 	"errors"
-
 	"github.com/jmoiron/sqlx"
-	log "github.com/sirupsen/logrus"
-
+	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -81,7 +79,7 @@ func (blockRepository BlockRepository) MissingBlockNumbers(startingBlockNumber i
 		startingBlockNumber,
 		highestBlockNumber, nodeId)
 	if err != nil {
-		log.Error("MissingBlockNumbers: error getting blocks: ", err)
+		logrus.Error("MissingBlockNumbers: error getting blocks: ", err)
 	}
 	return numbers
 }
@@ -112,7 +110,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
 		case sql.ErrNoRows:
 			return core.Block{}, datastore.ErrBlockDoesNotExist(blockNumber)
 		default:
-			log.Error("GetBlock: error loading blocks: ", err)
+			logrus.Error("GetBlock: error loading blocks: ", err)
 			return savedBlock, err
 		}
 	}
@@ -151,7 +149,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 	if insertBlockErr != nil {
 		rollbackErr := tx.Rollback()
 		if rollbackErr != nil {
-			log.Error("failed to rollback transaction: ", rollbackErr)
+			logrus.Error("failed to rollback transaction: ", rollbackErr)
 		}
 		return 0, postgres.ErrDBInsertFailed(insertBlockErr)
 	}
@@ -167,7 +165,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 		if insertTxErr != nil {
 			rollbackErr := tx.Rollback()
 			if rollbackErr != nil {
-				log.Warn("failed to rollback transaction: ", rollbackErr)
+				logrus.Warn("failed to rollback transaction: ", rollbackErr)
 			}
 			return 0, postgres.ErrDBInsertFailed(insertTxErr)
 		}
@@ -176,7 +174,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 	if commitErr != nil {
 		rollbackErr := tx.Rollback()
 		if rollbackErr != nil {
-			log.Warn("failed to rollback transaction: ", rollbackErr)
+			logrus.Warn("failed to rollback transaction: ", rollbackErr)
 		}
 		return 0, commitErr
 	}
@@ -268,7 +266,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
 	return retrievedBlockHash, blockExists(retrievedBlockHash)
 }
 
-func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int64) error {
+func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.FullSyncLog, receiptId int64) error {
 	for _, tlog := range logs {
 		_, err := tx.Exec(
 			`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -305,7 +303,7 @@ func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Bloc
 	var block b
 	err := blockRows.StructScan(&block)
 	if err != nil {
-		log.Error("loadBlock: error loading block: ", err)
+		logrus.Error("loadBlock: error loading block: ", err)
 		return core.Block{}, err
 	}
 	transactionRows, err := blockRepository.database.Queryx(`
@@ -323,7 +321,7 @@ func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Bloc
 		WHERE block_id = $1
 		ORDER BY hash`, block.ID)
 	if err != nil {
-		log.Error("loadBlock: error fetting transactions: ", err)
+		logrus.Error("loadBlock: error fetting transactions: ", err)
 		return core.Block{}, err
 	}
 	block.Transactions = blockRepository.LoadTransactions(transactionRows)
@@ -336,7 +334,7 @@ func (blockRepository BlockRepository) LoadTransactions(transactionRows *sqlx.Ro
 		var transaction core.TransactionModel
 		err := transactionRows.StructScan(&transaction)
 		if err != nil {
-			log.Fatal(err)
+			logrus.Fatal(err)
 		}
 		transactions = append(transactions, transaction)
 	}

pkg/datastore/postgres/repositories/checked_headers_repository.go

@@ -0,0 +1,70 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+const (
+	insertCheckedHeaderQuery = `INSERT INTO public.checked_headers (header_id) VALUES ($1)
+		ON CONFLICT (header_id) DO UPDATE
+		SET check_count = (SELECT check_count FROM public.checked_headers WHERE header_id = $1) + 1`
+)
+
+type CheckedHeadersRepository struct {
+	db *postgres.DB
+}
+
+func NewCheckedHeadersRepository(db *postgres.DB) CheckedHeadersRepository {
+	return CheckedHeadersRepository{db: db}
+}
+
+// Adds header_id to the checked_headers table, or increment check_count if header_id already present
+func (repo CheckedHeadersRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repo.db.Exec(insertCheckedHeaderQuery, headerID)
+	return err
+}
+
+// Return header_id if not present in checked_headers or its check_count is < passed checkCount
+func (repo CheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) {
+	var result []core.Header
+	var query string
+	var err error
+
+	if endingBlockNumber == -1 {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE (header_id ISNULL OR check_count < $2)
+				AND headers.block_number >= $1
+				AND headers.eth_node_fingerprint = $3
+				LIMIT 100`
+		err = repo.db.Select(&result, query, startingBlockNumber, checkCount, repo.db.Node.ID)
+	} else {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE (header_id ISNULL OR check_count < $3)
+				AND headers.block_number >= $1
+				AND headers.block_number <= $2
+				AND headers.eth_node_fingerprint = $4
+				LIMIT 100`
+		err = repo.db.Select(&result, query, startingBlockNumber, endingBlockNumber, checkCount, repo.db.Node.ID)
+	}
+
+	return result, err
+}

pkg/datastore/postgres/repositories/checked_headers_repository_test.go

@@ -0,0 +1,270 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+	"math/rand"
+)
+
+var _ = Describe("Checked Headers repository", func() {
+	var (
+		db   *postgres.DB
+		repo datastore.CheckedHeadersRepository
+	)
+
+	Describe("MarkHeaderChecked", func() {
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+			repo = repositories.NewCheckedHeadersRepository(db)
+		})
+
+		It("marks passed header as checked on insert", func() {
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(headerErr).NotTo(HaveOccurred())
+
+			err := repo.MarkHeaderChecked(headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var checkedCount int
+			fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(fetchErr).NotTo(HaveOccurred())
+			Expect(checkedCount).To(Equal(1))
+		})
+
+		It("increments check count on update", func() {
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(headerErr).NotTo(HaveOccurred())
+
+			insertErr := repo.MarkHeaderChecked(headerID)
+			Expect(insertErr).NotTo(HaveOccurred())
+
+			updateErr := repo.MarkHeaderChecked(headerID)
+			Expect(updateErr).NotTo(HaveOccurred())
+
+			var checkedCount int
+			fetchErr := db.Get(&checkedCount, `SELECT check_count FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(fetchErr).NotTo(HaveOccurred())
+			Expect(checkedCount).To(Equal(2))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		var (
+			headerRepository      datastore.HeaderRepository
+			startingBlockNumber   int64
+			endingBlockNumber     int64
+			middleBlockNumber     int64
+			outOfRangeBlockNumber int64
+			blockNumbers          []int64
+			headerIDs             []int64
+			err                   error
+			uncheckedCheckCount   = int64(1)
+			recheckCheckCount     = int64(2)
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+			headerRepository = repositories.NewHeaderRepository(db)
+			repo = repositories.NewCheckedHeadersRepository(db)
+
+			startingBlockNumber = rand.Int63()
+			middleBlockNumber = startingBlockNumber + 1
+			endingBlockNumber = startingBlockNumber + 2
+			outOfRangeBlockNumber = endingBlockNumber + 1
+
+			blockNumbers = []int64{startingBlockNumber, middleBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
+
+			headerIDs = []int64{}
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		Describe("when ending block is specified", func() {
+			It("excludes headers that are out of range", func() {
+				headers, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, uncheckedCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				// doesn't include outOfRangeBlockNumber
+				Expect(len(headers)).To(Equal(3))
+				Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber)))
+				Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber)))
+				Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber)))
+			})
+
+			It("excludes headers that have been checked more than the check count", func() {
+				_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+				Expect(err).NotTo(HaveOccurred())
+
+				headers, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, uncheckedCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				// doesn't include middleBlockNumber
+				Expect(len(headers)).To(Equal(2))
+				Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+				Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			})
+
+			It("does not exclude headers that have been checked less than the check count", func() {
+				_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+				Expect(err).NotTo(HaveOccurred())
+
+				headers, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, recheckCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(headers)).To(Equal(3))
+				Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
+				Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
+				Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
+			})
+
+			It("only returns headers associated with the current node", func() {
+				dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+				headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+				repoTwo := repositories.NewCheckedHeadersRepository(dbTwo)
+				for _, n := range blockNumbers {
+					_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
+					Expect(err).NotTo(HaveOccurred())
+				}
+
+				Expect(err).NotTo(HaveOccurred())
+				nodeOneMissingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber, uncheckedCheckCount)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(nodeOneMissingHeaders)).To(Equal(3))
+				Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
+				Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
+				Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber)))
+
+				nodeTwoMissingHeaders, err := repoTwo.MissingHeaders(startingBlockNumber, endingBlockNumber+10, uncheckedCheckCount)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
+				Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
+				Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
+				Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
+			})
+
+			It("only returns 100 results to prevent blocking log delegation", func() {
+				for n := outOfRangeBlockNumber + 1; n < outOfRangeBlockNumber+100; n++ {
+					_, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
+					Expect(err).NotTo(HaveOccurred())
+				}
+
+				missingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber+200, uncheckedCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(missingHeaders)).To(Equal(100))
+			})
+		})
+
+		Describe("when ending block is -1", func() {
+			var endingBlock = int64(-1)
+
+			It("includes all non-checked headers when ending block is -1 ", func() {
+				headers, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(headers)).To(Equal(4))
+				Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(middleBlockNumber), Equal(outOfRangeBlockNumber)))
+			})
+
+			It("excludes headers that have been checked more than the check count", func() {
+				_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+				Expect(err).NotTo(HaveOccurred())
+
+				headers, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				// doesn't include middleBlockNumber
+				Expect(len(headers)).To(Equal(3))
+				Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+			})
+
+			It("does not exclude headers that have been checked less than the check count", func() {
+				_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+				Expect(err).NotTo(HaveOccurred())
+
+				headers, err := repo.MissingHeaders(startingBlockNumber, endingBlock, recheckCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(headers)).To(Equal(4))
+				Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+			})
+
+			It("only returns headers associated with the current node", func() {
+				dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+				headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+				repoTwo := repositories.NewCheckedHeadersRepository(dbTwo)
+				for _, n := range blockNumbers {
+					_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
+					Expect(err).NotTo(HaveOccurred())
+				}
+
+				Expect(err).NotTo(HaveOccurred())
+				nodeOneMissingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(nodeOneMissingHeaders)).To(Equal(4))
+				Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+				Expect(nodeOneMissingHeaders[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(middleBlockNumber), Equal(endingBlockNumber), Equal(outOfRangeBlockNumber)))
+
+				nodeTwoMissingHeaders, err := repoTwo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(nodeTwoMissingHeaders)).To(Equal(4))
+				Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
+				Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
+				Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
+				Expect(nodeTwoMissingHeaders[3].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
+			})
+
+			It("only returns 100 results to prevent blocking log delegation", func() {
+				for n := outOfRangeBlockNumber + 1; n < outOfRangeBlockNumber+100; n++ {
+					_, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
+					Expect(err).NotTo(HaveOccurred())
+				}
+
+				missingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(missingHeaders)).To(Equal(100))
+			})
+		})
+
+	})
+})

pkg/datastore/postgres/repositories/full_sync_log_repository.go

@@ -17,20 +17,18 @@
 package repositories
 
 import (
-	"github.com/sirupsen/logrus"
-
 	"database/sql"
-
+	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-type LogRepository struct {
+type FullSyncLogRepository struct {
 	*postgres.DB
 }
 
-func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) error {
-	tx, _ := logRepository.DB.Beginx()
+func (repository FullSyncLogRepository) CreateLogs(lgs []core.FullSyncLog, receiptId int64) error {
+	tx, _ := repository.DB.Beginx()
 	for _, tlog := range lgs {
 		_, insertLogErr := tx.Exec(
 			`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -57,8 +55,8 @@ func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) e
 	return nil
 }
 
-func (logRepository LogRepository) GetLogs(address string, blockNumber int64) ([]core.Log, error) {
-	logRows, err := logRepository.DB.Query(
+func (repository FullSyncLogRepository) GetLogs(address string, blockNumber int64) ([]core.FullSyncLog, error) {
+	logRows, err := repository.DB.Query(
 		`SELECT block_number,
 					  address,
 					  tx_hash,
@@ -72,13 +70,13 @@ func (logRepository LogRepository) GetLogs(address string, blockNumber int64) ([
 				WHERE address = $1 AND block_number = $2
 				ORDER BY block_number DESC`, address, blockNumber)
 	if err != nil {
-		return []core.Log{}, err
+		return []core.FullSyncLog{}, err
 	}
-	return logRepository.loadLogs(logRows)
+	return repository.loadLogs(logRows)
 }
 
-func (logRepository LogRepository) loadLogs(logsRows *sql.Rows) ([]core.Log, error) {
-	var lgs []core.Log
+func (repository FullSyncLogRepository) loadLogs(logsRows *sql.Rows) ([]core.FullSyncLog, error) {
+	var lgs []core.FullSyncLog
 	for logsRows.Next() {
 		var blockNumber int64
 		var address string
@@ -89,9 +87,9 @@ func (logRepository LogRepository) loadLogs(logsRows *sql.Rows) ([]core.Log, err
 		err := logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data)
 		if err != nil {
 			logrus.Error("loadLogs: Error scanning a row in logRows: ", err)
-			return []core.Log{}, err
+			return []core.FullSyncLog{}, err
 		}
-		lg := core.Log{
+		lg := core.FullSyncLog{
 			BlockNumber: blockNumber,
 			TxHash:      txHash,
 			Address:     address,

pkg/datastore/postgres/repositories/full_sync_log_repository_test.go

@@ -29,11 +29,11 @@ import (
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
-var _ = Describe("Logs Repository", func() {
+var _ = Describe("Full sync log Repository", func() {
 	Describe("Saving logs", func() {
 		var db *postgres.DB
 		var blockRepository datastore.BlockRepository
-		var logsRepository datastore.LogRepository
+		var logsRepository datastore.FullSyncLogRepository
 		var receiptRepository datastore.FullSyncReceiptRepository
 		var node core.Node
 
@@ -47,7 +47,7 @@ var _ = Describe("Logs Repository", func() {
 			db = test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			blockRepository = repositories.NewBlockRepository(db)
-			logsRepository = repositories.LogRepository{DB: db}
+			logsRepository = repositories.FullSyncLogRepository{DB: db}
 			receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
 		})
 
@@ -59,7 +59,7 @@ var _ = Describe("Logs Repository", func() {
 			receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
 			tx.Commit()
 			Expect(err).NotTo(HaveOccurred())
-			err = logsRepository.CreateLogs([]core.Log{{
+			err = logsRepository.CreateLogs([]core.FullSyncLog{{
 				BlockNumber: blockNumber,
 				Index:       0,
 				Address:     "x123",
@@ -98,7 +98,7 @@ var _ = Describe("Logs Repository", func() {
 			tx.Commit()
 			Expect(err).NotTo(HaveOccurred())
 
-			err = logsRepository.CreateLogs([]core.Log{{
+			err = logsRepository.CreateLogs([]core.FullSyncLog{{
 				BlockNumber: blockNumber,
 				Index:       0,
 				Address:     "x123",
@@ -108,7 +108,7 @@ var _ = Describe("Logs Repository", func() {
 			}}, receiptId)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = logsRepository.CreateLogs([]core.Log{{
+			err = logsRepository.CreateLogs([]core.FullSyncLog{{
 				BlockNumber: blockNumber,
 				Index:       1,
 				Address:     "x123",
@@ -118,7 +118,7 @@ var _ = Describe("Logs Repository", func() {
 			}}, receiptId)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = logsRepository.CreateLogs([]core.Log{{
+			err = logsRepository.CreateLogs([]core.FullSyncLog{{
 				BlockNumber: 2,
 				Index:       0,
 				Address:     "x123",
@@ -162,7 +162,7 @@ var _ = Describe("Logs Repository", func() {
 
 		It("saves the logs attached to a receipt", func() {
 
-			logs := []core.Log{{
+			logs := []core.FullSyncLog{{
 				Address:     "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973",
 				BlockNumber: 4745407,
 				Data:        "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08",

pkg/datastore/postgres/repositories/full_sync_receipt_repository.go

@@ -20,7 +20,6 @@ import (
 	"database/sql"
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
-
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -53,7 +52,22 @@ func (receiptRepository FullSyncReceiptRepository) CreateReceiptsAndLogs(blockId
 	return nil
 }
 
-func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
+func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
+	var receiptId int64
+	err := tx.QueryRow(
+		`INSERT INTO full_sync_receipts
+		               (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
+		               VALUES ($1, $2, $3, $4, $5, $6, $7)
+		               RETURNING id`,
+		receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId,
+	).Scan(&receiptId)
+	if err != nil {
+		logrus.Error("createReceipt: Error inserting: ", err)
+	}
+	return receiptId, err
+}
+
+func createLogs(logs []core.FullSyncLog, receiptId int64, tx *sqlx.Tx) error {
 	for _, log := range logs {
 		_, err := tx.Exec(
 			`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

pkg/datastore/postgres/repositories/full_sync_receipt_repository_test.go

@@ -29,7 +29,7 @@ import (
 
 var _ = Describe("Receipt Repository", func() {
 	var blockRepository datastore.BlockRepository
-	var logRepository datastore.LogRepository
+	var logRepository datastore.FullSyncLogRepository
 	var receiptRepository datastore.FullSyncReceiptRepository
 	var db *postgres.DB
 	var node core.Node
@@ -43,7 +43,7 @@ var _ = Describe("Receipt Repository", func() {
 		db = test_config.NewTestDB(node)
 		test_config.CleanTestDB(db)
 		blockRepository = repositories.NewBlockRepository(db)
-		logRepository = repositories.LogRepository{DB: db}
+		logRepository = repositories.FullSyncLogRepository{DB: db}
 		receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
 	})
 
@@ -56,7 +56,7 @@ var _ = Describe("Receipt Repository", func() {
 			txHashTwo := "0xTxHashTwo"
 			addressOne := "0xAddressOne"
 			addressTwo := "0xAddressTwo"
-			logsOne := []core.Log{{
+			logsOne := []core.FullSyncLog{{
 				Address:     addressOne,
 				BlockNumber: blockNumber,
 				TxHash:      txHashOne,
@@ -65,7 +65,7 @@ var _ = Describe("Receipt Repository", func() {
 				BlockNumber: blockNumber,
 				TxHash:      txHashOne,
 			}}
-			logsTwo := []core.Log{{
+			logsTwo := []core.FullSyncLog{{
 				BlockNumber: blockNumber,
 				TxHash:      txHashTwo,
 				Address:     addressTwo,
@@ -112,7 +112,7 @@ var _ = Describe("Receipt Repository", func() {
 				ContractAddress:   "0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae",
 				CumulativeGasUsed: 7996119,
 				GasUsed:           21000,
-				Logs:              []core.Log{},
+				Logs:              []core.FullSyncLog{},
 				StateRoot:         "0x88abf7e73128227370aa7baa3dd4e18d0af70e92ef1f9ef426942fbe2dddb733",
 				Status:            1,
 				TxHash:            "0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223",

pkg/datastore/postgres/repositories/header_sync_log_repository.go

@@ -0,0 +1,134 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+const insertHeaderSyncLogQuery = `INSERT INTO header_sync_logs
+		(header_id, address, topics, data, block_number, block_hash, tx_index, tx_hash, log_index, raw)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING`
+
+type HeaderSyncLogRepository struct {
+	db *postgres.DB
+}
+
+func NewHeaderSyncLogRepository(db *postgres.DB) HeaderSyncLogRepository {
+	return HeaderSyncLogRepository{db: db}
+}
+
+type headerSyncLog struct {
+	ID          int64
+	HeaderID    int64 `db:"header_id"`
+	Address     string
+	Topics      pq.ByteaArray
+	Data        []byte
+	BlockNumber uint64 `db:"block_number"`
+	BlockHash   string `db:"block_hash"`
+	TxHash      string `db:"tx_hash"`
+	TxIndex     uint   `db:"tx_index"`
+	LogIndex    uint   `db:"log_index"`
+	Transformed bool
+	Raw         []byte
+}
+
+func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) {
+	rows, queryErr := repository.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`)
+	if queryErr != nil {
+		return nil, queryErr
+	}
+
+	var results []core.HeaderSyncLog
+	for rows.Next() {
+		var rawLog headerSyncLog
+		scanErr := rows.StructScan(&rawLog)
+		if scanErr != nil {
+			return nil, scanErr
+		}
+		var logTopics []common.Hash
+		for _, topic := range rawLog.Topics {
+			logTopics = append(logTopics, common.BytesToHash(topic))
+		}
+		reconstructedLog := types.Log{
+			Address:     common.HexToAddress(rawLog.Address),
+			Topics:      logTopics,
+			Data:        rawLog.Data,
+			BlockNumber: rawLog.BlockNumber,
+			TxHash:      common.HexToHash(rawLog.TxHash),
+			TxIndex:     rawLog.TxIndex,
+			BlockHash:   common.HexToHash(rawLog.BlockHash),
+			Index:       rawLog.LogIndex,
+			// TODO: revisit if not cascade deleting logs when header removed
+			// currently, fetched logs are cascade deleted if removed
+			Removed: false,
+		}
+		result := core.HeaderSyncLog{
+			ID:          rawLog.ID,
+			HeaderID:    rawLog.HeaderID,
+			Log:         reconstructedLog,
+			Transformed: rawLog.Transformed,
+		}
+		// TODO: Consider returning each result async to avoid keeping large result sets in memory
+		results = append(results, result)
+	}
+
+	return results, nil
+}
+
+func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error {
+	tx, txErr := repository.db.Beginx()
+	if txErr != nil {
+		return txErr
+	}
+	for _, log := range logs {
+		err := insertLog(headerID, log, tx)
+		if err != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Errorf("failed to rollback header sync log insert: %s", rollbackErr.Error())
+			}
+			return err
+		}
+	}
+	return tx.Commit()
+}
+
+func insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error {
+	topics := buildTopics(log)
+	raw, jsonErr := log.MarshalJSON()
+	if jsonErr != nil {
+		return jsonErr
+	}
+	_, insertErr := tx.Exec(insertHeaderSyncLogQuery, headerID, log.Address.Hex(), topics, log.Data, log.BlockNumber,
+		log.BlockHash.Hex(), log.TxIndex, log.TxHash.Hex(), log.Index, raw)
+	return insertErr
+}
+
+func buildTopics(log types.Log) pq.ByteaArray {
+	var topics pq.ByteaArray
+	for _, topic := range log.Topics {
+		topics = append(topics, topic.Bytes())
+	}
+	return topics
+}

pkg/datastore/postgres/repositories/header_sync_log_repository_test.go

@@ -0,0 +1,203 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/lib/pq"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Header sync log repository", func() {
+	var (
+		db         *postgres.DB
+		headerID   int64
+		repository datastore.HeaderSyncLogRepository
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		headerRepository := repositories.NewHeaderRepository(db)
+		var headerErr error
+		headerID, headerErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+		Expect(headerErr).NotTo(HaveOccurred())
+		repository = repositories.NewHeaderSyncLogRepository(db)
+	})
+
+	Describe("CreateHeaderSyncLogs", func() {
+		type HeaderSyncLog struct {
+			ID          int64
+			HeaderID    int64 `db:"header_id"`
+			Address     string
+			Topics      pq.ByteaArray
+			Data        []byte
+			BlockNumber uint64 `db:"block_number"`
+			BlockHash   string `db:"block_hash"`
+			TxHash      string `db:"tx_hash"`
+			TxIndex     uint   `db:"tx_index"`
+			LogIndex    uint   `db:"log_index"`
+			Transformed bool
+			Raw         []byte
+		}
+
+		It("writes a log to the db", func() {
+			log := test_data.GenericTestLog()
+
+			err := repository.CreateHeaderSyncLogs(headerID, []types.Log{log})
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbLog HeaderSyncLog
+			lookupErr := db.Get(&dbLog, `SELECT * FROM header_sync_logs`)
+			Expect(lookupErr).NotTo(HaveOccurred())
+			Expect(dbLog.ID).NotTo(BeZero())
+			Expect(dbLog.HeaderID).To(Equal(headerID))
+			Expect(dbLog.Address).To(Equal(log.Address.Hex()))
+			Expect(dbLog.Topics[0]).To(Equal(log.Topics[0].Bytes()))
+			Expect(dbLog.Topics[1]).To(Equal(log.Topics[1].Bytes()))
+			Expect(dbLog.Data).To(Equal(log.Data))
+			Expect(dbLog.BlockNumber).To(Equal(log.BlockNumber))
+			Expect(dbLog.BlockHash).To(Equal(log.BlockHash.Hex()))
+			Expect(dbLog.TxIndex).To(Equal(log.TxIndex))
+			Expect(dbLog.TxHash).To(Equal(log.TxHash.Hex()))
+			Expect(dbLog.LogIndex).To(Equal(log.Index))
+			expectedRaw, jsonErr := log.MarshalJSON()
+			Expect(jsonErr).NotTo(HaveOccurred())
+			Expect(dbLog.Raw).To(MatchJSON(expectedRaw))
+			Expect(dbLog.Transformed).To(BeFalse())
+		})
+
+		It("writes several logs to the db", func() {
+			log1 := test_data.GenericTestLog()
+			log2 := test_data.GenericTestLog()
+			logs := []types.Log{log1, log2}
+
+			err := repository.CreateHeaderSyncLogs(headerID, logs)
+
+			Expect(err).NotTo(HaveOccurred())
+			var count int
+			lookupErr := db.Get(&count, `SELECT COUNT(*) FROM header_sync_logs`)
+			Expect(lookupErr).NotTo(HaveOccurred())
+			Expect(count).To(Equal(len(logs)))
+		})
+
+		It("persists record that can be unpacked into types.Log", func() {
+			// important if we want to decouple log persistence from transforming and still make use of
+			// tools on types.Log like abi.Unpack
+			log := test_data.GenericTestLog()
+
+			err := repository.CreateHeaderSyncLogs(headerID, []types.Log{log})
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbLog HeaderSyncLog
+			lookupErr := db.Get(&dbLog, `SELECT * FROM header_sync_logs`)
+			Expect(lookupErr).NotTo(HaveOccurred())
+
+			var logTopics []common.Hash
+			for _, topic := range dbLog.Topics {
+				logTopics = append(logTopics, common.BytesToHash(topic))
+			}
+
+			reconstructedLog := types.Log{
+				Address:     common.HexToAddress(dbLog.Address),
+				Topics:      logTopics,
+				Data:        dbLog.Data,
+				BlockNumber: dbLog.BlockNumber,
+				TxHash:      common.HexToHash(dbLog.TxHash),
+				TxIndex:     dbLog.TxIndex,
+				BlockHash:   common.HexToHash(dbLog.BlockHash),
+				Index:       dbLog.LogIndex,
+				Removed:     false,
+			}
+			Expect(reconstructedLog).To(Equal(log))
+		})
+
+		It("does not duplicate logs", func() {
+			log := test_data.GenericTestLog()
+
+			err := repository.CreateHeaderSyncLogs(headerID, []types.Log{log, log})
+
+			Expect(err).NotTo(HaveOccurred())
+			var count int
+			lookupErr := db.Get(&count, `SELECT COUNT(*) FROM header_sync_logs`)
+			Expect(lookupErr).NotTo(HaveOccurred())
+			Expect(count).To(Equal(1))
+		})
+	})
+
+	Describe("GetFullSyncLogs", func() {
+		Describe("when there are no logs", func() {
+			It("returns empty collection", func() {
+				result, err := repository.GetUntransformedHeaderSyncLogs()
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(result)).To(BeZero())
+			})
+		})
+
+		Describe("when there are logs", func() {
+			var log1, log2 types.Log
+
+			BeforeEach(func() {
+				log1 = test_data.GenericTestLog()
+				log2 = test_data.GenericTestLog()
+				logs := []types.Log{log1, log2}
+				logsErr := repository.CreateHeaderSyncLogs(headerID, logs)
+				Expect(logsErr).NotTo(HaveOccurred())
+			})
+
+			It("returns persisted logs", func() {
+				result, err := repository.GetUntransformedHeaderSyncLogs()
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(result)).To(Equal(2))
+				Expect(result[0].Log).To(Or(Equal(log1), Equal(log2)))
+				Expect(result[1].Log).To(Or(Equal(log1), Equal(log2)))
+				Expect(result[0].Log).NotTo(Equal(result[1].Log))
+			})
+
+			It("excludes logs that have been transformed", func() {
+				_, insertErr := db.Exec(`UPDATE public.header_sync_logs SET transformed = true WHERE tx_hash = $1`, log1.TxHash.Hex())
+				Expect(insertErr).NotTo(HaveOccurred())
+
+				result, err := repository.GetUntransformedHeaderSyncLogs()
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(result)).To(Equal(1))
+				Expect(result[0].Log).To(Equal(log2))
+			})
+
+			It("returns empty collection if all logs transformed", func() {
+				_, insertErr := db.Exec(`UPDATE public.header_sync_logs SET transformed = true WHERE header_id = $1`, headerID)
+				Expect(insertErr).NotTo(HaveOccurred())
+
+				result, err := repository.GetUntransformedHeaderSyncLogs()
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(result)).To(BeZero())
+			})
+		})
+	})
+})

pkg/datastore/postgres/repositories/watched_events_repository_test.go

@@ -32,7 +32,7 @@ var _ = Describe("Watched Events Repository", func() {
 	var db *postgres.DB
 	var blocksRepository datastore.BlockRepository
 	var filterRepository datastore.FilterRepository
-	var logRepository datastore.LogRepository
+	var logRepository datastore.FullSyncLogRepository
 	var receiptRepository datastore.FullSyncReceiptRepository
 	var watchedEventRepository datastore.WatchedEventRepository
 
@@ -41,7 +41,7 @@ var _ = Describe("Watched Events Repository", func() {
 		test_config.CleanTestDB(db)
 		blocksRepository = repositories.NewBlockRepository(db)
 		filterRepository = repositories.FilterRepository{DB: db}
-		logRepository = repositories.LogRepository{DB: db}
+		logRepository = repositories.FullSyncLogRepository{DB: db}
 		receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
 		watchedEventRepository = repositories.WatchedEventRepository{DB: db}
 	})
@@ -54,7 +54,7 @@ var _ = Describe("Watched Events Repository", func() {
 			Address:   "0x123",
 			Topics:    core.Topics{0: "event1=10", 2: "event3=hello"},
 		}
-		logs := []core.Log{
+		logs := []core.FullSyncLog{
 			{
 				BlockNumber: 0,
 				TxHash:      "0x1",
@@ -108,7 +108,7 @@ var _ = Describe("Watched Events Repository", func() {
 			Address:   "0x123",
 			Topics:    core.Topics{0: "event1=10", 2: "event3=hello"},
 		}
-		logs := []core.Log{
+		logs := []core.FullSyncLog{
 			{
 				BlockNumber: 0,
 				TxHash:      "0x1",