Decouple log extraction from transformer delegation

- limit missing headers results set to 100 so that extraction doesn't
  excessively block delegation
- wrap checked headers functions in repository struct
- move storage repository to factory, to correspond with event
  repository path
- remove unused files
- reformat sql
- remove line breaks in imports

libraries/shared/logs/delegator.go

@@ -0,0 +1,78 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs
+
+import (
+	"errors"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+var ErrNoTransformers = errors.New("no event transformers configured in the log delegator")
+
+type ILogDelegator interface {
+	AddTransformer(t transformer.EventTransformer)
+	DelegateLogs() error
+}
+
+type LogDelegator struct {
+	Chunker       chunker.Chunker
+	LogRepository datastore.HeaderSyncLogRepository
+	Transformers  []transformer.EventTransformer
+}
+
+func (delegator *LogDelegator) AddTransformer(t transformer.EventTransformer) {
+	delegator.Transformers = append(delegator.Transformers, t)
+	delegator.Chunker.AddConfig(t.GetConfig())
+}
+
+func (delegator LogDelegator) DelegateLogs() error {
+	if len(delegator.Transformers) < 1 {
+		return ErrNoTransformers
+	}
+
+	persistedLogs, fetchErr := delegator.LogRepository.GetUntransformedHeaderSyncLogs()
+	if fetchErr != nil {
+		logrus.Errorf("error loading logs from db: %s", fetchErr.Error())
+		return fetchErr
+	}
+
+	transformErr := delegator.delegateLogs(persistedLogs)
+	if transformErr != nil {
+		logrus.Errorf("error transforming logs: %s", transformErr)
+		return transformErr
+	}
+
+	return nil
+}
+
+func (delegator *LogDelegator) delegateLogs(logs []core.HeaderSyncLog) error {
+	chunkedLogs := delegator.Chunker.ChunkLogs(logs)
+	for _, t := range delegator.Transformers {
+		transformerName := t.GetConfig().TransformerName
+		logChunk := chunkedLogs[transformerName]
+		err := t.Execute(logChunk)
+		if err != nil {
+			logrus.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
+			return err
+		}
+	}
+	return nil
+}

libraries/shared/logs/delegator_test.go

@@ -0,0 +1,134 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/logs"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"strings"
+)
+
+var _ = Describe("Log delegator", func() {
+	Describe("AddTransformer", func() {
+		It("adds transformers to the delegator", func() {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			delegator := logs.LogDelegator{Chunker: chunker.NewLogChunker()}
+
+			delegator.AddTransformer(fakeTransformer)
+
+			Expect(delegator.Transformers).To(Equal([]transformer.EventTransformer{fakeTransformer}))
+		})
+
+		It("passes transformers' configs to the chunker", func() {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			fakeConfig := mocks.FakeTransformerConfig
+			fakeTransformer.SetTransformerConfig(fakeConfig)
+			chunker := chunker.NewLogChunker()
+			delegator := logs.LogDelegator{Chunker: chunker}
+
+			delegator.AddTransformer(fakeTransformer)
+
+			expectedName := fakeConfig.TransformerName
+			expectedTopic := common.HexToHash(fakeConfig.Topic)
+			Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{expectedName: expectedTopic}))
+			expectedAddress := strings.ToLower(fakeConfig.ContractAddresses[0])
+			Expect(chunker.AddressToNames).To(Equal(map[string][]string{expectedAddress: {expectedName}}))
+		})
+	})
+
+	Describe("DelegateLogs", func() {
+		It("returns an error if no transformers configured", func() {
+			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(logs.ErrNoTransformers))
+		})
+
+		It("gets untransformed logs", func() {
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(&mocks.MockEventTransformer{})
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockLogRepository.GetCalled).To(BeTrue())
+		})
+
+		It("returns error if getting untransformed logs fails", func() {
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.GetError = fakes.FakeError
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(&mocks.MockEventTransformer{})
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		It("delegates chunked logs to transformers", func() {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
+			fakeGethLog := types.Log{
+				Address: common.HexToAddress(fakeTransformer.GetConfig().ContractAddresses[0]),
+				Topics:  []common.Hash{common.HexToHash(fakeTransformer.GetConfig().Topic)},
+			}
+			fakeHeaderSyncLog := core.HeaderSyncLog{Log: fakeGethLog}
+			fakeHeaderSyncLogs := []core.HeaderSyncLog{fakeHeaderSyncLog}
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.ReturnLogs = fakeHeaderSyncLogs
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(fakeTransformer)
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
+			Expect(fakeTransformer.PassedLogs).To(Equal(fakeHeaderSyncLogs))
+		})
+
+		It("returns an error if transformer returns an error", func() {
+			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+			fakeTransformer := &mocks.MockEventTransformer{ExecuteError: fakes.FakeError}
+			delegator.AddTransformer(fakeTransformer)
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
+
+})
+
+func newDelegator(headerSyncLogRepository *fakes.MockHeaderSyncLogRepository) logs.LogDelegator {
+	return logs.LogDelegator{
+		Chunker:       chunker.NewLogChunker(),
+		LogRepository: headerSyncLogRepository,
+	}
+}

libraries/shared/logs/extractor.go

@@ -0,0 +1,122 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs
+
+import (
+	"errors"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transactions"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+var ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor")
+
+type ILogExtractor interface {
+	AddTransformerConfig(config transformer.EventTransformerConfig)
+	ExtractLogs(recheckHeaders constants.TransformerExecution) error
+}
+
+type LogExtractor struct {
+	Addresses                []common.Address
+	CheckedHeadersRepository datastore.CheckedHeadersRepository
+	Fetcher                  fetcher.ILogFetcher
+	LogRepository            datastore.HeaderSyncLogRepository
+	StartingBlock            *int64
+	Syncer                   transactions.ITransactionsSyncer
+	Topics                   []common.Hash
+}
+
+// Add additional logs to extract
+func (extractor *LogExtractor) AddTransformerConfig(config transformer.EventTransformerConfig) {
+	if extractor.StartingBlock == nil {
+		extractor.StartingBlock = &config.StartingBlockNumber
+	} else if earlierStartingBlockNumber(config.StartingBlockNumber, *extractor.StartingBlock) {
+		extractor.StartingBlock = &config.StartingBlockNumber
+	}
+
+	addresses := transformer.HexStringsToAddresses(config.ContractAddresses)
+	extractor.Addresses = append(extractor.Addresses, addresses...)
+	extractor.Topics = append(extractor.Topics, common.HexToHash(config.Topic))
+}
+
+// Fetch and persist watched logs
+func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) error {
+	if len(extractor.Addresses) < 1 {
+		logrus.Errorf("error extracting logs: %s", ErrNoWatchedAddresses.Error())
+		return ErrNoWatchedAddresses
+	}
+
+	missingHeaders, missingHeadersErr := extractor.CheckedHeadersRepository.MissingHeaders(*extractor.StartingBlock, -1, getCheckCount(recheckHeaders))
+	if missingHeadersErr != nil {
+		logrus.Errorf("error fetching missing headers: %s", missingHeadersErr)
+		return missingHeadersErr
+	}
+
+	for _, header := range missingHeaders {
+		logs, fetchLogsErr := extractor.Fetcher.FetchLogs(extractor.Addresses, extractor.Topics, header)
+		if fetchLogsErr != nil {
+			logError("error fetching logs for header: %s", fetchLogsErr, header)
+			return fetchLogsErr
+		}
+
+		if len(logs) > 0 {
+			transactionsSyncErr := extractor.Syncer.SyncTransactions(header.Id, logs)
+			if transactionsSyncErr != nil {
+				logError("error syncing transactions: %s", transactionsSyncErr, header)
+				return transactionsSyncErr
+			}
+
+			createLogsErr := extractor.LogRepository.CreateHeaderSyncLogs(header.Id, logs)
+			if createLogsErr != nil {
+				logError("error persisting logs: %s", createLogsErr, header)
+				return createLogsErr
+			}
+		}
+
+		markHeaderCheckedErr := extractor.CheckedHeadersRepository.MarkHeaderChecked(header.Id)
+		if markHeaderCheckedErr != nil {
+			logError("error marking header checked: %s", markHeaderCheckedErr, header)
+			return markHeaderCheckedErr
+		}
+	}
+	return nil
+}
+
+func earlierStartingBlockNumber(transformerBlock, watcherBlock int64) bool {
+	return transformerBlock < watcherBlock
+}
+
+func logError(description string, err error, header core.Header) {
+	logrus.WithFields(logrus.Fields{
+		"headerId":    header.Id,
+		"headerHash":  header.Hash,
+		"blockNumber": header.BlockNumber,
+	}).Errorf(description, err.Error())
+}
+
+func getCheckCount(recheckHeaders constants.TransformerExecution) int64 {
+	if recheckHeaders == constants.HeaderMissing {
+		return 1
+	} else {
+		return constants.RecheckHeaderCap
+	}
+}

libraries/shared/logs/extractor_test.go

@@ -0,0 +1,310 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/logs"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"math/rand"
+)
+
+var _ = Describe("Log extractor", func() {
+	var extractor *logs.LogExtractor
+
+	BeforeEach(func() {
+		extractor = &logs.LogExtractor{
+			Fetcher:                  &mocks.MockLogFetcher{},
+			CheckedHeadersRepository: &fakes.MockCheckedHeadersRepository{},
+			LogRepository:            &fakes.MockHeaderSyncLogRepository{},
+			Syncer:                   &fakes.MockTransactionSyncer{},
+		}
+	})
+
+	Describe("AddTransformerConfig", func() {
+		It("it includes earliest starting block number in fetch logs query", func() {
+			earlierStartingBlockNumber := rand.Int63()
+			laterStartingBlockNumber := earlierStartingBlockNumber + 1
+
+			extractor.AddTransformerConfig(getTransformerConfig(laterStartingBlockNumber))
+			extractor.AddTransformerConfig(getTransformerConfig(earlierStartingBlockNumber))
+
+			Expect(*extractor.StartingBlock).To(Equal(earlierStartingBlockNumber))
+		})
+
+		It("includes added addresses in fetch logs query", func() {
+			addresses := []string{"0xA", "0xB"}
+			configWithAddresses := transformer.EventTransformerConfig{
+				ContractAddresses:   addresses,
+				StartingBlockNumber: rand.Int63(),
+			}
+
+			extractor.AddTransformerConfig(configWithAddresses)
+
+			expectedAddresses := transformer.HexStringsToAddresses(addresses)
+			Expect(extractor.Addresses).To(Equal(expectedAddresses))
+		})
+
+		It("includes added topics in fetch logs query", func() {
+			topic := "0x1"
+			configWithTopic := transformer.EventTransformerConfig{
+				ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+				Topic:               topic,
+				StartingBlockNumber: rand.Int63(),
+			}
+
+			extractor.AddTransformerConfig(configWithTopic)
+
+			Expect(extractor.Topics).To(Equal([]common.Hash{common.HexToHash(topic)}))
+		})
+	})
+
+	Describe("ExtractLogs", func() {
+		It("returns error if no watched addresses configured", func() {
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(logs.ErrNoWatchedAddresses))
+		})
+
+		Describe("when checking missing headers", func() {
+			It("gets missing headers since configured starting block with check count < 1", func() {
+				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+				mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{}}
+				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+				startingBlockNumber := rand.Int63()
+				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
+
+				err := extractor.ExtractLogs(constants.HeaderMissing)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(mockCheckedHeadersRepository.StartingBlockNumber).To(Equal(startingBlockNumber))
+				Expect(mockCheckedHeadersRepository.EndingBlockNumber).To(Equal(int64(-1)))
+				Expect(mockCheckedHeadersRepository.CheckCount).To(Equal(int64(1)))
+			})
+		})
+
+		Describe("when rechecking headers", func() {
+			It("gets missing headers since configured starting block with check count < 1", func() {
+				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+				mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{}}
+				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+				startingBlockNumber := rand.Int63()
+				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
+
+				err := extractor.ExtractLogs(constants.HeaderRecheck)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(mockCheckedHeadersRepository.StartingBlockNumber).To(Equal(startingBlockNumber))
+				Expect(mockCheckedHeadersRepository.EndingBlockNumber).To(Equal(int64(-1)))
+				Expect(mockCheckedHeadersRepository.CheckCount).To(Equal(constants.RecheckHeaderCap))
+			})
+		})
+
+		It("returns error if getting missing headers fails", func() {
+			addTransformerConfig(extractor)
+			mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+			mockCheckedHeadersRepository.MissingHeadersReturnError = fakes.FakeError
+			extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		It("does not fetch logs if no missing headers", func() {
+			addTransformerConfig(extractor)
+			mockLogFetcher := &mocks.MockLogFetcher{}
+			extractor.Fetcher = mockLogFetcher
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockLogFetcher.FetchCalled).To(BeFalse())
+		})
+
+		It("fetches logs for missing headers", func() {
+			addMissingHeader(extractor)
+			config := transformer.EventTransformerConfig{
+				ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+				Topic:               fakes.FakeHash.Hex(),
+				StartingBlockNumber: rand.Int63(),
+			}
+			extractor.AddTransformerConfig(config)
+			mockLogFetcher := &mocks.MockLogFetcher{}
+			extractor.Fetcher = mockLogFetcher
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockLogFetcher.FetchCalled).To(BeTrue())
+			Expect(mockLogFetcher.Topics).To(Equal([]common.Hash{common.HexToHash(config.Topic)}))
+			Expect(mockLogFetcher.ContractAddresses).To(Equal(transformer.HexStringsToAddresses(config.ContractAddresses)))
+		})
+
+		It("returns error if fetching logs fails", func() {
+			addMissingHeader(extractor)
+			addTransformerConfig(extractor)
+			mockLogFetcher := &mocks.MockLogFetcher{}
+			mockLogFetcher.ReturnError = fakes.FakeError
+			extractor.Fetcher = mockLogFetcher
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		It("does not sync transactions if no fetched logs", func() {
+			addMissingHeader(extractor)
+			addTransformerConfig(extractor)
+			mockTransactionSyncer := &fakes.MockTransactionSyncer{}
+			extractor.Syncer = mockTransactionSyncer
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse())
+		})
+
+		It("syncs transactions for fetched logs", func() {
+			addMissingHeader(extractor)
+			addFetchedLog(extractor)
+			addTransformerConfig(extractor)
+			mockTransactionSyncer := &fakes.MockTransactionSyncer{}
+			extractor.Syncer = mockTransactionSyncer
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue())
+		})
+
+		It("returns error if syncing transactions fails", func() {
+			addMissingHeader(extractor)
+			addFetchedLog(extractor)
+			addTransformerConfig(extractor)
+			mockTransactionSyncer := &fakes.MockTransactionSyncer{}
+			mockTransactionSyncer.SyncTransactionsError = fakes.FakeError
+			extractor.Syncer = mockTransactionSyncer
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		It("persists fetched logs", func() {
+			addMissingHeader(extractor)
+			addTransformerConfig(extractor)
+			fakeLogs := []types.Log{{
+				Address: common.HexToAddress("0xA"),
+				Topics:  []common.Hash{common.HexToHash("0xA")},
+				Data:    []byte{},
+				Index:   0,
+			}}
+			mockLogFetcher := &mocks.MockLogFetcher{ReturnLogs: fakeLogs}
+			extractor.Fetcher = mockLogFetcher
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			extractor.LogRepository = mockLogRepository
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs))
+		})
+
+		It("returns error if persisting logs fails", func() {
+			addMissingHeader(extractor)
+			addFetchedLog(extractor)
+			addTransformerConfig(extractor)
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.CreateError = fakes.FakeError
+			extractor.LogRepository = mockLogRepository
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		It("marks header checked", func() {
+			addFetchedLog(extractor)
+			addTransformerConfig(extractor)
+			mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+			headerID := rand.Int63()
+			mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{Id: headerID}}
+			extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockCheckedHeadersRepository.HeaderID).To(Equal(headerID))
+		})
+
+		It("returns error if marking header checked fails", func() {
+			addFetchedLog(extractor)
+			addTransformerConfig(extractor)
+			mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+			mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{Id: rand.Int63()}}
+			mockCheckedHeadersRepository.MarkHeaderCheckedReturnError = fakes.FakeError
+			extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+
+			err := extractor.ExtractLogs(constants.HeaderMissing)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
+})
+
+func addTransformerConfig(extractor *logs.LogExtractor) {
+	fakeConfig := transformer.EventTransformerConfig{
+		ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+		Topic:               fakes.FakeHash.Hex(),
+		StartingBlockNumber: rand.Int63(),
+	}
+	extractor.AddTransformerConfig(fakeConfig)
+}
+
+func addMissingHeader(extractor *logs.LogExtractor) {
+	mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+	mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{}}
+	extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+}
+
+func addFetchedLog(extractor *logs.LogExtractor) {
+	mockLogFetcher := &mocks.MockLogFetcher{}
+	mockLogFetcher.ReturnLogs = []types.Log{{}}
+	extractor.Fetcher = mockLogFetcher
+}
+
+func getTransformerConfig(startingBlockNumber int64) transformer.EventTransformerConfig {
+	return transformer.EventTransformerConfig{
+		ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+		Topic:               fakes.FakeHash.Hex(),
+		StartingBlockNumber: startingBlockNumber,
+	}
+}

libraries/shared/logs/logs_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs_test
+
+import (
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestLogs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Logs Suite")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})