Extract and delegate logs concurrently

cmd/execute.go

@@ -157,10 +157,13 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 	} else {
 		recheck = constants.HeaderMissing
 	}
-	ticker := time.NewTicker(pollingInterval)
+	errs := make(chan error)
-	defer ticker.Stop()
+	go w.Execute(recheck, errs)
-	for range ticker.C {
+	for {
-		w.Execute(recheck)
+		select {
+		case err := <-errs:
+			LogWithCommand.Fatalf("error executing event watcher: %s", err.Error())
+		}
 	}
 }
 

libraries/shared/logs/delegator.go

@@ -29,7 +29,7 @@ var ErrNoTransformers = errors.New("no event transformers configured in the log
 
 type ILogDelegator interface {
 	AddTransformer(t transformer.EventTransformer)
-	DelegateLogs() error
+	DelegateLogs(errs chan error, logsFound chan bool)
 }
 
 type LogDelegator struct {
@@ -43,24 +43,31 @@ func (delegator *LogDelegator) AddTransformer(t transformer.EventTransformer) {
 	delegator.Chunker.AddConfig(t.GetConfig())
 }
 
-func (delegator LogDelegator) DelegateLogs() error {
+func (delegator *LogDelegator) DelegateLogs(errs chan error, logsFound chan bool) {
 	if len(delegator.Transformers) < 1 {
-		return ErrNoTransformers
+		errs <- ErrNoTransformers
+		return
 	}
 
 	persistedLogs, fetchErr := delegator.LogRepository.GetUntransformedHeaderSyncLogs()
 	if fetchErr != nil {
 		logrus.Errorf("error loading logs from db: %s", fetchErr.Error())
-		return fetchErr
+		errs <- fetchErr
+		return
+	}
+
+	if len(persistedLogs) < 1 {
+		logsFound <- false
 	}
 
 	transformErr := delegator.delegateLogs(persistedLogs)
 	if transformErr != nil {
 		logrus.Errorf("error transforming logs: %s", transformErr)
-		return transformErr
+		errs <- transformErr
+		return
 	}
 
-	return nil
+	logsFound <- true
 }
 
 func (delegator *LogDelegator) delegateLogs(logs []core.HeaderSyncLog) error {

libraries/shared/logs/delegator_test.go

@@ -59,75 +59,123 @@ var _ = Describe("Log delegator", func() {
 	})
 
 	Describe("DelegateLogs", func() {
-		It("returns an error if no transformers configured", func() {
+		var (
-			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+			errsChan  chan error
+			logsFound chan bool
+		)
 
-			err := delegator.DelegateLogs()
+		BeforeEach(func() {
-
+			errsChan = make(chan error)
-			Expect(err).To(HaveOccurred())
+			logsFound = make(chan bool)
-			Expect(err).To(MatchError(logs.ErrNoTransformers))
 		})
 
-		It("gets untransformed logs", func() {
+		It("returns an error if no transformers configured", func(done Done) {
+			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+
+			go delegator.DelegateLogs(errsChan, logsFound)
+
+			Expect(<-errsChan).To(MatchError(logs.ErrNoTransformers))
+			close(done)
+		})
+
+		It("gets untransformed logs", func(done Done) {
 			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
 			delegator := newDelegator(mockLogRepository)
 			delegator.AddTransformer(&mocks.MockEventTransformer{})
 
-			err := delegator.DelegateLogs()
+			go delegator.DelegateLogs(errsChan, logsFound)
 
-			Expect(err).NotTo(HaveOccurred())
+			Eventually(func() bool {
-			Expect(mockLogRepository.GetCalled).To(BeTrue())
+				return mockLogRepository.GetCalled
+			}).Should(BeTrue())
+			close(done)
 		})
 
-		It("returns error if getting untransformed logs fails", func() {
+		It("emits error if getting untransformed logs fails", func(done Done) {
 			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
 			mockLogRepository.GetError = fakes.FakeError
 			delegator := newDelegator(mockLogRepository)
 			delegator.AddTransformer(&mocks.MockEventTransformer{})
 
-			err := delegator.DelegateLogs()
+			go delegator.DelegateLogs(errsChan, logsFound)
 
-			Expect(err).To(HaveOccurred())
+			Expect(<-errsChan).To(MatchError(fakes.FakeError))
-			Expect(err).To(MatchError(fakes.FakeError))
+			close(done)
 		})
 
-		It("delegates chunked logs to transformers", func() {
+		It("emits that no logs were found if no logs returned", func(done Done) {
+			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+			delegator.AddTransformer(&mocks.MockEventTransformer{})
+
+			go delegator.DelegateLogs(errsChan, logsFound)
+
+			Expect(<-logsFound).To(BeFalse())
+			close(done)
+		})
+
+		It("delegates chunked logs to transformers", func(done Done) {
 			fakeTransformer := &mocks.MockEventTransformer{}
-			fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig)
+			config := mocks.FakeTransformerConfig
+			fakeTransformer.SetTransformerConfig(config)
 			fakeGethLog := types.Log{
-				Address: common.HexToAddress(fakeTransformer.GetConfig().ContractAddresses[0]),
+				Address: common.HexToAddress(config.ContractAddresses[0]),
-				Topics:  []common.Hash{common.HexToHash(fakeTransformer.GetConfig().Topic)},
+				Topics:  []common.Hash{common.HexToHash(config.Topic)},
 			}
-			fakeHeaderSyncLog := core.HeaderSyncLog{Log: fakeGethLog}
+			fakeHeaderSyncLogs := []core.HeaderSyncLog{{Log: fakeGethLog}}
-			fakeHeaderSyncLogs := []core.HeaderSyncLog{fakeHeaderSyncLog}
 			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
 			mockLogRepository.ReturnLogs = fakeHeaderSyncLogs
 			delegator := newDelegator(mockLogRepository)
 			delegator.AddTransformer(fakeTransformer)
 
-			err := delegator.DelegateLogs()
+			go delegator.DelegateLogs(errsChan, logsFound)
 
-			Expect(err).NotTo(HaveOccurred())
+			Eventually(func() bool {
-			Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
+				return fakeTransformer.ExecuteWasCalled
-			Expect(fakeTransformer.PassedLogs).To(Equal(fakeHeaderSyncLogs))
+			}).Should(BeTrue())
+			Eventually(func() []core.HeaderSyncLog {
+				return fakeTransformer.PassedLogs
+			}).Should(Equal(fakeHeaderSyncLogs))
+			close(done)
 		})
 
-		It("returns an error if transformer returns an error", func() {
+		It("emits error if transformer returns an error", func(done Done) {
-			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.ReturnLogs = []core.HeaderSyncLog{{}}
+			delegator := newDelegator(mockLogRepository)
 			fakeTransformer := &mocks.MockEventTransformer{ExecuteError: fakes.FakeError}
 			delegator.AddTransformer(fakeTransformer)
 
-			err := delegator.DelegateLogs()
+			go delegator.DelegateLogs(errsChan, logsFound)
 
-			Expect(err).To(HaveOccurred())
+			Expect(<-errsChan).To(MatchError(fakes.FakeError))
-			Expect(err).To(MatchError(fakes.FakeError))
+			close(done)
+		})
+
+		It("emits logs found when logs returned and delegated", func(done Done) {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			config := mocks.FakeTransformerConfig
+			fakeTransformer.SetTransformerConfig(config)
+			fakeGethLog := types.Log{
+				Address: common.HexToAddress(config.ContractAddresses[0]),
+				Topics:  []common.Hash{common.HexToHash(config.Topic)},
+			}
+			fakeHeaderSyncLogs := []core.HeaderSyncLog{{Log: fakeGethLog}}
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.ReturnLogs = fakeHeaderSyncLogs
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(fakeTransformer)
+
+			go delegator.DelegateLogs(errsChan, logsFound)
+
+			Expect(<-logsFound).To(BeTrue())
+			close(done)
 		})
 	})
 
 })
 
-func newDelegator(headerSyncLogRepository *fakes.MockHeaderSyncLogRepository) logs.LogDelegator {
+func newDelegator(headerSyncLogRepository *fakes.MockHeaderSyncLogRepository) *logs.LogDelegator {
-	return logs.LogDelegator{
+	return &logs.LogDelegator{
 		Chunker:       chunker.NewLogChunker(),
 		LogRepository: headerSyncLogRepository,
 	}

libraries/shared/logs/extractor.go

@@ -32,7 +32,7 @@ var ErrNoWatchedAddresses = errors.New("no watched addresses configured in the l
 
 type ILogExtractor interface {
 	AddTransformerConfig(config transformer.EventTransformerConfig)
-	ExtractLogs(recheckHeaders constants.TransformerExecution) error
+	ExtractLogs(recheckHeaders constants.TransformerExecution, errs chan error, missingHeadersFound chan bool)
 }
 
 type LogExtractor struct {
@@ -59,46 +59,56 @@ func (extractor *LogExtractor) AddTransformerConfig(config transformer.EventTran
 }
 
 // Fetch and persist watched logs
-func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) error {
+func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution, errs chan error, missingHeadersFound chan bool) {
 	if len(extractor.Addresses) < 1 {
 		logrus.Errorf("error extracting logs: %s", ErrNoWatchedAddresses.Error())
-		return ErrNoWatchedAddresses
+		errs <- ErrNoWatchedAddresses
+		return
 	}
 
 	missingHeaders, missingHeadersErr := extractor.CheckedHeadersRepository.MissingHeaders(*extractor.StartingBlock, -1, getCheckCount(recheckHeaders))
 	if missingHeadersErr != nil {
 		logrus.Errorf("error fetching missing headers: %s", missingHeadersErr)
-		return missingHeadersErr
+		errs <- missingHeadersErr
+		return
+	}
+
+	if len(missingHeaders) < 1 {
+		missingHeadersFound <- false
+		return
 	}
 
 	for _, header := range missingHeaders {
 		logs, fetchLogsErr := extractor.Fetcher.FetchLogs(extractor.Addresses, extractor.Topics, header)
 		if fetchLogsErr != nil {
 			logError("error fetching logs for header: %s", fetchLogsErr, header)
-			return fetchLogsErr
+			errs <- fetchLogsErr
+			return
 		}
 
 		if len(logs) > 0 {
 			transactionsSyncErr := extractor.Syncer.SyncTransactions(header.Id, logs)
 			if transactionsSyncErr != nil {
 				logError("error syncing transactions: %s", transactionsSyncErr, header)
-				return transactionsSyncErr
+				errs <- transactionsSyncErr
+				return
 			}
 
 			createLogsErr := extractor.LogRepository.CreateHeaderSyncLogs(header.Id, logs)
 			if createLogsErr != nil {
 				logError("error persisting logs: %s", createLogsErr, header)
-				return createLogsErr
+				errs <- createLogsErr
+				return
 			}
 		}
 
 		markHeaderCheckedErr := extractor.CheckedHeadersRepository.MarkHeaderChecked(header.Id)
 		if markHeaderCheckedErr != nil {
 			logError("error marking header checked: %s", markHeaderCheckedErr, header)
-			return markHeaderCheckedErr
+			errs <- markHeaderCheckedErr
 		}
 	}
-	return nil
+	missingHeadersFound <- true
 }
 
 func earlierStartingBlockNumber(transformerBlock, watcherBlock int64) bool {

libraries/shared/logs/extractor_test.go

@@ -81,71 +81,109 @@ var _ = Describe("Log extractor", func() {
 	})
 
 	Describe("ExtractLogs", func() {
-		It("returns error if no watched addresses configured", func() {
+		var (
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+			errsChan            chan error
+			missingHeadersFound chan bool
+		)
 
-			Expect(err).To(HaveOccurred())
+		BeforeEach(func() {
-			Expect(err).To(MatchError(logs.ErrNoWatchedAddresses))
+			errsChan = make(chan error)
+			missingHeadersFound = make(chan bool)
+		})
+
+		It("returns error if no watched addresses configured", func(done Done) {
+			go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
+
+			Expect(<-errsChan).To(MatchError(logs.ErrNoWatchedAddresses))
+			close(done)
 		})
 
 		Describe("when checking missing headers", func() {
-			It("gets missing headers since configured starting block with check count < 1", func() {
+			It("gets missing headers since configured starting block with check_count < 1", func(done Done) {
 				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
 				mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{}}
 				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
 				startingBlockNumber := rand.Int63()
 				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
 
-				err := extractor.ExtractLogs(constants.HeaderMissing)
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-				Expect(err).NotTo(HaveOccurred())
+				Eventually(func() int64 {
-				Expect(mockCheckedHeadersRepository.StartingBlockNumber).To(Equal(startingBlockNumber))
+					return mockCheckedHeadersRepository.StartingBlockNumber
-				Expect(mockCheckedHeadersRepository.EndingBlockNumber).To(Equal(int64(-1)))
+				}).Should(Equal(startingBlockNumber))
-				Expect(mockCheckedHeadersRepository.CheckCount).To(Equal(int64(1)))
+				Eventually(func() int64 {
+					return mockCheckedHeadersRepository.EndingBlockNumber
+				}).Should(Equal(int64(-1)))
+				Eventually(func() int64 {
+					return mockCheckedHeadersRepository.CheckCount
+				}).Should(Equal(int64(1)))
+				close(done)
 			})
 		})
 
 		Describe("when rechecking headers", func() {
-			It("gets missing headers since configured starting block with check count < 1", func() {
+			It("gets missing headers since configured starting block with check_count < RecheckHeaderCap", func(done Done) {
 				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
 				mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{}}
 				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
 				startingBlockNumber := rand.Int63()
 				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
 
-				err := extractor.ExtractLogs(constants.HeaderRecheck)
+				go extractor.ExtractLogs(constants.HeaderRecheck, errsChan, missingHeadersFound)
 
-				Expect(err).NotTo(HaveOccurred())
+				Eventually(func() int64 {
-				Expect(mockCheckedHeadersRepository.StartingBlockNumber).To(Equal(startingBlockNumber))
+					return mockCheckedHeadersRepository.StartingBlockNumber
-				Expect(mockCheckedHeadersRepository.EndingBlockNumber).To(Equal(int64(-1)))
+				}).Should(Equal(startingBlockNumber))
-				Expect(mockCheckedHeadersRepository.CheckCount).To(Equal(constants.RecheckHeaderCap))
+				Eventually(func() int64 {
+					return mockCheckedHeadersRepository.EndingBlockNumber
+				}).Should(Equal(int64(-1)))
+				Eventually(func() int64 {
+					return mockCheckedHeadersRepository.CheckCount
+				}).Should(Equal(constants.RecheckHeaderCap))
+				close(done)
 			})
 		})
 
-		It("returns error if getting missing headers fails", func() {
+		It("emits error if getting missing headers fails", func(done Done) {
 			addTransformerConfig(extractor)
 			mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
 			mockCheckedHeadersRepository.MissingHeadersReturnError = fakes.FakeError
 			extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+			go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).To(HaveOccurred())
+			Expect(<-errsChan).To(MatchError(fakes.FakeError))
-			Expect(err).To(MatchError(fakes.FakeError))
+			close(done)
 		})
 
-		It("does not fetch logs if no missing headers", func() {
+		Describe("when no missing headers", func() {
+			It("does not fetch logs", func(done Done) {
 				addTransformerConfig(extractor)
 				mockLogFetcher := &mocks.MockLogFetcher{}
 				extractor.Fetcher = mockLogFetcher
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).NotTo(HaveOccurred())
+				Consistently(func() bool {
-			Expect(mockLogFetcher.FetchCalled).To(BeFalse())
+					return mockLogFetcher.FetchCalled
+				}).Should(BeFalse())
+				close(done)
 			})
 
-		It("fetches logs for missing headers", func() {
+			It("emits that no missing headers were found", func(done Done) {
+				addTransformerConfig(extractor)
+				mockLogFetcher := &mocks.MockLogFetcher{}
+				extractor.Fetcher = mockLogFetcher
+
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
+
+				Expect(<-missingHeadersFound).To(BeFalse())
+				close(done)
+			})
+		})
+
+		Describe("when there are missing headers", func() {
+			It("fetches logs for missing headers", func(done Done) {
 				addMissingHeader(extractor)
 				config := transformer.EventTransformerConfig{
 					ContractAddresses:   []string{fakes.FakeAddress.Hex()},
@@ -156,53 +194,68 @@ var _ = Describe("Log extractor", func() {
 				mockLogFetcher := &mocks.MockLogFetcher{}
 				extractor.Fetcher = mockLogFetcher
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).NotTo(HaveOccurred())
+				Eventually(func() bool {
-			Expect(mockLogFetcher.FetchCalled).To(BeTrue())
+					return mockLogFetcher.FetchCalled
-			Expect(mockLogFetcher.Topics).To(Equal([]common.Hash{common.HexToHash(config.Topic)}))
+				}).Should(BeTrue())
-			Expect(mockLogFetcher.ContractAddresses).To(Equal(transformer.HexStringsToAddresses(config.ContractAddresses)))
+				expectedTopics := []common.Hash{common.HexToHash(config.Topic)}
+				Eventually(func() []common.Hash {
+					return mockLogFetcher.Topics
+				}).Should(Equal(expectedTopics))
+				expectedAddresses := transformer.HexStringsToAddresses(config.ContractAddresses)
+				Eventually(func() []common.Address {
+					return mockLogFetcher.ContractAddresses
+				}).Should(Equal(expectedAddresses))
+				close(done)
 			})
 
-		It("returns error if fetching logs fails", func() {
+			It("returns error if fetching logs fails", func(done Done) {
 				addMissingHeader(extractor)
 				addTransformerConfig(extractor)
 				mockLogFetcher := &mocks.MockLogFetcher{}
 				mockLogFetcher.ReturnError = fakes.FakeError
 				extractor.Fetcher = mockLogFetcher
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).To(HaveOccurred())
+				Expect(<-errsChan).To(MatchError(fakes.FakeError))
-			Expect(err).To(MatchError(fakes.FakeError))
+				close(done)
 			})
 
-		It("does not sync transactions if no fetched logs", func() {
+			Describe("when no fetched logs", func() {
+				It("does not sync transactions", func(done Done) {
 					addMissingHeader(extractor)
 					addTransformerConfig(extractor)
 					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
 					extractor.Syncer = mockTransactionSyncer
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+					go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).NotTo(HaveOccurred())
+					Consistently(func() bool {
-			Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse())
+						return mockTransactionSyncer.SyncTransactionsCalled
+					}).Should(BeFalse())
+					close(done)
+				})
 			})
 
-		It("syncs transactions for fetched logs", func() {
+			Describe("when there are fetched logs", func() {
+				It("syncs transactions", func(done Done) {
 					addMissingHeader(extractor)
 					addFetchedLog(extractor)
 					addTransformerConfig(extractor)
 					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
 					extractor.Syncer = mockTransactionSyncer
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+					go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).NotTo(HaveOccurred())
+					Eventually(func() bool {
-			Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue())
+						return mockTransactionSyncer.SyncTransactionsCalled
+					}).Should(BeTrue())
+					close(done)
 				})
 
-		It("returns error if syncing transactions fails", func() {
+				It("returns error if syncing transactions fails", func(done Done) {
 					addMissingHeader(extractor)
 					addFetchedLog(extractor)
 					addTransformerConfig(extractor)
@@ -210,13 +263,13 @@ var _ = Describe("Log extractor", func() {
 					mockTransactionSyncer.SyncTransactionsError = fakes.FakeError
 					extractor.Syncer = mockTransactionSyncer
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+					go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).To(HaveOccurred())
+					Expect(<-errsChan).To(MatchError(fakes.FakeError))
-			Expect(err).To(MatchError(fakes.FakeError))
+					close(done)
 				})
 
-		It("persists fetched logs", func() {
+				It("persists fetched logs", func(done Done) {
 					addMissingHeader(extractor)
 					addTransformerConfig(extractor)
 					fakeLogs := []types.Log{{
@@ -230,13 +283,15 @@ var _ = Describe("Log extractor", func() {
 					mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
 					extractor.LogRepository = mockLogRepository
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+					go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).NotTo(HaveOccurred())
+					Eventually(func() []types.Log {
-			Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs))
+						return mockLogRepository.PassedLogs
+					}).Should(Equal(fakeLogs))
+					close(done)
 				})
 
-		It("returns error if persisting logs fails", func() {
+				It("returns error if persisting logs fails", func(done Done) {
 					addMissingHeader(extractor)
 					addFetchedLog(extractor)
 					addTransformerConfig(extractor)
@@ -244,13 +299,14 @@ var _ = Describe("Log extractor", func() {
 					mockLogRepository.CreateError = fakes.FakeError
 					extractor.LogRepository = mockLogRepository
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+					go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).To(HaveOccurred())
+					Expect(<-errsChan).To(MatchError(fakes.FakeError))
-			Expect(err).To(MatchError(fakes.FakeError))
+					close(done)
+				})
 			})
 
-		It("marks header checked", func() {
+			It("marks header checked", func(done Done) {
 				addFetchedLog(extractor)
 				addTransformerConfig(extractor)
 				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
@@ -258,13 +314,15 @@ var _ = Describe("Log extractor", func() {
 				mockCheckedHeadersRepository.ReturnHeaders = []core.Header{{Id: headerID}}
 				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).NotTo(HaveOccurred())
+				Eventually(func() int64 {
-			Expect(mockCheckedHeadersRepository.HeaderID).To(Equal(headerID))
+					return mockCheckedHeadersRepository.HeaderID
+				}).Should(Equal(headerID))
+				close(done)
 			})
 
-		It("returns error if marking header checked fails", func() {
+			It("returns error if marking header checked fails", func(done Done) {
 				addFetchedLog(extractor)
 				addTransformerConfig(extractor)
 				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
@@ -272,10 +330,21 @@ var _ = Describe("Log extractor", func() {
 				mockCheckedHeadersRepository.MarkHeaderCheckedReturnError = fakes.FakeError
 				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
 
-			err := extractor.ExtractLogs(constants.HeaderMissing)
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
 
-			Expect(err).To(HaveOccurred())
+				Expect(<-errsChan).To(MatchError(fakes.FakeError))
-			Expect(err).To(MatchError(fakes.FakeError))
+				close(done)
+			})
+
+			It("emits that missing headers were found", func(done Done) {
+				addMissingHeader(extractor)
+				addTransformerConfig(extractor)
+
+				go extractor.ExtractLogs(constants.HeaderMissing, errsChan, missingHeadersFound)
+
+				Expect(<-missingHeadersFound).To(BeTrue())
+				close(done)
+			})
 		})
 	})
 })

libraries/shared/mocks/log_delegator.go

@@ -22,15 +22,23 @@ import (
 
 type MockLogDelegator struct {
 	AddedTransformers []transformer.EventTransformer
-	DelegateCalled    bool
+	DelegateCallCount int
-	DelegateError     error
+	DelegateErrors    []error
+	LogsFound         []bool
 }
 
 func (delegator *MockLogDelegator) AddTransformer(t transformer.EventTransformer) {
 	delegator.AddedTransformers = append(delegator.AddedTransformers, t)
 }
 
-func (delegator *MockLogDelegator) DelegateLogs() error {
+func (delegator *MockLogDelegator) DelegateLogs(errs chan error, logsFound chan bool) {
-	delegator.DelegateCalled = true
+	delegator.DelegateCallCount++
-	return delegator.DelegateError
+	var delegateErrorThisRun error
+	delegateErrorThisRun, delegator.DelegateErrors = delegator.DelegateErrors[0], delegator.DelegateErrors[1:]
+	if delegateErrorThisRun != nil {
+		errs <- delegateErrorThisRun
+	}
+	var logsFoundThisRun bool
+	logsFoundThisRun, delegator.LogsFound = delegator.LogsFound[0], delegator.LogsFound[1:]
+	logsFound <- logsFoundThisRun
 }

libraries/shared/mocks/log_extractor.go

@@ -23,15 +23,23 @@ import (
 
 type MockLogExtractor struct {
 	AddedConfigs        []transformer.EventTransformerConfig
-	ExtractLogsCalled bool
+	ExtractLogsCount    int
-	ExtractLogsError  error
+	ExtractLogsErrors   []error
+	MissingHeadersExist []bool
 }
 
 func (extractor *MockLogExtractor) AddTransformerConfig(config transformer.EventTransformerConfig) {
 	extractor.AddedConfigs = append(extractor.AddedConfigs, config)
 }
 
-func (extractor *MockLogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) error {
+func (extractor *MockLogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution, errs chan error, missingHeadersFound chan bool) {
-	extractor.ExtractLogsCalled = true
+	extractor.ExtractLogsCount++
-	return extractor.ExtractLogsError
+	var errorThisRun error
+	errorThisRun, extractor.ExtractLogsErrors = extractor.ExtractLogsErrors[0], extractor.ExtractLogsErrors[1:]
+	if errorThisRun != nil {
+		errs <- errorThisRun
+	}
+	var missingHeadersExist bool
+	missingHeadersExist, extractor.MissingHeadersExist = extractor.MissingHeadersExist[0], extractor.MissingHeadersExist[1:]
+	missingHeadersFound <- missingHeadersExist
 }

libraries/shared/watcher/event_watcher.go

@@ -27,8 +27,11 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"time"
 )
 
+const NoNewDataPause = time.Second * 7
+
 type EventWatcher struct {
 	blockChain   core.BlockChain
 	db           *postgres.DB
@@ -66,18 +69,62 @@ func (watcher *EventWatcher) AddTransformers(initializers []transformer.EventTra
 }
 
 // Extracts and delegates watched log events.
-func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecution) error {
+func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecution, errsChan chan error) {
-	extractErr := watcher.LogExtractor.ExtractLogs(recheckHeaders)
+	extractErrsChan := make(chan error)
-	if extractErr != nil {
+	delegateErrsChan := make(chan error)
+
+	go watcher.extractLogs(recheckHeaders, extractErrsChan)
+	go watcher.delegateLogs(delegateErrsChan)
+
+	for {
+		select {
+		case extractErr := <-extractErrsChan:
 			logrus.Errorf("error extracting logs in event watcher: %s", extractErr.Error())
-		return extractErr
+			errsChan <- extractErr
-	}
+		case delegateErr := <-delegateErrsChan:
-
-	delegateErr := watcher.LogDelegator.DelegateLogs()
-	if delegateErr != nil {
 			logrus.Errorf("error delegating logs in event watcher: %s", delegateErr.Error())
-		return delegateErr
+			errsChan <- delegateErr
+		}
+	}
+}
+
+func (watcher *EventWatcher) extractLogs(recheckHeaders constants.TransformerExecution, errs chan error) {
+	extractLogsErr := make(chan error)
+	missingHeadersFound := make(chan bool)
+	go watcher.LogExtractor.ExtractLogs(recheckHeaders, extractLogsErr, missingHeadersFound)
+
+	for {
+		select {
+		case err := <-extractLogsErr:
+			errs <- err
+		case missingHeaders := <-missingHeadersFound:
+			if missingHeaders {
+				go watcher.extractLogs(recheckHeaders, errs)
+			} else {
+				time.Sleep(NoNewDataPause)
+				go watcher.extractLogs(recheckHeaders, errs)
+			}
+		}
+	}
+}
+
+func (watcher *EventWatcher) delegateLogs(errs chan error) {
+	delegateLogsErr := make(chan error)
+	logsFound := make(chan bool)
+	go watcher.LogDelegator.DelegateLogs(delegateLogsErr, logsFound)
+
+	for {
+		select {
+		case err := <-delegateLogsErr:
+			errs <- err
+		case logs := <-logsFound:
+			if logs {
+				go watcher.delegateLogs(errs)
+			} else {
+				time.Sleep(NoNewDataPause)
+				go watcher.delegateLogs(errs)
+			}
+		}
 	}
 
-	return nil
 }

libraries/shared/watcher/event_watcher_test.go

@@ -30,13 +30,13 @@ var _ = Describe("Event Watcher", func() {
 	var (
 		delegator    *mocks.MockLogDelegator
 		extractor    *mocks.MockLogExtractor
-		eventWatcher watcher.EventWatcher
+		eventWatcher *watcher.EventWatcher
 	)
 
 	BeforeEach(func() {
 		delegator = &mocks.MockLogDelegator{}
 		extractor = &mocks.MockLogExtractor{}
-		eventWatcher = watcher.EventWatcher{
+		eventWatcher = &watcher.EventWatcher{
 			LogDelegator: delegator,
 			LogExtractor: extractor,
 		}
@@ -78,36 +78,115 @@ var _ = Describe("Event Watcher", func() {
 	})
 
 	Describe("Execute", func() {
-		It("extracts watched logs", func() {
+		var errsChan chan error
-			err := eventWatcher.Execute(constants.HeaderMissing)
 
-			Expect(err).NotTo(HaveOccurred())
+		BeforeEach(func() {
-			Expect(extractor.ExtractLogsCalled).To(BeTrue())
+			errsChan = make(chan error)
 		})
 
-		It("returns error if extracting logs fails", func() {
+		It("extracts watched logs", func(done Done) {
-			extractor.ExtractLogsError = fakes.FakeError
+			delegator.DelegateErrors = []error{nil}
+			delegator.LogsFound = []bool{false}
+			extractor.ExtractLogsErrors = []error{nil}
+			extractor.MissingHeadersExist = []bool{false}
 
-			err := eventWatcher.Execute(constants.HeaderMissing)
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
 
-			Expect(err).To(HaveOccurred())
+			Eventually(func() int {
-			Expect(err).To(MatchError(fakes.FakeError))
+				return extractor.ExtractLogsCount
+			}).Should(Equal(1))
+			close(done)
 		})
 
-		It("delegates untransformed logs", func() {
+		It("returns error if extracting logs fails", func(done Done) {
-			err := eventWatcher.Execute(constants.HeaderMissing)
+			delegator.DelegateErrors = []error{nil}
+			delegator.LogsFound = []bool{false}
+			extractor.ExtractLogsErrors = []error{fakes.FakeError}
+			extractor.MissingHeadersExist = []bool{false}
 
-			Expect(err).NotTo(HaveOccurred())
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
-			Expect(delegator.DelegateCalled).To(BeTrue())
+
+			Expect(<-errsChan).To(MatchError(fakes.FakeError))
+			close(done)
 		})
 
-		It("returns error if delegating logs fails", func() {
+		It("extracts watched logs again if missing headers found", func(done Done) {
-			delegator.DelegateError = fakes.FakeError
+			delegator.DelegateErrors = []error{nil}
+			delegator.LogsFound = []bool{false}
+			extractor.ExtractLogsErrors = []error{nil, nil}
+			extractor.MissingHeadersExist = []bool{true, false}
 
-			err := eventWatcher.Execute(constants.HeaderMissing)
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
 
-			Expect(err).To(HaveOccurred())
+			Eventually(func() int {
-			Expect(err).To(MatchError(fakes.FakeError))
+				return extractor.ExtractLogsCount
+			}).Should(Equal(2))
+			close(done)
+		})
+
+		It("returns error if extracting logs fails on subsequent run", func(done Done) {
+			delegator.DelegateErrors = []error{nil}
+			delegator.LogsFound = []bool{false}
+			extractor.ExtractLogsErrors = []error{nil, fakes.FakeError}
+			extractor.MissingHeadersExist = []bool{true, false}
+
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
+
+			Expect(<-errsChan).To(MatchError(fakes.FakeError))
+			close(done)
+
+		})
+
+		It("delegates untransformed logs", func(done Done) {
+			delegator.DelegateErrors = []error{nil}
+			delegator.LogsFound = []bool{false}
+			extractor.ExtractLogsErrors = []error{nil}
+			extractor.MissingHeadersExist = []bool{false}
+
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
+
+			Eventually(func() int {
+				return delegator.DelegateCallCount
+			}).Should(Equal(1))
+			close(done)
+		})
+
+		It("returns error if delegating logs fails", func(done Done) {
+			delegator.LogsFound = []bool{false}
+			delegator.DelegateErrors = []error{fakes.FakeError}
+			extractor.ExtractLogsErrors = []error{nil}
+			extractor.MissingHeadersExist = []bool{false}
+
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
+
+			Expect(<-errsChan).To(MatchError(fakes.FakeError))
+			close(done)
+		})
+
+		It("delegates logs again if untransformed logs found", func(done Done) {
+			delegator.DelegateErrors = []error{nil, nil}
+			delegator.LogsFound = []bool{true, false}
+			extractor.ExtractLogsErrors = []error{nil}
+			extractor.MissingHeadersExist = []bool{false}
+
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
+
+			Eventually(func() int {
+				return delegator.DelegateCallCount
+			}).Should(Equal(2))
+			close(done)
+		})
+
+		It("returns error if delegating logs fails on subsequent run", func(done Done) {
+			delegator.DelegateErrors = []error{nil, fakes.FakeError}
+			delegator.LogsFound = []bool{true, false}
+			extractor.ExtractLogsErrors = []error{nil}
+			extractor.MissingHeadersExist = []bool{false}
+
+			go eventWatcher.Execute(constants.HeaderMissing, errsChan)
+
+			Expect(<-errsChan).To(MatchError(fakes.FakeError))
+			close(done)
 		})
 	})
 })

pkg/datastore/postgres/repositories/checked_headers_repository.go

@@ -52,8 +52,7 @@ func (repo CheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingB
 				LEFT JOIN checked_headers on headers.id = header_id
 				WHERE (header_id ISNULL OR check_count < $2)
 				AND headers.block_number >= $1
-				AND headers.eth_node_fingerprint = $3
+				AND headers.eth_node_fingerprint = $3`
-				LIMIT 100`
 		err = repo.db.Select(&result, query, startingBlockNumber, checkCount, repo.db.Node.ID)
 	} else {
 		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
@@ -61,8 +60,7 @@ func (repo CheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingB
 				WHERE (header_id ISNULL OR check_count < $3)
 				AND headers.block_number >= $1
 				AND headers.block_number <= $2
-				AND headers.eth_node_fingerprint = $4
+				AND headers.eth_node_fingerprint = $4`
-				LIMIT 100`
 		err = repo.db.Select(&result, query, startingBlockNumber, endingBlockNumber, checkCount, repo.db.Node.ID)
 	}
 

pkg/datastore/postgres/repositories/checked_headers_repository_test.go

@@ -170,18 +170,6 @@ var _ = Describe("Checked Headers repository", func() {
 				Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
 				Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10)))
 			})
-
-			It("only returns 100 results to prevent blocking log delegation", func() {
-				for n := outOfRangeBlockNumber + 1; n < outOfRangeBlockNumber+100; n++ {
-					_, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
-					Expect(err).NotTo(HaveOccurred())
-				}
-
-				missingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlockNumber+200, uncheckedCheckCount)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(missingHeaders)).To(Equal(100))
-			})
 		})
 
 		Describe("when ending block is -1", func() {
@@ -252,19 +240,6 @@ var _ = Describe("Checked Headers repository", func() {
 				Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
 				Expect(nodeTwoMissingHeaders[3].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(middleBlockNumber+10), Equal(endingBlockNumber+10), Equal(outOfRangeBlockNumber+10)))
 			})
-
-			It("only returns 100 results to prevent blocking log delegation", func() {
-				for n := outOfRangeBlockNumber + 1; n < outOfRangeBlockNumber+100; n++ {
-					_, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
-					Expect(err).NotTo(HaveOccurred())
-				}
-
-				missingHeaders, err := repo.MissingHeaders(startingBlockNumber, endingBlock, uncheckedCheckCount)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(missingHeaders)).To(Equal(100))
 		})
 	})
-
-	})
 })