Distinguish between missing and unchecked headers

- Missing == not in DB
- Unchecked == logs haven't been fetched
This commit is contained in:
Rob Mulholand 2019-09-10 14:39:45 -05:00
parent c568fedd89
commit 13d503b851
8 changed files with 89 additions and 88 deletions

View File

@ -158,7 +158,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
if recheckHeadersArg {
recheck = constants.HeaderRecheck
} else {
recheck = constants.HeaderMissing
recheck = constants.HeaderUnchecked
}
errs := make(chan error)
go w.Execute(recheck, errs)

View File

@ -20,6 +20,6 @@ type TransformerExecution bool
const (
HeaderRecheck TransformerExecution = true
HeaderMissing TransformerExecution = false
HeaderUnchecked TransformerExecution = false
RecheckHeaderCap = int64(5)
)

View File

@ -31,8 +31,8 @@ import (
var ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor")
const (
missingHeadersFound = true
noMissingHeadersFound = false
uncheckedHeadersFound = true
noUncheckedHeadersFound = false
)
type ILogExtractor interface {
@ -74,47 +74,47 @@ func (extractor *LogExtractor) AddTransformerConfig(config transformer.EventTran
func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) (error, bool) {
if len(extractor.Addresses) < 1 {
logrus.Errorf("error extracting logs: %s", ErrNoWatchedAddresses.Error())
return ErrNoWatchedAddresses, noMissingHeadersFound
return ErrNoWatchedAddresses, noUncheckedHeadersFound
}
uncheckedHeaders, uncheckedHeadersErr := extractor.CheckedHeadersRepository.UncheckedHeaders(*extractor.StartingBlock, -1, getCheckCount(recheckHeaders))
if uncheckedHeadersErr != nil {
logrus.Errorf("error fetching missing headers: %s", uncheckedHeadersErr)
return uncheckedHeadersErr, noMissingHeadersFound
return uncheckedHeadersErr, noUncheckedHeadersFound
}
if len(uncheckedHeaders) < 1 {
return nil, noMissingHeadersFound
return nil, noUncheckedHeadersFound
}
for _, header := range uncheckedHeaders {
logs, fetchLogsErr := extractor.Fetcher.FetchLogs(extractor.Addresses, extractor.Topics, header)
if fetchLogsErr != nil {
logError("error fetching logs for header: %s", fetchLogsErr, header)
return fetchLogsErr, missingHeadersFound
return fetchLogsErr, uncheckedHeadersFound
}
if len(logs) > 0 {
transactionsSyncErr := extractor.Syncer.SyncTransactions(header.Id, logs)
if transactionsSyncErr != nil {
logError("error syncing transactions: %s", transactionsSyncErr, header)
return transactionsSyncErr, missingHeadersFound
return transactionsSyncErr, uncheckedHeadersFound
}
createLogsErr := extractor.LogRepository.CreateHeaderSyncLogs(header.Id, logs)
if createLogsErr != nil {
logError("error persisting logs: %s", createLogsErr, header)
return createLogsErr, missingHeadersFound
return createLogsErr, uncheckedHeadersFound
}
}
markHeaderCheckedErr := extractor.CheckedHeadersRepository.MarkHeaderChecked(header.Id)
if markHeaderCheckedErr != nil {
logError("error marking header checked: %s", markHeaderCheckedErr, header)
return markHeaderCheckedErr, missingHeadersFound
return markHeaderCheckedErr, uncheckedHeadersFound
}
}
return nil, missingHeadersFound
return nil, uncheckedHeadersFound
}
func earlierStartingBlockNumber(transformerBlock, watcherBlock int64) bool {
@ -130,7 +130,7 @@ func logError(description string, err error, header core.Header) {
}
func getCheckCount(recheckHeaders constants.TransformerExecution) int64 {
if recheckHeaders == constants.HeaderMissing {
if recheckHeaders == constants.HeaderUnchecked {
return 1
} else {
return constants.RecheckHeaderCap

View File

@ -157,33 +157,33 @@ var _ = Describe("Log extractor", func() {
Describe("ExtractLogs", func() {
It("returns error if no watched addresses configured", func() {
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(logs.ErrNoWatchedAddresses))
})
Describe("when checking missing headers", func() {
It("gets missing headers since configured starting block with check_count < 1", func() {
Describe("when checking unchecked headers", func() {
It("gets headers since configured starting block with check_count < 1", func() {
mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
mockCheckedHeadersRepository.MissingHeadersReturnHeaders = []core.Header{{}}
mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
startingBlockNumber := rand.Int63()
extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockCheckedHeadersRepository.MissingHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
Expect(mockCheckedHeadersRepository.MissingHeadersEndingBlockNumber).To(Equal(int64(-1)))
Expect(mockCheckedHeadersRepository.MissingHeadersCheckCount).To(Equal(int64(1)))
Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
Expect(mockCheckedHeadersRepository.UncheckedHeadersEndingBlockNumber).To(Equal(int64(-1)))
Expect(mockCheckedHeadersRepository.UncheckedHeadersCheckCount).To(Equal(int64(1)))
})
})
Describe("when rechecking headers", func() {
It("gets missing headers since configured starting block with check_count < RecheckHeaderCap", func() {
It("gets headers since configured starting block with check_count < RecheckHeaderCap", func() {
mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
mockCheckedHeadersRepository.MissingHeadersReturnHeaders = []core.Header{{}}
mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
startingBlockNumber := rand.Int63()
extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
@ -191,60 +191,61 @@ var _ = Describe("Log extractor", func() {
err, _ := extractor.ExtractLogs(constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
Expect(mockCheckedHeadersRepository.MissingHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
Expect(mockCheckedHeadersRepository.MissingHeadersEndingBlockNumber).To(Equal(int64(-1)))
Expect(mockCheckedHeadersRepository.MissingHeadersCheckCount).To(Equal(constants.RecheckHeaderCap))
Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
Expect(mockCheckedHeadersRepository.UncheckedHeadersEndingBlockNumber).To(Equal(int64(-1)))
Expect(mockCheckedHeadersRepository.UncheckedHeadersCheckCount).To(Equal(constants.RecheckHeaderCap))
})
})
It("emits error if getting missing headers fails", func() {
It("emits error if getting unchecked headers fails", func() {
addTransformerConfig(extractor)
mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
mockCheckedHeadersRepository.MissingHeadersReturnError = fakes.FakeError
mockCheckedHeadersRepository.UncheckedHeadersReturnError = fakes.FakeError
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
Describe("when no missing headers", func() {
Describe("when no unchecked headers", func() {
It("does not fetch logs", func() {
addTransformerConfig(extractor)
mockLogFetcher := &mocks.MockLogFetcher{}
extractor.Fetcher = mockLogFetcher
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockLogFetcher.FetchCalled).To(BeFalse())
})
It("emits that no missing headers were found", func() {
It("emits that no unchecked headers were found", func() {
addTransformerConfig(extractor)
mockLogFetcher := &mocks.MockLogFetcher{}
extractor.Fetcher = mockLogFetcher
_, missingHeadersFound := extractor.ExtractLogs(constants.HeaderMissing)
_, uncheckedHeadersFound := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(missingHeadersFound).To(BeFalse())
Expect(uncheckedHeadersFound).To(BeFalse())
})
})
Describe("when there are missing headers", func() {
It("fetches logs for missing headers", func() {
addMissingHeader(extractor)
Describe("when there are unchecked headers", func() {
It("fetches logs for unchecked headers", func() {
addUncheckedHeader(extractor)
config := transformer.EventTransformerConfig{
ContractAddresses: []string{fakes.FakeAddress.Hex()},
Topic: fakes.FakeHash.Hex(),
StartingBlockNumber: rand.Int63(),
}
extractor.AddTransformerConfig(config)
addTransformerErr := extractor.AddTransformerConfig(config)
Expect(addTransformerErr).NotTo(HaveOccurred())
mockLogFetcher := &mocks.MockLogFetcher{}
extractor.Fetcher = mockLogFetcher
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockLogFetcher.FetchCalled).To(BeTrue())
@ -255,13 +256,13 @@ var _ = Describe("Log extractor", func() {
})
It("returns error if fetching logs fails", func() {
addMissingHeader(extractor)
addUncheckedHeader(extractor)
addTransformerConfig(extractor)
mockLogFetcher := &mocks.MockLogFetcher{}
mockLogFetcher.ReturnError = fakes.FakeError
extractor.Fetcher = mockLogFetcher
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@ -269,12 +270,12 @@ var _ = Describe("Log extractor", func() {
Describe("when no fetched logs", func() {
It("does not sync transactions", func() {
addMissingHeader(extractor)
addUncheckedHeader(extractor)
addTransformerConfig(extractor)
mockTransactionSyncer := &fakes.MockTransactionSyncer{}
extractor.Syncer = mockTransactionSyncer
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse())
@ -283,34 +284,34 @@ var _ = Describe("Log extractor", func() {
Describe("when there are fetched logs", func() {
It("syncs transactions", func() {
addMissingHeader(extractor)
addUncheckedHeader(extractor)
addFetchedLog(extractor)
addTransformerConfig(extractor)
mockTransactionSyncer := &fakes.MockTransactionSyncer{}
extractor.Syncer = mockTransactionSyncer
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue())
})
It("returns error if syncing transactions fails", func() {
addMissingHeader(extractor)
addUncheckedHeader(extractor)
addFetchedLog(extractor)
addTransformerConfig(extractor)
mockTransactionSyncer := &fakes.MockTransactionSyncer{}
mockTransactionSyncer.SyncTransactionsError = fakes.FakeError
extractor.Syncer = mockTransactionSyncer
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists fetched logs", func() {
addMissingHeader(extractor)
addUncheckedHeader(extractor)
addTransformerConfig(extractor)
fakeLogs := []types.Log{{
Address: common.HexToAddress("0xA"),
@ -323,21 +324,21 @@ var _ = Describe("Log extractor", func() {
mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
extractor.LogRepository = mockLogRepository
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs))
})
It("returns error if persisting logs fails", func() {
addMissingHeader(extractor)
addUncheckedHeader(extractor)
addFetchedLog(extractor)
addTransformerConfig(extractor)
mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
mockLogRepository.CreateError = fakes.FakeError
extractor.LogRepository = mockLogRepository
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@ -349,10 +350,10 @@ var _ = Describe("Log extractor", func() {
addTransformerConfig(extractor)
mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
headerID := rand.Int63()
mockCheckedHeadersRepository.MissingHeadersReturnHeaders = []core.Header{{Id: headerID}}
mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{Id: headerID}}
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockCheckedHeadersRepository.MarkHeaderCheckedHeaderID).To(Equal(headerID))
@ -362,21 +363,21 @@ var _ = Describe("Log extractor", func() {
addFetchedLog(extractor)
addTransformerConfig(extractor)
mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
mockCheckedHeadersRepository.MissingHeadersReturnHeaders = []core.Header{{Id: rand.Int63()}}
mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{Id: rand.Int63()}}
mockCheckedHeadersRepository.MarkHeaderCheckedReturnError = fakes.FakeError
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
err, _ := extractor.ExtractLogs(constants.HeaderMissing)
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("emits that missing headers were found", func() {
addMissingHeader(extractor)
addUncheckedHeader(extractor)
addTransformerConfig(extractor)
err, missingHeadersFound := extractor.ExtractLogs(constants.HeaderMissing)
err, missingHeadersFound := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(missingHeadersFound).To(BeTrue())
@ -394,9 +395,9 @@ func addTransformerConfig(extractor *logs.LogExtractor) {
extractor.AddTransformerConfig(fakeConfig)
}
func addMissingHeader(extractor *logs.LogExtractor) {
func addUncheckedHeader(extractor *logs.LogExtractor) {
mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
mockCheckedHeadersRepository.MissingHeadersReturnHeaders = []core.Header{{}}
mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
}

View File

@ -26,7 +26,7 @@ type MockLogExtractor struct {
AddTransformerConfigError error
ExtractLogsCount int
ExtractLogsErrors []error
MissingHeadersExist []bool
UncheckedHeadersExist []bool
}
func (extractor *MockLogExtractor) AddTransformerConfig(config transformer.EventTransformerConfig) error {
@ -42,6 +42,6 @@ func (extractor *MockLogExtractor) ExtractLogs(recheckHeaders constants.Transfor
return errorThisRun, false
}
var missingHeadersExist bool
missingHeadersExist, extractor.MissingHeadersExist = extractor.MissingHeadersExist[0], extractor.MissingHeadersExist[1:]
missingHeadersExist, extractor.UncheckedHeadersExist = extractor.UncheckedHeadersExist[0], extractor.UncheckedHeadersExist[1:]
return nil, missingHeadersExist
}

View File

@ -94,12 +94,12 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
}
func (watcher *EventWatcher) extractLogs(recheckHeaders constants.TransformerExecution, errs chan error) {
err, missingHeadersFound := watcher.LogExtractor.ExtractLogs(recheckHeaders)
err, uncheckedHeadersFound := watcher.LogExtractor.ExtractLogs(recheckHeaders)
if err != nil {
errs <- err
}
if missingHeadersFound {
if uncheckedHeadersFound {
watcher.extractLogs(recheckHeaders, errs)
} else {
time.Sleep(NoNewDataPause)

View File

@ -88,9 +88,9 @@ var _ = Describe("Event Watcher", func() {
delegator.DelegateErrors = []error{nil}
delegator.LogsFound = []bool{false}
extractor.ExtractLogsErrors = []error{nil}
extractor.MissingHeadersExist = []bool{false}
extractor.UncheckedHeadersExist = []bool{false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Eventually(func() int {
return extractor.ExtractLogsCount
@ -102,9 +102,9 @@ var _ = Describe("Event Watcher", func() {
delegator.DelegateErrors = []error{nil}
delegator.LogsFound = []bool{false}
extractor.ExtractLogsErrors = []error{fakes.FakeError}
extractor.MissingHeadersExist = []bool{false}
extractor.UncheckedHeadersExist = []bool{false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Expect(<-errsChan).To(MatchError(fakes.FakeError))
close(done)
@ -114,9 +114,9 @@ var _ = Describe("Event Watcher", func() {
delegator.DelegateErrors = []error{nil}
delegator.LogsFound = []bool{false}
extractor.ExtractLogsErrors = []error{nil, nil}
extractor.MissingHeadersExist = []bool{true, false}
extractor.UncheckedHeadersExist = []bool{true, false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Eventually(func() int {
return extractor.ExtractLogsCount
@ -128,9 +128,9 @@ var _ = Describe("Event Watcher", func() {
delegator.DelegateErrors = []error{nil}
delegator.LogsFound = []bool{false}
extractor.ExtractLogsErrors = []error{nil, fakes.FakeError}
extractor.MissingHeadersExist = []bool{true, false}
extractor.UncheckedHeadersExist = []bool{true, false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Expect(<-errsChan).To(MatchError(fakes.FakeError))
close(done)
@ -141,9 +141,9 @@ var _ = Describe("Event Watcher", func() {
delegator.DelegateErrors = []error{nil}
delegator.LogsFound = []bool{false}
extractor.ExtractLogsErrors = []error{nil}
extractor.MissingHeadersExist = []bool{false}
extractor.UncheckedHeadersExist = []bool{false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Eventually(func() int {
return delegator.DelegateCallCount
@ -155,9 +155,9 @@ var _ = Describe("Event Watcher", func() {
delegator.LogsFound = []bool{false}
delegator.DelegateErrors = []error{fakes.FakeError}
extractor.ExtractLogsErrors = []error{nil}
extractor.MissingHeadersExist = []bool{false}
extractor.UncheckedHeadersExist = []bool{false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Expect(<-errsChan).To(MatchError(fakes.FakeError))
close(done)
@ -167,9 +167,9 @@ var _ = Describe("Event Watcher", func() {
delegator.DelegateErrors = []error{nil, nil}
delegator.LogsFound = []bool{true, false}
extractor.ExtractLogsErrors = []error{nil}
extractor.MissingHeadersExist = []bool{false}
extractor.UncheckedHeadersExist = []bool{false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Eventually(func() int {
return delegator.DelegateCallCount
@ -181,9 +181,9 @@ var _ = Describe("Event Watcher", func() {
delegator.DelegateErrors = []error{nil, fakes.FakeError}
delegator.LogsFound = []bool{true, false}
extractor.ExtractLogsErrors = []error{nil}
extractor.MissingHeadersExist = []bool{false}
extractor.UncheckedHeadersExist = []bool{false}
go eventWatcher.Execute(constants.HeaderMissing, errsChan)
go eventWatcher.Execute(constants.HeaderUnchecked, errsChan)
Expect(<-errsChan).To(MatchError(fakes.FakeError))
close(done)

View File

@ -26,11 +26,11 @@ type MockCheckedHeadersRepository struct {
MarkHeadersUncheckedCalled bool
MarkHeadersUncheckedReturnError error
MarkHeadersUncheckedStartingBlockNumber int64
MissingHeadersCheckCount int64
MissingHeadersEndingBlockNumber int64
MissingHeadersReturnError error
MissingHeadersReturnHeaders []core.Header
MissingHeadersStartingBlockNumber int64
UncheckedHeadersCheckCount int64
UncheckedHeadersEndingBlockNumber int64
UncheckedHeadersReturnError error
UncheckedHeadersReturnHeaders []core.Header
UncheckedHeadersStartingBlockNumber int64
}
func (repository *MockCheckedHeadersRepository) MarkHeadersUnchecked(startingBlockNumber int64) error {
@ -45,8 +45,8 @@ func (repository *MockCheckedHeadersRepository) MarkHeaderChecked(headerID int64
}
func (repository *MockCheckedHeadersRepository) UncheckedHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) {
repository.MissingHeadersStartingBlockNumber = startingBlockNumber
repository.MissingHeadersEndingBlockNumber = endingBlockNumber
repository.MissingHeadersCheckCount = checkCount
return repository.MissingHeadersReturnHeaders, repository.MissingHeadersReturnError
repository.UncheckedHeadersStartingBlockNumber = startingBlockNumber
repository.UncheckedHeadersEndingBlockNumber = endingBlockNumber
repository.UncheckedHeadersCheckCount = checkCount
return repository.UncheckedHeadersReturnHeaders, repository.UncheckedHeadersReturnError
}