deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Return error when no logs/headers available

Browse Source 
- Replaces bool and moots question of error/bool ordering
- Also make event watcher execution synchronous
This commit is contained in:
Rob Mulholand
2019-09-18 20:55:15 -05:00
parent 2b798e00e0
commit 4fa19be90a
9 changed files with 137 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/logs/delegator.go
+10 -12
View File
@@ -25,16 +25,14 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore"
)
var ErrNoTransformers = errors.New("no event transformers configured in the log delegator")
const (
logsFound = true
noLogsFound = false
var (
ErrNoLogs = errors.New("no logs available for transforming")
ErrNoTransformers = errors.New("no event transformers configured in the log delegator")
)
type ILogDelegator interface {
AddTransformer(t transformer.EventTransformer)
DelegateLogs() (error, bool)
DelegateLogs() error
}
type LogDelegator struct {
@@ -48,28 +46,28 @@ func (delegator *LogDelegator) AddTransformer(t transformer.EventTransformer) {
delegator.Chunker.AddConfig(t.GetConfig())
}
func (delegator *LogDelegator) DelegateLogs() (error, bool) {
func (delegator *LogDelegator) DelegateLogs() error {
if len(delegator.Transformers) < 1 {
return ErrNoTransformers, noLogsFound
return ErrNoTransformers
}
persistedLogs, fetchErr := delegator.LogRepository.GetUntransformedHeaderSyncLogs()
if fetchErr != nil {
logrus.Errorf("error loading logs from db: %s", fetchErr.Error())
return fetchErr, noLogsFound
return fetchErr
}
if len(persistedLogs) < 1 {
return nil, noLogsFound
return ErrNoLogs
}
transformErr := delegator.delegateLogs(persistedLogs)
if transformErr != nil {
logrus.Errorf("error transforming logs: %s", transformErr)
return transformErr, logsFound
return transformErr
}
return nil, logsFound
return nil
}
func (delegator *LogDelegator) delegateLogs(logs []core.HeaderSyncLog) error {
libraries/shared/logs/delegator_test.go
+15 -15
View File
@@ -59,10 +59,10 @@ var _ = Describe("Log delegator", func() {
})
Describe("DelegateLogs", func() {
It("returns an error if no transformers configured", func() {
It("returns error if no transformers configured", func() {
delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
err, _ := delegator.DelegateLogs()
err := delegator.DelegateLogs()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(logs.ErrNoTransformers))
@@ -70,35 +70,36 @@ var _ = Describe("Log delegator", func() {
It("gets untransformed logs", func() {
mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
mockLogRepository.ReturnLogs = []core.HeaderSyncLog{{}}
delegator := newDelegator(mockLogRepository)
delegator.AddTransformer(&mocks.MockEventTransformer{})
err, _ := delegator.DelegateLogs()
err := delegator.DelegateLogs()
Expect(err).NotTo(HaveOccurred())
Expect(mockLogRepository.GetCalled).To(BeTrue())
})
It("emits error if getting untransformed logs fails", func() {
It("returns error if getting untransformed logs fails", func() {
mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
mockLogRepository.GetError = fakes.FakeError
delegator := newDelegator(mockLogRepository)
delegator.AddTransformer(&mocks.MockEventTransformer{})
err, _ := delegator.DelegateLogs()
err := delegator.DelegateLogs()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("emits that no logs were found if no logs returned", func() {
It("returns error that no logs were found if no logs returned", func() {
delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
delegator.AddTransformer(&mocks.MockEventTransformer{})
err, logsFound := delegator.DelegateLogs()
err := delegator.DelegateLogs()
Expect(err).NotTo(HaveOccurred())
Expect(logsFound).To(BeFalse())
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(logs.ErrNoLogs))
})
It("delegates chunked logs to transformers", func() {
@@ -115,27 +116,27 @@ var _ = Describe("Log delegator", func() {
delegator := newDelegator(mockLogRepository)
delegator.AddTransformer(fakeTransformer)
err, _ := delegator.DelegateLogs()
err := delegator.DelegateLogs()
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
Expect(fakeTransformer.PassedLogs).To(Equal(fakeHeaderSyncLogs))
})
It("emits error if transformer returns an error", func() {
It("returns error if transformer returns an error", func() {
mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
mockLogRepository.ReturnLogs = []core.HeaderSyncLog{{}}
delegator := newDelegator(mockLogRepository)
fakeTransformer := &mocks.MockEventTransformer{ExecuteError: fakes.FakeError}
delegator.AddTransformer(fakeTransformer)
err, _ := delegator.DelegateLogs()
err := delegator.DelegateLogs()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("emits logs found when logs returned and delegated", func() {
It("returns nil for error when logs returned and delegated", func() {
fakeTransformer := &mocks.MockEventTransformer{}
config := mocks.FakeTransformerConfig
fakeTransformer.SetTransformerConfig(config)
@@ -149,10 +150,9 @@ var _ = Describe("Log delegator", func() {
delegator := newDelegator(mockLogRepository)
delegator.AddTransformer(fakeTransformer)
err, logsFound := delegator.DelegateLogs()
err := delegator.DelegateLogs()
Expect(err).NotTo(HaveOccurred())
Expect(logsFound).To(BeTrue())
})
})
})
libraries/shared/logs/extractor.go
+13 -15
View File
@@ -28,16 +28,14 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore"
)
var ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor")
const (
uncheckedHeadersFound = true
noUncheckedHeadersFound = false
var (
ErrNoUncheckedHeaders = errors.New("no unchecked headers available for log fetching")
ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor")
)
type ILogExtractor interface {
AddTransformerConfig(config transformer.EventTransformerConfig) error
ExtractLogs(recheckHeaders constants.TransformerExecution) (error, bool)
ExtractLogs(recheckHeaders constants.TransformerExecution) error
}
type LogExtractor struct {
@@ -71,50 +69,50 @@ func (extractor *LogExtractor) AddTransformerConfig(config transformer.EventTran
}
// Fetch and persist watched logs
func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) (error, bool) {
func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) error {
if len(extractor.Addresses) < 1 {
logrus.Errorf("error extracting logs: %s", ErrNoWatchedAddresses.Error())
return ErrNoWatchedAddresses, noUncheckedHeadersFound
return ErrNoWatchedAddresses
}
uncheckedHeaders, uncheckedHeadersErr := extractor.CheckedHeadersRepository.UncheckedHeaders(*extractor.StartingBlock, -1, getCheckCount(recheckHeaders))
if uncheckedHeadersErr != nil {
logrus.Errorf("error fetching missing headers: %s", uncheckedHeadersErr)
return uncheckedHeadersErr, noUncheckedHeadersFound
return uncheckedHeadersErr
}
if len(uncheckedHeaders) < 1 {
return nil, noUncheckedHeadersFound
return ErrNoUncheckedHeaders
}
for _, header := range uncheckedHeaders {
logs, fetchLogsErr := extractor.Fetcher.FetchLogs(extractor.Addresses, extractor.Topics, header)
if fetchLogsErr != nil {
logError("error fetching logs for header: %s", fetchLogsErr, header)
return fetchLogsErr, uncheckedHeadersFound
return fetchLogsErr
}
if len(logs) > 0 {
transactionsSyncErr := extractor.Syncer.SyncTransactions(header.Id, logs)
if transactionsSyncErr != nil {
logError("error syncing transactions: %s", transactionsSyncErr, header)
return transactionsSyncErr, uncheckedHeadersFound
return transactionsSyncErr
}
createLogsErr := extractor.LogRepository.CreateHeaderSyncLogs(header.Id, logs)
if createLogsErr != nil {
logError("error persisting logs: %s", createLogsErr, header)
return createLogsErr, uncheckedHeadersFound
return createLogsErr
}
}
markHeaderCheckedErr := extractor.CheckedHeadersRepository.MarkHeaderChecked(header.Id)
if markHeaderCheckedErr != nil {
logError("error marking header checked: %s", markHeaderCheckedErr, header)
return markHeaderCheckedErr, uncheckedHeadersFound
return markHeaderCheckedErr
}
}
return nil, uncheckedHeadersFound
return nil
}
func earlierStartingBlockNumber(transformerBlock, watcherBlock int64) bool {
libraries/shared/logs/extractor_test.go
+20 -22
View File
@@ -157,7 +157,7 @@ var _ = Describe("Log extractor", func() {
Describe("ExtractLogs", func() {
It("returns error if no watched addresses configured", func() {
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(logs.ErrNoWatchedAddresses))
@@ -171,7 +171,7 @@ var _ = Describe("Log extractor", func() {
startingBlockNumber := rand.Int63()
extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
@@ -188,7 +188,7 @@ var _ = Describe("Log extractor", func() {
startingBlockNumber := rand.Int63()
extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
err, _ := extractor.ExtractLogs(constants.HeaderRecheck)
err := extractor.ExtractLogs(constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
@@ -197,13 +197,13 @@ var _ = Describe("Log extractor", func() {
})
})
It("emits error if getting unchecked headers fails", func() {
It("returns error if getting unchecked headers fails", func() {
addTransformerConfig(extractor)
mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
mockCheckedHeadersRepository.UncheckedHeadersReturnError = fakes.FakeError
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -215,20 +215,19 @@ var _ = Describe("Log extractor", func() {
mockLogFetcher := &mocks.MockLogFetcher{}
extractor.Fetcher = mockLogFetcher
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
_ = extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockLogFetcher.FetchCalled).To(BeFalse())
})
It("emits that no unchecked headers were found", func() {
It("returns error that no unchecked headers were found", func() {
addTransformerConfig(extractor)
mockLogFetcher := &mocks.MockLogFetcher{}
extractor.Fetcher = mockLogFetcher
_, uncheckedHeadersFound := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(uncheckedHeadersFound).To(BeFalse())
Expect(err).To(MatchError(logs.ErrNoUncheckedHeaders))
})
})
@@ -245,7 +244,7 @@ var _ = Describe("Log extractor", func() {
mockLogFetcher := &mocks.MockLogFetcher{}
extractor.Fetcher = mockLogFetcher
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockLogFetcher.FetchCalled).To(BeTrue())
@@ -262,7 +261,7 @@ var _ = Describe("Log extractor", func() {
mockLogFetcher.ReturnError = fakes.FakeError
extractor.Fetcher = mockLogFetcher
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -275,7 +274,7 @@ var _ = Describe("Log extractor", func() {
mockTransactionSyncer := &fakes.MockTransactionSyncer{}
extractor.Syncer = mockTransactionSyncer
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse())
@@ -290,7 +289,7 @@ var _ = Describe("Log extractor", func() {
mockTransactionSyncer := &fakes.MockTransactionSyncer{}
extractor.Syncer = mockTransactionSyncer
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue())
@@ -304,7 +303,7 @@ var _ = Describe("Log extractor", func() {
mockTransactionSyncer.SyncTransactionsError = fakes.FakeError
extractor.Syncer = mockTransactionSyncer
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -324,7 +323,7 @@ var _ = Describe("Log extractor", func() {
mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
extractor.LogRepository = mockLogRepository
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs))
@@ -338,7 +337,7 @@ var _ = Describe("Log extractor", func() {
mockLogRepository.CreateError = fakes.FakeError
extractor.LogRepository = mockLogRepository
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -353,7 +352,7 @@ var _ = Describe("Log extractor", func() {
mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{Id: headerID}}
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(mockCheckedHeadersRepository.MarkHeaderCheckedHeaderID).To(Equal(headerID))
@@ -367,20 +366,19 @@ var _ = Describe("Log extractor", func() {
mockCheckedHeadersRepository.MarkHeaderCheckedReturnError = fakes.FakeError
extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
err, _ := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("emits that missing headers were found", func() {
It("returns nil for error if everything succeeds", func() {
addUncheckedHeader(extractor)
addTransformerConfig(extractor)
err, missingHeadersFound := extractor.ExtractLogs(constants.HeaderUnchecked)
err := extractor.ExtractLogs(constants.HeaderUnchecked)
Expect(err).NotTo(HaveOccurred())
Expect(missingHeadersFound).To(BeTrue())
})
})
})