deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Write event logs to database before transforming

Browse Source 
- enables decoupling event extraction/persistence from transformation
- modifies event transformer, converter, and log chunker to accept
  payload that includes internal log database ID with log data
- remove alias for transformer pkg as shared_t
- remove unused mock watcher repository
This commit is contained in:
Rob Mulholand
2019-08-28 09:13:44 -05:00
parent 66a4e20b20
commit cb819fa9a6
18 changed files with 468 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/watcher/event_watcher.go
+7 -3
View File
@@ -21,7 +21,6 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/transactions"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
@@ -122,7 +121,12 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
return transactionsSyncErr
}
transformErr := watcher.transformLogs(logs, header.Id)
persistedLogs, createLogsErr := repository.CreateLogs(header.Id, logs, watcher.DB)
if createLogsErr != nil {
logrus.Errorf("error persisting logs: %s", createLogsErr.Error())
}
transformErr := watcher.transformLogs(persistedLogs, header.Id)
if transformErr != nil {
logrus.Error("Could not transform logs: ", transformErr)
return transformErr
@@ -131,7 +135,7 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
return err
}
func (watcher *EventWatcher) transformLogs(logs []types.Log, headerID int64) error {
func (watcher *EventWatcher) transformLogs(logs []core.HeaderSyncLog, headerID int64) error {
chunkedLogs := watcher.Chunker.ChunkLogs(logs)
// Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync,
libraries/shared/watcher/event_watcher_test.go
+31 -13
View File
@@ -18,7 +18,6 @@ package watcher_test
import (
"errors"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
@@ -106,7 +105,6 @@ var _ = Describe("Watcher", func() {
w watcher.EventWatcher
mockBlockChain fakes.MockBlockChain
headerRepository repositories.HeaderRepository
repository mocks.MockWatcherRepository
)
BeforeEach(func() {
@@ -117,14 +115,12 @@ var _ = Describe("Watcher", func() {
_, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
repository = mocks.MockWatcherRepository{}
w = watcher.NewEventWatcher(db, &mockBlockChain)
})
It("syncs transactions for fetched logs", func() {
fakeTransformer := &mocks.MockTransformer{}
w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
mockTransactionSyncer := &fakes.MockTransactionSyncer{}
w.Syncer = mockTransactionSyncer
@@ -137,7 +133,6 @@ var _ = Describe("Watcher", func() {
It("returns error if syncing transactions fails", func() {
fakeTransformer := &mocks.MockTransformer{}
w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
mockTransactionSyncer := &fakes.MockTransactionSyncer{}
mockTransactionSyncer.SyncTransactionsError = fakes.FakeError
w.Syncer = mockTransactionSyncer
@@ -148,10 +143,30 @@ var _ = Describe("Watcher", func() {
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists fetched logs", func() {
fakeTransformer := &mocks.MockTransformer{}
transformerConfig := transformer.EventTransformerConfig{TransformerName: "transformerA",
ContractAddresses: []string{"0x000000000000000000000000000000000000000A"},
Topic: "0xA"}
fakeTransformer.SetTransformerConfig(transformerConfig)
w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
log := types.Log{Address: common.HexToAddress("0xA"),
Topics: []common.Hash{common.HexToHash("0xA")},
Index: 0,
}
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{log})
err := w.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
Expect(len(fakeTransformer.PassedLogs)).NotTo(BeZero())
Expect(fakeTransformer.PassedLogs[0].ID).NotTo(BeZero())
Expect(fakeTransformer.PassedLogs[0].Log).To(Equal(log))
})
It("executes each transformer", func() {
fakeTransformer := &mocks.MockTransformer{}
w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := w.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
@@ -161,7 +176,6 @@ var _ = Describe("Watcher", func() {
It("returns an error if transformer returns an error", func() {
fakeTransformer := &mocks.MockTransformer{ExecuteError: errors.New("Something bad happened")}
w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := w.Execute(constants.HeaderMissing)
Expect(err).To(HaveOccurred())
@@ -183,26 +197,30 @@ var _ = Describe("Watcher", func() {
transformerB.SetTransformerConfig(configB)
logA := types.Log{Address: common.HexToAddress("0xA"),
Topics: []common.Hash{common.HexToHash("0xA")}}
Topics: []common.Hash{common.HexToHash("0xA")},
Index: 0,
}
logB := types.Log{Address: common.HexToAddress("0xB"),
Topics: []common.Hash{common.HexToHash("0xB")}}
Topics: []common.Hash{common.HexToHash("0xB")},
Index: 1,
}
mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
w = watcher.NewEventWatcher(db, &mockBlockChain)
w.AddTransformers([]transformer.EventTransformerInitializer{
transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})
err := w.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA}))
Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB}))
Expect(len(transformerA.PassedLogs)).NotTo(BeZero())
Expect(transformerA.PassedLogs[0].Log).To(Equal(logA))
Expect(len(transformerB.PassedLogs)).NotTo(BeZero())
Expect(transformerB.PassedLogs[0].Log).To(Equal(logB))
})
Describe("uses the LogFetcher correctly:", func() {
var fakeTransformer mocks.MockTransformer
BeforeEach(func() {
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
fakeTransformer = mocks.MockTransformer{}
})