deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Decouple log extraction from transformer delegation

Browse Source 
- limit missing headers results set to 100 so that extraction doesn't
  excessively block delegation
- wrap checked headers functions in repository struct
- move storage repository to factory, to correspond with event
  repository path
- remove unused files
- reformat sql
- remove line breaks in imports
This commit is contained in:
Rob Mulholand
2019-08-28 09:25:13 -05:00
parent cb819fa9a6
commit d496dad33c
81 changed files with 2049 additions and 1328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/factories/event/repository.go
+2 -5
View File
@@ -16,12 +16,9 @@
package event
import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
type Repository interface {
Create(headerID int64, models []interface{}) error
MarkHeaderChecked(headerID int64) error
Create(models []interface{}) error
SetDB(db *postgres.DB)
}
libraries/shared/factories/event/transformer.go
+5 -11
View File
@@ -18,7 +18,6 @@ package event
import (
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -35,34 +34,29 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.Event
return transformer
}
func (transformer Transformer) Execute(logs []core.HeaderSyncLog, headerID int64) error {
func (transformer Transformer) Execute(logs []core.HeaderSyncLog) error {
transformerName := transformer.Config.TransformerName
config := transformer.Config
if len(logs) < 1 {
err := transformer.Repository.MarkHeaderChecked(headerID)
if err != nil {
logrus.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
return nil
}
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
if err != nil {
logrus.Printf("Error converting logs to entities in %v: %v", transformerName, err)
logrus.Errorf("error converting logs to entities in %v: %v", transformerName, err)
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
logrus.Printf("Error converting entities to models in %v: %v", transformerName, err)
logrus.Errorf("error converting entities to models in %v: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(headerID, models)
err = transformer.Repository.Create(models)
if err != nil {
logrus.Printf("Error persisting %v record: %v", transformerName, err)
logrus.Errorf("error persisting %v record: %v", transformerName, err)
return err
}
libraries/shared/factories/event/transformer_test.go
+10 -36
View File
@@ -17,22 +17,20 @@
package event_test
import (
"math/rand"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"math/rand"
)
var _ = Describe("Transformer", func() {
var (
repository mocks.MockRepository
repository mocks.MockEventRepository
converter mocks.MockConverter
t transformer.EventTransformer
headerOne core.Header
@@ -41,7 +39,7 @@ var _ = Describe("Transformer", func() {
)
BeforeEach(func() {
repository = mocks.MockRepository{}
repository = mocks.MockEventRepository{}
converter = mocks.MockConverter{}
t = event.Transformer{
@@ -64,15 +62,8 @@ var _ = Describe("Transformer", func() {
Expect(repository.SetDbCalled).To(BeTrue())
})
It("marks header checked if no logs returned", func() {
err := t.Execute([]core.HeaderSyncLog{}, headerOne.Id)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
})
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
err := t.Execute([]core.HeaderSyncLog{}, headerOne.Id)
err := t.Execute([]core.HeaderSyncLog{})
Expect(err).NotTo(HaveOccurred())
Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
@@ -80,24 +71,8 @@ var _ = Describe("Transformer", func() {
Expect(repository.CreateCalledCounter).To(Equal(0))
})
It("does not call repository.MarkCheckedHeader when there are logs", func() {
err := t.Execute(logs, headerOne.Id)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled()
})
It("returns error if marking header checked returns err", func() {
repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := t.Execute([]core.HeaderSyncLog{}, headerOne.Id)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts an eth log to an entity", func() {
err := t.Execute(logs, headerOne.Id)
err := t.Execute(logs)
Expect(err).NotTo(HaveOccurred())
Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
@@ -107,7 +82,7 @@ var _ = Describe("Transformer", func() {
It("returns an error if converter fails", func() {
converter.ToEntitiesError = fakes.FakeError
err := t.Execute(logs, headerOne.Id)
err := t.Execute(logs)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -116,7 +91,7 @@ var _ = Describe("Transformer", func() {
It("converts an entity to a model", func() {
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
err := t.Execute(logs, headerOne.Id)
err := t.Execute(logs)
Expect(err).NotTo(HaveOccurred())
Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
@@ -126,7 +101,7 @@ var _ = Describe("Transformer", func() {
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
converter.ToModelsError = fakes.FakeError
err := t.Execute(logs, headerOne.Id)
err := t.Execute(logs)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -135,16 +110,15 @@ var _ = Describe("Transformer", func() {
It("persists the record", func() {
converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
err := t.Execute(logs, headerOne.Id)
err := t.Execute(logs)
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
Expect(repository.PassedModels[0]).To(Equal(test_data.GenericModel{}))
})
It("returns error if persisting the record fails", func() {
repository.SetCreateError(fakes.FakeError)
err := t.Execute(logs, headerOne.Id)
err := t.Execute(logs)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
libraries/shared/factories/storage/repository.go
+27
View File
@@ -0,0 +1,27 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error
SetDB(db *postgres.DB)
}
libraries/shared/factories/storage/transformer.go
+1 -2
View File
@@ -19,7 +19,6 @@ package storage
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
@@ -29,7 +28,7 @@ import (
type Transformer struct {
Address common.Address
Mappings storage.Mappings
Repository repository.StorageRepository
Repository Repository
}
func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {