deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Frob transformer factory (#95)

Browse Source 
* Updates to previous transformers

* Use transformer factory for Frob

* Move bite integration test to integrationt est pkg
This commit is contained in:
Elizabeth
2018-10-29 15:08:00 -05:00
committed by GitHub
parent 78b8173b37
commit ab4e904713
15 changed files with 155 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/transformers/frob/config.go
+3 -2
View File
@@ -16,10 +16,11 @@ package frob
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var FrobConfig = shared.TransformerConfig{
var FrobConfig = shared.SingleTransformerConfig{
TransformerName: shared.FrobLabel,
ContractAddresses: []string{shared.PitContractAddress},
ContractAbi: shared.PitABI,
Topics: []string{shared.FrobSignature},
Topic: shared.FrobSignature,
StartingBlockNumber: 0,
EndingBlockNumber: 10000000,
}
pkg/transformers/frob/converter.go
+24 -22
View File
@@ -15,24 +15,21 @@
package frob
import (
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/core/types"
"bytes"
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/geth"
)
type Converter interface {
ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error)
ToModels(entities []FrobEntity) ([]FrobModel, error)
}
type FrobConverter struct{}
func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error) {
var entities []FrobEntity
func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
var entities []interface{}
for _, ethLog := range ethLogs {
entity := FrobEntity{}
address := ethLog.Address
@@ -54,23 +51,28 @@ func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]Frob
return entities, nil
}
func (FrobConverter) ToModels(entities []FrobEntity) ([]FrobModel, error) {
var models []FrobModel
func (FrobConverter) ToModels(entities []interface{}) ([]interface{}, error) {
var models []interface{}
for _, entity := range entities {
rawLog, err := json.Marshal(entity.Raw)
frobEntity, ok := entity.(FrobEntity)
if !ok {
return nil, fmt.Errorf("entity of type %T, not %T", entity, FrobEntity{})
}
rawLog, err := json.Marshal(frobEntity.Raw)
if err != nil {
return nil, err
}
model := FrobModel{
Ilk: string(bytes.Trim(entity.Ilk[:], "\x00)")),
Urn: common.BytesToAddress(entity.Urn[:]).String(),
Ink: entity.Ink.String(),
Art: entity.Art.String(),
Dink: entity.Dink.String(),
Dart: entity.Dart.String(),
IArt: entity.IArt.String(),
LogIndex: entity.LogIndex,
TransactionIndex: entity.TransactionIndex,
Ilk: string(bytes.Trim(frobEntity.Ilk[:], "\x00)")),
Urn: common.BytesToAddress(frobEntity.Urn[:]).String(),
Ink: frobEntity.Ink.String(),
Art: frobEntity.Art.String(),
Dink: frobEntity.Dink.String(),
Dart: frobEntity.Dart.String(),
IArt: frobEntity.IArt.String(),
LogIndex: frobEntity.LogIndex,
TransactionIndex: frobEntity.TransactionIndex,
Raw: rawLog,
}
models = append(models, model)
pkg/transformers/frob/converter_test.go
+15 -5
View File
@@ -25,9 +25,8 @@ import (
)
var _ = Describe("Frob converter", func() {
var converter = frob.FrobConverter{}
It("converts a log to an entity", func() {
converter := frob.FrobConverter{}
entities, err := converter.ToEntities(shared.PitABI, []types.Log{test_data.EthFrobLog})
Expect(err).NotTo(HaveOccurred())
@@ -35,13 +34,24 @@ var _ = Describe("Frob converter", func() {
Expect(entities[0]).To(Equal(test_data.FrobEntity))
})
It("converts an entity to a model", func() {
converter := frob.FrobConverter{}
It("returns an error if converting to an entity fails", func() {
_, err := converter.ToEntities("bad abi", []types.Log{test_data.EthFrobLog})
models, err := converter.ToModels([]frob.FrobEntity{test_data.FrobEntity})
Expect(err).To(HaveOccurred())
})
It("converts an entity to a model", func() {
models, err := converter.ToModels([]interface{}{test_data.FrobEntity})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.FrobModel))
})
It("returns an error if the entity type is wrong", func() {
_, err := converter.ToModels([]interface{}{test_data.WrongEntity{}})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("entity of type test_data.WrongEntity, not frob.FrobEntity"))
})
})
pkg/transformers/frob/repository.go
+13 -12
View File
@@ -15,33 +15,30 @@
package frob
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerID int64, models []FrobModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
type FrobRepository struct {
db *postgres.DB
}
func NewFrobRepository(db *postgres.DB) FrobRepository {
return FrobRepository{db: db}
}
func (repository FrobRepository) Create(headerID int64, models []FrobModel) error {
func (repository FrobRepository) Create(headerID int64, models []interface{}) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
frobModel, ok := model.(FrobModel)
if !ok {
tx.Rollback()
return fmt.Errorf("model of type %T, not %T", model, FrobModel{})
}
_, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)`,
headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.LogIndex, model.TransactionIndex)
headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex)
if err != nil {
tx.Rollback()
return err
@@ -82,3 +79,7 @@ func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlock
)
return result, err
}
func (repository *FrobRepository) SetDB(db *postgres.DB) {
repository.db = db
}
pkg/transformers/frob/repository_test.go
+17 -9
View File
@@ -33,7 +33,7 @@ import (
var _ = Describe("Frob repository", func() {
var (
db *postgres.DB
frobRepository frob.Repository
frobRepository frob.FrobRepository
err error
headerRepository datastore.HeaderRepository
)
@@ -42,7 +42,8 @@ var _ = Describe("Frob repository", func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
frobRepository = frob.NewFrobRepository(db)
frobRepository = frob.FrobRepository{}
frobRepository.SetDB(db)
})
Describe("Create", func() {
@@ -54,7 +55,7 @@ var _ = Describe("Frob repository", func() {
})
It("adds a frob", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
err = frobRepository.Create(headerID, []interface{}{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
var dbFrob frob.FrobModel
@@ -73,7 +74,7 @@ var _ = Describe("Frob repository", func() {
})
It("marks header as checked for logs", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
err = frobRepository.Create(headerID, []interface{}{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
@@ -86,7 +87,7 @@ var _ = Describe("Frob repository", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
Expect(err).NotTo(HaveOccurred())
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
err = frobRepository.Create(headerID, []interface{}{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
@@ -96,17 +97,23 @@ var _ = Describe("Frob repository", func() {
})
It("does not duplicate frob events", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
err = frobRepository.Create(headerID, []interface{}{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
err = frobRepository.Create(headerID, []interface{}{test_data.FrobModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("returns an error if the wrong model type is passed in", func() {
err = frobRepository.Create(headerID, []interface{}{test_data.WrongModel{}})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("model of type test_data.WrongModel, not frob.FrobModel"))
})
It("removes frob if corresponding header is deleted", func() {
err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel})
err = frobRepository.Create(headerID, []interface{}{test_data.FrobModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@@ -203,7 +210,8 @@ var _ = Describe("Frob repository", func() {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
}
frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
frobRepositoryTwo := frob.FrobRepository{}
frobRepositoryTwo.SetDB(dbTwo)
err := frobRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())
pkg/transformers/frob/transformer.go
-82
View File
@@ -1,82 +0,0 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package frob
import (
"log"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type FrobTransformer struct {
Config shared.TransformerConfig
Converter Converter
Fetcher shared.LogFetcher
Repository Repository
}
type FrobTransformerInitializer struct {
Config shared.TransformerConfig
}
func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := FrobConverter{}
fetcher := shared.NewFetcher(blockChain)
repository := NewFrobRepository(db)
return FrobTransformer{
Config: initializer.Config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
}
}
func (transformer FrobTransformer) Execute() error {
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
if err != nil {
return err
}
log.Printf("Fetching frob event logs for %d headers \n", len(missingHeaders))
for _, header := range missingHeaders {
topics := [][]common.Hash{{common.HexToHash(shared.FrobSignature)}}
matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddresses, topics, header.BlockNumber)
if err != nil {
return err
}
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
}
entities, err := transformer.Converter.ToEntities(FrobConfig.ContractAbi, matchingLogs)
if err != nil {
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}
pkg/transformers/frob/transformer_test.go
+35 -106
View File
@@ -15,30 +15,41 @@
package frob_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
frob_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/frob"
)
var _ = Describe("Frob transformer", func() {
It("gets missing headers for block numbers specified in config", func() {
repository := &frob_mocks.MockFrobRepository{}
transformer := frob.FrobTransformer{
Config: frob.FrobConfig,
Fetcher: &mocks.MockLogFetcher{},
Converter: &frob_mocks.MockFrobConverter{},
Repository: repository,
}
var (
repository mocks.MockRepository
transformer shared.Transformer
fetcher mocks.MockLogFetcher
converter mocks.MockConverter
config = frob.FrobConfig
)
BeforeEach(func() {
repository = mocks.MockRepository{}
fetcher = mocks.MockLogFetcher{}
converter = mocks.MockConverter{}
transformer = factories.Transformer{
Config: config,
Fetcher: &fetcher,
Converter: &converter,
Repository: &repository,
}.NewTransformer(nil, nil)
})
It("gets missing headers for block numbers specified in config", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
@@ -47,13 +58,7 @@ var _ = Describe("Frob transformer", func() {
})
It("returns error if repository returns error for missing headers", func() {
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeadersErr(fakes.FakeError)
transformer := frob.FrobTransformer{
Fetcher: &mocks.MockLogFetcher{},
Converter: &frob_mocks.MockFrobConverter{},
Repository: repository,
}
repository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
@@ -62,14 +67,7 @@ var _ = Describe("Frob transformer", func() {
})
It("fetches logs for missing headers", func() {
fetcher := &mocks.MockLogFetcher{}
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: &frob_mocks.MockFrobConverter{},
Repository: repository,
}
err := transformer.Execute()
@@ -80,15 +78,8 @@ var _ = Describe("Frob transformer", func() {
})
It("returns error if fetcher returns error", func() {
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetcherError(fakes.FakeError)
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: &frob_mocks.MockFrobConverter{},
Repository: repository,
}
err := transformer.Execute()
@@ -97,34 +88,18 @@ var _ = Describe("Frob transformer", func() {
})
It("marks header checked if no logs returned", func() {
mockConverter := &frob_mocks.MockFrobConverter{}
mockRepository := &frob_mocks.MockFrobRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := frob.FrobTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
repository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &frob_mocks.MockFrobConverter{}
mockRepository := &frob_mocks.MockFrobRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := frob.FrobTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
repository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute()
@@ -133,36 +108,20 @@ var _ = Describe("Frob transformer", func() {
})
It("converts matching logs to entity", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthFrobLog}))
Expect(converter.ContractAbi).To(Equal(frob.FrobConfig.ContractAbi))
Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthFrobLog}))
})
It("returns error if converting to entity returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetToEntitiesError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
converter.ToEntitiesError = fakes.FakeError
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
@@ -171,35 +130,20 @@ var _ = Describe("Frob transformer", func() {
})
It("converts frob entity to model", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
converter.EntitiesToReturn = []interface{}{test_data.FrobEntity}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedEntities).To(Equal([]frob.FrobEntity{test_data.FrobEntity}))
Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.FrobEntity))
})
It("returns error if converting to model returns error", func() {
converter := &frob_mocks.MockFrobConverter{}
converter.SetToModelsError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
converter.ToModelsError = fakes.FakeError
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
@@ -208,37 +152,22 @@ var _ = Describe("Frob transformer", func() {
})
It("persists frob model", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
converter.ModelsToReturn = []interface{}{test_data.FrobModel}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
fakeHeader := core.Header{BlockNumber: 1, Id: 2}
repository.SetMissingHeaders([]core.Header{fakeHeader})
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModels).To(Equal([]frob.FrobModel{test_data.FrobModel}))
Expect(repository.PassedModels[0]).To(Equal(test_data.FrobModel))
})
It("returns error if repository returns error for create", func() {
converter := &frob_mocks.MockFrobConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
repository := &frob_mocks.MockFrobRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
repository.SetCreateError(fakes.FakeError)
transformer := frob.FrobTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()