cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Update vat grab to use transformer factory (#85)

Browse Source 
* Update vat grab to use transformer factory
This commit is contained in:
Elizabeth 2018-10-26 11:02:39 -05:00 committed by GitHub
parent 745b3a460e
commit bd428e752a
11 changed files with 88 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/transformers/integration_tests/vat_grab.go
View File

@ -20,6 +20,8 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab"
"github.com/vulcanize/vulcanizedb/test_config"
)
@ -42,8 +44,13 @@ var _ = Describe("Vat Grab Transformer", func() {
err = persistHeader(db, blockNumber)
Expect(err).NotTo(HaveOccurred())
initializer := vat_grab.VatGrabTransformerInitializer{Config: config}
transformer := initializer.NewVatGrabTransformer(db, blockchain)
transformer := factories.Transformer{
Config: vat_grab.VatGrabConfig,
Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer(db, blockchain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred())

22
pkg/transformers/test_data/mocks/vat_grab/converter.go
View File

@ -1,22 +0,0 @@
package vat_grab
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab"
)
type MockVatGrabConverter struct {
err error
PassedLogs []types.Log
}
func (converter *MockVatGrabConverter) ToModels(ethLogs []types.Log) ([]vat_grab.VatGrabModel, error) {
converter.PassedLogs = ethLogs
return []vat_grab.VatGrabModel{test_data.VatGrabModel}, converter.err
}
func (converter *MockVatGrabConverter) SetConverterError(e error) {
converter.err = e
}

56
pkg/transformers/test_data/mocks/vat_grab/repository.go
View File

@ -1,56 +0,0 @@
package vat_grab
import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab"
)
type MockVatGrabRepository struct {
createErr error
markHeaderCheckedErr error
markHeaderCheckedPassedHeaderID int64
missingHeaders []core.Header
missingHeadersErr error
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedHeaderID int64
PassedModels []vat_grab.VatGrabModel
}
func (repository *MockVatGrabRepository) Create(headerID int64, models []vat_grab.VatGrabModel) error {
repository.PassedHeaderID = headerID
repository.PassedModels = models
return repository.createErr
}
func (repository *MockVatGrabRepository) MarkHeaderChecked(headerID int64) error {
repository.markHeaderCheckedPassedHeaderID = headerID
return repository.markHeaderCheckedErr
}
func (repository *MockVatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersErr
}
func (repository *MockVatGrabRepository) SetMissingHeadersErr(e error) {
repository.missingHeadersErr = e
}
func (repository *MockVatGrabRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockVatGrabRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i))
}
func (repository *MockVatGrabRepository) SetMarkHeaderCheckedErr(e error) {
repository.markHeaderCheckedErr = e
}
func (repository *MockVatGrabRepository) SetCreateError(e error) {
repository.createErr = e
}

7
pkg/transformers/transformers.go
View File

@ -123,7 +123,12 @@ var (
Repository: &vat_init.VatInitRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer
VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer
VatGrabTransformerInitializer = factories.Transformer{
Config: vat_grab.VatGrabConfig,
Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{},
Fetcher: &shared.Fetcher{},
}.NewTransformer
VatHealTransformerInitializer = vat_heal.VatHealTransformerInitializer{Config: vat_heal.VatHealConfig}.NewVatHealTransformer
VatFoldTransformerInitializer = factories.Transformer{
Config: vat_fold.VatFoldConfig,

5
pkg/transformers/vat_grab/config.go
View File

@ -2,10 +2,11 @@ package vat_grab
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var VatGrabConfig = shared.TransformerConfig{
var VatGrabConfig = shared.SingleTransformerConfig{
TransformerName: shared.VatGrabLabel,
ContractAddresses: []string{shared.VatContractAddress},
ContractAbi: shared.VatABI,
Topics: []string{shared.VatGrabSignature},
Topic: shared.VatGrabSignature,
StartingBlockNumber: 0,
EndingBlockNumber: 10000000,
}

8
pkg/transformers/vat_grab/converter.go
View File

@ -10,14 +10,10 @@ import (
"math/big"
)
type Converter interface {
ToModels(ethLogs []types.Log) ([]VatGrabModel, error)
}
type VatGrabConverter struct{}
func (VatGrabConverter) ToModels(ethLogs []types.Log) ([]VatGrabModel, error) {
var models []VatGrabModel
func (VatGrabConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var models []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {

3
pkg/transformers/vat_grab/converter_test.go
View File

@ -6,12 +6,13 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab"
)
var _ = Describe("Vat grab converter", func() {
var converter vat_grab.Converter
var converter factories.Converter
BeforeEach(func() {
converter = vat_grab.VatGrabConverter{}

27
pkg/transformers/vat_grab/repository.go
View File

@ -1,36 +1,31 @@
package vat_grab
import (
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerID int64, models []VatGrabModel) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
type VatGrabRepository struct {
db *postgres.DB
}
func NewVatGrabRepository(db *postgres.DB) VatGrabRepository {
return VatGrabRepository{
db: db,
}
}
func (repository VatGrabRepository) Create(headerID int64, models []VatGrabModel) error {
func (repository VatGrabRepository) Create(headerID int64, models []interface{}) error {
tx, err := repository.db.Begin()
if err != nil {
return err
}
for _, model := range models {
vatGrab, ok := model.(VatGrabModel)
if !ok {
tx.Rollback()
return fmt.Errorf("model of type %T, not %T", model, VatGrabModel{})
}
_, err = tx.Exec(
`INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`,
headerID, model.Ilk, model.Urn, model.V, model.W, model.Dink, model.Dart, model.LogIndex, model.TransactionIndex, model.Raw,
headerID, vatGrab.Ilk, vatGrab.Urn, vatGrab.V, vatGrab.W, vatGrab.Dink, vatGrab.Dart, vatGrab.LogIndex, vatGrab.TransactionIndex, vatGrab.Raw,
)
if err != nil {
tx.Rollback()
@ -72,3 +67,7 @@ func (repository VatGrabRepository) MissingHeaders(startingBlockNumber, endingBl
)
return result, err
}
func (repository *VatGrabRepository) SetDB(db *postgres.DB) {
repository.db = db
}

30
pkg/transformers/vat_grab/repository_test.go
View File

@ -2,7 +2,6 @@ package vat_grab_test
import (
"database/sql"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -11,6 +10,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab"
"github.com/vulcanize/vulcanizedb/test_config"
@ -19,7 +19,7 @@ import (
var _ = Describe("Vat grab repository", func() {
var (
db *postgres.DB
vatGrabRepository vat_grab.Repository
vatGrabRepository factories.Repository
err error
headerRepository datastore.HeaderRepository
)
@ -28,7 +28,8 @@ var _ = Describe("Vat grab repository", func() {
db = test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
vatGrabRepository = vat_grab.NewVatGrabRepository(db)
vatGrabRepository = &vat_grab.VatGrabRepository{}
vatGrabRepository.SetDB(db)
})
Describe("Create", func() {
@ -40,7 +41,7 @@ var _ = Describe("Vat grab repository", func() {
})
It("adds a vat grab event", func() {
err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
err = vatGrabRepository.Create(headerID, []interface{}{test_data.VatGrabModel})
Expect(err).NotTo(HaveOccurred())
var dbVatGrab vat_grab.VatGrabModel
@ -58,7 +59,7 @@ var _ = Describe("Vat grab repository", func() {
})
It("marks header as checked for logs", func() {
err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
err = vatGrabRepository.Create(headerID, []interface{}{test_data.VatGrabModel})
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
@ -71,7 +72,7 @@ var _ = Describe("Vat grab repository", func() {
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
Expect(err).NotTo(HaveOccurred())
err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
err = vatGrabRepository.Create(headerID, []interface{}{test_data.VatGrabModel})
Expect(err).NotTo(HaveOccurred())
var headerChecked bool
@ -81,17 +82,17 @@ var _ = Describe("Vat grab repository", func() {
})
It("does not duplicate vat grab events", func() {
err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
err = vatGrabRepository.Create(headerID, []interface{}{test_data.VatGrabModel})
Expect(err).NotTo(HaveOccurred())
err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
err = vatGrabRepository.Create(headerID, []interface{}{test_data.VatGrabModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes vat grab if corresponding header is deleted", func() {
err = vatGrabRepository.Create(headerID, []vat_grab.VatGrabModel{test_data.VatGrabModel})
err = vatGrabRepository.Create(headerID, []interface{}{test_data.VatGrabModel})
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
@ -102,6 +103,14 @@ var _ = Describe("Vat grab repository", func() {
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(sql.ErrNoRows))
})
It("returns an error if the wrong model is passed in", func() {
badModel := struct{}{}
err = vatGrabRepository.Create(headerID, []interface{}{badModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("model of type struct {}, not vat_grab.VatGrabModel"))
})
})
Describe("MarkHeaderChecked", func() {
@ -187,7 +196,8 @@ var _ = Describe("Vat grab repository", func() {
_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
Expect(err).NotTo(HaveOccurred())
}
vatGrabRepositoryTwo := vat_grab.NewVatGrabRepository(dbTwo)
vatGrabRepositoryTwo := vat_grab.VatGrabRepository{}
vatGrabRepositoryTwo.SetDB(dbTwo)
err := vatGrabRepository.MarkHeaderChecked(headerIDs[0])
Expect(err).NotTo(HaveOccurred())

63
pkg/transformers/vat_grab/tranformer.go
View File

@ -1,63 +0,0 @@
package vat_grab
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"log"
)
type VatGrabTransformerInitializer struct {
Config shared.TransformerConfig
}
func (initializer VatGrabTransformerInitializer) NewVatGrabTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := VatGrabConverter{}
fetcher := shared.NewFetcher(blockChain)
repository := NewVatGrabRepository(db)
return VatGrabTransformer{
Config: initializer.Config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
}
}
type VatGrabTransformer struct {
Config shared.TransformerConfig
Converter Converter
Fetcher shared.LogFetcher
Repository Repository
}
func (transformer VatGrabTransformer) Execute() error {
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
if err != nil {
return err
}
log.Printf("Fetching vat grab event logs for %d headers \n", len(missingHeaders))
for _, header := range missingHeaders {
topics := [][]common.Hash{{common.HexToHash(shared.VatGrabSignature)}}
matchingLogs, err := transformer.Fetcher.FetchLogs(VatGrabConfig.ContractAddresses, topics, header.BlockNumber)
if err != nil {
return err
}
if len(matchingLogs) < 1 {
err = transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
return err
}
continue
}
models, err := transformer.Converter.ToModels(matchingLogs)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
return err
}
}
return nil
}

116
pkg/transformers/vat_grab/transformer_test.go
View File

@ -8,23 +8,40 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
vat_grab_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_grab"
"github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab"
)
var _ = Describe("Vat grab transformer", func() {
It("gets missing headers for block numbers specified in config", func() {
repository := &vat_grab_mocks.MockVatGrabRepository{}
transformer := vat_grab.VatGrabTransformer{
Config: vat_grab.VatGrabConfig,
Fetcher: &mocks.MockLogFetcher{},
Converter: &vat_grab_mocks.MockVatGrabConverter{},
Repository: repository,
}
var (
config = vat_grab.VatGrabConfig
converter mocks.MockConverter
repository mocks.MockRepository
fetcher mocks.MockLogFetcher
transformer shared.Transformer
)
BeforeEach(func() {
repository = mocks.MockRepository{}
fetcher = mocks.MockLogFetcher{}
converter = mocks.MockConverter{}
transformer = factories.Transformer{
Config: config,
Fetcher: &fetcher,
Converter: &converter,
Repository: &repository,
}.NewTransformer(nil, nil)
})
It("sets the blockchain and database", func() {
Expect(fetcher.SetBcCalled).To(BeTrue())
Expect(repository.SetDbCalled).To(BeTrue())
})
It("gets missing headers for block numbers specified in config", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
@ -33,13 +50,7 @@ var _ = Describe("Vat grab transformer", func() {
})
It("returns error if repository returns error for missing headers", func() {
repository := &vat_grab_mocks.MockVatGrabRepository{}
repository.SetMissingHeadersErr(fakes.FakeError)
transformer := vat_grab.VatGrabTransformer{
Fetcher: &mocks.MockLogFetcher{},
Converter: &vat_grab_mocks.MockVatGrabConverter{},
Repository: repository,
}
repository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
@ -48,14 +59,7 @@ var _ = Describe("Vat grab transformer", func() {
})
It("fetches logs for missing headers", func() {
fetcher := &mocks.MockLogFetcher{}
repository := &vat_grab_mocks.MockVatGrabRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
transformer := vat_grab.VatGrabTransformer{
Fetcher: fetcher,
Converter: &vat_grab_mocks.MockVatGrabConverter{},
Repository: repository,
}
err := transformer.Execute()
@ -66,15 +70,8 @@ var _ = Describe("Vat grab transformer", func() {
})
It("returns error if fetcher returns error", func() {
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetcherError(fakes.FakeError)
repository := &vat_grab_mocks.MockVatGrabRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := vat_grab.VatGrabTransformer{
Fetcher: fetcher,
Converter: &vat_grab_mocks.MockVatGrabConverter{},
Repository: repository,
}
err := transformer.Execute()
@ -83,34 +80,18 @@ var _ = Describe("Vat grab transformer", func() {
})
It("marks header checked if no logs returned", func() {
mockConverter := &vat_grab_mocks.MockVatGrabConverter{}
mockRepository := &vat_grab_mocks.MockVatGrabRepository{}
headerID := int64(123)
mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}})
mockFetcher := &mocks.MockLogFetcher{}
transformer := vat_grab.VatGrabTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
repository.SetMissingHeaders([]core.Header{{Id: headerID}})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
mockRepository.AssertMarkHeaderCheckedCalledWith(headerID)
repository.AssertMarkHeaderCheckedCalledWith(headerID)
})
It("returns error if marking header checked returns err", func() {
mockConverter := &vat_grab_mocks.MockVatGrabConverter{}
mockRepository := &vat_grab_mocks.MockVatGrabRepository{}
mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError)
mockFetcher := &mocks.MockLogFetcher{}
transformer := vat_grab.VatGrabTransformer{
Converter: mockConverter,
Fetcher: mockFetcher,
Repository: mockRepository,
}
repository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute()
@ -119,16 +100,8 @@ var _ = Describe("Vat grab transformer", func() {
})
It("converts matching logs", func() {
converter := &vat_grab_mocks.MockVatGrabConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog})
repository := &vat_grab_mocks.MockVatGrabRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := vat_grab.VatGrabTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
@ -137,17 +110,9 @@ var _ = Describe("Vat grab transformer", func() {
})
It("returns error if converter returns error", func() {
converter := &vat_grab_mocks.MockVatGrabConverter{}
converter.SetConverterError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog})
repository := &vat_grab_mocks.MockVatGrabRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := vat_grab.VatGrabTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
@ -156,37 +121,22 @@ var _ = Describe("Vat grab transformer", func() {
})
It("persists vat grab model", func() {
converter := &vat_grab_mocks.MockVatGrabConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog})
repository := &vat_grab_mocks.MockVatGrabRepository{}
fakeHeader := core.Header{BlockNumber: 1, Id: 2}
repository.SetMissingHeaders([]core.Header{fakeHeader})
transformer := vat_grab.VatGrabTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
converter.SetReturnModels([]interface{}{test_data.VatGrabModel})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModels).To(Equal([]vat_grab.VatGrabModel{test_data.VatGrabModel}))
Expect(repository.PassedModels[0]).To(Equal(test_data.VatGrabModel))
})
It("returns error if repository returns error for create", func() {
converter := &vat_grab_mocks.MockVatGrabConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthVatGrabLog})
repository := &vat_grab_mocks.MockVatGrabRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
repository.SetCreateError(fakes.FakeError)
transformer := vat_grab.VatGrabTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()