Use transformer factory for Bite (#89)

* Use transformer factory for Bite

- introduces separate transformer factory for non-LogNote events
- converter includes `ToEntities` for events defined via ABI

* Updates after rebasing with staging

pkg/transformers/bite/config.go

@@ -20,10 +20,11 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
-var BiteConfig = shared.TransformerConfig{
+var BiteConfig = shared.SingleTransformerConfig{
+	TransformerName:     shared.BiteLabel,
 	ContractAddresses:   []string{shared.CatContractAddress},
 	ContractAbi:         shared.CatABI,
-	Topics:              []string{shared.BiteSignature},
+	Topic:               shared.BiteSignature,
 	StartingBlockNumber: 0,
 	EndingBlockNumber:   10000000,
 }

pkg/transformers/bite/converter.go

@@ -19,6 +19,7 @@ package bite
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -26,15 +27,10 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
-type Converter interface {
-	ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error)
-	ToModels(biteEntities []BiteEntity) ([]BiteModel, error)
-}
-
 type BiteConverter struct{}
 
-func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) {
-	var entities []BiteEntity
+func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
+	var entities []interface{}
 	for _, ethLog := range ethLogs {
 		entity := &BiteEntity{}
 		address := ethLog.Address
@@ -60,19 +56,24 @@ func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]Bite
 	return entities, nil
 }
 
-func (converter BiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) {
-	var models []BiteModel
+func (converter BiteConverter) ToModels(entities []interface{}) ([]interface{}, error) {
+	var models []interface{}
 	for _, entity := range entities {
-		ilk := string(bytes.Trim(entity.Ilk[:], "\x00"))
-		urn := common.BytesToAddress(entity.Urn[:]).String()
-		ink := entity.Ink
-		art := entity.Art
-		iArt := entity.IArt
-		tab := entity.Tab
-		flip := entity.Flip
-		logIdx := entity.LogIndex
-		txIdx := entity.TransactionIndex
-		rawLogJson, err := json.Marshal(entity.Raw)
+		biteEntity, ok := entity.(BiteEntity)
+		if !ok {
+			return nil, fmt.Errorf("entity of type %T, not %T", entity, BiteEntity{})
+		}
+
+		ilk := string(bytes.Trim(biteEntity.Ilk[:], "\x00"))
+		urn := common.BytesToAddress(biteEntity.Urn[:]).String()
+		ink := biteEntity.Ink
+		art := biteEntity.Art
+		iArt := biteEntity.IArt
+		tab := biteEntity.Tab
+		flip := biteEntity.Flip
+		logIdx := biteEntity.LogIndex
+		txIdx := biteEntity.TransactionIndex
+		rawLogJson, err := json.Marshal(biteEntity.Raw)
 		rawLogString := string(rawLogJson)
 		if err != nil {
 			return nil, err

pkg/transformers/bite/converter_test.go

@@ -37,15 +37,7 @@ var _ = Describe("Bite Converter", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(entities)).To(Equal(1))
 			entity := entities[0]
-			Expect(entity.Ilk).To(Equal(test_data.BiteEntity.Ilk))
-			Expect(entity.Urn).To(Equal(test_data.BiteEntity.Urn))
-			Expect(entity.Ink).To(Equal(test_data.BiteEntity.Ink))
-			Expect(entity.Art).To(Equal(test_data.BiteEntity.Art))
-			Expect(entity.Tab).To(Equal(test_data.BiteEntity.Tab))
-			Expect(entity.Flip).To(Equal(test_data.BiteEntity.Flip))
-			Expect(entity.IArt).To(Equal(test_data.BiteEntity.IArt))
-			Expect(entity.TransactionIndex).To(Equal(test_data.BiteEntity.TransactionIndex))
-			Expect(entity.Raw).To(Equal(test_data.BiteEntity.Raw))
+			Expect(entity).To(Equal(test_data.BiteEntity))
 		})
 
 		It("returns an error if converting log to entity fails", func() {
@@ -59,7 +51,7 @@ var _ = Describe("Bite Converter", func() {
 		var emptyEntity = bite.BiteEntity{}
 
 		It("converts an Entity to a Model", func() {
-			models, err := converter.ToModels([]bite.BiteEntity{test_data.BiteEntity})
+			models, err := converter.ToModels([]interface{}{test_data.BiteEntity})
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(models)).To(Equal(1))
@@ -81,7 +73,7 @@ var _ = Describe("Bite Converter", func() {
 				TransactionIndex: 0,
 				Raw:              string(emptyLog),
 			}
-			models, err := converter.ToModels([]bite.BiteEntity{emptyEntity})
+			models, err := converter.ToModels([]interface{}{emptyEntity})
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(models)).To(Equal(1))

pkg/transformers/bite/repository.go

@@ -15,34 +15,34 @@
 package bite
 
 import (
+	"fmt"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-type Repository interface {
-	Create(headerID int64, models []BiteModel) error
-	MarkHeaderChecked(headerID int64) error
-	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
-}
-
 type BiteRepository struct {
 	db *postgres.DB
 }
 
-func NewBiteRepository(db *postgres.DB) Repository {
-	return BiteRepository{db: db}
+func (repository *BiteRepository) SetDB(db *postgres.DB) {
+	repository.db = db
 }
 
-func (repository BiteRepository) Create(headerID int64, models []BiteModel) error {
+func (repository BiteRepository) Create(headerID int64, models []interface{}) error {
 	tx, err := repository.db.Begin()
 	if err != nil {
 		return err
 	}
 	for _, model := range models {
+		biteModel, ok := model.(BiteModel)
+		if !ok {
+			tx.Rollback()
+			return fmt.Errorf("model of type %T, not %T", model, BiteModel{})
+		}
 		_, err := tx.Exec(
 			`INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log)
         VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)`,
-			headerID, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.NFlip, model.LogIndex, model.TransactionIndex, model.Raw,
+			headerID, biteModel.Ilk, biteModel.Urn, biteModel.Ink, biteModel.Art, biteModel.IArt, biteModel.Tab, biteModel.NFlip, biteModel.LogIndex, biteModel.TransactionIndex, biteModel.Raw,
 		)
 		if err != nil {
 			tx.Rollback()
@@ -50,13 +50,14 @@ func (repository BiteRepository) Create(headerID int64, models []BiteModel) erro
 		}
 	}
 	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
-		VALUES ($1, $2) 
-	ON CONFLICT (header_id) DO
-		UPDATE SET bite_checked = $2`, headerID, true)
+			VALUES ($1, $2) 
+		ON CONFLICT (header_id) DO
+			UPDATE SET bite_checked = $2`, headerID, true)
 	if err != nil {
 		tx.Rollback()
 		return err
 	}
+
 	return tx.Commit()
 }
 

pkg/transformers/bite/repository_test.go

@@ -16,6 +16,7 @@ package bite_test
 
 import (
 	"database/sql"
+	"math/rand"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -26,13 +27,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Bite repository", func() {
 	var (
-		biteRepository   bite.Repository
+		biteRepository   factories.Repository
 		db               *postgres.DB
 		err              error
 		headerRepository datastore.HeaderRepository
@@ -42,7 +44,8 @@ var _ = Describe("Bite repository", func() {
 		db = test_config.NewTestDB(test_config.NewTestNode())
 		test_config.CleanTestDB(db)
 		headerRepository = repositories.NewHeaderRepository(db)
-		biteRepository = bite.NewBiteRepository(db)
+		biteRepository = &bite.BiteRepository{}
+		biteRepository.SetDB(db)
 	})
 
 	Describe("Create", func() {
@@ -51,12 +54,12 @@ var _ = Describe("Bite repository", func() {
 		BeforeEach(func() {
 			headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
 			Expect(err).NotTo(HaveOccurred())
-
-			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
-			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("persists a bite record", func() {
+			err = biteRepository.Create(headerID, []interface{}{test_data.BiteModel})
+
+			Expect(err).NotTo(HaveOccurred())
 			var dbBite bite.BiteModel
 			err = db.Get(&dbBite, `SELECT ilk, urn, ink, art, tab, nflip, iart, log_idx, tx_idx, raw_log FROM maker.bite WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -73,6 +76,21 @@ var _ = Describe("Bite repository", func() {
 		})
 
 		It("marks header as checked for logs", func() {
+			err = biteRepository.Create(headerID, []interface{}{test_data.BiteModel})
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerChecked bool
+			err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(headerChecked).To(BeTrue())
+		})
+
+		It("updates a header as checked if row already exists", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			err = biteRepository.Create(headerID, []interface{}{test_data.BiteModel})
+
+			Expect(err).NotTo(HaveOccurred())
 			var headerChecked bool
 			err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
@@ -80,13 +98,19 @@ var _ = Describe("Bite repository", func() {
 		})
 
 		It("does not duplicate bite events", func() {
-			err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel})
+			err = biteRepository.Create(headerID, []interface{}{test_data.BiteModel})
+			Expect(err).NotTo(HaveOccurred())
+
+			err = biteRepository.Create(headerID, []interface{}{test_data.BiteModel})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes bite if corresponding header is deleted", func() {
+			err = biteRepository.Create(headerID, []interface{}{test_data.BiteModel})
+			Expect(err).NotTo(HaveOccurred())
+
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -95,6 +119,13 @@ var _ = Describe("Bite repository", func() {
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
+
+		It("returns an error if the model type is not a Bite", func() {
+			err = biteRepository.Create(headerID, []interface{}{test_data.WrongModel{}})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("model of type test_data.WrongModel, not bite.BiteModel"))
+		})
 	})
 
 	Describe("MarkHeaderChecked", func() {
@@ -135,7 +166,7 @@ var _ = Describe("Bite repository", func() {
 		)
 
 		BeforeEach(func() {
-			startingBlock = GinkgoRandomSeed()
+			startingBlock = rand.Int63()
 			biteBlock = startingBlock + 1
 			endingBlock = startingBlock + 2
 
@@ -183,7 +214,8 @@ var _ = Describe("Bite repository", func() {
 				_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
 				Expect(err).NotTo(HaveOccurred())
 			}
-			biteRepositoryTwo := bite.NewBiteRepository(dbTwo)
+			biteRepositoryTwo := bite.BiteRepository{}
+			biteRepositoryTwo.SetDB(dbTwo)
 
 			nodeOneMissingHeaders, err := biteRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/bite/transformer.go

@@ -1,97 +0,0 @@
-/*
- *  Copyright 2018 Vulcanize
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package bite
-
-import (
-	"log"
-
-	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
-)
-
-type BiteTransformer struct {
-	Repository Repository
-	Fetcher    shared.LogFetcher
-	Converter  Converter
-	Config     shared.TransformerConfig
-}
-
-type BiteTransformerInitializer struct {
-	Config shared.TransformerConfig
-}
-
-func (i BiteTransformerInitializer) NewBiteTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
-	fetcher := shared.NewFetcher(blockChain)
-	repository := NewBiteRepository(db)
-	transformer := BiteTransformer{
-		Fetcher:    fetcher,
-		Repository: repository,
-		Converter:  BiteConverter{},
-		Config:     i.Config,
-	}
-
-	return transformer
-}
-
-func (b BiteTransformer) Execute() error {
-	config := b.Config
-	topics := [][]common.Hash{{common.HexToHash(shared.BiteSignature)}}
-
-	missingHeaders, err := b.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
-	if err != nil {
-		log.Println("Error fetching missing headers:", err)
-		return err
-	}
-
-	log.Printf("Fetching bite event logs for %d headers \n", len(missingHeaders))
-	for _, header := range missingHeaders {
-		ethLogs, err := b.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber)
-		if err != nil {
-			log.Println("Error fetching matching logs:", err)
-			return err
-		}
-		if len(ethLogs) < 1 {
-			err = b.Repository.MarkHeaderChecked(header.Id)
-			if err != nil {
-				return err
-			}
-		}
-		entities, err := b.Converter.ToEntities(config.ContractAbi, ethLogs)
-		if err != nil {
-			return err
-		}
-		model, err := b.Converter.ToModels(entities)
-		if err != nil {
-			log.Println("Error converting logs:", err)
-			return err
-		}
-
-		err = b.Repository.Create(header.Id, model)
-		if err != nil {
-			log.Println("Error persisting bite record:", err)
-			return err
-		}
-	}
-
-	return nil
-}
-func (b BiteTransformer) SetConfig(config shared.TransformerConfig) {
-	b.Config = config
-}

pkg/transformers/bite/transformer_test.go

@@ -27,33 +27,36 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
-	bite_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/bite"
 )
 
 var _ = Describe("Bite Transformer", func() {
-	var repository bite_mocks.MockBiteRepository
+	var repository mocks.MockRepository
 	var fetcher mocks.MockLogFetcher
-	var converter bite_mocks.MockBiteConverter
-	var transformer bite.BiteTransformer
+	var converter mocks.MockConverter
+	var transformer shared.Transformer
 	var blockNumber1 = rand.Int63()
 	var blockNumber2 = rand.Int63()
 
 	BeforeEach(func() {
-		repository = bite_mocks.MockBiteRepository{}
+		repository = mocks.MockRepository{}
 		fetcher = mocks.MockLogFetcher{}
-		converter = bite_mocks.MockBiteConverter{}
+		converter = mocks.MockConverter{}
 
-		transformer = bite.BiteTransformer{
+		transformer = factories.Transformer{
 			Repository: &repository,
 			Fetcher:    &fetcher,
 			Converter:  &converter,
 			Config:     bite.BiteConfig,
-		}
+		}.NewTransformer(nil, nil)
+	})
 
-		transformer.SetConfig(bite.BiteConfig)
+	It("sets the blockchain and db", func() {
+		Expect(fetcher.SetBcCalled).To(BeTrue())
+		Expect(repository.SetDbCalled).To(BeTrue())
 	})
 
 	It("gets missing headers for blocks in the configured range", func() {
@@ -65,7 +68,7 @@ var _ = Describe("Bite Transformer", func() {
 	})
 
 	It("returns an error if it fails to get missing headers", func() {
-		repository.SetMissingHeadersErr(fakes.FakeError)
+		repository.SetMissingHeadersError(fakes.FakeError)
 		err := transformer.Execute()
 
 		Expect(err).To(HaveOccurred())
@@ -103,7 +106,7 @@ var _ = Describe("Bite Transformer", func() {
 
 	It("returns error if marking header checked returns err", func() {
 		repository.SetMissingHeaders([]core.Header{{Id: int64(123)}})
-		repository.SetMarkHeaderCheckedErr(fakes.FakeError)
+		repository.SetMarkHeaderCheckedError(fakes.FakeError)
 
 		err := transformer.Execute()
 
@@ -117,7 +120,7 @@ var _ = Describe("Bite Transformer", func() {
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ConverterAbi).To(Equal(bite.BiteConfig.ContractAbi))
+		Expect(converter.ContractAbi).To(Equal(bite.BiteConfig.ContractAbi))
 		Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthBiteLog}))
 	})
 
@@ -125,7 +128,30 @@ var _ = Describe("Bite Transformer", func() {
 		headerId := int64(1)
 		repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1, Id: headerId}})
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
-		converter.SetConverterError(fakes.FakeError)
+		converter.ToEntitiesError = fakes.FakeError
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts an entity to a BiteModel", func() {
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
+		converter.EntitiesToReturn = []interface{}{test_data.BiteEntity}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.BiteEntity))
+	})
+
+	It("returns an error if converting to models fails", func() {
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
+		converter.EntitiesToReturn = []interface{}{test_data.BiteEntity}
+		converter.ToModelsError = fakes.FakeError
 
 		err := transformer.Execute()
 
@@ -137,12 +163,13 @@ var _ = Describe("Bite Transformer", func() {
 		headerId := int64(1)
 		repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1, Id: headerId}})
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog})
+		converter.ModelsToReturn = []interface{}{test_data.BiteModel}
 
 		err := transformer.Execute()
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(headerId))
-		Expect(repository.PassedBiteModels).To(Equal([]bite.BiteModel{test_data.BiteModel}))
+		Expect(repository.PassedModels[0]).To(Equal(test_data.BiteModel))
 	})
 
 	It("returns error if persisting bite record fails", func() {