Add raw log to Frob record in DB

- Enables parsing additional values off the log if necessary
- Also update Frob to reflect DSS changes (lad => urn)
- Move Frob constants to shared directory

db/migrations/1533844125_create_frob_table.up.sql

@@ -1,13 +1,14 @@
 CREATE TABLE maker.frob (
   id        SERIAL PRIMARY KEY,
   header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-  tx_idx    INTEGER,
   ilk       bytea,
-  lad       bytea,
+  urn       bytea,
   dink      NUMERIC,
   dart      NUMERIC,
   ink       NUMERIC,
   art       NUMERIC,
-  iart       NUMERIC,
+  iart      NUMERIC,
+  tx_idx    INTEGER NOT NUll,
+  raw_log   JSONB,
   UNIQUE (header_id, tx_idx)
 );

db/schema.sql

@@ -124,14 +124,15 @@ ALTER SEQUENCE maker.flip_kick_db_id_seq OWNED BY maker.flip_kick.db_id;
 CREATE TABLE maker.frob (
     id integer NOT NULL,
     header_id integer NOT NULL,
-    tx_idx integer,
     ilk bytea,
-    lad bytea,
+    urn bytea,
     dink numeric,
     dart numeric,
     ink numeric,
     art numeric,
-    iart numeric
+    iart numeric,
+    tx_idx integer NOT NULL,
+    raw_log jsonb
 );
 
 

pkg/transformers/frob/config.go

@@ -18,8 +18,8 @@ import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 
 var FrobConfig = shared.TransformerConfig{
 	ContractAddress:     "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally
-	ContractAbi:         FrobABI,
+	ContractAbi:         shared.PitABI,
-	Topics:              []string{FrobEventSignature},
+	Topics:              []string{shared.FrobSignature},
 	StartingBlockNumber: 0,
 	EndingBlockNumber:   100,
 }

pkg/transformers/frob/constants.go

@@ -1,20 +0,0 @@
-// Copyright 2018 Vulcanize
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package frob
-
-var (
-	FrobABI            = `[{"constant":true,"inputs":[],"name":"vat","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"live","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"drip","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"Line","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"wards","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"ilks","outputs":[{"name":"spot","type":"uint256"},{"name":"line","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"inputs":[{"name":"vat_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"ilk","type":"bytes32"},{"indexed":true,"name":"lad","type":"bytes32"},{"indexed":false,"name":"ink","type":"uint256"},{"indexed":false,"name":"art","type":"uint256"},{"indexed":false,"name":"dink","type":"int256"},{"indexed":false,"name":"dart","type":"int256"},{"indexed":false,"name":"iArt","type":"uint256"}],"name":"Frob","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"rely","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"deny","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"address"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"dink","type":"int256"},{"name":"dart","type":"int256"}],"name":"frob","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}]`
-	FrobEventSignature = "0x6cedf1d3a466a3d6bab04887b1642177bf6dbf1daa737c2e8f639cd0b020d9d0"
-)

pkg/transformers/frob/converter.go

@@ -19,12 +19,13 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
+	"encoding/json"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
 )
 
 type Converter interface {
 	ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error)
-	ToModel(flipKick FrobEntity) FrobModel
+	ToModel(flipKick FrobEntity) (FrobModel, error)
 }
 
 type FrobConverter struct {
@@ -39,17 +40,25 @@ func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog
 	}
 	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
 	err = contract.UnpackLog(&entity, "Frob", ethLog)
+	entity.TransactionIndex = ethLog.TxIndex
+	entity.Raw = ethLog
 	return entity, err
 }
 
-func (FrobConverter) ToModel(frob FrobEntity) FrobModel {
+func (FrobConverter) ToModel(frob FrobEntity) (FrobModel, error) {
-	return FrobModel{
+	rawLog, err := json.Marshal(frob.Raw)
-		Ilk:  frob.Ilk[:],
+	if err != nil {
-		Lad:  frob.Lad[:],
+		return FrobModel{}, err
-		Dink: frob.Dink.String(),
-		Dart: frob.Dart.String(),
-		Ink:  frob.Ink.String(),
-		Art:  frob.Art.String(),
-		IArt: frob.IArt.String(),
 	}
+	return FrobModel{
+		Ilk:              frob.Ilk[:],
+		Urn:              frob.Urn[:],
+		Ink:              frob.Ink.String(),
+		Art:              frob.Art.String(),
+		Dink:             frob.Dink.String(),
+		Dart:             frob.Dart.String(),
+		IArt:             frob.IArt.String(),
+		TransactionIndex: frob.TransactionIndex,
+		Raw:              rawLog,
+	}, nil
 }

pkg/transformers/frob/converter_test.go

@@ -19,6 +19,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 )
 
@@ -26,7 +27,7 @@ var _ = Describe("Frob converter", func() {
 	It("converts a log to an entity", func() {
 		converter := frob.FrobConverter{}
 
-		entity, err := converter.ToEntity(test_data.TemporaryFrobAddress, frob.FrobABI, test_data.EthFrobLog)
+		entity, err := converter.ToEntity(test_data.TemporaryFrobAddress, shared.PitABI, test_data.EthFrobLog)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(entity).To(Equal(test_data.FrobEntity))
@@ -35,8 +36,9 @@ var _ = Describe("Frob converter", func() {
 	It("converts an entity to a model", func() {
 		converter := frob.FrobConverter{}
 
-		model := converter.ToModel(test_data.FrobEntity)
+		model, err := converter.ToModel(test_data.FrobEntity)
 
+		Expect(err).NotTo(HaveOccurred())
 		Expect(model).To(Equal(test_data.FrobModel))
 	})
 })

pkg/transformers/frob/entity.go

@@ -16,14 +16,18 @@ package frob
 
 import (
 	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 type FrobEntity struct {
-	Ilk  [32]byte
+	Ilk              [32]byte
-	Lad  [32]byte
+	Urn              [32]byte
-	Ink  *big.Int
+	Ink              *big.Int
-	Art  *big.Int
+	Art              *big.Int
-	Dink *big.Int
+	Dink             *big.Int
-	Dart *big.Int
+	Dart             *big.Int
-	IArt *big.Int
+	IArt             *big.Int
+	TransactionIndex uint
+	Raw              types.Log
 }

pkg/transformers/frob/integration_test.go

@@ -46,7 +46,7 @@ var _ = Describe("Integration tests", func() {
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
 		realBlockChain := geth.NewBlockChain(blockChainClient, realNode, transactionConverter)
 		realFetcher := shared.NewFetcher(realBlockChain)
-		topic0 := common.HexToHash(frob.FrobEventSignature)
+		topic0 := common.HexToHash(shared.FrobSignature)
 		topics := [][]common.Hash{{topic0}}
 
 		result, err := realFetcher.FetchLogs(test_data.TemporaryFrobAddress, topics, int64(12))
@@ -62,7 +62,7 @@ var _ = Describe("Integration tests", func() {
 
 	It("unpacks an event log", func() {
 		address := common.HexToAddress(test_data.TemporaryFrobAddress)
-		abi, err := geth.ParseAbi(frob.FrobABI)
+		abi, err := geth.ParseAbi(shared.PitABI)
 		Expect(err).NotTo(HaveOccurred())
 
 		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
@@ -78,6 +78,6 @@ var _ = Describe("Integration tests", func() {
 		Expect(entity.IArt).To(Equal(expectedEntity.IArt))
 		Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
 		Expect(entity.Ink).To(Equal(expectedEntity.Ink))
-		Expect(entity.Lad).To(Equal(expectedEntity.Lad))
+		Expect(entity.Urn).To(Equal(expectedEntity.Urn))
 	})
 })

pkg/transformers/frob/model.go

@@ -15,11 +15,13 @@
 package frob
 
 type FrobModel struct {
-	Ilk  []byte
+	Ilk              []byte
-	Lad  []byte
+	Urn              []byte
-	Ink  string
+	Ink              string
-	Art  string
+	Art              string
-	Dink string
+	Dink             string
-	Dart string
+	Dart             string
-	IArt string
+	IArt             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
 }

pkg/transformers/frob/repository.go

@@ -20,7 +20,7 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, transactionIndex uint, model FrobModel) error
+	Create(headerID int64, model FrobModel) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,10 +32,10 @@ func NewFrobRepository(db *postgres.DB) FrobRepository {
 	return FrobRepository{db: db}
 }
 
-func (repository FrobRepository) Create(headerID int64, transactionIndex uint, model FrobModel) error {
+func (repository FrobRepository) Create(headerID int64, model FrobModel) error {
-	_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, tx_idx, art, dart, dink, iart, ilk, ink, lad)
+	_, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx)
-		VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7, $8::NUMERIC, $9)`,
+		VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10)`,
-		headerID, transactionIndex, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Lad)
+		headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex)
 	return err
 }
 

pkg/transformers/frob/repository_test.go

@@ -30,56 +30,61 @@ import (
 var _ = Describe("Frob repository", func() {
 	Describe("Create", func() {
 		It("adds a frob", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			frobRepository := frob.NewFrobRepository(db)
 
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbFrob frob.FrobModel
-			err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
+			err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, urn, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(dbFrob).To(Equal(test_data.FrobModel))
+			Expect(dbFrob.Ilk).To(Equal(test_data.FrobModel.Ilk))
+			Expect(dbFrob.Urn).To(Equal(test_data.FrobModel.Urn))
+			Expect(dbFrob.Ink).To(Equal(test_data.FrobModel.Ink))
+			Expect(dbFrob.Art).To(Equal(test_data.FrobModel.Art))
+			Expect(dbFrob.Dink).To(Equal(test_data.FrobModel.Dink))
+			Expect(dbFrob.Dart).To(Equal(test_data.FrobModel.Dart))
+			Expect(dbFrob.IArt).To(Equal(test_data.FrobModel.IArt))
+			Expect(dbFrob.TransactionIndex).To(Equal(test_data.FrobModel.TransactionIndex))
+			Expect(dbFrob.Raw).To(MatchJSON(test_data.FrobModel.Raw))
 		})
 
 		It("does not duplicate frob events", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			frobRepository := frob.NewFrobRepository(db)
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
 		It("removes frob if corresponding header is deleted", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
 			Expect(err).NotTo(HaveOccurred())
 			frobRepository := frob.NewFrobRepository(db)
-			err = frobRepository.Create(headerID, 123, test_data.FrobModel)
+			err = frobRepository.Create(headerID, test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbFrob frob.FrobModel
-			err = db.Get(&dbFrob, `SELECT art, iart, ilk, ink, lad FROM maker.frob WHERE header_id = $1`, headerID)
+			err = db.Get(&dbFrob, `SELECT art, iart, ilk, ink, urn, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID)
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(sql.ErrNoRows))
 		})
@@ -87,8 +92,7 @@ var _ = Describe("Frob repository", func() {
 
 	Describe("MissingHeaders", func() {
 		It("returns headers with no associated frob event", func() {
-			node := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(node)
 			test_config.CleanTestDB(db)
 			headerRepository := repositories.NewHeaderRepository(db)
 			startingBlockNumber := int64(1)
@@ -102,7 +106,7 @@ var _ = Describe("Frob repository", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 			frobRepository := frob.NewFrobRepository(db)
-			err := frobRepository.Create(headerIDs[1], 123, test_data.FrobModel)
+			err := frobRepository.Create(headerIDs[1], test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
@@ -114,13 +118,11 @@ var _ = Describe("Frob repository", func() {
 		})
 
 		It("only returns headers associated with the current node", func() {
-			nodeOne := core.Node{}
+			db := test_config.NewTestDB(core.Node{})
-			db := test_config.NewTestDB(nodeOne)
 			test_config.CleanTestDB(db)
 			blockNumbers := []int64{1, 2, 3}
 			headerRepository := repositories.NewHeaderRepository(db)
-			nodeTwo := core.Node{ID: "second"}
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			dbTwo := test_config.NewTestDB(nodeTwo)
 			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
 			var headerIDs []int64
 			for _, n := range blockNumbers {
@@ -132,7 +134,7 @@ var _ = Describe("Frob repository", func() {
 			}
 			frobRepository := frob.NewFrobRepository(db)
 			frobRepositoryTwo := frob.NewFrobRepository(dbTwo)
-			err := frobRepository.Create(headerIDs[0], 0, test_data.FrobModel)
+			err := frobRepository.Create(headerIDs[0], test_data.FrobModel)
 			Expect(err).NotTo(HaveOccurred())
 
 			nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])

pkg/transformers/frob/transformer.go

@@ -51,7 +51,7 @@ func (transformer FrobTransformer) Execute() error {
 		return err
 	}
 	for _, header := range missingHeaders {
-		topics := [][]common.Hash{{common.HexToHash(FrobEventSignature)}}
+		topics := [][]common.Hash{{common.HexToHash(shared.FrobSignature)}}
 		matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddress, topics, header.BlockNumber)
 		if err != nil {
 			return err
@@ -61,8 +61,11 @@ func (transformer FrobTransformer) Execute() error {
 			if err != nil {
 				return err
 			}
-			model := transformer.Converter.ToModel(entity)
+			model, err := transformer.Converter.ToModel(entity)
-			err = transformer.Repository.Create(header.Id, log.TxIndex, model)
+			if err != nil {
+				return err
+			}
+			err = transformer.Repository.Create(header.Id, model)
 			if err != nil {
 				return err
 			}

pkg/transformers/frob/transformer_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
 	frob_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/frob"
@@ -75,7 +76,7 @@ var _ = Describe("Frob transformer", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
 		Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
-		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(frob.FrobEventSignature)}}))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.FrobSignature)}}))
 	})
 
 	It("returns error if fetcher returns error", func() {
@@ -95,7 +96,7 @@ var _ = Describe("Frob transformer", func() {
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
-	It("converts matching logs", func() {
+	It("converts matching logs to entity", func() {
 		converter := &frob_mocks.MockFrobConverter{}
 		fetcher := &mocks.MockLogFetcher{}
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
@@ -113,12 +114,48 @@ var _ = Describe("Frob transformer", func() {
 		Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddress))
 		Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi))
 		Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog))
+	})
+
+	It("returns error if converting to entity returns error", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		converter.SetToEntityError(fakes.FakeError)
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts frob entity to model", func() {
+		converter := &frob_mocks.MockFrobConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
+		repository := &frob_mocks.MockFrobRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := frob.FrobTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity))
 	})
 
-	It("returns error if converter returns error", func() {
+	It("returns error if converting to model returns error", func() {
 		converter := &frob_mocks.MockFrobConverter{}
-		converter.SetConverterError(fakes.FakeError)
+		converter.SetToModelError(fakes.FakeError)
 		fetcher := &mocks.MockLogFetcher{}
 		fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog})
 		repository := &frob_mocks.MockFrobRepository{}
@@ -152,7 +189,6 @@ var _ = Describe("Frob transformer", func() {
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
-		Expect(repository.PassedTransactionIndex).To(Equal(test_data.EthFrobLog.TxIndex))
 		Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel))
 	})
 

pkg/transformers/pit_file/repository_test.go

@@ -31,7 +31,6 @@ var _ = Describe("Pit file repository", func() {
 			Expect(dbPitFile.What).To(Equal(test_data.PitFileModel.What))
 			Expect(dbPitFile.Risk).To(Equal(test_data.PitFileModel.Risk))
 			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileModel.TransactionIndex))
-			// The raw bytes scanned out of the DB are compressed (missing whitespace), such that they don't strictly equal the input bytes
 			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileModel.Raw))
 		})
 

pkg/transformers/shared/constants.go

@@ -18,7 +18,8 @@ var (
 	FlipperABI             = `[{"constant":true,"inputs":[{"name":"","type":"uint256"}],"name":"bids","outputs":[{"name":"bid","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"guy","type":"address"},{"name":"tic","type":"uint48"},{"name":"end","type":"uint48"},{"name":"urn","type":"bytes32"},{"name":"gal","type":"address"},{"name":"tab","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x4423c5f1"},{"constant":true,"inputs":[],"name":"ttl","outputs":[{"name":"","type":"uint48"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x4e8b1dd5"},{"constant":true,"inputs":[],"name":"gem","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x7bd2bea7"},{"constant":true,"inputs":[],"name":"beg","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x7d780d82"},{"constant":true,"inputs":[],"name":"tau","outputs":[{"name":"","type":"uint48"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xcfc4af55"},{"constant":true,"inputs":[],"name":"kicks","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xcfdd3302"},{"constant":true,"inputs":[],"name":"dai","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xf4b9fa75"},{"inputs":[{"name":"dai_","type":"address"},{"name":"gem_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor","signature":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"id","type":"uint256"},{"indexed":false,"name":"lot","type":"uint256"},{"indexed":false,"name":"bid","type":"uint256"},{"indexed":false,"name":"gal","type":"address"},{"indexed":false,"name":"end","type":"uint48"},{"indexed":true,"name":"urn","type":"bytes32"},{"indexed":false,"name":"tab","type":"uint256"}],"name":"Kick","type":"event","signature":"0xbac86238bdba81d21995024470425ecb370078fa62b7271b90cf28cbd1e3e87e"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event","signature":"0x644843f351d3fba4abcd60109eaff9f54bac8fb8ccf0bab941009c21df21cf31"},{"constant":true,"inputs":[],"name":"era","outputs":[{"name":"","type":"uint48"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x143e55e0"},{"constant":false,"inputs":[{"name":"urn","type":"bytes32"},{"name":"gal","type":"address"},{"name":"tab","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"bid","type":"uint256"}],"name":"kick","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0xeae19d9e"},{"constant":false,"inputs":[{"name":"id","type":"uint256"}],"name":"tick","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0xfc7b6aee"},{"constant":false,"inputs":[{"name":"id","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"bid","type":"uint256"}],"name":"tend","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x4b43ed12"},{"constant":false,"inputs":[{"name":"id","type":"uint256"},{"name":"lot","type":"uint256"},{"name":"bid","type":"uint256"}],"name":"dent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x5ff3a382"},{"constant":false,"inputs":[{"name":"id","type":"uint256"}],"name":"deal","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0xc959c42b"}]`
 	FlipperContractAddress = "0x6b59c42097e2fff7cad96cb08ceefd601081ad9c" //this is a temporary address deployed locally
 	FlipKickSignature      = "0x8828a22eb6a18623309ad55592866c4b077989e9e8a25e1b85f9bf6f7282520f"
-	PitABI                 = `[{"constant":true,"inputs":[],"name":"vat","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"live","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"drip","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"Line","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"wards","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"ilks","outputs":[{"name":"spot","type":"uint256"},{"name":"line","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"inputs":[{"name":"vat_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"ilk","type":"bytes32"},{"indexed":true,"name":"lad","type":"bytes32"},{"indexed":false,"name":"ink","type":"uint256"},{"indexed":false,"name":"art","type":"uint256"},{"indexed":false,"name":"dink","type":"int256"},{"indexed":false,"name":"dart","type":"int256"},{"indexed":false,"name":"iArt","type":"uint256"}],"name":"Frob","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"rely","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"deny","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"address"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"dink","type":"int256"},{"name":"dart","type":"int256"}],"name":"frob","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}]`
+	FrobSignature          = "0x6cedf1d3a466a3d6bab04887b1642177bf6dbf1daa737c2e8f639cd0b020d9d0"
+	PitABI                 = `[{"constant":true,"inputs":[],"name":"vat","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x36569e77"},{"constant":true,"inputs":[],"name":"live","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x957aa58c"},{"constant":true,"inputs":[],"name":"drip","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function","signature":"0x9f678cca"},{"constant":true,"inputs":[],"name":"Line","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xbabe8a3f"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"wards","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xbf353dbb"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"ilks","outputs":[{"name":"spot","type":"uint256"},{"name":"line","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function","signature":"0xd9638d36"},{"inputs":[{"name":"vat_","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor","signature":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"ilk","type":"bytes32"},{"indexed":true,"name":"urn","type":"bytes32"},{"indexed":false,"name":"ink","type":"uint256"},{"indexed":false,"name":"art","type":"uint256"},{"indexed":false,"name":"dink","type":"int256"},{"indexed":false,"name":"dart","type":"int256"},{"indexed":false,"name":"iArt","type":"uint256"}],"name":"Frob","type":"event","signature":"0xb2afa28318bcc689926b52835d844de174ef8de97e982a85c0199d584920791b"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event","signature":"0x644843f351d3fba4abcd60109eaff9f54bac8fb8ccf0bab941009c21df21cf31"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"rely","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x65fae35e"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"deny","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x9c52a7f1"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x1a0b287e"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"uint256"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x29ae8114"},{"constant":false,"inputs":[{"name":"what","type":"bytes32"},{"name":"data","type":"address"}],"name":"file","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0xd4e8be83"},{"constant":false,"inputs":[{"name":"ilk","type":"bytes32"},{"name":"dink","type":"int256"},{"name":"dart","type":"int256"}],"name":"frob","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function","signature":"0x5a984ded"}]`
 	PitFileSignature       = "0x1a0b287e00000000000000000000000000000000000000000000000000000000"
 	TendFunctionSignature  = "0x4b43ed1200000000000000000000000000000000000000000000000000000000"
 	BiteSignature          = "0x44a8f29dfcf27e5dd7a4db50b390f851bf3ecf2bf221243be3e853f586134455"

pkg/transformers/test_data/frob.go

@@ -15,6 +15,7 @@
 package test_data
 
 import (
+	"encoding/json"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -59,21 +60,26 @@ var EthFrobLog = types.Log{
 }
 
 var FrobEntity = frob.FrobEntity{
-	Ilk:  ilk,
+	Ilk:              ilk,
-	Lad:  frobLad,
+	Urn:              frobLad,
-	Dink: dink,
+	Ink:              ink,
-	Dart: dart,
+	Art:              art,
-	Ink:  ink,
+	Dink:             dink,
-	Art:  art,
+	Dart:             dart,
-	IArt: iArt,
+	IArt:             iArt,
+	TransactionIndex: EthFrobLog.TxIndex,
+	Raw:              EthFrobLog,
 }
 
+var rawFrobLog, _ = json.Marshal(EthFrobLog)
 var FrobModel = frob.FrobModel{
-	Ilk:  ilk[:],
+	Ilk:              ilk[:],
-	Lad:  frobLad[:],
+	Urn:              frobLad[:],
-	Dink: dink.String(),
+	Ink:              ink.String(),
-	Dart: dart.String(),
+	Art:              art.String(),
-	Ink:  ink.String(),
+	Dink:             dink.String(),
-	Art:  art.String(),
+	Dart:             dart.String(),
-	IArt: iArt.String(),
+	IArt:             iArt.String(),
+	TransactionIndex: EthFrobLog.TxIndex,
+	Raw:              rawFrobLog,
 }

pkg/transformers/test_data/mocks/frob/converter.go

@@ -25,21 +25,26 @@ type MockFrobConverter struct {
 	PassedContractABI     string
 	PassedLog             types.Log
 	PassedEntity          frob.FrobEntity
-	converterError        error
+	toEntityError         error
+	toModelError          error
 }
 
-func (converter *MockFrobConverter) SetConverterError(err error) {
+func (converter *MockFrobConverter) SetToEntityError(err error) {
-	converter.converterError = err
+	converter.toEntityError = err
+}
+
+func (converter *MockFrobConverter) SetToModelError(err error) {
+	converter.toModelError = err
 }
 
 func (converter *MockFrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (frob.FrobEntity, error) {
 	converter.PassedContractAddress = contractAddress
 	converter.PassedContractABI = contractAbi
 	converter.PassedLog = ethLog
-	return test_data.FrobEntity, converter.converterError
+	return test_data.FrobEntity, converter.toEntityError
 }
 
-func (converter *MockFrobConverter) ToModel(frobEntity frob.FrobEntity) frob.FrobModel {
+func (converter *MockFrobConverter) ToModel(frobEntity frob.FrobEntity) (frob.FrobModel, error) {
 	converter.PassedEntity = frobEntity
-	return test_data.FrobModel
+	return test_data.FrobModel, converter.toModelError
 }

pkg/transformers/test_data/mocks/frob/repository.go

@@ -25,7 +25,6 @@ type MockFrobRepository struct {
 	PassedFrobModel           frob.FrobModel
 	PassedHeaderID            int64
 	PassedStartingBlockNumber int64
-	PassedTransactionIndex    uint
 	missingHeaders            []core.Header
 	missingHeadersErr         error
 }
@@ -42,9 +41,8 @@ func (repository *MockFrobRepository) SetMissingHeaders(headers []core.Header) {
 	repository.missingHeaders = headers
 }
 
-func (repository *MockFrobRepository) Create(headerID int64, transactionIndex uint, model frob.FrobModel) error {
+func (repository *MockFrobRepository) Create(headerID int64, model frob.FrobModel) error {
 	repository.PassedHeaderID = headerID
-	repository.PassedTransactionIndex = transactionIndex
 	repository.PassedFrobModel = model
 	return repository.createError
 }

pkg/transformers/test_data/pit_file.go

@@ -30,12 +30,11 @@ var EthPitFileLog = types.Log{
 	Removed:     false,
 }
 
-var raw, _ = json.Marshal(EthPitFileLog)
+var rawPitFileLog, _ = json.Marshal(EthPitFileLog)
-
 var PitFileModel = pit_file.PitFileModel{
 	Ilk:              "fake ilk",
 	What:             "spot",
 	Risk:             big.NewInt(123).String(),
 	TransactionIndex: EthPitFileLog.TxIndex,
-	Raw:              raw,
+	Raw:              rawPitFileLog,
 }