From 82fd73ba3f00ce870c936326ef94843bdd111950 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Mon, 12 Nov 2018 14:59:16 -0600 Subject: [PATCH] (VDB-68) Verify log block hash matches header block hash - Delete header on conflict to prompt data refresh (cascade deletes all data associated with that block) - Derive header hash from rpc payload rather than computing it from data (prevents hash mismatch between blockchain and cache) --- cmd/continuousLogSync.go | 2 +- pkg/fakes/data.go | 8 ++- pkg/geth/blockchain.go | 4 +- .../converters/common/header_converter.go | 4 +- .../common/header_converter_test.go | 8 +-- pkg/transformers/bite/converter.go | 5 +- pkg/transformers/bite/converter_test.go | 2 +- pkg/transformers/bite/model.go | 2 +- pkg/transformers/bite/repository.go | 10 +++- .../cat_file/chop_lump/repository.go | 8 ++- pkg/transformers/cat_file/flip/repository.go | 6 +++ .../cat_file/pit_vow/repository.go | 6 +++ pkg/transformers/deal/repository.go | 12 +++-- pkg/transformers/dent/repository.go | 22 +++++--- pkg/transformers/drip_drip/repository.go | 6 +++ pkg/transformers/drip_file/ilk/repository.go | 6 +++ pkg/transformers/drip_file/repo/repository.go | 6 +++ pkg/transformers/drip_file/vow/repository.go | 6 +++ pkg/transformers/flap_kick/repository.go | 9 +++- pkg/transformers/flip_kick/converter.go | 5 +- pkg/transformers/flip_kick/converter_test.go | 9 ++-- pkg/transformers/flip_kick/model.go | 2 +- pkg/transformers/flip_kick/repository.go | 29 ++++++---- pkg/transformers/flop_kick/repository.go | 13 +++-- pkg/transformers/frob/repository.go | 7 +++ pkg/transformers/integration_tests/bite.go | 8 +-- .../integration_tests/cat_file.go | 18 +++---- pkg/transformers/integration_tests/deal.go | 25 ++++----- pkg/transformers/integration_tests/dent.go | 21 +++----- .../integration_tests/drip_drip.go | 10 ++-- .../integration_tests/drip_file_vow.go | 8 +-- .../integration_tests/flap_kick.go | 10 ++-- .../integration_tests/flip_kick.go | 10 ++-- .../integration_tests/flop_kick.go | 44 +++++++-------- pkg/transformers/integration_tests/frob.go | 30 +++++++---- pkg/transformers/integration_tests/helpers.go | 10 ++-- .../pit_file_debt_ceiling.go | 8 +-- .../integration_tests/pit_file_ilk.go | 8 +-- .../pit_file_stability_fee.go | 8 +-- .../integration_tests/price_feeds.go | 6 +-- pkg/transformers/integration_tests/tend.go | 54 ++++++++----------- .../integration_tests/vat_flux.go | 8 +-- .../integration_tests/vat_fold.go | 8 +-- .../integration_tests/vat_grab.go | 10 ++-- .../integration_tests/vat_heal.go | 10 ++-- .../integration_tests/vat_init.go | 8 +-- .../integration_tests/vat_move.go | 8 +-- .../integration_tests/vat_slip.go | 2 +- .../integration_tests/vat_tune.go | 8 +-- .../pit_file/debt_ceiling/repository.go | 6 +++ pkg/transformers/pit_file/ilk/repository.go | 6 +++ .../pit_file/stability_fee/repository.go | 6 +++ pkg/transformers/price_feeds/repository.go | 6 +++ .../shared/constants/checked_headers.go | 2 +- pkg/transformers/shared/constants/label.go | 2 +- pkg/transformers/shared/repository.go | 31 +++++++++++ pkg/transformers/tend/converter.go | 5 +- pkg/transformers/tend/model.go | 2 +- pkg/transformers/tend/repository.go | 12 +++-- pkg/transformers/tend/repository_test.go | 2 +- pkg/transformers/test_data/bite.go | 7 ++- pkg/transformers/test_data/cat_file.go | 9 ++-- pkg/transformers/test_data/deal.go | 3 +- pkg/transformers/test_data/dent.go | 4 +- pkg/transformers/test_data/drip_drip.go | 3 +- pkg/transformers/test_data/drip_file.go | 7 +-- pkg/transformers/test_data/flap_kick.go | 3 +- pkg/transformers/test_data/flip_kick.go | 33 ++++++------ pkg/transformers/test_data/flop_kick.go | 3 +- pkg/transformers/test_data/frob.go | 5 +- pkg/transformers/test_data/pit_file.go | 7 +-- pkg/transformers/test_data/price_feed.go | 3 +- .../shared_behaviors/repository_behaviors.go | 31 +++++++++++ pkg/transformers/test_data/tend.go | 8 +-- pkg/transformers/test_data/vat_flux.go | 3 +- pkg/transformers/test_data/vat_fold.go | 3 +- pkg/transformers/test_data/vat_grab.go | 3 +- pkg/transformers/test_data/vat_heal.go | 3 +- pkg/transformers/test_data/vat_init.go | 3 +- pkg/transformers/test_data/vat_move.go | 3 +- pkg/transformers/test_data/vat_slip.go | 3 +- pkg/transformers/test_data/vat_toll.go | 3 +- pkg/transformers/test_data/vat_tune.go | 3 +- pkg/transformers/test_data/vow_flog.go | 3 +- pkg/transformers/vat_flux/repository.go | 14 +++-- pkg/transformers/vat_flux/repository_test.go | 2 +- pkg/transformers/vat_fold/repository.go | 6 +++ pkg/transformers/vat_grab/repository.go | 7 +++ pkg/transformers/vat_heal/repository.go | 13 +++-- pkg/transformers/vat_init/repository.go | 6 +++ pkg/transformers/vat_move/repository.go | 6 +++ pkg/transformers/vat_slip/repository.go | 6 +++ pkg/transformers/vat_toll/repository.go | 7 +++ pkg/transformers/vat_tune/repository.go | 9 ++++ pkg/transformers/vow_flog/repository.go | 6 +++ 95 files changed, 557 insertions(+), 298 deletions(-) diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 840708b6..d4249724 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -40,7 +40,7 @@ Available transformers for (optional) selection with --transformers: %v This command expects a light sync to have been run, and the presence of header records in the Vulcanize database.`, - constants.AllTransformerLabels()), + constants.AllTransformerLabels()), Run: func(cmd *cobra.Command, args []string) { syncMakerLogs() }, diff --git a/pkg/fakes/data.go b/pkg/fakes/data.go index dde5445b..b949440e 100644 --- a/pkg/fakes/data.go +++ b/pkg/fakes/data.go @@ -3,20 +3,26 @@ package fakes import ( "encoding/json" "errors" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" ) -var FakeError = errors.New("failed") +var ( + FakeError = errors.New("failed") + FakeHash = common.BytesToHash([]byte{1, 2, 3, 4, 5}) +) var rawFakeHeader, _ = json.Marshal(types.Header{}) var FakeHeader = core.Header{ + Hash: FakeHash.String(), Raw: rawFakeHeader, Timestamp: "111111111", } func GetFakeHeader(blockNumber int64) core.Header { return core.Header{ + Hash: FakeHash.String(), BlockNumber: blockNumber, Raw: rawFakeHeader, Timestamp: "111111111", diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index 0f4d62bb..d749e701 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -54,7 +54,7 @@ func (blockChain *BlockChain) getPOWHeader(blockNumber int64) (header core.Heade if err != nil { return header, err } - return blockChain.headerConverter.Convert(gethHeader) + return blockChain.headerConverter.Convert(gethHeader, gethHeader.Hash().String()) } func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Header, err error) { @@ -82,7 +82,7 @@ func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Heade GasUsed: uint64(POAHeader.GasUsed), Time: POAHeader.Time.ToInt(), Extra: POAHeader.Extra, - }) + }, POAHeader.Hash.String()) } func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) { diff --git a/pkg/geth/converters/common/header_converter.go b/pkg/geth/converters/common/header_converter.go index e2e1846d..a6cd2eaf 100644 --- a/pkg/geth/converters/common/header_converter.go +++ b/pkg/geth/converters/common/header_converter.go @@ -8,13 +8,13 @@ import ( type HeaderConverter struct{} -func (converter HeaderConverter) Convert(gethHeader *types.Header) (core.Header, error) { +func (converter HeaderConverter) Convert(gethHeader *types.Header, blockHash string) (core.Header, error) { rawHeader, err := json.Marshal(gethHeader) if err != nil { panic(err) } coreHeader := core.Header{ - Hash: gethHeader.Hash().Hex(), + Hash: blockHash, BlockNumber: gethHeader.Number.Int64(), Raw: rawHeader, Timestamp: gethHeader.Time.String(), diff --git a/pkg/geth/converters/common/header_converter_test.go b/pkg/geth/converters/common/header_converter_test.go index 7e68ab2d..f25b02eb 100644 --- a/pkg/geth/converters/common/header_converter_test.go +++ b/pkg/geth/converters/common/header_converter_test.go @@ -2,6 +2,7 @@ package common_test import ( "encoding/json" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "math/big" "github.com/ethereum/go-ethereum/common" @@ -25,12 +26,13 @@ var _ = Describe("Block header converter", func() { UncleHash: common.HexToHash("0xUncle"), } converter := common2.HeaderConverter{} + hash := fakes.FakeHash.String() - coreHeader, err := converter.Convert(gethHeader) + coreHeader, err := converter.Convert(gethHeader, hash) Expect(err).NotTo(HaveOccurred()) Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64())) - Expect(coreHeader.Hash).To(Equal(gethHeader.Hash().Hex())) + Expect(coreHeader.Hash).To(Equal(hash)) Expect(coreHeader.Timestamp).To(Equal(gethHeader.Time.String())) }) @@ -38,7 +40,7 @@ var _ = Describe("Block header converter", func() { gethHeader := types.Header{Number: big.NewInt(123)} converter := common2.HeaderConverter{} - coreHeader, err := converter.Convert(&gethHeader) + coreHeader, err := converter.Convert(&gethHeader, fakes.FakeHash.String()) Expect(err).NotTo(HaveOccurred()) expectedJSON, err := json.Marshal(gethHeader) diff --git a/pkg/transformers/bite/converter.go b/pkg/transformers/bite/converter.go index fe0007a9..e346024e 100644 --- a/pkg/transformers/bite/converter.go +++ b/pkg/transformers/bite/converter.go @@ -73,8 +73,7 @@ func (converter BiteConverter) ToModels(entities []interface{}) ([]interface{}, flip := biteEntity.Flip logIdx := biteEntity.LogIndex txIdx := biteEntity.TransactionIndex - rawLogJson, err := json.Marshal(biteEntity.Raw) - rawLogString := string(rawLogJson) + rawLog, err := json.Marshal(biteEntity.Raw) if err != nil { return nil, err } @@ -89,7 +88,7 @@ func (converter BiteConverter) ToModels(entities []interface{}) ([]interface{}, NFlip: shared.BigIntToString(flip), LogIndex: logIdx, TransactionIndex: txIdx, - Raw: rawLogString, + Raw: rawLog, } models = append(models, model) } diff --git a/pkg/transformers/bite/converter_test.go b/pkg/transformers/bite/converter_test.go index 10928bfb..a62477d7 100644 --- a/pkg/transformers/bite/converter_test.go +++ b/pkg/transformers/bite/converter_test.go @@ -79,7 +79,7 @@ var _ = Describe("Bite Converter", func() { Tab: "", NFlip: "", TransactionIndex: 0, - Raw: string(emptyLog), + Raw: emptyLog, } models, err := converter.ToModels([]interface{}{emptyEntity}) diff --git a/pkg/transformers/bite/model.go b/pkg/transformers/bite/model.go index b7b3890c..a0580c89 100644 --- a/pkg/transformers/bite/model.go +++ b/pkg/transformers/bite/model.go @@ -24,5 +24,5 @@ type BiteModel struct { NFlip string LogIndex uint `db:"log_idx"` TransactionIndex uint `db:"tx_idx"` - Raw string `db:"raw_log"` + Raw []byte `db:"raw_log"` } diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 231cf8e6..7c8c8da7 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -41,7 +41,14 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er tx.Rollback() return fmt.Errorf("model of type %T, not %T", model, BiteModel{}) } - _, err := tx.Exec( + + err = shared.ValidateHeaderConsistency(headerID, biteModel.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + + _, err = tx.Exec( `INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)`, headerID, biteModel.Ilk, biteModel.Urn, biteModel.Ink, biteModel.Art, biteModel.IArt, biteModel.Tab, biteModel.NFlip, biteModel.LogIndex, biteModel.TransactionIndex, biteModel.Raw, @@ -51,6 +58,7 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er return err } } + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.BiteChecked) if err != nil { tx.Rollback() diff --git a/pkg/transformers/cat_file/chop_lump/repository.go b/pkg/transformers/cat_file/chop_lump/repository.go index 20c00f02..9db0d987 100644 --- a/pkg/transformers/cat_file/chop_lump/repository.go +++ b/pkg/transformers/cat_file/chop_lump/repository.go @@ -39,7 +39,13 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte return fmt.Errorf("model of type %T, not %T", model, CatFileChopLumpModel{}) } - _, err := tx.Exec( + err = shared.ValidateHeaderConsistency(headerID, chopLump.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + + _, err = tx.Exec( `INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, headerID, chopLump.Ilk, chopLump.What, chopLump.Data, chopLump.TransactionIndex, chopLump.LogIndex, chopLump.Raw, diff --git a/pkg/transformers/cat_file/flip/repository.go b/pkg/transformers/cat_file/flip/repository.go index 8f59c20f..76c60b8c 100644 --- a/pkg/transformers/cat_file/flip/repository.go +++ b/pkg/transformers/cat_file/flip/repository.go @@ -38,6 +38,12 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac return fmt.Errorf("model of type %T, not %T", model, CatFileFlipModel{}) } + err = shared.ValidateHeaderConsistency(headerID, flip.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = repository.db.Exec( `INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6, $7)`, diff --git a/pkg/transformers/cat_file/pit_vow/repository.go b/pkg/transformers/cat_file/pit_vow/repository.go index 6aa7a108..a13ad927 100644 --- a/pkg/transformers/cat_file/pit_vow/repository.go +++ b/pkg/transformers/cat_file/pit_vow/repository.go @@ -38,6 +38,12 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf return fmt.Errorf("model of type %T, not %T", model, CatFilePitVowModel{}) } + err = shared.ValidateHeaderConsistency(headerID, vow.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = repository.db.Exec( `INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6)`, diff --git a/pkg/transformers/deal/repository.go b/pkg/transformers/deal/repository.go index 1bfa420b..d109b338 100644 --- a/pkg/transformers/deal/repository.go +++ b/pkg/transformers/deal/repository.go @@ -26,7 +26,7 @@ type DealRepository struct { db *postgres.DB } -func (repository DealRepository) Create(headerId int64, models []interface{}) error { +func (repository DealRepository) Create(headerID int64, models []interface{}) error { tx, err := repository.db.Begin() if err != nil { return err @@ -39,10 +39,16 @@ func (repository DealRepository) Create(headerId int64, models []interface{}) er return fmt.Errorf("model of type %T, not %T", model, DealModel{}) } + err = shared.ValidateHeaderConsistency(headerID, dealModel.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6)`, - headerId, dealModel.BidId, dealModel.ContractAddress, dealModel.LogIndex, dealModel.TransactionIndex, dealModel.Raw, + headerID, dealModel.BidId, dealModel.ContractAddress, dealModel.LogIndex, dealModel.TransactionIndex, dealModel.Raw, ) if err != nil { tx.Rollback() @@ -50,7 +56,7 @@ func (repository DealRepository) Create(headerId int64, models []interface{}) er } } - err = shared.MarkHeaderCheckedInTransaction(headerId, tx, constants.DealChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DealChecked) if err != nil { tx.Rollback() return err diff --git a/pkg/transformers/dent/repository.go b/pkg/transformers/dent/repository.go index a2a27ee8..e86e6aac 100644 --- a/pkg/transformers/dent/repository.go +++ b/pkg/transformers/dent/repository.go @@ -26,8 +26,8 @@ type DentRepository struct { db *postgres.DB } -func (r DentRepository) Create(headerId int64, models []interface{}) error { - tx, err := r.db.Begin() +func (repository DentRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.db.Begin() if err != nil { return err } @@ -39,10 +39,16 @@ func (r DentRepository) Create(headerId int64, models []interface{}) error { return fmt.Errorf("model of type %T, not %T", model, DentModel{}) } + err = shared.ValidateHeaderConsistency(headerID, dent.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - headerId, dent.BidId, dent.Lot, dent.Bid, dent.Guy, dent.Tic, dent.LogIndex, dent.TransactionIndex, dent.Raw, + headerID, dent.BidId, dent.Lot, dent.Bid, dent.Guy, dent.Tic, dent.LogIndex, dent.TransactionIndex, dent.Raw, ) if err != nil { tx.Rollback() @@ -50,7 +56,7 @@ func (r DentRepository) Create(headerId int64, models []interface{}) error { } } - err = shared.MarkHeaderCheckedInTransaction(headerId, tx, constants.DentChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) if err != nil { tx.Rollback() return err @@ -58,12 +64,12 @@ func (r DentRepository) Create(headerId int64, models []interface{}) error { return tx.Commit() } -func (r DentRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, r.db, constants.DentChecked) +func (repository DentRepository) MarkHeaderChecked(headerId int64) error { + return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) } -func (r DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, r.db, constants.DentChecked) +func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked) } func (repository *DentRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index 95780c74..a26c9a53 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -38,6 +38,12 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{} return fmt.Errorf("model of type %T, not %T", model, DripDripModel{}) } + err = shared.ValidateHeaderConsistency(headerID, dripDrip.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5)`, diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 8c602ef9..fc8ec129 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -39,6 +39,12 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac return fmt.Errorf("model of type %T, not %T", model, DripFileIlkModel{}) } + err = shared.ValidateHeaderConsistency(headerID, ilk.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 592e92e6..d461ec7f 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -39,6 +39,12 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa return fmt.Errorf("model of type %T, not %T", model, DripFileRepoModel{}) } + err = shared.ValidateHeaderConsistency(headerID, repo.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.drip_file_repo (header_id, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index 35b089d0..3e54fb83 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -39,6 +39,12 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac return fmt.Errorf("model of type %T, not %T", model, DripFileVowModel{}) } + err = shared.ValidateHeaderConsistency(headerID, vow.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6)`, diff --git a/pkg/transformers/flap_kick/repository.go b/pkg/transformers/flap_kick/repository.go index 1e623f7a..9e6d5751 100644 --- a/pkg/transformers/flap_kick/repository.go +++ b/pkg/transformers/flap_kick/repository.go @@ -36,7 +36,14 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{ if !ok { return fmt.Errorf("model of type %T, not %T", model, FlapKickModel{}) } - _, err := tx.Exec( + + err = shared.ValidateHeaderConsistency(headerID, flapKickModel.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + + _, err = tx.Exec( `INSERT into maker.flap_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, headerID, flapKickModel.BidId, flapKickModel.Lot, flapKickModel.Bid, flapKickModel.Gal, flapKickModel.End, flapKickModel.TransactionIndex, flapKickModel.LogIndex, flapKickModel.Raw, diff --git a/pkg/transformers/flip_kick/converter.go b/pkg/transformers/flip_kick/converter.go index 0702c868..cf0a0a87 100644 --- a/pkg/transformers/flip_kick/converter.go +++ b/pkg/transformers/flip_kick/converter.go @@ -75,11 +75,10 @@ func (FlipKickConverter) ToModels(entities []interface{}) ([]interface{}, error) end := time.Unix(endValue, 0) urn := common.BytesToAddress(flipKickEntity.Urn[:common.AddressLength]).String() tab := shared.BigIntToString(flipKickEntity.Tab) - rawLogJson, err := json.Marshal(flipKickEntity.Raw) + rawLog, err := json.Marshal(flipKickEntity.Raw) if err != nil { return nil, err } - rawLogString := string(rawLogJson) model := FlipKickModel{ BidId: id, @@ -91,7 +90,7 @@ func (FlipKickConverter) ToModels(entities []interface{}) ([]interface{}, error) Tab: tab, TransactionIndex: flipKickEntity.TransactionIndex, LogIndex: flipKickEntity.LogIndex, - Raw: rawLogString, + Raw: rawLog, } models = append(models, model) } diff --git a/pkg/transformers/flip_kick/converter_test.go b/pkg/transformers/flip_kick/converter_test.go index 233dc590..bd2cff9b 100644 --- a/pkg/transformers/flip_kick/converter_test.go +++ b/pkg/transformers/flip_kick/converter_test.go @@ -53,16 +53,13 @@ var _ = Describe("FlipKick Converter", func() { var emptyString = "" var emptyTime = time.Unix(0, 0) var emptyEntity = flip_kick.FlipKickEntity{} - var emptyRawLog string + var emptyRawLog []byte + var err error BeforeEach(func() { emptyEntity.Id = big.NewInt(1) - var emptyRawLogJson, err = json.Marshal(types.Log{}) + emptyRawLog, err = json.Marshal(types.Log{}) Expect(err).NotTo(HaveOccurred()) - - emptyRawLogJson, err = json.Marshal(types.Log{}) - Expect(err).NotTo(HaveOccurred()) - emptyRawLog = string(emptyRawLogJson) }) It("converts an Entity to a Model", func() { diff --git a/pkg/transformers/flip_kick/model.go b/pkg/transformers/flip_kick/model.go index 5077e5c3..0d1ae2f5 100644 --- a/pkg/transformers/flip_kick/model.go +++ b/pkg/transformers/flip_kick/model.go @@ -26,5 +26,5 @@ type FlipKickModel struct { Tab string TransactionIndex uint `db:"tx_idx"` LogIndex uint `db:"log_idx"` - Raw string `db:"raw_log"` + Raw []byte `db:"raw_log"` } diff --git a/pkg/transformers/flip_kick/repository.go b/pkg/transformers/flip_kick/repository.go index 4c11f060..488483e7 100644 --- a/pkg/transformers/flip_kick/repository.go +++ b/pkg/transformers/flip_kick/repository.go @@ -27,8 +27,8 @@ type FlipKickRepository struct { db *postgres.DB } -func (fkr FlipKickRepository) Create(headerId int64, models []interface{}) error { - tx, err := fkr.db.Begin() +func (repository FlipKickRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.db.Begin() if err != nil { return err } @@ -37,17 +37,24 @@ func (fkr FlipKickRepository) Create(headerId int64, models []interface{}) error if !ok { return fmt.Errorf("model of type %T, not %T", model, FlipKickModel{}) } - _, err := tx.Exec( + + err = shared.ValidateHeaderConsistency(headerID, flipKickModel.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + + _, err = tx.Exec( `INSERT into maker.flip_kick (header_id, bid_id, lot, bid, gal, "end", urn, tab, tx_idx, log_idx, raw_log) VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11)`, - headerId, flipKickModel.BidId, flipKickModel.Lot, flipKickModel.Bid, flipKickModel.Gal, flipKickModel.End, flipKickModel.Urn, flipKickModel.Tab, flipKickModel.TransactionIndex, flipKickModel.LogIndex, flipKickModel.Raw, + headerID, flipKickModel.BidId, flipKickModel.Lot, flipKickModel.Bid, flipKickModel.Gal, flipKickModel.End, flipKickModel.Urn, flipKickModel.Tab, flipKickModel.TransactionIndex, flipKickModel.LogIndex, flipKickModel.Raw, ) if err != nil { tx.Rollback() return err } } - err = shared.MarkHeaderCheckedInTransaction(headerId, tx, constants.FlipKickChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlipKickChecked) if err != nil { tx.Rollback() return err @@ -55,14 +62,14 @@ func (fkr FlipKickRepository) Create(headerId int64, models []interface{}) error return tx.Commit() } -func (fkr FlipKickRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, fkr.db, constants.FlipKickChecked) +func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error { + return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) } -func (fkr FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, fkr.db, constants.FlipKickChecked) +func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked) } -func (fkr *FlipKickRepository) SetDB(db *postgres.DB) { - fkr.db = db +func (repository *FlipKickRepository) SetDB(db *postgres.DB) { + repository.db = db } diff --git a/pkg/transformers/flop_kick/repository.go b/pkg/transformers/flop_kick/repository.go index 467d58ad..ee6c9d4b 100644 --- a/pkg/transformers/flop_kick/repository.go +++ b/pkg/transformers/flop_kick/repository.go @@ -26,7 +26,7 @@ type FlopKickRepository struct { db *postgres.DB } -func (repository FlopKickRepository) Create(headerId int64, models []interface{}) error { +func (repository FlopKickRepository) Create(headerID int64, models []interface{}) error { tx, err := repository.db.Begin() if err != nil { return err @@ -37,10 +37,17 @@ func (repository FlopKickRepository) Create(headerId int64, models []interface{} if !ok { return fmt.Errorf("model of type %T, not %T", flopKick, Model{}) } + + err = shared.ValidateHeaderConsistency(headerID, flopKickModel.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.flop_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, - headerId, flopKickModel.BidId, flopKickModel.Lot, flopKickModel.Bid, flopKickModel.Gal, flopKickModel.End, flopKickModel.TransactionIndex, flopKickModel.LogIndex, flopKickModel.Raw, + headerID, flopKickModel.BidId, flopKickModel.Lot, flopKickModel.Bid, flopKickModel.Gal, flopKickModel.End, flopKickModel.TransactionIndex, flopKickModel.LogIndex, flopKickModel.Raw, ) if err != nil { tx.Rollback() @@ -48,7 +55,7 @@ func (repository FlopKickRepository) Create(headerId int64, models []interface{} } } - err = shared.MarkHeaderCheckedInTransaction(headerId, tx, constants.FlopKickChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlopKickChecked) if err != nil { tx.Rollback() return err diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index 0aa16ed9..2df57940 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -38,6 +38,13 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er tx.Rollback() return fmt.Errorf("model of type %T, not %T", model, FrobModel{}) } + + err = shared.ValidateHeaderConsistency(headerID, frobModel.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx) VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)`, headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex) diff --git a/pkg/transformers/integration_tests/bite.go b/pkg/transformers/integration_tests/bite.go index 90f2a640..86c9f7ff 100644 --- a/pkg/transformers/integration_tests/bite.go +++ b/pkg/transformers/integration_tests/bite.go @@ -39,13 +39,13 @@ var _ = Describe("Bite Transformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ @@ -54,7 +54,7 @@ var _ = Describe("Bite Transformer", func() { Repository: &bite.BiteRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockchain) + transformer := initializer.NewTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index 35d24015..68983f88 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -35,7 +35,7 @@ import ( var _ = Describe("Cat File transformer", func() { var ( db *postgres.DB - blockchain core.BlockChain + blockChain core.BlockChain rpcClient client.RpcClient err error ethClient *ethclient.Client @@ -44,9 +44,9 @@ var _ = Describe("Cat File transformer", func() { BeforeEach(func() { rpcClient, ethClient, err = getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err = getBlockChain(rpcClient, ethClient) + blockChain, err = getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db = test_config.NewTestDB(blockchain.Node()) + db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) }) @@ -54,7 +54,7 @@ var _ = Describe("Cat File transformer", func() { It("persists a chop lump event", func() { // transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014 chopLumpBlockNumber := int64(8762253) - err = persistHeader(db, chopLumpBlockNumber) + err = persistHeader(db, chopLumpBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := chop_lump.CatFileChopLumpConfig @@ -67,7 +67,7 @@ var _ = Describe("Cat File transformer", func() { Repository: &chop_lump.CatFileChopLumpRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -92,7 +92,7 @@ var _ = Describe("Cat File transformer", func() { It("persists a flip event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe flipBlockNumber := int64(8751794) - err = persistHeader(db, flipBlockNumber) + err = persistHeader(db, flipBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := flip.CatFileFlipConfig @@ -105,7 +105,7 @@ var _ = Describe("Cat File transformer", func() { Repository: &flip.CatFileFlipRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -122,7 +122,7 @@ var _ = Describe("Cat File transformer", func() { It("persists a pit vow event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe pitVowBlockNumber := int64(8751794) - err = persistHeader(db, pitVowBlockNumber) + err = persistHeader(db, pitVowBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := pit_vow.CatFilePitVowConfig @@ -135,7 +135,7 @@ var _ = Describe("Cat File transformer", func() { Repository: &pit_vow.CatFilePitVowRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/deal.go b/pkg/transformers/integration_tests/deal.go index 995e3f62..a43e04cf 100644 --- a/pkg/transformers/integration_tests/deal.go +++ b/pkg/transformers/integration_tests/deal.go @@ -15,13 +15,11 @@ package integration_tests import ( - "github.com/ethereum/go-ethereum/ethclient" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/geth/client" "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -32,25 +30,22 @@ import ( var _ = Describe("Deal transformer", func() { var ( db *postgres.DB - blockchain core.BlockChain - rpcClient client.RpcClient - err error - ethClient *ethclient.Client + blockChain core.BlockChain ) BeforeEach(func() { - rpcClient, ethClient, err = getClients(ipc) + rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err = getBlockChain(rpcClient, ethClient) + blockChain, err = getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db = test_config.NewTestDB(blockchain.Node()) + db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) }) It("persists a flip deal log event", func() { // transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58 flipBlockNumber := int64(8958007) - err = persistHeader(db, flipBlockNumber) + err := persistHeader(db, flipBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := deal.DealConfig @@ -63,8 +58,8 @@ var _ = Describe("Deal transformer", func() { Repository: &deal.DealRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) - err := transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db, blockChain) + err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel @@ -82,7 +77,7 @@ var _ = Describe("Deal transformer", func() { It("persists a flap deal log event", func() { flapBlockNumber := int64(9004628) - err = persistHeader(db, flapBlockNumber) + err := persistHeader(db, flapBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := deal.DealConfig @@ -95,8 +90,8 @@ var _ = Describe("Deal transformer", func() { Repository: &deal.DealRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) - err := transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db, blockChain) + err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel diff --git a/pkg/transformers/integration_tests/dent.go b/pkg/transformers/integration_tests/dent.go index 9703d308..716b257e 100644 --- a/pkg/transformers/integration_tests/dent.go +++ b/pkg/transformers/integration_tests/dent.go @@ -1,15 +1,13 @@ package integration_tests import ( - "github.com/ethereum/go-ethereum/ethclient" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/geth/client" "github.com/vulcanize/vulcanizedb/pkg/transformers/dent" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -17,24 +15,21 @@ import ( var _ = Describe("Dent transformer", func() { var ( db *postgres.DB - blockchain core.BlockChain - rpcClient client.RpcClient - err error - ethClient *ethclient.Client + blockChain core.BlockChain ) BeforeEach(func() { - rpcClient, ethClient, err = getClients(ipc) + rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err = getBlockChain(rpcClient, ethClient) + blockChain, err = getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db = test_config.NewTestDB(blockchain.Node()) + db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) }) It("persists a flop dent log event", func() { blockNumber := int64(8955613) - err = persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := dent.DentConfig @@ -47,8 +42,8 @@ var _ = Describe("Dent transformer", func() { Repository: &dent.DentRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) - err := transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db, blockChain) + err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) var dbResult []dent.DentModel diff --git a/pkg/transformers/integration_tests/drip_drip.go b/pkg/transformers/integration_tests/drip_drip.go index 191576e0..6203b2dc 100644 --- a/pkg/transformers/integration_tests/drip_drip.go +++ b/pkg/transformers/integration_tests/drip_drip.go @@ -33,22 +33,22 @@ var _ = Describe("DripDrip Transformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ - Config: drip_drip.DripDripConfig, + Config: config, Converter: &drip_drip.DripDripConverter{}, Repository: &drip_drip.DripDripRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/drip_file_vow.go b/pkg/transformers/integration_tests/drip_file_vow.go index f6aae07c..08630370 100644 --- a/pkg/transformers/integration_tests/drip_file_vow.go +++ b/pkg/transformers/integration_tests/drip_file_vow.go @@ -33,13 +33,13 @@ var _ = Describe("Drip File Vow LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -48,7 +48,7 @@ var _ = Describe("Drip File Vow LogNoteTransformer", func() { Converter: &vow.DripFileVowConverter{}, Repository: &vow.DripFileVowRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/flap_kick.go b/pkg/transformers/integration_tests/flap_kick.go index 7bf6d82f..6b317f6e 100644 --- a/pkg/transformers/integration_tests/flap_kick.go +++ b/pkg/transformers/integration_tests/flap_kick.go @@ -35,22 +35,22 @@ var _ = Describe("FlapKick Transformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ - Config: flap_kick.FlapKickConfig, + Config: config, Converter: &flap_kick.FlapKickConverter{}, Repository: &flap_kick.FlapKickRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockchain) + transformer := initializer.NewTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/flip_kick.go b/pkg/transformers/integration_tests/flip_kick.go index 2ff2ba5f..17076b47 100644 --- a/pkg/transformers/integration_tests/flip_kick.go +++ b/pkg/transformers/integration_tests/flip_kick.go @@ -63,22 +63,22 @@ var _ = Describe("FlipKick Transformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ - Config: flip_kick.FlipKickConfig, + Config: config, Converter: &flip_kick.FlipKickConverter{}, Repository: &flip_kick.FlipKickRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockchain) + transformer := initializer.NewTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/flop_kick.go b/pkg/transformers/integration_tests/flop_kick.go index cca936fd..93690d43 100644 --- a/pkg/transformers/integration_tests/flop_kick.go +++ b/pkg/transformers/integration_tests/flop_kick.go @@ -22,6 +22,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/flop_kick" @@ -32,30 +34,36 @@ import ( ) var _ = Describe("FlopKick Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("fetches and transforms a FlopKick event from Kovan chain", func() { blockNumber := int64(8672119) config := flop_kick.Config config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockchain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ - Config: flop_kick.Config, + Config: config, Converter: &flop_kick.FlopKickConverter{}, Repository: &flop_kick.FlopKickRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockchain) + transformer := initializer.NewTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -78,24 +86,16 @@ var _ = Describe("FlopKick Transformer", func() { config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockchain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ - Config: flop_kick.Config, + Config: config, Converter: &flop_kick.FlopKickConverter{}, Repository: &flop_kick.FlopKickRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockchain) + transformer := initializer.NewTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/frob.go b/pkg/transformers/integration_tests/frob.go index 32aac5a9..00f31df4 100644 --- a/pkg/transformers/integration_tests/frob.go +++ b/pkg/transformers/integration_tests/frob.go @@ -20,6 +20,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" @@ -30,30 +32,36 @@ import ( ) var _ = Describe("Frob Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("fetches and transforms a Frob event from Kovan chain", func() { blockNumber := int64(8935258) config := frob.FrobConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockchain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ - Config: frob.FrobConfig, + Config: config, Converter: &frob.FrobConverter{}, Repository: &frob.FrobRepository{}, Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockchain) + transformer := initializer.NewTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/helpers.go b/pkg/transformers/integration_tests/helpers.go index 2cd56375..19078a14 100644 --- a/pkg/transformers/integration_tests/helpers.go +++ b/pkg/transformers/integration_tests/helpers.go @@ -20,7 +20,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" @@ -34,6 +33,7 @@ func getClients(ipc string) (client.RpcClient, *ethclient.Client, error) { } return client.NewRpcClient(raw, ipc), ethclient.NewClient(raw), nil } + func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (core.BlockChain, error) { client := client.NewEthClient(ethClient) node := node.MakeNode(rpcClient) @@ -42,8 +42,12 @@ func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (cor return blockChain, nil } -func persistHeader(db *postgres.DB, blockNumber int64) error { +func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) error { + header, err := blockChain.GetHeaderByNumber(blockNumber) + if err != nil { + return err + } headerRepository := repositories.NewHeaderRepository(db) - _, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) + _, err = headerRepository.CreateOrUpdateHeader(header) return err } diff --git a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go index 1a9c3507..f3eaff08 100644 --- a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go +++ b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go @@ -33,13 +33,13 @@ var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -48,7 +48,7 @@ var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() { Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index be174871..7a255141 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -33,13 +33,13 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -48,7 +48,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/pit_file_stability_fee.go b/pkg/transformers/integration_tests/pit_file_stability_fee.go index 446000cb..1ca226ff 100644 --- a/pkg/transformers/integration_tests/pit_file_stability_fee.go +++ b/pkg/transformers/integration_tests/pit_file_stability_fee.go @@ -33,13 +33,13 @@ var _ = Describe("PitFileStabilityFee LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -48,7 +48,7 @@ var _ = Describe("PitFileStabilityFee LogNoteTransformer", func() { Converter: &stability_fee.PitFileStabilityFeeConverter{}, Repository: &stability_fee.PitFileStabilityFeeRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/price_feeds.go b/pkg/transformers/integration_tests/price_feeds.go index da5e9a5b..fa8a13e2 100644 --- a/pkg/transformers/integration_tests/price_feeds.go +++ b/pkg/transformers/integration_tests/price_feeds.go @@ -44,7 +44,7 @@ var _ = Describe("Price feeds transformer", func() { It("persists a ETH/USD price feed event", func() { blockNumber := int64(8763054) - err := persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := price_feeds.PriceFeedConfig config.ContractAddresses = []string{constants.PipContractAddress} @@ -71,7 +71,7 @@ var _ = Describe("Price feeds transformer", func() { It("persists a MKR/USD price feed event", func() { blockNumber := int64(8763059) - err := persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := price_feeds.PriceFeedConfig config.ContractAddresses = []string{constants.PepContractAddress} @@ -98,7 +98,7 @@ var _ = Describe("Price feeds transformer", func() { It("persists a REP/USD price feed event", func() { blockNumber := int64(8763062) - err := persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := price_feeds.PriceFeedConfig config.ContractAddresses = []string{constants.RepContractAddress} diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index 1d976d0e..4e146446 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -17,29 +17,37 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("Tend LogNoteTransformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("fetches and transforms a Flip Tend event from Kovan chain", func() { blockNumber := int64(8935601) config := tend.TendConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockchain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -48,7 +56,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -70,15 +78,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockchain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -87,7 +87,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -109,15 +109,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockchain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber) + err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -126,7 +118,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_flux.go b/pkg/transformers/integration_tests/vat_flux.go index 2cd8ae17..901e294d 100644 --- a/pkg/transformers/integration_tests/vat_flux.go +++ b/pkg/transformers/integration_tests/vat_flux.go @@ -33,13 +33,13 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -48,7 +48,7 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { Converter: &vat_flux.VatFluxConverter{}, Repository: &vat_flux.VatFluxRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_fold.go b/pkg/transformers/integration_tests/vat_fold.go index ddb699d8..6331a89c 100644 --- a/pkg/transformers/integration_tests/vat_fold.go +++ b/pkg/transformers/integration_tests/vat_fold.go @@ -34,13 +34,13 @@ var _ = Describe("VatFold Transformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -49,7 +49,7 @@ var _ = Describe("VatFold Transformer", func() { Converter: &vat_fold.VatFoldConverter{}, Repository: &vat_fold.VatFoldRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_grab.go b/pkg/transformers/integration_tests/vat_grab.go index a942cbc5..f33f0918 100644 --- a/pkg/transformers/integration_tests/vat_grab.go +++ b/pkg/transformers/integration_tests/vat_grab.go @@ -35,21 +35,21 @@ var _ = Describe("Vat Grab Transformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) transformer := factories.LogNoteTransformer{ - Config: vat_grab.VatGrabConfig, + Config: config, Converter: &vat_grab.VatGrabConverter{}, Repository: &vat_grab.VatGrabRepository{}, Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer(db, blockchain) + }.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_heal.go b/pkg/transformers/integration_tests/vat_heal.go index de98fbb3..61c8ec0d 100644 --- a/pkg/transformers/integration_tests/vat_heal.go +++ b/pkg/transformers/integration_tests/vat_heal.go @@ -34,21 +34,21 @@ var _ = Describe("VatHeal Transformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) transformer := factories.LogNoteTransformer{ - Config: vat_heal.VatHealConfig, + Config: config, Converter: &vat_heal.VatHealConverter{}, Repository: &vat_heal.VatHealRepository{}, Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer(db, blockchain) + }.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_init.go b/pkg/transformers/integration_tests/vat_init.go index fa4f2d8d..010f7429 100644 --- a/pkg/transformers/integration_tests/vat_init.go +++ b/pkg/transformers/integration_tests/vat_init.go @@ -33,13 +33,13 @@ var _ = Describe("VatInit LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -48,7 +48,7 @@ var _ = Describe("VatInit LogNoteTransformer", func() { Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_move.go b/pkg/transformers/integration_tests/vat_move.go index ab1bec54..c977e5cc 100644 --- a/pkg/transformers/integration_tests/vat_move.go +++ b/pkg/transformers/integration_tests/vat_move.go @@ -34,13 +34,13 @@ var _ = Describe("VatMove LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -49,7 +49,7 @@ var _ = Describe("VatMove LogNoteTransformer", func() { Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_slip.go b/pkg/transformers/integration_tests/vat_slip.go index 0dcc6494..0b82914a 100644 --- a/pkg/transformers/integration_tests/vat_slip.go +++ b/pkg/transformers/integration_tests/vat_slip.go @@ -31,7 +31,7 @@ var _ = Describe("Vat slip transformer", func() { config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ diff --git a/pkg/transformers/integration_tests/vat_tune.go b/pkg/transformers/integration_tests/vat_tune.go index 44480210..59001f80 100644 --- a/pkg/transformers/integration_tests/vat_tune.go +++ b/pkg/transformers/integration_tests/vat_tune.go @@ -35,13 +35,13 @@ var _ = Describe("VatTune LogNoteTransformer", func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockchain, err := getBlockChain(rpcClient, ethClient) + blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockchain.Node()) + db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber) + err = persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ @@ -50,7 +50,7 @@ var _ = Describe("VatTune LogNoteTransformer", func() { Converter: &vat_tune.VatTuneConverter{}, Repository: &vat_tune.VatTuneRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockchain) + transformer := initializer.NewLogNoteTransformer(db, blockChain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 34e398f8..d3d1f9bf 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -39,6 +39,12 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i return fmt.Errorf("model of type %T, not %T", model, PitFileDebtCeilingModel{}) } + err = shared.ValidateHeaderConsistency(headerID, pitFileDC.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index c186a638..c38eca10 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -39,6 +39,12 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface return fmt.Errorf("model of type %T, not %T", model, PitFileIlkModel{}) } + err = shared.ValidateHeaderConsistency(headerID, pitFileIlk.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, diff --git a/pkg/transformers/pit_file/stability_fee/repository.go b/pkg/transformers/pit_file/stability_fee/repository.go index 43ec8b3f..17c2063a 100644 --- a/pkg/transformers/pit_file/stability_fee/repository.go +++ b/pkg/transformers/pit_file/stability_fee/repository.go @@ -39,6 +39,12 @@ func (repository PitFileStabilityFeeRepository) Create(headerID int64, models [] return fmt.Errorf("model of type %T, not %T", model, PitFileStabilityFeeModel{}) } + err = shared.ValidateHeaderConsistency(headerID, pitFileSF.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.pit_file_stability_fee (header_id, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6)`, diff --git a/pkg/transformers/price_feeds/repository.go b/pkg/transformers/price_feeds/repository.go index 59dbab8c..1c110259 100644 --- a/pkg/transformers/price_feeds/repository.go +++ b/pkg/transformers/price_feeds/repository.go @@ -38,6 +38,12 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{ return fmt.Errorf("model of type %T, not %T", model, PriceFeedModel{}) } + err = shared.ValidateHeaderConsistency(headerID, priceUpdate.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log) VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw) if err != nil { diff --git a/pkg/transformers/shared/constants/checked_headers.go b/pkg/transformers/shared/constants/checked_headers.go index 0e17c648..5ffe031d 100644 --- a/pkg/transformers/shared/constants/checked_headers.go +++ b/pkg/transformers/shared/constants/checked_headers.go @@ -20,7 +20,7 @@ var ( PitFileStabilityFeeChecked = "pit_file_stability_fee_checked" PriceFeedsChecked = "price_feeds_checked" TendChecked = "tend_checked" - VatFluxChecked = "vat_flux_checked" + VatFluxChecked = "vat_flux_checked" VatFoldChecked = "vat_fold_checked" VatGrabChecked = "vat_grab_checked" VatHealChecked = "vat_heal_checked" diff --git a/pkg/transformers/shared/constants/label.go b/pkg/transformers/shared/constants/label.go index c86e86c1..5434425b 100644 --- a/pkg/transformers/shared/constants/label.go +++ b/pkg/transformers/shared/constants/label.go @@ -64,4 +64,4 @@ func AllTransformerLabels() []string { VatTuneLabel, VowFlogLabel, } -} \ No newline at end of file +} diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index d74e09e5..10303298 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -2,10 +2,41 @@ package shared import ( "database/sql" + "encoding/json" + "errors" + "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) +var ErrHeaderMismatch = errors.New("log's block hash does not match associated header's block hash") + +func ValidateHeaderConsistency(headerID int64, rawLog []byte, db *postgres.DB) error { + var log types.Log + err := json.Unmarshal(rawLog, &log) + if err != nil { + return err + } + var hash string + err = db.Get(&hash, `SELECT hash FROM public.headers WHERE id = $1`, headerID) + if err != nil { + return err + } + if hash != log.BlockHash.String() { + err = deleteHeader(headerID, db) + if err != nil { + return err + } + return ErrHeaderMismatch + } + return nil +} + +func deleteHeader(headerID int64, db *postgres.DB) error { + _, err := db.Exec(`DELETE FROM public.headers WHERE id = $1`, headerID) + return err +} + func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { _, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2) diff --git a/pkg/transformers/tend/converter.go b/pkg/transformers/tend/converter.go index ba46ee9d..632d718b 100644 --- a/pkg/transformers/tend/converter.go +++ b/pkg/transformers/tend/converter.go @@ -46,11 +46,10 @@ func (TendConverter) ToModels(ethLogs []types.Log) (results []interface{}, err e transactionIndex := ethLog.TxIndex logIndex := ethLog.Index - rawJson, err := json.Marshal(ethLog) + rawLog, err := json.Marshal(ethLog) if err != nil { return nil, err } - raw := string(rawJson) model := TendModel{ BidId: bidId.String(), @@ -60,7 +59,7 @@ func (TendConverter) ToModels(ethLogs []types.Log) (results []interface{}, err e Tic: tic, LogIndex: logIndex, TransactionIndex: transactionIndex, - Raw: raw, + Raw: rawLog, } results = append(results, model) } diff --git a/pkg/transformers/tend/model.go b/pkg/transformers/tend/model.go index 4a8c6b43..8dedcadc 100644 --- a/pkg/transformers/tend/model.go +++ b/pkg/transformers/tend/model.go @@ -22,5 +22,5 @@ type TendModel struct { Tic string LogIndex uint `db:"log_idx"` TransactionIndex uint `db:"tx_idx"` - Raw string `db:"raw_log"` + Raw []byte `db:"raw_log"` } diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 0b0bcd72..91437658 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -26,7 +26,7 @@ type TendRepository struct { db *postgres.DB } -func (repository TendRepository) Create(headerId int64, models []interface{}) error { +func (repository TendRepository) Create(headerID int64, models []interface{}) error { tx, err := repository.db.Begin() if err != nil { return err @@ -39,10 +39,16 @@ func (repository TendRepository) Create(headerId int64, models []interface{}) er return fmt.Errorf("model of type %T, not %T", model, TendModel{}) } + err = shared.ValidateHeaderConsistency(headerID, tend.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9)`, - headerId, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tend.Tic, tend.LogIndex, tend.TransactionIndex, tend.Raw, + headerID, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tend.Tic, tend.LogIndex, tend.TransactionIndex, tend.Raw, ) if err != nil { @@ -51,7 +57,7 @@ func (repository TendRepository) Create(headerId int64, models []interface{}) er } } - err = shared.MarkHeaderCheckedInTransaction(headerId, tx, constants.TendChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.TendChecked) if err != nil { tx.Rollback() return err diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index 8cd61f2f..79465cee 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -78,7 +78,7 @@ var _ = Describe("TendRepository", func() { Expect(dbResult.Tic).To(Equal(test_data.TendModel.Tic)) Expect(dbResult.LogIndex).To(Equal(test_data.TendModel.LogIndex)) Expect(dbResult.TransactionIndex).To(Equal(test_data.TendModel.TransactionIndex)) - Expect(dbResult.Raw).To(MatchJSON(test_data.RawLogNoteJson)) + Expect(dbResult.Raw).To(MatchJSON(test_data.TendModel.Raw)) }) }) diff --git a/pkg/transformers/test_data/bite.go b/pkg/transformers/test_data/bite.go index 9ed56039..f82b2456 100644 --- a/pkg/transformers/test_data/bite.go +++ b/pkg/transformers/test_data/bite.go @@ -19,13 +19,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/bite" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "math/big" ) var ( - TemporaryBiteBlockHash = common.HexToHash("0xd130caaccc9203ca63eb149faeb013aed21f0317ce23489c0486da2f9adcd0eb") TemporaryBiteBlockNumber = int64(26) TemporaryBiteData = "0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000005" TemporaryBiteTransaction = "0x5c698f13940a2153440c6d19660878bc90219d9298fdcf37365aa8d88d40fc42" @@ -38,7 +38,6 @@ var ( biteFlip = big.NewInt(4) biteIArt = big.NewInt(5) biteRawJson, _ = json.Marshal(EthBiteLog) - biteRawString = string(biteRawJson) biteIlk = [32]byte{69, 84, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} biteLad = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 180, 20, 126, 218, 128, 254, 199, 18, 42, 225, 109, 162, 71, 156, 189, 127, 251} biteIlkString = "ETH" @@ -56,7 +55,7 @@ var EthBiteLog = types.Log{ BlockNumber: uint64(TemporaryBiteBlockNumber), TxHash: common.HexToHash(TemporaryBiteTransaction), TxIndex: 111, - BlockHash: TemporaryBiteBlockHash, + BlockHash: fakes.FakeHash, Index: 7, Removed: false, } @@ -84,5 +83,5 @@ var BiteModel = bite.BiteModel{ NFlip: biteFlip.String(), LogIndex: EthBiteLog.Index, TransactionIndex: EthBiteLog.TxIndex, - Raw: biteRawString, + Raw: biteRawJson, } diff --git a/pkg/transformers/test_data/cat_file.go b/pkg/transformers/test_data/cat_file.go index 4e12c4d2..4c501647 100644 --- a/pkg/transformers/test_data/cat_file.go +++ b/pkg/transformers/test_data/cat_file.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/chop_lump" "github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/flip" "github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/pit_vow" @@ -37,7 +38,7 @@ var EthCatFileChopLog = types.Log{ BlockNumber: 110, TxHash: common.HexToHash("0xe32dfe6afd7ea28475569756fc30f0eea6ad4cfd32f67436ff1d1c805e4382df"), TxIndex: 13, - BlockHash: common.HexToHash("0x2764998a4e048d4c4ba45ea40fd5efaa8e2d4f1dd2b15425a6c6a3dea7f1064a"), + BlockHash: fakes.FakeHash, Index: 1, Removed: false, } @@ -63,7 +64,7 @@ var EthCatFileLumpLog = types.Log{ BlockNumber: 110, TxHash: common.HexToHash("0xe32dfe6afd7ea28475569756fc30f0eea6ad4cfd32f67436ff1d1c805e4382df"), TxIndex: 15, - BlockHash: common.HexToHash("0x2764998a4e048d4c4ba45ea40fd5efaa8e2d4f1dd2b15425a6c6a3dea7f1064a"), + BlockHash: fakes.FakeHash, Index: 3, Removed: false, } @@ -89,7 +90,7 @@ var EthCatFileFlipLog = types.Log{ BlockNumber: 88, TxHash: common.HexToHash("0xc71ef3e9999595913d31e89446cab35319bd4289520e55611a1b42fc2a8463b6"), TxIndex: 12, - BlockHash: common.HexToHash("0xe5fcc1b65dd901e003e3768c1d4ce58d72f5f3a31e6a5d27d9cbdc7dca4bb405"), + BlockHash: fakes.FakeHash, Index: 1, Removed: false, } @@ -116,7 +117,7 @@ var EthCatFilePitVowLog = types.Log{ BlockNumber: 87, TxHash: common.HexToHash("0x6515c7dfe53f0ad83ce1173fa99032c24a07cfd8b5d5a1c1f80486c99dd52800"), TxIndex: 11, - BlockHash: common.HexToHash("0xae75936bc6b6a3383e8c991686747ef2221984b0ec8a5d4a6350989ec0ddbd67"), + BlockHash: fakes.FakeHash, Index: 2, Removed: false, } diff --git a/pkg/transformers/test_data/deal.go b/pkg/transformers/test_data/deal.go index 6dc42814..9e90f67e 100644 --- a/pkg/transformers/test_data/deal.go +++ b/pkg/transformers/test_data/deal.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) @@ -35,7 +36,7 @@ var DealLogNote = types.Log{ BlockNumber: 16, TxHash: common.HexToHash("0xc6ff19de9299e5b290ba2d52fdb4662360ca86376613d78ee546244866a0be2d"), TxIndex: 74, - BlockHash: common.HexToHash("0x6454844075164a1d264c86d2a2c31ac1b64eb2f4ebdbbaeb4d44388fdf74470b"), + BlockHash: fakes.FakeHash, Index: 75, Removed: false, } diff --git a/pkg/transformers/test_data/dent.go b/pkg/transformers/test_data/dent.go index 9c2cd887..0acd5972 100644 --- a/pkg/transformers/test_data/dent.go +++ b/pkg/transformers/test_data/dent.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/dent" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "strconv" @@ -27,7 +28,6 @@ import ( var ( DentData = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645ff3a382000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000098a7d9b8314c000000000000000000000000000000000000000000000000000029a2241af62c0000" DentTransactionHash = "0x5a210319fcd31eea5959fedb4a1b20881c21a21976e23ff19dff3b44cc1c71e8" - DentBlockHash = "0x105b771e04d7b8516f9291b1f006c46c09cfbff9efa8bc52498b171ff99d28b5" dentBidId = int64(1) dentLot = "11000000000000000000" dentBid = "3000000000000000000" @@ -48,7 +48,7 @@ var DentLog = types.Log{ BlockNumber: 15, TxHash: common.HexToHash(DentTransactionHash), TxIndex: 5, - BlockHash: common.HexToHash(DentBlockHash), + BlockHash: fakes.FakeHash, Index: 2, Removed: false, } diff --git a/pkg/transformers/test_data/drip_drip.go b/pkg/transformers/test_data/drip_drip.go index 2792c9b6..147d1056 100644 --- a/pkg/transformers/test_data/drip_drip.go +++ b/pkg/transformers/test_data/drip_drip.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) @@ -35,7 +36,7 @@ var EthDripDripLog = types.Log{ BlockNumber: 62, TxHash: common.HexToHash("0xa34fd5cfcb125ebfc81d33633495701b531753669712092bdb8aa6159a240040"), TxIndex: 10, - BlockHash: common.HexToHash("0x7a2aa72468986774d90dc8c80d436956a5b6a7e5acea430fb8a79f9217ef00a3"), + BlockHash: fakes.FakeHash, Index: 11, Removed: false, } diff --git a/pkg/transformers/test_data/drip_file.go b/pkg/transformers/test_data/drip_file.go index 54c06156..9be218cc 100644 --- a/pkg/transformers/test_data/drip_file.go +++ b/pkg/transformers/test_data/drip_file.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" ilk2 "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" @@ -38,7 +39,7 @@ var EthDripFileIlkLog = types.Log{ BlockNumber: 35, TxHash: common.HexToHash("0xa1c31b7e6389470902237161263558615e60b40f2e63060b2f4aeafe92d57e5f"), TxIndex: 12, - BlockHash: common.HexToHash("0x0188f3ee3cc05aa72457fa328e6a461de31e4cbd429fc37f9a52da4e9773c0b4"), + BlockHash: fakes.FakeHash, Index: 15, Removed: false, } @@ -65,7 +66,7 @@ var EthDripFileRepoLog = types.Log{ BlockNumber: 36, TxHash: common.HexToHash("0xeeaa16de1d91c239b66773e8c2116a26cfeaaf5d962b31466c9bf047a5caa20f"), TxIndex: 13, - BlockHash: common.HexToHash("0x89de4145ea8e34dfd9db9a7ea34f5be6f1f402e812fd389acca342513b353288"), + BlockHash: fakes.FakeHash, Index: 16, Removed: false, } @@ -91,7 +92,7 @@ var EthDripFileVowLog = types.Log{ BlockNumber: 51, TxHash: common.HexToHash("0x586e26b71b41fcd6905044dbe8f0cca300517542278f74a9b925c4f800fed85c"), TxIndex: 14, - BlockHash: common.HexToHash("0xbec69b1e93503679c9c006819477b86fe16aaff3a418da1e916c431b68be5522"), + BlockHash: fakes.FakeHash, Index: 17, Removed: false, } diff --git a/pkg/transformers/test_data/flap_kick.go b/pkg/transformers/test_data/flap_kick.go index f9513642..d7384adf 100644 --- a/pkg/transformers/test_data/flap_kick.go +++ b/pkg/transformers/test_data/flap_kick.go @@ -5,6 +5,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "math/big" @@ -21,7 +22,7 @@ var EthFlapKickLog = types.Log{ BlockNumber: 65, TxHash: common.HexToHash("0xee7930b76b6e93974bd3f37824644ae42a89a3887a1131a7bcb3267ab4dc0169"), TxIndex: 66, - BlockHash: common.HexToHash("0x051525dfb9a524739ae09ec240ecd5ddbc172ccdc2f65670786fa9f37806c378"), + BlockHash: fakes.FakeHash, Index: 67, Removed: false, } diff --git a/pkg/transformers/test_data/flip_kick.go b/pkg/transformers/test_data/flip_kick.go index 53cabd12..79183995 100644 --- a/pkg/transformers/test_data/flip_kick.go +++ b/pkg/transformers/test_data/flip_kick.go @@ -16,6 +16,7 @@ package test_data import ( "encoding/json" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "math/big" "time" @@ -28,26 +29,24 @@ import ( ) var ( - idString = "1" - id, _ = new(big.Int).SetString(idString, 10) - lotString = "100" - lot, _ = new(big.Int).SetString(lotString, 10) - bidString = "0" - bid = new(big.Int).SetBytes([]byte{0}) - gal = "0x07Fa9eF6609cA7921112231F8f195138ebbA2977" - end = int64(1535991025) - urn = [32]byte{115, 64, 224, 6, 244, 19, 91, 166, 151, 13, 67, 191, 67, 216, 141, 202, 212, 231, 168, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} - urnString = "0x7340e006f4135BA6970D43bf43d88DCAD4e7a8CA" - tabString = "50" - tab, _ = new(big.Int).SetString(tabString, 10) - rawLogJson, _ = json.Marshal(EthFlipKickLog) - rawLogString = string(rawLogJson) + idString = "1" + id, _ = new(big.Int).SetString(idString, 10) + lotString = "100" + lot, _ = new(big.Int).SetString(lotString, 10) + bidString = "0" + bid = new(big.Int).SetBytes([]byte{0}) + gal = "0x07Fa9eF6609cA7921112231F8f195138ebbA2977" + end = int64(1535991025) + urn = [32]byte{115, 64, 224, 6, 244, 19, 91, 166, 151, 13, 67, 191, 67, 216, 141, 202, 212, 231, 168, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + urnString = "0x7340e006f4135BA6970D43bf43d88DCAD4e7a8CA" + tabString = "50" + tab, _ = new(big.Int).SetString(tabString, 10) + rawLog, _ = json.Marshal(EthFlipKickLog) ) var ( flipKickTransactionHash = "0xd11ab35cfb1ad71f790d3dd488cc1a2046080e765b150e8997aa0200947d4a9b" flipKickData = "0x0000000000000000000000000000000000000000000000000000000000000064000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007fa9ef6609ca7921112231f8f195138ebba2977000000000000000000000000000000000000000000000000000000005b8d5cf10000000000000000000000000000000000000000000000000000000000000032" - flipKickBlockHash = "0x40fcad7863ab4bef421d638b7ad6116e81577f14a62ef847b07f8527944466fd" FlipKickBlockNumber = int64(10) ) @@ -62,7 +61,7 @@ var EthFlipKickLog = types.Log{ BlockNumber: uint64(FlipKickBlockNumber), TxHash: common.HexToHash(flipKickTransactionHash), TxIndex: 999, - BlockHash: common.HexToHash(flipKickBlockHash), + BlockHash: fakes.FakeHash, Index: 1, Removed: false, } @@ -90,7 +89,7 @@ var FlipKickModel = flip_kick.FlipKickModel{ Tab: tabString, TransactionIndex: EthFlipKickLog.TxIndex, LogIndex: EthFlipKickLog.Index, - Raw: rawLogString, + Raw: rawLog, } type FlipKickDBRow struct { diff --git a/pkg/transformers/test_data/flop_kick.go b/pkg/transformers/test_data/flop_kick.go index a265560c..452d0ec2 100644 --- a/pkg/transformers/test_data/flop_kick.go +++ b/pkg/transformers/test_data/flop_kick.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/flop_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "math/big" @@ -37,7 +38,7 @@ var ( BlockNumber: 19, TxHash: common.HexToHash("0xd8fd67b37a6aa64a3cef4937204765183b180d8dc92eecd0d233f445526d31b5"), TxIndex: flopTxIndex, - BlockHash: common.HexToHash("0x7891fe725691abc5b94113dc2bfa8c3509e1e1d0e0bb74289e3b996c30d00c8c"), + BlockHash: fakes.FakeHash, Index: 32, Removed: false, } diff --git a/pkg/transformers/test_data/frob.go b/pkg/transformers/test_data/frob.go index 2cd11d0e..4f5faf3d 100644 --- a/pkg/transformers/test_data/frob.go +++ b/pkg/transformers/test_data/frob.go @@ -19,13 +19,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "math/big" ) var ( - TemporaryFrobBlockHash = common.HexToHash("0x67ae45eace52de052a0fc58598974b101733f823fc191329ace7aded9a72b84b") TemporaryFrobBlockNumber = int64(13) TemporaryFrobData = "0x000000000000000000000000000000000000000000000000000000000000000f0000000000000000000000000000000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000019" TemporaryFrobTransaction = "0xbcff98316acb5732891d1a7e02f23ec12fbf8c231ca4b5530fa7a21c1e9b6aa9" @@ -38,7 +38,6 @@ var ( dart = big.NewInt(0).SetBytes([]byte{0}) iArt = big.NewInt(25) frobLad = [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 217, 34, 137, 65, 83, 190, 158, 239, 123, 114, 24, 220, 86, 93, 29, 12, 226, 160, 146} - gem, _ = big.NewInt(0).SetString("115792089237316195423570985008687907853269984665640564039457584007913129639926", 10) ink = big.NewInt(15) ilk = [32]byte{102, 97, 107, 101, 32, 105, 108, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} frobIlkString = "fake ilk" @@ -56,7 +55,7 @@ var EthFrobLog = types.Log{ BlockNumber: uint64(TemporaryFrobBlockNumber), TxHash: common.HexToHash(TemporaryFrobTransaction), TxIndex: 123, - BlockHash: TemporaryFrobBlockHash, + BlockHash: fakes.FakeHash, Index: 7, Removed: false, } diff --git a/pkg/transformers/test_data/pit_file.go b/pkg/transformers/test_data/pit_file.go index 772e1ae1..01881495 100644 --- a/pkg/transformers/test_data/pit_file.go +++ b/pkg/transformers/test_data/pit_file.go @@ -16,6 +16,7 @@ package test_data import ( "encoding/json" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "math/big" "github.com/ethereum/go-ethereum/common" @@ -40,7 +41,7 @@ var EthPitFileDebtCeilingLog = types.Log{ BlockNumber: 22, TxHash: common.HexToHash("0xd744878a0b6655e3ba729e1019f56b563b4a16750196b8ad6104f3977db43f42"), TxIndex: 333, - BlockHash: common.HexToHash("0xa54d9d99c315bea3dda7256a36e51773ed009a01c0859295c5382d4b83d7eeb9"), + BlockHash: fakes.FakeHash, Index: 15, Removed: false, } @@ -66,7 +67,7 @@ var EthPitFileIlkLog = types.Log{ BlockNumber: 11, TxHash: common.HexToHash("0x1ba8125f60fa045c85b35df3983bee37db8627fbc32e3442a5cf17c85bb83f09"), TxIndex: 111, - BlockHash: common.HexToHash("0x6dc284247c524b22b10a75ef1c9d1709a509208d04c15fa2b675a293db637d21"), + BlockHash: fakes.FakeHash, Index: 14, Removed: false, } @@ -93,7 +94,7 @@ var EthPitFileStabilityFeeLog = types.Log{ BlockNumber: 12, TxHash: common.HexToHash("0x78cdc62316ccf8e31515d09745cc724f557569f01a557d0d09b1066bf7079fd2"), TxIndex: 222, - BlockHash: common.HexToHash("0xe3d8e458421533170871b4033f978a3793ef10b7e33a9328a13c09e2fd90208d"), + BlockHash: fakes.FakeHash, Index: 13, Removed: false, } diff --git a/pkg/transformers/test_data/price_feed.go b/pkg/transformers/test_data/price_feed.go index e4ce5a5e..cab4dc5d 100644 --- a/pkg/transformers/test_data/price_feed.go +++ b/pkg/transformers/test_data/price_feed.go @@ -18,6 +18,7 @@ import ( "encoding/json" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) @@ -36,7 +37,7 @@ var EthPriceFeedLog = types.Log{ BlockNumber: blockNumber, TxHash: common.HexToHash("0xa51a50a2adbfba4e2ab3d72dfd67a21c769f1bc8d2b180663a15500a56cde58f"), TxIndex: txIndex, - BlockHash: common.HexToHash("0x27ecebbf69eefa3bb3cf65f472322a80ff4946653a50a2171dc605f49829467d"), + BlockHash: fakes.FakeHash, Index: 8, Removed: false, } diff --git a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go index 65ba206c..26b7b52d 100644 --- a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go +++ b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go @@ -15,6 +15,7 @@ package shared_behaviors import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -23,6 +24,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/test_config" "math/rand" @@ -101,6 +103,35 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) + Describe("when log's block hash does not match associated header's block hash", func() { + var ( + differentHeaderID int64 + err error + ) + + BeforeEach(func() { + differentHeader := fakes.FakeHeader + differentHeader.BlockNumber = fakes.FakeHeader.BlockNumber + 1 + differentHeader.Hash = common.BytesToHash(append(fakes.FakeHash.Bytes(), 1)).String() + differentHeaderID, err = headerRepository.CreateOrUpdateHeader(differentHeader) + Expect(err).NotTo(HaveOccurred()) + + err = repository.Create(differentHeaderID, []interface{}{logEventModel}) + }) + + It("returns error", func() { + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(shared.ErrHeaderMismatch)) + }) + + It("removes header", func() { + var headerIDs []int64 + err = db.Select(&headerIDs, `SELECT id FROM public.headers`) + Expect(err).NotTo(HaveOccurred()) + Expect(headerIDs).NotTo(ContainElement(differentHeaderID)) + }) + }) + It("allows for multiple log events of the same type in one transaction if they have different log indexes", func() { err = repository.Create(headerID, []interface{}{logEventModel}) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/test_data/tend.go b/pkg/transformers/test_data/tend.go index 7166af22..3ea850f5 100644 --- a/pkg/transformers/test_data/tend.go +++ b/pkg/transformers/test_data/tend.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" ) @@ -33,7 +34,6 @@ var ( tendGuy = "0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6" tendData = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000644b43ed12000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000049b9ca9a6943400000000000000000000000000000000000000000000000000000de0b6b3a7640000" tendTransactionHash = "0x7909c8793ded2b8348f5db623044fbc26bb7ab78ad5792897abdf68ddc1df63d" - tendBlockHash = "0xa8ea87147c0a68daeb6b1d9f8c0937ba975a650809cab80d19c969e8d0df452c" TendTic = "0" ) @@ -49,12 +49,12 @@ var TendLogNote = types.Log{ BlockNumber: 11, TxHash: common.HexToHash(tendTransactionHash), TxIndex: 10, - BlockHash: common.HexToHash(tendBlockHash), + BlockHash: fakes.FakeHash, Index: 1, Removed: false, } -var RawLogNoteJson, _ = json.Marshal(TendLogNote) +var rawTendLog, _ = json.Marshal(TendLogNote) var TendModel = tend.TendModel{ BidId: strconv.FormatInt(tendBidId, 10), Lot: tendLot, @@ -63,5 +63,5 @@ var TendModel = tend.TendModel{ Tic: TendTic, LogIndex: TendLogNote.Index, TransactionIndex: TendLogNote.TxIndex, - Raw: string(RawLogNoteJson), + Raw: rawTendLog, } diff --git a/pkg/transformers/test_data/vat_flux.go b/pkg/transformers/test_data/vat_flux.go index 619cdd69..b311ac74 100644 --- a/pkg/transformers/test_data/vat_flux.go +++ b/pkg/transformers/test_data/vat_flux.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" ) @@ -37,7 +38,7 @@ var VatFluxLog = types.Log{ BlockNumber: 23, TxHash: common.HexToHash("0xf98681bab9b8c75bd8aa4a7d0a8142ff527c5ea8fa54f3c2835d4533838b2e6f"), TxIndex: 0, - BlockHash: common.HexToHash("0xc3fe212ad4f81ade1265af6de2b4bb50d962b1a4db06aabc982e7f9cb0972c2d"), + BlockHash: fakes.FakeHash, Index: 3, Removed: false, } diff --git a/pkg/transformers/test_data/vat_fold.go b/pkg/transformers/test_data/vat_fold.go index fd6ccbd1..49634f07 100644 --- a/pkg/transformers/test_data/vat_fold.go +++ b/pkg/transformers/test_data/vat_fold.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" ) @@ -37,7 +38,7 @@ var EthVatFoldLog = types.Log{ BlockNumber: 8940380, TxHash: common.HexToHash("0xfb37b7a88aa8ad14538d1e244a55939fa07c1828e5ca8168bf4edd56f5fc4d57"), TxIndex: 8, - BlockHash: common.HexToHash("0xf43ab2fd3cf0a7e08fcc16ec17bbc7f67417a37a4cd978d1d7ca32130c7f64be"), + BlockHash: fakes.FakeHash, Index: 5, Removed: false, } diff --git a/pkg/transformers/test_data/vat_grab.go b/pkg/transformers/test_data/vat_grab.go index faf19175..4c78a113 100644 --- a/pkg/transformers/test_data/vat_grab.go +++ b/pkg/transformers/test_data/vat_grab.go @@ -5,6 +5,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" ) @@ -21,7 +22,7 @@ var EthVatGrabLog = types.Log{ BlockNumber: 23, TxHash: common.HexToHash("0x7cb84c750ce4985f7811abf641d52ffcb35306d943081475226484cf1470c6fa"), TxIndex: 4, - BlockHash: common.HexToHash("0xf5a367d560e14c4658ef85e4877e08b5560a4773b69b39f6b8025910b666fade"), + BlockHash: fakes.FakeHash, Index: 5, Removed: false, } diff --git a/pkg/transformers/test_data/vat_heal.go b/pkg/transformers/test_data/vat_heal.go index 900fc0ad..c5350ba3 100644 --- a/pkg/transformers/test_data/vat_heal.go +++ b/pkg/transformers/test_data/vat_heal.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_heal" ) @@ -35,7 +36,7 @@ var VatHealLog = types.Log{ BlockNumber: 10, TxHash: common.HexToHash("0x991b8079b1333024000dcaf2b00c24c5db0315e112a4ac4d912aa96a602e12b9"), TxIndex: 2, - BlockHash: common.HexToHash("0x0c54bdadb149691a103cfa5ad57329bda1e0fe55f1ba1ce5dd49dc1ecfe03daa"), + BlockHash: fakes.FakeHash, Index: 3, Removed: false, } diff --git a/pkg/transformers/test_data/vat_init.go b/pkg/transformers/test_data/vat_init.go index c6a61838..f267cf12 100644 --- a/pkg/transformers/test_data/vat_init.go +++ b/pkg/transformers/test_data/vat_init.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" ) @@ -37,7 +38,7 @@ var EthVatInitLog = types.Log{ BlockNumber: 24, TxHash: common.HexToHash("0xe8f39fbb7fea3621f543868f19b1114e305aff6a063a30d32835ff1012526f91"), TxIndex: 7, - BlockHash: common.HexToHash("0xe3dd2e05bd8b92833e20ed83e2171bbc06a9ec823232eca1730a807bd8f5edc0"), + BlockHash: fakes.FakeHash, Index: 8, Removed: false, } diff --git a/pkg/transformers/test_data/vat_move.go b/pkg/transformers/test_data/vat_move.go index b7ef7c1c..36682275 100644 --- a/pkg/transformers/test_data/vat_move.go +++ b/pkg/transformers/test_data/vat_move.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_move" ) @@ -37,7 +38,7 @@ var EthVatMoveLog = types.Log{ BlockNumber: 10, TxHash: common.HexToHash("0xe8f39fbb7fea3621f543868f19b1114e305aff6a063a30d32835ff1012526f91"), TxIndex: 7, - BlockHash: common.HexToHash("0xe3dd2e05bd8b92833e20ed83e2171bbc06a9ec823232eca1730a807bd8f5edc0"), + BlockHash: fakes.FakeHash, Index: 8, Removed: false, } diff --git a/pkg/transformers/test_data/vat_slip.go b/pkg/transformers/test_data/vat_slip.go index 5870f3d5..0acd1771 100644 --- a/pkg/transformers/test_data/vat_slip.go +++ b/pkg/transformers/test_data/vat_slip.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_slip" ) @@ -37,7 +38,7 @@ var EthVatSlipLog = types.Log{ BlockNumber: 10, TxHash: common.HexToHash("0xb114ba306c80c86d51bdbf4a5ac8ed151020cd81b70cfa1dc9822f4a1f73930b"), TxIndex: 3, - BlockHash: common.HexToHash("0x34b7e5ddb3be73257a5a0087f10b8bf68d4df5c8831ec04c63ecae4094de72ad"), + BlockHash: fakes.FakeHash, Index: 2, Removed: false, } diff --git a/pkg/transformers/test_data/vat_toll.go b/pkg/transformers/test_data/vat_toll.go index d4d6f230..55ca700e 100644 --- a/pkg/transformers/test_data/vat_toll.go +++ b/pkg/transformers/test_data/vat_toll.go @@ -5,6 +5,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_toll" "math/big" @@ -22,7 +23,7 @@ var EthVatTollLog = types.Log{ BlockNumber: 21, TxHash: common.HexToHash("0x0d59cb158b033ffdfb9a021d1e80bfbbcd99594c62c501897ccee446bcd33828"), TxIndex: 2, - BlockHash: common.HexToHash("0xdbfda0ecb4ac6267c50be21db97874dc1203c22033e19733a6f67ca00b51dfc5"), + BlockHash: fakes.FakeHash, Index: 4, Removed: false, } diff --git a/pkg/transformers/test_data/vat_tune.go b/pkg/transformers/test_data/vat_tune.go index ce36c826..d2ac9ab5 100644 --- a/pkg/transformers/test_data/vat_tune.go +++ b/pkg/transformers/test_data/vat_tune.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_tune" ) @@ -24,7 +25,7 @@ var EthVatTuneLog = types.Log{ BlockNumber: 8761670, TxHash: common.HexToHash("0x95eb3d6cbd83032efa29714d4a391ce163d7d215db668aadd7d33dd5c20b1ec7"), TxIndex: 0, - BlockHash: common.HexToHash("0xd35188df40a741f4bf4ccacc6be0154e338652660d0b56cc9058c1bca53b09ee"), + BlockHash: fakes.FakeHash, Index: 6, Removed: false, } diff --git a/pkg/transformers/test_data/vow_flog.go b/pkg/transformers/test_data/vow_flog.go index 8fee35a4..53309689 100644 --- a/pkg/transformers/test_data/vow_flog.go +++ b/pkg/transformers/test_data/vow_flog.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog" ) @@ -35,7 +36,7 @@ var EthVowFlogLog = types.Log{ BlockNumber: 11, TxHash: common.HexToHash("0x47ffd75c1cda1d5c08219755743663a3790e4f5ae9e1fcb85bb3fe0d74bb7109"), TxIndex: 4, - BlockHash: common.HexToHash("0x6fd1980ab4af87ce371599a3ef37d4f8fba03718a1f06d127a245b068492c65d"), + BlockHash: fakes.FakeHash, Index: 3, Removed: false, } diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index af77d2ae..2aa28399 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -26,7 +26,7 @@ type VatFluxRepository struct { db *postgres.DB } -func (repository VatFluxRepository) Create(headerId int64, models []interface{}) error { +func (repository VatFluxRepository) Create(headerID int64, models []interface{}) error { tx, err := repository.db.Begin() if err != nil { return err @@ -39,16 +39,22 @@ func (repository VatFluxRepository) Create(headerId int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatFluxModel{}) } - _, err := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log) + err = shared.ValidateHeaderConsistency(headerID, vatFlux.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + + _, err = tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8)`, - headerId, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw) + headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw) if err != nil { tx.Rollback() return err } } - err = shared.MarkHeaderCheckedInTransaction(headerId, tx, constants.VatFluxChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFluxChecked) if err != nil { tx.Rollback() return err diff --git a/pkg/transformers/vat_flux/repository_test.go b/pkg/transformers/vat_flux/repository_test.go index 0289b59b..14b3f403 100644 --- a/pkg/transformers/vat_flux/repository_test.go +++ b/pkg/transformers/vat_flux/repository_test.go @@ -21,11 +21,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" "github.com/vulcanize/vulcanizedb/test_config" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) var _ = Describe("VatFlux Repository", func() { diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index bb14f172..cd6c18b0 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -39,6 +39,12 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatFoldModel{}) } + err = shared.ValidateHeaderConsistency(headerID, vatFold.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go index fe8e39a5..c613702b 100644 --- a/pkg/transformers/vat_grab/repository.go +++ b/pkg/transformers/vat_grab/repository.go @@ -24,6 +24,13 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{}) tx.Rollback() return fmt.Errorf("model of type %T, not %T", model, VatGrabModel{}) } + + err = shared.ValidateHeaderConsistency(headerID, vatGrab.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, diff --git a/pkg/transformers/vat_heal/repository.go b/pkg/transformers/vat_heal/repository.go index 84d2a980..f7658dd3 100644 --- a/pkg/transformers/vat_heal/repository.go +++ b/pkg/transformers/vat_heal/repository.go @@ -31,7 +31,7 @@ func (repository *VatHealRepository) SetDB(db *postgres.DB) { repository.db = db } -func (repository VatHealRepository) Create(headerId int64, models []interface{}) error { +func (repository VatHealRepository) Create(headerID int64, models []interface{}) error { tx, err := repository.db.Begin() if err != nil { return err @@ -43,16 +43,23 @@ func (repository VatHealRepository) Create(headerId int64, models []interface{}) tx.Rollback() return fmt.Errorf("model of type %T, not %T", model, VatHealModel{}) } + + err = shared.ValidateHeaderConsistency(headerID, vatHeal.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err := tx.Exec(`INSERT INTO maker.vat_heal (header_id, urn, v, rad, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, - headerId, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw) + headerID, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw) if err != nil { tx.Rollback() return err } } - err = shared.MarkHeaderCheckedInTransaction(headerId, tx, constants.VatHealChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatHealChecked) if err != nil { tx.Rollback() return err diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index cee9e44a..353dba45 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -39,6 +39,12 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatInitModel{}) } + err = shared.ValidateHeaderConsistency(headerID, vatInit.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5)`, diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index f47db4d6..c114c8b4 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -39,6 +39,12 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatMoveModel{}) } + err = shared.ValidateHeaderConsistency(headerID, vatMove.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT INTO maker.vat_move (header_id, src, dst, rad, log_idx, tx_idx, raw_log) VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, diff --git a/pkg/transformers/vat_slip/repository.go b/pkg/transformers/vat_slip/repository.go index 1c6b8d06..bda0c242 100644 --- a/pkg/transformers/vat_slip/repository.go +++ b/pkg/transformers/vat_slip/repository.go @@ -24,6 +24,12 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatSlipModel{}) } + err = shared.ValidateHeaderConsistency(headerID, vatSlip.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go index d115c742..d01d6941 100644 --- a/pkg/transformers/vat_toll/repository.go +++ b/pkg/transformers/vat_toll/repository.go @@ -23,6 +23,13 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{}) tx.Rollback() return fmt.Errorf("model of type %T, not %T", model, VatTollModel{}) } + + err = shared.ValidateHeaderConsistency(headerID, vatToll.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.vat_toll (header_id, ilk, urn, take, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go index 191dbdfd..0ea2d589 100644 --- a/pkg/transformers/vat_tune/repository.go +++ b/pkg/transformers/vat_tune/repository.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) type VatTuneRepository struct { @@ -21,6 +22,13 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) tx.Rollback() return fmt.Errorf("model of type %T, not %T", model, VatTuneModel{}) } + + err = shared.ValidateHeaderConsistency(headerID, vatTune.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, @@ -31,6 +39,7 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) return err } } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_tune_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO diff --git a/pkg/transformers/vow_flog/repository.go b/pkg/transformers/vow_flog/repository.go index 0754fd5f..2edb1359 100644 --- a/pkg/transformers/vow_flog/repository.go +++ b/pkg/transformers/vow_flog/repository.go @@ -39,6 +39,12 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VowFlogModel{}) } + err = shared.ValidateHeaderConsistency(headerID, flog.Raw, repository.db) + if err != nil { + tx.Rollback() + return err + } + _, err = tx.Exec( `INSERT into maker.vow_flog (header_id, era, log_idx, tx_idx, raw_log) VALUES($1, $2::NUMERIC, $3, $4, $5)`,