From bf44ec614a754f2851fb0de63f67b71ed4957671 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 2 Oct 2018 17:25:38 -0500 Subject: [PATCH] Log checked headers for MCD events (#43) * Log checked headers for Bite events * Log checked headers for Drip drip events * Log checked headers for Drip file events - mark headers checked for 3 implementations of file on the Drip contract * Log checked headers for Frob frob events * Log checked headers for Pit file events - mark headers checked for 3 implementations of file on the Pit contract * Log checked headers for Vat init events --- ...64962_add_bite_to_checked_headers.down.sql | 2 + ...8064962_add_bite_to_checked_headers.up.sql | 2 + ..._add_drip_drip_to_checked_headers.down.sql | 2 + ...90_add_drip_drip_to_checked_headers.up.sql | 2 + ..._add_drip_file_to_checked_headers.down.sql | 6 + ...24_add_drip_file_to_checked_headers.up.sql | 6 + ...75501_add_frob_to_checked_headers.down.sql | 2 + ...8075501_add_frob_to_checked_headers.up.sql | 2 + ...2_add_pit_file_to_checked_headers.down.sql | 6 + ...432_add_pit_file_to_checked_headers.up.sql | 6 + ...8_add_vat_init_to_checked_headers.down.sql | 2 + ...658_add_vat_init_to_checked_headers.up.sql | 2 + db/schema.sql | 12 +- pkg/transformers/bite/converter.go | 103 ++++++++------- pkg/transformers/bite/converter_test.go | 14 +- pkg/transformers/bite/repository.go | 47 +++++-- pkg/transformers/bite/repository_test.go | 92 +++++++++---- pkg/transformers/bite/transformer.go | 26 ++-- pkg/transformers/bite/transfromer_test.go | 27 +++- pkg/transformers/drip_drip/converter.go | 32 +++-- pkg/transformers/drip_drip/converter_test.go | 6 +- pkg/transformers/drip_drip/repository.go | 43 ++++-- pkg/transformers/drip_drip/repository_test.go | 121 +++++++++++++---- pkg/transformers/drip_drip/transformer.go | 16 ++- .../drip_drip/transformer_test.go | 40 +++++- pkg/transformers/drip_file/ilk/converter.go | 42 +++--- .../drip_file/ilk/converter_test.go | 9 +- pkg/transformers/drip_file/ilk/repository.go | 43 ++++-- .../drip_file/ilk/repository_test.go | 119 ++++++++++++++--- pkg/transformers/drip_file/ilk/transformer.go | 16 ++- .../drip_file/ilk/transformer_test.go | 41 +++++- pkg/transformers/drip_file/repo/converter.go | 36 +++-- .../drip_file/repo/converter_test.go | 7 +- pkg/transformers/drip_file/repo/repository.go | 43 ++++-- .../drip_file/repo/repository_test.go | 121 +++++++++++++---- .../drip_file/repo/transformer.go | 16 ++- .../drip_file/repo/transformer_test.go | 40 +++++- pkg/transformers/drip_file/vow/converter.go | 36 +++-- .../drip_file/vow/converter_test.go | 7 +- pkg/transformers/drip_file/vow/repository.go | 43 ++++-- .../drip_file/vow/repository_test.go | 125 ++++++++++++++---- pkg/transformers/drip_file/vow/transformer.go | 16 ++- .../drip_file/vow/transformer_test.go | 40 +++++- pkg/transformers/frob/converter.go | 67 ++++++---- pkg/transformers/frob/converter_test.go | 11 +- pkg/transformers/frob/repository.go | 39 +++++- pkg/transformers/frob/repository_test.go | 121 +++++++++++++---- pkg/transformers/frob/transformer.go | 24 ++-- pkg/transformers/frob/transformer_test.go | 47 ++++++- .../pit_file/debt_ceiling/converter.go | 38 +++--- .../pit_file/debt_ceiling/converter_test.go | 9 +- .../pit_file/debt_ceiling/repository.go | 44 ++++-- .../pit_file/debt_ceiling/repository_test.go | 121 +++++++++++++---- .../pit_file/debt_ceiling/transformer.go | 16 ++- .../pit_file/debt_ceiling/transformer_test.go | 40 +++++- pkg/transformers/pit_file/ilk/converter.go | 44 +++--- .../pit_file/ilk/converter_test.go | 9 +- pkg/transformers/pit_file/ilk/repository.go | 44 ++++-- .../pit_file/ilk/repository_test.go | 121 +++++++++++++---- pkg/transformers/pit_file/ilk/transformer.go | 16 ++- .../pit_file/ilk/transformer_test.go | 40 +++++- .../pit_file/stability_fee/converter.go | 38 +++--- .../pit_file/stability_fee/converter_test.go | 7 +- .../pit_file/stability_fee/repository.go | 44 ++++-- .../pit_file/stability_fee/repository_test.go | 121 +++++++++++++---- .../pit_file/stability_fee/transformer.go | 16 ++- .../stability_fee/transformer_test.go | 40 +++++- .../test_data/mocks/bite/converter.go | 15 +-- .../test_data/mocks/bite/repository.go | 37 ++++-- .../test_data/mocks/drip_drip/converter.go | 10 +- .../test_data/mocks/drip_drip/repository.go | 34 +++-- .../mocks/drip_file/ilk/converter.go | 8 +- .../mocks/drip_file/ilk/repository.go | 34 +++-- .../mocks/drip_file/repo/converter.go | 8 +- .../mocks/drip_file/repo/repository.go | 34 +++-- .../mocks/drip_file/vow/converter.go | 8 +- .../mocks/drip_file/vow/repository.go | 34 +++-- .../test_data/mocks/frob/converter.go | 28 ++-- .../test_data/mocks/frob/repository.go | 34 +++-- .../mocks/pit_file/debt_ceiling/converter.go | 8 +- .../mocks/pit_file/debt_ceiling/repository.go | 35 +++-- .../test_data/mocks/pit_file/ilk/converter.go | 8 +- .../mocks/pit_file/ilk/repository.go | 35 +++-- .../mocks/pit_file/stability_fee/converter.go | 9 +- .../pit_file/stability_fee/repository.go | 37 ++++-- .../test_data/mocks/vat_init/converter.go | 8 +- .../test_data/mocks/vat_init/repository.go | 35 +++-- pkg/transformers/vat_init/converter.go | 32 +++-- pkg/transformers/vat_init/converter_test.go | 7 +- pkg/transformers/vat_init/repository.go | 37 ++++-- pkg/transformers/vat_init/repository_test.go | 84 ++++++++++-- pkg/transformers/vat_init/transformer.go | 16 ++- pkg/transformers/vat_init/transformer_test.go | 40 +++++- 93 files changed, 2439 insertions(+), 772 deletions(-) create mode 100644 db/migrations/1538064962_add_bite_to_checked_headers.down.sql create mode 100644 db/migrations/1538064962_add_bite_to_checked_headers.up.sql create mode 100644 db/migrations/1538067790_add_drip_drip_to_checked_headers.down.sql create mode 100644 db/migrations/1538067790_add_drip_drip_to_checked_headers.up.sql create mode 100644 db/migrations/1538070524_add_drip_file_to_checked_headers.down.sql create mode 100644 db/migrations/1538070524_add_drip_file_to_checked_headers.up.sql create mode 100644 db/migrations/1538075501_add_frob_to_checked_headers.down.sql create mode 100644 db/migrations/1538075501_add_frob_to_checked_headers.up.sql create mode 100644 db/migrations/1538078432_add_pit_file_to_checked_headers.down.sql create mode 100644 db/migrations/1538078432_add_pit_file_to_checked_headers.up.sql create mode 100644 db/migrations/1538082658_add_vat_init_to_checked_headers.down.sql create mode 100644 db/migrations/1538082658_add_vat_init_to_checked_headers.up.sql diff --git a/db/migrations/1538064962_add_bite_to_checked_headers.down.sql b/db/migrations/1538064962_add_bite_to_checked_headers.down.sql new file mode 100644 index 00000000..673e1677 --- /dev/null +++ b/db/migrations/1538064962_add_bite_to_checked_headers.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + DROP COLUMN bite_checked; \ No newline at end of file diff --git a/db/migrations/1538064962_add_bite_to_checked_headers.up.sql b/db/migrations/1538064962_add_bite_to_checked_headers.up.sql new file mode 100644 index 00000000..07209852 --- /dev/null +++ b/db/migrations/1538064962_add_bite_to_checked_headers.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + ADD COLUMN bite_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/1538067790_add_drip_drip_to_checked_headers.down.sql b/db/migrations/1538067790_add_drip_drip_to_checked_headers.down.sql new file mode 100644 index 00000000..0da752a1 --- /dev/null +++ b/db/migrations/1538067790_add_drip_drip_to_checked_headers.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + DROP COLUMN drip_drip_checked; \ No newline at end of file diff --git a/db/migrations/1538067790_add_drip_drip_to_checked_headers.up.sql b/db/migrations/1538067790_add_drip_drip_to_checked_headers.up.sql new file mode 100644 index 00000000..852e61b5 --- /dev/null +++ b/db/migrations/1538067790_add_drip_drip_to_checked_headers.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + ADD COLUMN drip_drip_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/1538070524_add_drip_file_to_checked_headers.down.sql b/db/migrations/1538070524_add_drip_file_to_checked_headers.down.sql new file mode 100644 index 00000000..8f8de9f9 --- /dev/null +++ b/db/migrations/1538070524_add_drip_file_to_checked_headers.down.sql @@ -0,0 +1,6 @@ +ALTER TABLE public.checked_headers + DROP COLUMN drip_file_ilk_checked; +ALTER TABLE public.checked_headers + DROP COLUMN drip_file_repo_checked; +ALTER TABLE public.checked_headers + DROP COLUMN drip_file_vow_checked; \ No newline at end of file diff --git a/db/migrations/1538070524_add_drip_file_to_checked_headers.up.sql b/db/migrations/1538070524_add_drip_file_to_checked_headers.up.sql new file mode 100644 index 00000000..1f2c454e --- /dev/null +++ b/db/migrations/1538070524_add_drip_file_to_checked_headers.up.sql @@ -0,0 +1,6 @@ +ALTER TABLE public.checked_headers + ADD COLUMN drip_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE public.checked_headers + ADD COLUMN drip_file_repo_checked BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE public.checked_headers + ADD COLUMN drip_file_vow_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/1538075501_add_frob_to_checked_headers.down.sql b/db/migrations/1538075501_add_frob_to_checked_headers.down.sql new file mode 100644 index 00000000..8549624c --- /dev/null +++ b/db/migrations/1538075501_add_frob_to_checked_headers.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + DROP COLUMN frob_checked; \ No newline at end of file diff --git a/db/migrations/1538075501_add_frob_to_checked_headers.up.sql b/db/migrations/1538075501_add_frob_to_checked_headers.up.sql new file mode 100644 index 00000000..b6332392 --- /dev/null +++ b/db/migrations/1538075501_add_frob_to_checked_headers.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + ADD COLUMN frob_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/1538078432_add_pit_file_to_checked_headers.down.sql b/db/migrations/1538078432_add_pit_file_to_checked_headers.down.sql new file mode 100644 index 00000000..3629d652 --- /dev/null +++ b/db/migrations/1538078432_add_pit_file_to_checked_headers.down.sql @@ -0,0 +1,6 @@ +ALTER TABLE public.checked_headers + DROP COLUMN pit_file_debt_ceiling_checked; +ALTER TABLE public.checked_headers + DROP COLUMN pit_file_ilk_checked; +ALTER TABLE public.checked_headers + DROP COLUMN pit_file_stability_fee_checked; \ No newline at end of file diff --git a/db/migrations/1538078432_add_pit_file_to_checked_headers.up.sql b/db/migrations/1538078432_add_pit_file_to_checked_headers.up.sql new file mode 100644 index 00000000..c90e2488 --- /dev/null +++ b/db/migrations/1538078432_add_pit_file_to_checked_headers.up.sql @@ -0,0 +1,6 @@ +ALTER TABLE public.checked_headers + ADD COLUMN pit_file_debt_ceiling_checked BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE public.checked_headers + ADD COLUMN pit_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE public.checked_headers + ADD COLUMN pit_file_stability_fee_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/migrations/1538082658_add_vat_init_to_checked_headers.down.sql b/db/migrations/1538082658_add_vat_init_to_checked_headers.down.sql new file mode 100644 index 00000000..ce6abf0c --- /dev/null +++ b/db/migrations/1538082658_add_vat_init_to_checked_headers.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + DROP COLUMN vat_init_checked; \ No newline at end of file diff --git a/db/migrations/1538082658_add_vat_init_to_checked_headers.up.sql b/db/migrations/1538082658_add_vat_init_to_checked_headers.up.sql new file mode 100644 index 00000000..4aba5cd2 --- /dev/null +++ b/db/migrations/1538082658_add_vat_init_to_checked_headers.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.checked_headers + ADD COLUMN vat_init_checked BOOLEAN NOT NULL DEFAULT FALSE; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 4a00afc8..7e5ac73f 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -820,7 +820,17 @@ CREATE TABLE public.checked_headers ( tend_checked boolean DEFAULT false NOT NULL, cat_file_chop_lump_checked boolean DEFAULT false NOT NULL, cat_file_flip_checked boolean DEFAULT false NOT NULL, - cat_file_pit_vow_checked boolean DEFAULT false NOT NULL + cat_file_pit_vow_checked boolean DEFAULT false NOT NULL, + bite_checked boolean DEFAULT false NOT NULL, + drip_drip_checked boolean DEFAULT false NOT NULL, + drip_file_ilk_checked boolean DEFAULT false NOT NULL, + drip_file_repo_checked boolean DEFAULT false NOT NULL, + drip_file_vow_checked boolean DEFAULT false NOT NULL, + frob_checked boolean DEFAULT false NOT NULL, + pit_file_debt_ceiling_checked boolean DEFAULT false NOT NULL, + pit_file_ilk_checked boolean DEFAULT false NOT NULL, + pit_file_stability_fee_checked boolean DEFAULT false NOT NULL, + vat_init_checked boolean DEFAULT false NOT NULL ); diff --git a/pkg/transformers/bite/converter.go b/pkg/transformers/bite/converter.go index 2c145b3f..3d97ff20 100644 --- a/pkg/transformers/bite/converter.go +++ b/pkg/transformers/bite/converter.go @@ -19,66 +19,75 @@ package bite import ( "encoding/json" "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) type Converter interface { - ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) - ToModel(flipKick BiteEntity) (BiteModel, error) + ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) + ToModels(biteEntities []BiteEntity) ([]BiteModel, error) } type BiteConverter struct{} -func (BiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) { - entity := BiteEntity{} - address := common.HexToAddress(contractAddress) - abi, err := geth.ParseAbi(contractAbi) - if err != nil { - return entity, err +func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) { + var entities []BiteEntity + for _, ethLog := range ethLogs { + entity := BiteEntity{} + address := ethLog.Address + abi, err := geth.ParseAbi(contractAbi) + if err != nil { + return nil, err + } + + contract := bind.NewBoundContract(address, abi, nil, nil, nil) + + err = contract.UnpackLog(&entity, "Bite", ethLog) + if err != nil { + return nil, err + } + + entity.Raw = ethLog + entity.TransactionIndex = ethLog.TxIndex + + entities = append(entities, entity) } - contract := bind.NewBoundContract(address, abi, nil, nil, nil) - - err = contract.UnpackLog(&entity, "Bite", ethLog) - if err != nil { - return entity, err - } - - entity.Raw = ethLog - entity.TransactionIndex = ethLog.TxIndex - - return entity, nil + return entities, nil } -func (converter BiteConverter) ToModel(entity BiteEntity) (BiteModel, error) { - id := entity.Id - ilk := entity.Ilk[:] - urn := entity.Urn[:] - ink := entity.Ink - art := entity.Art - iArt := entity.IArt - tab := entity.Tab - flip := entity.Flip - txIdx := entity.TransactionIndex - rawLogJson, err := json.Marshal(entity.Raw) - rawLogString := string(rawLogJson) - if err != nil { - return BiteModel{}, err +func (converter BiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) { + var models []BiteModel + for _, entity := range entities { + id := entity.Id + ilk := entity.Ilk[:] + urn := entity.Urn[:] + ink := entity.Ink + art := entity.Art + iArt := entity.IArt + tab := entity.Tab + flip := entity.Flip + txIdx := entity.TransactionIndex + rawLogJson, err := json.Marshal(entity.Raw) + rawLogString := string(rawLogJson) + if err != nil { + return nil, err + } + + model := BiteModel{ + Id: shared.BigIntToString(id), + Ilk: ilk, + Urn: urn, + Ink: shared.BigIntToString(ink), + Art: shared.BigIntToString(art), + IArt: shared.BigIntToString(iArt), + Tab: shared.BigIntToString(tab), + Flip: shared.BigIntToString(flip), + TransactionIndex: txIdx, + Raw: rawLogString, + } + models = append(models, model) } - - return BiteModel{ - Id: shared.BigIntToString(id), - Ilk: ilk, - Urn: urn, - Ink: shared.BigIntToString(ink), - Art: shared.BigIntToString(art), - IArt: shared.BigIntToString(iArt), - Tab: shared.BigIntToString(tab), - Flip: shared.BigIntToString(flip), - TransactionIndex: txIdx, - Raw: rawLogString, - }, nil + return models, nil } diff --git a/pkg/transformers/bite/converter_test.go b/pkg/transformers/bite/converter_test.go index dd526ec9..e33b218a 100644 --- a/pkg/transformers/bite/converter_test.go +++ b/pkg/transformers/bite/converter_test.go @@ -34,9 +34,11 @@ var _ = Describe("Bite Converter", func() { Describe("ToEntity", func() { It("converts an eth log to a bite entity", func() { - entity, err := converter.ToEntity(shared.CatContractAddress, shared.CatABI, test_data.EthBiteLog) + entities, err := converter.ToEntities(shared.CatABI, []types.Log{test_data.EthBiteLog}) Expect(err).NotTo(HaveOccurred()) + Expect(len(entities)).To(Equal(1)) + entity := entities[0] Expect(entity.Ilk).To(Equal(test_data.BiteEntity.Ilk)) Expect(entity.Urn).To(Equal(test_data.BiteEntity.Urn)) Expect(entity.Ink).To(Equal(test_data.BiteEntity.Ink)) @@ -49,7 +51,7 @@ var _ = Describe("Bite Converter", func() { }) It("returns an error if converting log to entity fails", func() { - _, err := converter.ToEntity(shared.CatContractAddress, "error abi", test_data.EthBiteLog) + _, err := converter.ToEntities("error abi", []types.Log{test_data.EthBiteLog}) Expect(err).To(HaveOccurred()) }) @@ -63,9 +65,11 @@ var _ = Describe("Bite Converter", func() { }) It("converts an Entity to a Model", func() { - model, err := converter.ToModel(test_data.BiteEntity) + models, err := converter.ToModels([]bite.BiteEntity{test_data.BiteEntity}) Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + model := models[0] Expect(model).To(Equal(test_data.BiteModel)) Expect(model.TransactionIndex).To(Equal(test_data.BiteModel.TransactionIndex)) }) @@ -85,9 +89,11 @@ var _ = Describe("Bite Converter", func() { TransactionIndex: 0, Raw: string(emptyLog), } - model, err := converter.ToModel(emptyEntity) + models, err := converter.ToModels([]bite.BiteEntity{emptyEntity}) Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + model := models[0] Expect(model).To(Equal(expectedModel)) }) }) diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 8baa7ca6..ae84cec1 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model BiteModel) error + Create(headerID int64, models []BiteModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -32,26 +33,48 @@ func NewBiteRepository(db *postgres.DB) Repository { return BiteRepository{db: db} } -func (repository BiteRepository) Create(headerID int64, model BiteModel) error { - _, err := repository.db.Exec( - `INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw, - ) - +func (repository BiteRepository) Create(headerID int64, models []BiteModel) error { + tx, err := repository.db.Begin() if err != nil { return err } - - return nil + for _, model := range models { + _, err := tx.Exec( + `INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log) + VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET bite_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() } + +func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET bite_checked = $2`, headerID, true) + return err +} + func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.bite on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR bite_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, diff --git a/pkg/transformers/bite/repository_test.go b/pkg/transformers/bite/repository_test.go index 19f3d023..bc9a925d 100644 --- a/pkg/transformers/bite/repository_test.go +++ b/pkg/transformers/bite/repository_test.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/bite" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -28,16 +29,24 @@ import ( var _ = Describe("Bite repository", func() { Describe("Create", func() { - It("persists a bite record", func() { - node := core.Node{} - db := test_config.NewTestDB(node) + var ( + biteRepository bite.Repository + db *postgres.DB + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - biteRepository := bite.NewBiteRepository(db) + biteRepository = bite.NewBiteRepository(db) + }) - err = biteRepository.Create(headerID, test_data.BiteModel) + It("persists a bite record", func() { + err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel}) Expect(err).NotTo(HaveOccurred()) var dbBite bite.BiteModel @@ -53,15 +62,18 @@ var _ = Describe("Bite repository", func() { Expect(dbBite.Raw).To(MatchJSON(test_data.BiteModel.Raw)) }) - It("does not duplicate bite events", func() { - node := core.Node{} - db := test_config.NewTestDB(node) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel}) + Expect(err).NotTo(HaveOccurred()) - biteRepository := bite.NewBiteRepository(db) - err = biteRepository.Create(headerID, test_data.BiteModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate bite events", func() { + err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel}) Expect(err).NotTo(HaveOccurred()) var anotherBiteModel = bite.BiteModel{ @@ -77,21 +89,14 @@ var _ = Describe("Bite repository", func() { Raw: test_data.BiteModel.Raw, } - err = biteRepository.Create(headerID, anotherBiteModel) + err = biteRepository.Create(headerID, []bite.BiteModel{anotherBiteModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes bite if corresponding header is deleted", func() { - node := core.Node{} - db := test_config.NewTestDB(node) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - biteRepository := bite.NewBiteRepository(db) - err = biteRepository.Create(headerID, test_data.BiteModel) + err = biteRepository.Create(headerID, []bite.BiteModel{test_data.BiteModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -104,8 +109,45 @@ var _ = Describe("Bite repository", func() { }) }) + Describe("MarkHeaderChecked", func() { + It("creates a row for a new headerID", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + biteRepository := bite.NewBiteRepository(db) + + err = biteRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + biteRepository := bite.NewBiteRepository(db) + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = biteRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated bite event", func() { + It("returns headers that haven't been checked", func() { node := core.Node{} db := test_config.NewTestDB(node) test_config.CleanTestDB(db) @@ -121,7 +163,7 @@ var _ = Describe("Bite repository", func() { Expect(err).NotTo(HaveOccurred()) } biteRepository := bite.NewBiteRepository(db) - err := biteRepository.Create(headerIDs[1], test_data.BiteModel) + err := biteRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := biteRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) diff --git a/pkg/transformers/bite/transformer.go b/pkg/transformers/bite/transformer.go index afe793f3..101b7d44 100644 --- a/pkg/transformers/bite/transformer.go +++ b/pkg/transformers/bite/transformer.go @@ -67,20 +67,26 @@ func (b BiteTransformer) Execute() error { log.Println("Error fetching matching logs:", err) return err } - - for _, ethLog := range ethLogs { - entity, err := b.Converter.ToEntity(ethLog.Address.Hex(), config.ContractAbi, ethLog) - model, err := b.Converter.ToModel(entity) + if len(ethLogs) < 1 { + err = b.Repository.MarkHeaderChecked(header.Id) if err != nil { - log.Println("Error converting logs:", err) return err } + } + entities, err := b.Converter.ToEntities(config.ContractAbi, ethLogs) + if err != nil { + return err + } + model, err := b.Converter.ToModels(entities) + if err != nil { + log.Println("Error converting logs:", err) + return err + } - err = b.Repository.Create(header.Id, model) - if err != nil { - log.Println("Error persisting bite record:", err) - return err - } + err = b.Repository.Create(header.Id, model) + if err != nil { + log.Println("Error persisting bite record:", err) + return err } } diff --git a/pkg/transformers/bite/transfromer_test.go b/pkg/transformers/bite/transfromer_test.go index 6193c09b..49960739 100644 --- a/pkg/transformers/bite/transfromer_test.go +++ b/pkg/transformers/bite/transfromer_test.go @@ -17,10 +17,13 @@ package bite_test import ( + "math/rand" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/bite" @@ -28,7 +31,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" bite_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/bite" - "math/rand" ) var _ = Describe("Bite Transformer", func() { @@ -89,13 +91,32 @@ var _ = Describe("Bite Transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + headerID := int64(123) + repository.SetMissingHeaders([]core.Header{{Id: headerID}}) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + repository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + repository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + repository.SetMarkHeaderCheckedErr(fakes.FakeError) + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts an eth log to an Entity", func() { repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) fetcher.SetFetchedLogs([]types.Log{test_data.EthBiteLog}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.ConverterContract).To(Equal(test_data.EthBiteLog.Address.Hex())) Expect(converter.ConverterAbi).To(Equal(bite.BiteConfig.ContractAbi)) Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.EthBiteLog})) }) @@ -121,7 +142,7 @@ var _ = Describe("Bite Transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerId)) - Expect(repository.PassedBiteModel).To(Equal(test_data.BiteModel)) + Expect(repository.PassedBiteModels).To(Equal([]bite.BiteModel{test_data.BiteModel})) }) It("returns error if persisting bite record fails", func() { diff --git a/pkg/transformers/drip_drip/converter.go b/pkg/transformers/drip_drip/converter.go index 99e58199..0a7473a1 100644 --- a/pkg/transformers/drip_drip/converter.go +++ b/pkg/transformers/drip_drip/converter.go @@ -22,23 +22,31 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (DripDripModel, error) + ToModels(ethLogs []types.Log) ([]DripDripModel, error) } type DripDripConverter struct{} -func (DripDripConverter) ToModel(ethLog types.Log) (DripDripModel, error) { - err := verifyLog(ethLog) - if err != nil { - return DripDripModel{}, err +func (DripDripConverter) ToModels(ethLogs []types.Log) ([]DripDripModel, error) { + var models []DripDripModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := DripDripModel{ + Ilk: ilk, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - raw, err := json.Marshal(ethLog) - return DripDripModel{ - Ilk: ilk, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/drip_drip/converter_test.go b/pkg/transformers/drip_drip/converter_test.go index 454a3409..772e758d 100644 --- a/pkg/transformers/drip_drip/converter_test.go +++ b/pkg/transformers/drip_drip/converter_test.go @@ -28,7 +28,7 @@ var _ = Describe("Drip drip converter", func() { converter := drip_drip.DripDripConverter{} badLog := types.Log{} - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -36,9 +36,9 @@ var _ = Describe("Drip drip converter", func() { It("converts a log to an model", func() { converter := drip_drip.DripDripConverter{} - model, err := converter.ToModel(test_data.EthDripDripLog) + model, err := converter.ToModels([]types.Log{test_data.EthDripDripLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.DripDripModel)) + Expect(model).To(Equal([]drip_drip.DripDripModel{test_data.DripDripModel})) }) }) diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index 0221a4cf..23881f34 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model DripDripModel) error + Create(headerID int64, models []DripDripModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -32,12 +33,38 @@ func NewDripDripRepository(db *postgres.DB) DripDripRepository { return DripDripRepository{db: db} } -func (repository DripDripRepository) Create(headerID int64, model DripDripModel) error { - _, err := repository.db.Exec( - `INSERT into maker.drip_drip (header_id, ilk, tx_idx, raw_log) +func (repository DripDripRepository) Create(headerID int64, models []DripDripModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.drip_drip (header_id, ilk, tx_idx, raw_log) VALUES($1, $2, $3, $4)`, - headerID, model.Ilk, model.TransactionIndex, model.Raw, - ) + headerID, model.Ilk, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_drip_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_drip_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_drip_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_drip_checked = $2`, headerID, true) return err } @@ -46,8 +73,8 @@ func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingB err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.drip_drip on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR drip_drip_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, diff --git a/pkg/transformers/drip_drip/repository_test.go b/pkg/transformers/drip_drip/repository_test.go index f4ff5094..6286adf8 100644 --- a/pkg/transformers/drip_drip/repository_test.go +++ b/pkg/transformers/drip_drip/repository_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -29,15 +30,24 @@ import ( var _ = Describe("Drip drip repository", func() { Describe("Create", func() { - It("adds a drip drip event", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + dripDripRepository drip_drip.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - dripDripRepository := drip_drip.NewDripDripRepository(db) + dripDripRepository = drip_drip.NewDripDripRepository(db) + }) - err = dripDripRepository.Create(headerID, test_data.DripDripModel) + It("adds a drip drip event", func() { + err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel}) Expect(err).NotTo(HaveOccurred()) var dbPitFile drip_drip.DripDripModel @@ -48,30 +58,28 @@ var _ = Describe("Drip drip repository", func() { Expect(dbPitFile.Raw).To(MatchJSON(test_data.DripDripModel.Raw)) }) - It("does not duplicate drip drip events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel}) + Expect(err).NotTo(HaveOccurred()) - dripDripRepository := drip_drip.NewDripDripRepository(db) - err = dripDripRepository.Create(headerID, test_data.DripDripModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate drip drip events", func() { + err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel}) Expect(err).NotTo(HaveOccurred()) - err = dripDripRepository.Create(headerID, test_data.DripDripModel) + err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes drip drip if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - dripDripRepository := drip_drip.NewDripDripRepository(db) - err = dripDripRepository.Create(headerID, test_data.DripDripModel) + err = dripDripRepository.Create(headerID, []drip_drip.DripDripModel{test_data.DripDripModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -84,8 +92,48 @@ var _ = Describe("Drip drip repository", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + dripDripRepository drip_drip.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + dripDripRepository = drip_drip.NewDripDripRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = dripDripRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = dripDripRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated drip drip event", func() { + It("returns headers that haven't been checked", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) @@ -100,7 +148,7 @@ var _ = Describe("Drip drip repository", func() { Expect(err).NotTo(HaveOccurred()) } dripDripRepository := drip_drip.NewDripDripRepository(db) - err := dripDripRepository.Create(headerIDs[1], test_data.DripDripModel) + err := dripDripRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -111,6 +159,33 @@ var _ = Describe("Drip drip repository", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if drip drip logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + dripDripBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, dripDripBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + dripDripRepository := drip_drip.NewDripDripRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripDripBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -128,7 +203,7 @@ var _ = Describe("Drip drip repository", func() { } dripDripRepository := drip_drip.NewDripDripRepository(db) dripDripRepositoryTwo := drip_drip.NewDripDripRepository(dbTwo) - err := dripDripRepository.Create(headerIDs[0], test_data.DripDripModel) + err := dripDripRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := dripDripRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/drip_drip/transformer.go b/pkg/transformers/drip_drip/transformer.go index c8e981f1..84898825 100644 --- a/pkg/transformers/drip_drip/transformer.go +++ b/pkg/transformers/drip_drip/transformer.go @@ -59,16 +59,20 @@ func (transformer DripDripTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/drip_drip/transformer_test.go b/pkg/transformers/drip_drip/transformer_test.go index ff2241be..6283d737 100644 --- a/pkg/transformers/drip_drip/transformer_test.go +++ b/pkg/transformers/drip_drip/transformer_test.go @@ -97,6 +97,42 @@ var _ = Describe("Drip drip transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &drip_drip_mocks.MockDripDripConverter{} + mockRepository := &drip_drip_mocks.MockDripDripRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := drip_drip.DripDripTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &drip_drip_mocks.MockDripDripConverter{} + mockRepository := &drip_drip_mocks.MockDripDripRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := drip_drip.DripDripTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &drip_drip_mocks.MockDripDripConverter{} fetcher := &mocks.MockLogFetcher{} @@ -112,7 +148,7 @@ var _ = Describe("Drip drip transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthDripDripLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripDripLog})) }) It("returns error if converter returns error", func() { @@ -151,7 +187,7 @@ var _ = Describe("Drip drip transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.DripDripModel)) + Expect(repository.PassedModels).To(Equal([]drip_drip.DripDripModel{test_data.DripDripModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/drip_file/ilk/converter.go b/pkg/transformers/drip_file/ilk/converter.go index 7b03f958..7455cfdf 100644 --- a/pkg/transformers/drip_file/ilk/converter.go +++ b/pkg/transformers/drip_file/ilk/converter.go @@ -25,28 +25,36 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (DripFileIlkModel, error) + ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error) } type DripFileIlkConverter struct{} -func (DripFileIlkConverter) ToModel(ethLog types.Log) (DripFileIlkModel, error) { - err := verifyLog(ethLog) - if err != nil { - return DripFileIlkModel{}, err +func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error) { + var models []DripFileIlkModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) + taxBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:] + tax := big.NewInt(0).SetBytes(taxBytes).String() + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := DripFileIlkModel{ + Ilk: ilk, + Vow: vow, + Tax: tax, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) - taxBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:] - tax := big.NewInt(0).SetBytes(taxBytes).String() - raw, err := json.Marshal(ethLog) - return DripFileIlkModel{ - Ilk: ilk, - Vow: vow, - Tax: tax, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/drip_file/ilk/converter_test.go b/pkg/transformers/drip_file/ilk/converter_test.go index fbbeb574..f7f48c9c 100644 --- a/pkg/transformers/drip_file/ilk/converter_test.go +++ b/pkg/transformers/drip_file/ilk/converter_test.go @@ -32,7 +32,7 @@ var _ = Describe("Drip file ilk converter", func() { Data: []byte{1, 1, 1, 1, 1}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -43,7 +43,7 @@ var _ = Describe("Drip file ilk converter", func() { Topics: []common.Hash{{}, {}, {}, {}}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) @@ -52,9 +52,10 @@ var _ = Describe("Drip file ilk converter", func() { It("converts a log to a model", func() { converter := ilk.DripFileIlkConverter{} - model, err := converter.ToModel(test_data.EthDripFileIlkLog) + models, err := converter.ToModels([]types.Log{test_data.EthDripFileIlkLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.DripFileIlkModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.DripFileIlkModel)) }) }) diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index a0b3581d..9ab100f0 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model DripFileIlkModel) error + Create(headerID int64, models []DripFileIlkModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -32,12 +33,38 @@ func NewDripFileIlkRepository(db *postgres.DB) DripFileIlkRepository { return DripFileIlkRepository{db: db} } -func (repository DripFileIlkRepository) Create(headerID int64, model DripFileIlkModel) error { - _, err := repository.db.Exec( - `INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, tx_idx, raw_log) +func (repository DripFileIlkRepository) Create(headerID int64, models []DripFileIlkModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, - headerID, model.Ilk, model.Vow, model.Tax, model.TransactionIndex, model.Raw, - ) + headerID, model.Ilk, model.Vow, model.Tax, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_ilk_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_ilk_checked = $2`, headerID, true) return err } @@ -46,8 +73,8 @@ func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endi err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.drip_file_ilk on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR drip_file_ilk_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, diff --git a/pkg/transformers/drip_file/ilk/repository_test.go b/pkg/transformers/drip_file/ilk/repository_test.go index 595572e6..c10d579e 100644 --- a/pkg/transformers/drip_file/ilk/repository_test.go +++ b/pkg/transformers/drip_file/ilk/repository_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -29,15 +30,24 @@ import ( var _ = Describe("Drip file ilk repository", func() { Describe("Create", func() { - It("adds a drip file ilk event", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + dripFileIlkRepository ilk.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - dripFileIlkRepository := ilk.NewDripFileIlkRepository(db) + dripFileIlkRepository = ilk.NewDripFileIlkRepository(db) + }) - err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel) + It("adds a drip file ilk event", func() { + err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel}) Expect(err).NotTo(HaveOccurred()) var dbDripFileIlk ilk.DripFileIlkModel @@ -50,30 +60,28 @@ var _ = Describe("Drip file ilk repository", func() { Expect(dbDripFileIlk.Raw).To(MatchJSON(test_data.DripFileIlkModel.Raw)) }) - It("does not duplicate drip file events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel}) + Expect(err).NotTo(HaveOccurred()) - dripFileIlkRepository := ilk.NewDripFileIlkRepository(db) - err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate drip file events", func() { + err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel}) Expect(err).NotTo(HaveOccurred()) - err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel) + err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes drip file if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - dripFileIlkRepository := ilk.NewDripFileIlkRepository(db) - err = dripFileIlkRepository.Create(headerID, test_data.DripFileIlkModel) + err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -86,6 +94,46 @@ var _ = Describe("Drip file ilk repository", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + dripFileIlkRepository ilk.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + dripFileIlkRepository = ilk.NewDripFileIlkRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = dripFileIlkRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = dripFileIlkRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { It("returns headers with no associated drip file event", func() { db := test_config.NewTestDB(core.Node{}) @@ -102,7 +150,7 @@ var _ = Describe("Drip file ilk repository", func() { Expect(err).NotTo(HaveOccurred()) } dripFileIlkRepository := ilk.NewDripFileIlkRepository(db) - err := dripFileIlkRepository.Create(headerIDs[1], test_data.DripFileIlkModel) + err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -113,6 +161,33 @@ var _ = Describe("Drip file ilk repository", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if drip file ilk logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + dripFileIlkdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, dripFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + dripFileIlkRepository := ilk.NewDripFileIlkRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := dripFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileIlkdBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -130,7 +205,7 @@ var _ = Describe("Drip file ilk repository", func() { } dripFileIlkRepository := ilk.NewDripFileIlkRepository(db) dripFileIlkRepositoryTwo := ilk.NewDripFileIlkRepository(dbTwo) - err := dripFileIlkRepository.Create(headerIDs[0], test_data.DripFileIlkModel) + err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := dripFileIlkRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/drip_file/ilk/transformer.go b/pkg/transformers/drip_file/ilk/transformer.go index fc07ea1a..d8a518f2 100644 --- a/pkg/transformers/drip_file/ilk/transformer.go +++ b/pkg/transformers/drip_file/ilk/transformer.go @@ -60,16 +60,20 @@ func (transformer DripFileIlkTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + model, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, model) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/drip_file/ilk/transformer_test.go b/pkg/transformers/drip_file/ilk/transformer_test.go index d104cb0a..3ad7194a 100644 --- a/pkg/transformers/drip_file/ilk/transformer_test.go +++ b/pkg/transformers/drip_file/ilk/transformer_test.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" @@ -96,6 +97,42 @@ var _ = Describe("Drip file ilk transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &ilk_mocks.MockDripFileIlkConverter{} + mockRepository := &ilk_mocks.MockDripFileIlkRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := ilk.DripFileIlkTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &ilk_mocks.MockDripFileIlkConverter{} + mockRepository := &ilk_mocks.MockDripFileIlkRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := ilk.DripFileIlkTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &ilk_mocks.MockDripFileIlkConverter{} fetcher := &mocks.MockLogFetcher{} @@ -111,7 +148,7 @@ var _ = Describe("Drip file ilk transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthDripFileIlkLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileIlkLog})) }) It("returns error if converter returns error", func() { @@ -150,7 +187,7 @@ var _ = Describe("Drip file ilk transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.DripFileIlkModel)) + Expect(repository.PassedModels).To(Equal([]ilk.DripFileIlkModel{test_data.DripFileIlkModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/drip_file/repo/converter.go b/pkg/transformers/drip_file/repo/converter.go index d525b246..37590dcf 100644 --- a/pkg/transformers/drip_file/repo/converter.go +++ b/pkg/transformers/drip_file/repo/converter.go @@ -23,25 +23,33 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (DripFileRepoModel, error) + ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error) } type DripFileRepoConverter struct{} -func (DripFileRepoConverter) ToModel(ethLog types.Log) (DripFileRepoModel, error) { - err := verifyLog(ethLog) - if err != nil { - return DripFileRepoModel{}, err +func (DripFileRepoConverter) ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error) { + var models []DripFileRepoModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := DripFileRepoModel{ + What: what, + Data: data, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) } - what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() - raw, err := json.Marshal(ethLog) - return DripFileRepoModel{ - What: what, - Data: data, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/drip_file/repo/converter_test.go b/pkg/transformers/drip_file/repo/converter_test.go index eda236d6..21ebc9fc 100644 --- a/pkg/transformers/drip_file/repo/converter_test.go +++ b/pkg/transformers/drip_file/repo/converter_test.go @@ -32,7 +32,7 @@ var _ = Describe("Drip file repo converter", func() { Data: []byte{1, 1, 1, 1, 1}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -40,9 +40,10 @@ var _ = Describe("Drip file repo converter", func() { It("converts a log to a model", func() { converter := repo.DripFileRepoConverter{} - model, err := converter.ToModel(test_data.EthDripFileRepoLog) + models, err := converter.ToModels([]types.Log{test_data.EthDripFileRepoLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.DripFileRepoModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.DripFileRepoModel)) }) }) diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 1ab629ad..82e9d7c8 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model DripFileRepoModel) error + Create(headerID int64, models []DripFileRepoModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -32,12 +33,38 @@ func NewDripFileRepoRepository(db *postgres.DB) DripFileRepoRepository { return DripFileRepoRepository{db: db} } -func (repository DripFileRepoRepository) Create(headerID int64, model DripFileRepoModel) error { - _, err := repository.db.Exec( - `INSERT into maker.drip_file_repo (header_id, what, data, tx_idx, raw_log) +func (repository DripFileRepoRepository) Create(headerID int64, models []DripFileRepoModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.drip_file_repo (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4, $5)`, - headerID, model.What, model.Data, model.TransactionIndex, model.Raw, - ) + headerID, model.What, model.Data, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_repo_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_repo_checked = $2`, headerID, true) return err } @@ -46,8 +73,8 @@ func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, end err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.drip_file_repo on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR drip_file_repo_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, diff --git a/pkg/transformers/drip_file/repo/repository_test.go b/pkg/transformers/drip_file/repo/repository_test.go index 138d918a..61fd50ef 100644 --- a/pkg/transformers/drip_file/repo/repository_test.go +++ b/pkg/transformers/drip_file/repo/repository_test.go @@ -19,23 +19,33 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/test_config" ) -var _ = Describe("", func() { +var _ = Describe("Drip file repo repository", func() { Describe("Create", func() { - It("adds a drip file repo event", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + dripFileRepoRepository repo.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - dripFileRepoRepository := repo.NewDripFileRepoRepository(db) + dripFileRepoRepository = repo.NewDripFileRepoRepository(db) + }) - err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel) + It("adds a drip file repo event", func() { + err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel}) Expect(err).NotTo(HaveOccurred()) var dbDripFileRepo repo.DripFileRepoModel @@ -47,30 +57,28 @@ var _ = Describe("", func() { Expect(dbDripFileRepo.Raw).To(MatchJSON(test_data.DripFileRepoModel.Raw)) }) - It("does not duplicate drip file events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel}) + Expect(err).NotTo(HaveOccurred()) - dripFileRepoRepository := repo.NewDripFileRepoRepository(db) - err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate drip file events", func() { + err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel}) Expect(err).NotTo(HaveOccurred()) - err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel) + err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes drip file if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - dripFileRepoRepository := repo.NewDripFileRepoRepository(db) - err = dripFileRepoRepository.Create(headerID, test_data.DripFileRepoModel) + err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -83,6 +91,46 @@ var _ = Describe("", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + dripFileRepoRepository repo.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + dripFileRepoRepository = repo.NewDripFileRepoRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = dripFileRepoRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = dripFileRepoRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_repo_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { It("returns headers with no associated drip file event", func() { db := test_config.NewTestDB(core.Node{}) @@ -99,7 +147,7 @@ var _ = Describe("", func() { Expect(err).NotTo(HaveOccurred()) } dripFileRepoRepository := repo.NewDripFileRepoRepository(db) - err := dripFileRepoRepository.Create(headerIDs[1], test_data.DripFileRepoModel) + err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -110,6 +158,33 @@ var _ = Describe("", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if drip file repo logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + dripFileRepodBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, dripFileRepodBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + dripFileRepoRepository := repo.NewDripFileRepoRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := dripFileRepoRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileRepodBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -127,7 +202,7 @@ var _ = Describe("", func() { } dripFileRepoRepository := repo.NewDripFileRepoRepository(db) dripFileRepoRepositoryTwo := repo.NewDripFileRepoRepository(dbTwo) - err := dripFileRepoRepository.Create(headerIDs[0], test_data.DripFileRepoModel) + err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := dripFileRepoRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/drip_file/repo/transformer.go b/pkg/transformers/drip_file/repo/transformer.go index 53920444..d34593dc 100644 --- a/pkg/transformers/drip_file/repo/transformer.go +++ b/pkg/transformers/drip_file/repo/transformer.go @@ -60,16 +60,20 @@ func (transformer DripFileRepoTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + model, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, model) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/drip_file/repo/transformer_test.go b/pkg/transformers/drip_file/repo/transformer_test.go index d526975c..41fe09b8 100644 --- a/pkg/transformers/drip_file/repo/transformer_test.go +++ b/pkg/transformers/drip_file/repo/transformer_test.go @@ -97,6 +97,42 @@ var _ = Describe("Drip file repo transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &repo_mocks.MockDripFileRepoConverter{} + mockRepository := &repo_mocks.MockDripFileRepoRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := repo.DripFileRepoTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &repo_mocks.MockDripFileRepoConverter{} + mockRepository := &repo_mocks.MockDripFileRepoRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := repo.DripFileRepoTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &repo_mocks.MockDripFileRepoConverter{} fetcher := &mocks.MockLogFetcher{} @@ -112,7 +148,7 @@ var _ = Describe("Drip file repo transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthDripFileRepoLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileRepoLog})) }) It("returns error if converter returns error", func() { @@ -151,7 +187,7 @@ var _ = Describe("Drip file repo transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.DripFileRepoModel)) + Expect(repository.PassedModels).To(Equal([]repo.DripFileRepoModel{test_data.DripFileRepoModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/drip_file/vow/converter.go b/pkg/transformers/drip_file/vow/converter.go index 93d1200b..dc2d442b 100644 --- a/pkg/transformers/drip_file/vow/converter.go +++ b/pkg/transformers/drip_file/vow/converter.go @@ -22,25 +22,33 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (DripFileVowModel, error) + ToModels(ethLogs []types.Log) ([]DripFileVowModel, error) } type DripFileVowConverter struct{} -func (DripFileVowConverter) ToModel(ethLog types.Log) (DripFileVowModel, error) { - err := verifyLog(ethLog) - if err != nil { - return DripFileVowModel{}, err +func (DripFileVowConverter) ToModels(ethLogs []types.Log) ([]DripFileVowModel, error) { + var models []DripFileVowModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + data := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := DripFileVowModel{ + What: what, + Data: data, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) } - what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - data := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) - raw, err := json.Marshal(ethLog) - return DripFileVowModel{ - What: what, - Data: data, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/drip_file/vow/converter_test.go b/pkg/transformers/drip_file/vow/converter_test.go index 6c86ff82..0458c611 100644 --- a/pkg/transformers/drip_file/vow/converter_test.go +++ b/pkg/transformers/drip_file/vow/converter_test.go @@ -32,7 +32,7 @@ var _ = Describe("Drip file repo converter", func() { Data: []byte{1, 1, 1, 1, 1}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -40,9 +40,10 @@ var _ = Describe("Drip file repo converter", func() { It("converts a log to a model", func() { converter := vow.DripFileVowConverter{} - model, err := converter.ToModel(test_data.EthDripFileVowLog) + models, err := converter.ToModels([]types.Log{test_data.EthDripFileVowLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.DripFileVowModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.DripFileVowModel)) }) }) diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index d41c4a53..25ba0919 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model DripFileVowModel) error + Create(headerID int64, models []DripFileVowModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -32,12 +33,38 @@ func NewDripFileVowRepository(db *postgres.DB) DripFileVowRepository { return DripFileVowRepository{db: db} } -func (repository DripFileVowRepository) Create(headerID int64, model DripFileVowModel) error { - _, err := repository.db.Exec( - `INSERT into maker.drip_file_vow (header_id, what, data, tx_idx, raw_log) +func (repository DripFileVowRepository) Create(headerID int64, models []DripFileVowModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.drip_file_vow (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5)`, - headerID, model.What, model.Data, model.TransactionIndex, model.Raw, - ) + headerID, model.What, model.Data, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_vow_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_vow_checked = $2`, headerID, true) return err } @@ -46,8 +73,8 @@ func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endi err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.drip_file_vow on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR drip_file_vow_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, diff --git a/pkg/transformers/drip_file/vow/repository_test.go b/pkg/transformers/drip_file/vow/repository_test.go index d9ca1a9a..ebfc1626 100644 --- a/pkg/transformers/drip_file/vow/repository_test.go +++ b/pkg/transformers/drip_file/vow/repository_test.go @@ -16,26 +16,38 @@ package vow_test import ( "database/sql" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/test_config" ) -var _ = Describe("", func() { +var _ = Describe("Drip file vow repository", func() { Describe("Create", func() { - It("adds a drip file vow event", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + dripFileVowRepository vow.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - dripFileVowRepository := vow.NewDripFileVowRepository(db) + dripFileVowRepository = vow.NewDripFileVowRepository(db) + }) - err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel) + It("adds a drip file vow event", func() { + err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel}) Expect(err).NotTo(HaveOccurred()) var dbDripFileVow vow.DripFileVowModel @@ -47,30 +59,28 @@ var _ = Describe("", func() { Expect(dbDripFileVow.Raw).To(MatchJSON(test_data.DripFileVowModel.Raw)) }) - It("does not duplicate drip file events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel}) + Expect(err).NotTo(HaveOccurred()) - dripFileVowRepository := vow.NewDripFileVowRepository(db) - err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate drip file events", func() { + err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel}) Expect(err).NotTo(HaveOccurred()) - err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel) + err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes drip file if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - dripFileVowRepository := vow.NewDripFileVowRepository(db) - err = dripFileVowRepository.Create(headerID, test_data.DripFileVowModel) + err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -83,8 +93,48 @@ var _ = Describe("", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + dripFileVowRepository vow.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + dripFileVowRepository = vow.NewDripFileVowRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = dripFileVowRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = dripFileVowRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated drip file event", func() { + It("returns headers that haven't been checked", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) @@ -99,7 +149,7 @@ var _ = Describe("", func() { Expect(err).NotTo(HaveOccurred()) } dripFileVowRepository := vow.NewDripFileVowRepository(db) - err := dripFileVowRepository.Create(headerIDs[1], test_data.DripFileVowModel) + err := dripFileVowRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -110,6 +160,33 @@ var _ = Describe("", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if drip file vow logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + dripFileVowdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, dripFileVowdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + dripFileVowRepository := vow.NewDripFileVowRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := dripFileVowRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(dripFileVowdBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -127,7 +204,7 @@ var _ = Describe("", func() { } dripFileVowRepository := vow.NewDripFileVowRepository(db) dripFileVowRepositoryTwo := vow.NewDripFileVowRepository(dbTwo) - err := dripFileVowRepository.Create(headerIDs[0], test_data.DripFileVowModel) + err := dripFileVowRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := dripFileVowRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/drip_file/vow/transformer.go b/pkg/transformers/drip_file/vow/transformer.go index a37cb904..bca52247 100644 --- a/pkg/transformers/drip_file/vow/transformer.go +++ b/pkg/transformers/drip_file/vow/transformer.go @@ -60,16 +60,20 @@ func (transformer DripFileVowTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + model, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, model) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/drip_file/vow/transformer_test.go b/pkg/transformers/drip_file/vow/transformer_test.go index cb7450d7..eb55c3e1 100644 --- a/pkg/transformers/drip_file/vow/transformer_test.go +++ b/pkg/transformers/drip_file/vow/transformer_test.go @@ -97,6 +97,42 @@ var _ = Describe("Drip file vow transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &vow_mocks.MockDripFileVowConverter{} + mockRepository := &vow_mocks.MockDripFileVowRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vow.DripFileVowTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vow_mocks.MockDripFileVowConverter{} + mockRepository := &vow_mocks.MockDripFileVowRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vow.DripFileVowTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &vow_mocks.MockDripFileVowConverter{} fetcher := &mocks.MockLogFetcher{} @@ -112,7 +148,7 @@ var _ = Describe("Drip file vow transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthDripFileVowLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthDripFileVowLog})) }) It("returns error if converter returns error", func() { @@ -151,7 +187,7 @@ var _ = Describe("Drip file vow transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.DripFileVowModel)) + Expect(repository.PassedModels).To(Equal([]vow.DripFileVowModel{test_data.DripFileVowModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/frob/converter.go b/pkg/transformers/frob/converter.go index c35cdba5..a5d6f7cf 100644 --- a/pkg/transformers/frob/converter.go +++ b/pkg/transformers/frob/converter.go @@ -16,7 +16,6 @@ package frob import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "encoding/json" @@ -24,40 +23,50 @@ import ( ) type Converter interface { - ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error) - ToModel(flipKick FrobEntity) (FrobModel, error) + ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error) + ToModels(entities []FrobEntity) ([]FrobModel, error) } type FrobConverter struct{} -func (FrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (FrobEntity, error) { - entity := FrobEntity{} - address := common.HexToAddress(contractAddress) - abi, err := geth.ParseAbi(contractAbi) - if err != nil { - return entity, err +func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]FrobEntity, error) { + var entities []FrobEntity + for _, ethLog := range ethLogs { + entity := FrobEntity{} + address := ethLog.Address + abi, err := geth.ParseAbi(contractAbi) + if err != nil { + return nil, err + } + contract := bind.NewBoundContract(address, abi, nil, nil, nil) + err = contract.UnpackLog(&entity, "Frob", ethLog) + entity.TransactionIndex = ethLog.TxIndex + entity.Raw = ethLog + entities = append(entities, entity) } - contract := bind.NewBoundContract(address, abi, nil, nil, nil) - err = contract.UnpackLog(&entity, "Frob", ethLog) - entity.TransactionIndex = ethLog.TxIndex - entity.Raw = ethLog - return entity, err + + return entities, nil } -func (FrobConverter) ToModel(frob FrobEntity) (FrobModel, error) { - rawLog, err := json.Marshal(frob.Raw) - if err != nil { - return FrobModel{}, err +func (FrobConverter) ToModels(entities []FrobEntity) ([]FrobModel, error) { + var models []FrobModel + for _, entity := range entities { + rawLog, err := json.Marshal(entity.Raw) + if err != nil { + return nil, err + } + model := FrobModel{ + Ilk: entity.Ilk[:], + Urn: entity.Urn[:], + Ink: entity.Ink.String(), + Art: entity.Art.String(), + Dink: entity.Dink.String(), + Dart: entity.Dart.String(), + IArt: entity.IArt.String(), + TransactionIndex: entity.TransactionIndex, + Raw: rawLog, + } + models = append(models, model) } - return FrobModel{ - Ilk: frob.Ilk[:], - Urn: frob.Urn[:], - Ink: frob.Ink.String(), - Art: frob.Art.String(), - Dink: frob.Dink.String(), - Dart: frob.Dart.String(), - IArt: frob.IArt.String(), - TransactionIndex: frob.TransactionIndex, - Raw: rawLog, - }, nil + return models, nil } diff --git a/pkg/transformers/frob/converter_test.go b/pkg/transformers/frob/converter_test.go index 9c190fdc..3679bb3c 100644 --- a/pkg/transformers/frob/converter_test.go +++ b/pkg/transformers/frob/converter_test.go @@ -18,6 +18,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -27,18 +28,20 @@ var _ = Describe("Frob converter", func() { It("converts a log to an entity", func() { converter := frob.FrobConverter{} - entity, err := converter.ToEntity(shared.PitContractAddress, shared.PitABI, test_data.EthFrobLog) + entities, err := converter.ToEntities(shared.PitABI, []types.Log{test_data.EthFrobLog}) Expect(err).NotTo(HaveOccurred()) - Expect(entity).To(Equal(test_data.FrobEntity)) + Expect(len(entities)).To(Equal(1)) + Expect(entities[0]).To(Equal(test_data.FrobEntity)) }) It("converts an entity to a model", func() { converter := frob.FrobConverter{} - model, err := converter.ToModel(test_data.FrobEntity) + models, err := converter.ToModels([]frob.FrobEntity{test_data.FrobEntity}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.FrobModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.FrobModel)) }) }) diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index f2d60b31..e4e1dfb7 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model FrobModel) error + Create(headerID int64, models []FrobModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -32,10 +33,36 @@ func NewFrobRepository(db *postgres.DB) FrobRepository { return FrobRepository{db: db} } -func (repository FrobRepository) Create(headerID int64, model FrobModel) error { - _, err := repository.db.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx) +func (repository FrobRepository) Create(headerID int64, models []FrobModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, tx_idx) VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10)`, - headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex) + headerID, model.Art, model.Dart, model.Dink, model.IArt, model.Ilk, model.Ink, model.Urn, model.Raw, model.TransactionIndex) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET frob_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository FrobRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, frob_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET frob_checked = $2`, headerID, true) return err } @@ -44,8 +71,8 @@ func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlock err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.frob on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR frob_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, diff --git a/pkg/transformers/frob/repository_test.go b/pkg/transformers/frob/repository_test.go index bfba9eed..a660582a 100644 --- a/pkg/transformers/frob/repository_test.go +++ b/pkg/transformers/frob/repository_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -29,15 +30,24 @@ import ( var _ = Describe("Frob repository", func() { Describe("Create", func() { - It("adds a frob", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + frobRepository frob.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - frobRepository := frob.NewFrobRepository(db) + frobRepository = frob.NewFrobRepository(db) + }) - err = frobRepository.Create(headerID, test_data.FrobModel) + It("adds a frob", func() { + err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel}) Expect(err).NotTo(HaveOccurred()) var dbFrob frob.FrobModel @@ -54,30 +64,28 @@ var _ = Describe("Frob repository", func() { Expect(dbFrob.Raw).To(MatchJSON(test_data.FrobModel.Raw)) }) - It("does not duplicate frob events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel}) + Expect(err).NotTo(HaveOccurred()) - frobRepository := frob.NewFrobRepository(db) - err = frobRepository.Create(headerID, test_data.FrobModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate frob events", func() { + err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel}) Expect(err).NotTo(HaveOccurred()) - err = frobRepository.Create(headerID, test_data.FrobModel) + err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes frob if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - frobRepository := frob.NewFrobRepository(db) - err = frobRepository.Create(headerID, test_data.FrobModel) + err = frobRepository.Create(headerID, []frob.FrobModel{test_data.FrobModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -90,8 +98,48 @@ var _ = Describe("Frob repository", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + frobRepository frob.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + frobRepository = frob.NewFrobRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = frobRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = frobRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated frob event", func() { + It("returns headers that haven't been checked", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) @@ -106,7 +154,7 @@ var _ = Describe("Frob repository", func() { Expect(err).NotTo(HaveOccurred()) } frobRepository := frob.NewFrobRepository(db) - err := frobRepository.Create(headerIDs[1], test_data.FrobModel) + err := frobRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -117,6 +165,33 @@ var _ = Describe("Frob repository", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if frob logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + frobdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, frobdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + frobRepository := frob.NewFrobRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := frobRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(frobdBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -134,7 +209,7 @@ var _ = Describe("Frob repository", func() { } frobRepository := frob.NewFrobRepository(db) frobRepositoryTwo := frob.NewFrobRepository(dbTwo) - err := frobRepository.Create(headerIDs[0], test_data.FrobModel) + err := frobRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := frobRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/frob/transformer.go b/pkg/transformers/frob/transformer.go index 01073526..b3b2d32c 100644 --- a/pkg/transformers/frob/transformer.go +++ b/pkg/transformers/frob/transformer.go @@ -59,20 +59,24 @@ func (transformer FrobTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - entity, err := transformer.Converter.ToEntity(log.Address.Hex(), FrobConfig.ContractAbi, log) - if err != nil { - return err - } - model, err := transformer.Converter.ToModel(entity) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + entities, err := transformer.Converter.ToEntities(FrobConfig.ContractAbi, matchingLogs) + if err != nil { + return err + } + models, err := transformer.Converter.ToModels(entities) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/frob/transformer_test.go b/pkg/transformers/frob/transformer_test.go index a5579cb2..ddf9e6ea 100644 --- a/pkg/transformers/frob/transformer_test.go +++ b/pkg/transformers/frob/transformer_test.go @@ -96,6 +96,42 @@ var _ = Describe("Frob transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &frob_mocks.MockFrobConverter{} + mockRepository := &frob_mocks.MockFrobRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := frob.FrobTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &frob_mocks.MockFrobConverter{} + mockRepository := &frob_mocks.MockFrobRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := frob.FrobTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs to entity", func() { converter := &frob_mocks.MockFrobConverter{} fetcher := &mocks.MockLogFetcher{} @@ -111,14 +147,13 @@ var _ = Describe("Frob transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedContractAddress).To(Equal(test_data.EthFrobLog.Address.Hex())) Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi)) - Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthFrobLog})) }) It("returns error if converting to entity returns error", func() { converter := &frob_mocks.MockFrobConverter{} - converter.SetToEntityError(fakes.FakeError) + converter.SetToEntitiesError(fakes.FakeError) fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog}) repository := &frob_mocks.MockFrobRepository{} @@ -150,12 +185,12 @@ var _ = Describe("Frob transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity)) + Expect(converter.PassedEntities).To(Equal([]frob.FrobEntity{test_data.FrobEntity})) }) It("returns error if converting to model returns error", func() { converter := &frob_mocks.MockFrobConverter{} - converter.SetToModelError(fakes.FakeError) + converter.SetToModelsError(fakes.FakeError) fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthFrobLog}) repository := &frob_mocks.MockFrobRepository{} @@ -189,7 +224,7 @@ var _ = Describe("Frob transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedFrobModel).To(Equal(test_data.FrobModel)) + Expect(repository.PassedModels).To(Equal([]frob.FrobModel{test_data.FrobModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/pit_file/debt_ceiling/converter.go b/pkg/transformers/pit_file/debt_ceiling/converter.go index 90108411..f47044f3 100644 --- a/pkg/transformers/pit_file/debt_ceiling/converter.go +++ b/pkg/transformers/pit_file/debt_ceiling/converter.go @@ -25,26 +25,34 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error) + ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error) } type PitFileDebtCeilingConverter struct{} -func (PitFileDebtCeilingConverter) ToModel(ethLog types.Log) (PitFileDebtCeilingModel, error) { - err := verifyLog(ethLog) - if err != nil { - return PitFileDebtCeilingModel{}, err - } - what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() +func (PitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error) { + var models []PitFileDebtCeilingModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() - raw, err := json.Marshal(ethLog) - return PitFileDebtCeilingModel{ - What: what, - Data: data, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := PitFileDebtCeilingModel{ + What: what, + Data: data, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/pit_file/debt_ceiling/converter_test.go b/pkg/transformers/pit_file/debt_ceiling/converter_test.go index 217c37f4..fec50524 100644 --- a/pkg/transformers/pit_file/debt_ceiling/converter_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/converter_test.go @@ -31,7 +31,7 @@ var _ = Describe("Pit file debt ceiling converter", func() { Data: []byte{1, 1, 1, 1, 1}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -42,7 +42,7 @@ var _ = Describe("Pit file debt ceiling converter", func() { Topics: []common.Hash{{}, {}, {}, {}}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -50,9 +50,10 @@ var _ = Describe("Pit file debt ceiling converter", func() { It("converts a log to an model", func() { converter := debt_ceiling.PitFileDebtCeilingConverter{} - model, err := converter.ToModel(test_data.EthPitFileDebtCeilingLog) + models, err := converter.ToModels([]types.Log{test_data.EthPitFileDebtCeilingLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.PitFileDebtCeilingModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.PitFileDebtCeilingModel)) }) }) diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 81ddf35f..a5e86335 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model PitFileDebtCeilingModel) error + Create(headerID int64, models []PitFileDebtCeilingModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -34,12 +35,38 @@ func NewPitFileDebtCeilingRepository(db *postgres.DB) PitFileDebtCeilingReposito } } -func (repository PitFileDebtCeilingRepository) Create(headerID int64, model PitFileDebtCeilingModel) error { - _, err := repository.db.Exec( - `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log) +func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []PitFileDebtCeilingModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4, $5)`, - headerID, model.What, model.Data, model.TransactionIndex, model.Raw, - ) + headerID, model.What, model.Data, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true) return err } @@ -48,8 +75,8 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.pit_file_debt_ceiling on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR pit_file_debt_ceiling_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, @@ -57,6 +84,5 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe endingBlockNumber, repository.db.Node.ID, ) - return result, err } diff --git a/pkg/transformers/pit_file/debt_ceiling/repository_test.go b/pkg/transformers/pit_file/debt_ceiling/repository_test.go index 3c95be29..31916dae 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -29,15 +30,24 @@ import ( var _ = Describe("Pit file debt ceiling repository", func() { Describe("Create", func() { - It("adds a pit file debt ceiling event", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + pitFileDebtCeilingRepository debt_ceiling.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db) + pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db) + }) - err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel) + It("adds a pit file debt ceiling event", func() { + err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}) Expect(err).NotTo(HaveOccurred()) var dbPitFile debt_ceiling.PitFileDebtCeilingModel @@ -49,30 +59,28 @@ var _ = Describe("Pit file debt ceiling repository", func() { Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileDebtCeilingModel.Raw)) }) - It("does not duplicate pit file events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}) + Expect(err).NotTo(HaveOccurred()) - pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db) - err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate pit file events", func() { + err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}) Expect(err).NotTo(HaveOccurred()) - err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel) + err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes pit file if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db) - err = pitFileRepository.Create(headerID, test_data.PitFileDebtCeilingModel) + err = pitFileDebtCeilingRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -85,8 +93,48 @@ var _ = Describe("Pit file debt ceiling repository", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + pitFileDebtCeilingRepository debt_ceiling.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + pitFileDebtCeilingRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated pit file event", func() { + It("returns headers that haven't been checked", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) @@ -101,7 +149,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { Expect(err).NotTo(HaveOccurred()) } pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db) - err := pitFileRepository.Create(headerIDs[1], test_data.PitFileDebtCeilingModel) + err := pitFileRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -112,6 +160,33 @@ var _ = Describe("Pit file debt ceiling repository", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if pit file debt ceiling logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + pitFileDebtCeilingdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, pitFileDebtCeilingdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + pitFileDebtCeilingRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := pitFileDebtCeilingRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileDebtCeilingdBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -129,7 +204,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { } pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db) pitFileRepositoryTwo := debt_ceiling.NewPitFileDebtCeilingRepository(dbTwo) - err := pitFileRepository.Create(headerIDs[0], test_data.PitFileDebtCeilingModel) + err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/pit_file/debt_ceiling/transformer.go b/pkg/transformers/pit_file/debt_ceiling/transformer.go index cdbf5256..a82b2eff 100644 --- a/pkg/transformers/pit_file/debt_ceiling/transformer.go +++ b/pkg/transformers/pit_file/debt_ceiling/transformer.go @@ -60,16 +60,20 @@ func (transformer PitFileDebtCeilingTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + model, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, model) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go index 2c6c01af..89dc5b67 100644 --- a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go @@ -97,6 +97,42 @@ var _ = Describe("Pit file debt ceiling transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} + mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := debt_ceiling.PitFileDebtCeilingTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} + mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := debt_ceiling.PitFileDebtCeilingTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} fetcher := &mocks.MockLogFetcher{} @@ -112,7 +148,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthPitFileDebtCeilingLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileDebtCeilingLog})) }) It("returns error if converter returns error", func() { @@ -151,7 +187,7 @@ var _ = Describe("Pit file debt ceiling transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.PitFileDebtCeilingModel)) + Expect(repository.PassedModels).To(Equal([]debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/pit_file/ilk/converter.go b/pkg/transformers/pit_file/ilk/converter.go index c0ab6dde..94777525 100644 --- a/pkg/transformers/pit_file/ilk/converter.go +++ b/pkg/transformers/pit_file/ilk/converter.go @@ -25,29 +25,37 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (PitFileIlkModel, error) + ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error) } type PitFileIlkConverter struct{} -func (PitFileIlkConverter) ToModel(ethLog types.Log) (PitFileIlkModel, error) { - err := verifyLog(ethLog) - if err != nil { - return PitFileIlkModel{}, err - } - ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) - riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:] - risk := big.NewInt(0).SetBytes(riskBytes).String() +func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error) { + var models []PitFileIlkModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) + riskBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:] + risk := big.NewInt(0).SetBytes(riskBytes).String() - raw, err := json.Marshal(ethLog) - return PitFileIlkModel{ - Ilk: ilk, - What: what, - Data: risk, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := PitFileIlkModel{ + Ilk: ilk, + What: what, + Data: risk, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/pit_file/ilk/converter_test.go b/pkg/transformers/pit_file/ilk/converter_test.go index 867fbc37..6dba9b9b 100644 --- a/pkg/transformers/pit_file/ilk/converter_test.go +++ b/pkg/transformers/pit_file/ilk/converter_test.go @@ -31,7 +31,7 @@ var _ = Describe("Pit file ilk converter", func() { Data: []byte{1, 1, 1, 1, 1}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -42,7 +42,7 @@ var _ = Describe("Pit file ilk converter", func() { Topics: []common.Hash{{}, {}, {}, {}}, } - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -50,9 +50,10 @@ var _ = Describe("Pit file ilk converter", func() { It("converts a log to an model", func() { converter := ilk.PitFileIlkConverter{} - model, err := converter.ToModel(test_data.EthPitFileIlkLog) + models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.PitFileIlkModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.PitFileIlkModel)) }) }) diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index 8c9374f8..68ac396b 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model PitFileIlkModel) error + Create(headerID int64, models []PitFileIlkModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -34,12 +35,38 @@ func NewPitFileIlkRepository(db *postgres.DB) PitFileIlkRepository { } } -func (repository PitFileIlkRepository) Create(headerID int64, model PitFileIlkModel) error { - _, err := repository.db.Exec( - `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log) +func (repository PitFileIlkRepository) Create(headerID int64, models []PitFileIlkModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, - headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw, - ) + headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_ilk_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_ilk_checked = $2`, headerID, true) return err } @@ -48,8 +75,8 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.pit_file_ilk on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR pit_file_ilk_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, @@ -57,6 +84,5 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin endingBlockNumber, repository.db.Node.ID, ) - return result, err } diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index cbc77b61..da680f17 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -29,15 +30,24 @@ import ( var _ = Describe("Pit file ilk repository", func() { Describe("Create", func() { - It("adds a pit file ilk event", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + pitFileIlkRepository ilk.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - pitFileRepository := ilk.NewPitFileIlkRepository(db) + pitFileIlkRepository = ilk.NewPitFileIlkRepository(db) + }) - err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel) + It("adds a pit file ilk event", func() { + err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel}) Expect(err).NotTo(HaveOccurred()) var dbPitFile ilk.PitFileIlkModel @@ -50,30 +60,28 @@ var _ = Describe("Pit file ilk repository", func() { Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkModel.Raw)) }) - It("does not duplicate pit file ilk events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel}) + Expect(err).NotTo(HaveOccurred()) - pitFileRepository := ilk.NewPitFileIlkRepository(db) - err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate pit file ilk events", func() { + err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel}) Expect(err).NotTo(HaveOccurred()) - err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel) + err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes pit file ilk if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - pitFileRepository := ilk.NewPitFileIlkRepository(db) - err = pitFileRepository.Create(headerID, test_data.PitFileIlkModel) + err = pitFileIlkRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -86,8 +94,48 @@ var _ = Describe("Pit file ilk repository", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + pitFileIlkRepository ilk.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + pitFileIlkRepository = ilk.NewPitFileIlkRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = pitFileIlkRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = pitFileIlkRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated pit file ilk event", func() { + It("returns headers that haven't been checked", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) @@ -102,7 +150,7 @@ var _ = Describe("Pit file ilk repository", func() { Expect(err).NotTo(HaveOccurred()) } pitFileRepository := ilk.NewPitFileIlkRepository(db) - err := pitFileRepository.Create(headerIDs[1], test_data.PitFileIlkModel) + err := pitFileRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -113,6 +161,33 @@ var _ = Describe("Pit file ilk repository", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if pit file ilk logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + pitFileIlkdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, pitFileIlkdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + pitFileIlkRepository := ilk.NewPitFileIlkRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := pitFileIlkRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileIlkdBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -130,7 +205,7 @@ var _ = Describe("Pit file ilk repository", func() { } pitFileRepository := ilk.NewPitFileIlkRepository(db) pitFileRepositoryTwo := ilk.NewPitFileIlkRepository(dbTwo) - err := pitFileRepository.Create(headerIDs[0], test_data.PitFileIlkModel) + err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/pit_file/ilk/transformer.go b/pkg/transformers/pit_file/ilk/transformer.go index 80e9ecd7..2318dca0 100644 --- a/pkg/transformers/pit_file/ilk/transformer.go +++ b/pkg/transformers/pit_file/ilk/transformer.go @@ -60,16 +60,20 @@ func (transformer PitFileIlkTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/pit_file/ilk/transformer_test.go b/pkg/transformers/pit_file/ilk/transformer_test.go index 82955149..2b6cf806 100644 --- a/pkg/transformers/pit_file/ilk/transformer_test.go +++ b/pkg/transformers/pit_file/ilk/transformer_test.go @@ -97,6 +97,42 @@ var _ = Describe("Pit file ilk transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} + mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := ilk.PitFileIlkTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} + mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := ilk.PitFileIlkTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} fetcher := &mocks.MockLogFetcher{} @@ -112,7 +148,7 @@ var _ = Describe("Pit file ilk transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthPitFileIlkLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileIlkLog})) }) It("returns error if converter returns error", func() { @@ -151,7 +187,7 @@ var _ = Describe("Pit file ilk transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.PitFileIlkModel)) + Expect(repository.PassedModels).To(Equal([]ilk.PitFileIlkModel{test_data.PitFileIlkModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/pit_file/stability_fee/converter.go b/pkg/transformers/pit_file/stability_fee/converter.go index 0cd12db0..27d20155 100644 --- a/pkg/transformers/pit_file/stability_fee/converter.go +++ b/pkg/transformers/pit_file/stability_fee/converter.go @@ -24,26 +24,34 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error) + ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error) } type PitFileStabilityFeeConverter struct{} -func (PitFileStabilityFeeConverter) ToModel(ethLog types.Log) (PitFileStabilityFeeModel, error) { - err := verifyLog(ethLog) - if err != nil { - return PitFileStabilityFeeModel{}, err - } - what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - data := common.HexToAddress(ethLog.Topics[3].String()).Hex() +func (PitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error) { + var models []PitFileStabilityFeeModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + data := common.HexToAddress(ethLog.Topics[3].String()).Hex() - raw, err := json.Marshal(ethLog) - return PitFileStabilityFeeModel{ - What: what, - Data: data, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := PitFileStabilityFeeModel{ + What: what, + Data: data, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) + } + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/pit_file/stability_fee/converter_test.go b/pkg/transformers/pit_file/stability_fee/converter_test.go index 451af664..1b07605c 100644 --- a/pkg/transformers/pit_file/stability_fee/converter_test.go +++ b/pkg/transformers/pit_file/stability_fee/converter_test.go @@ -28,7 +28,7 @@ var _ = Describe("Pit file stability fee converter", func() { converter := stability_fee.PitFileStabilityFeeConverter{} badLog := types.Log{} - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -36,9 +36,10 @@ var _ = Describe("Pit file stability fee converter", func() { It("converts a log to an model", func() { converter := stability_fee.PitFileStabilityFeeConverter{} - model, err := converter.ToModel(test_data.EthPitFileStabilityFeeLog) + models, err := converter.ToModels([]types.Log{test_data.EthPitFileStabilityFeeLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.PitFileStabilityFeeModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.PitFileStabilityFeeModel)) }) }) diff --git a/pkg/transformers/pit_file/stability_fee/repository.go b/pkg/transformers/pit_file/stability_fee/repository.go index 51376a9d..6942512f 100644 --- a/pkg/transformers/pit_file/stability_fee/repository.go +++ b/pkg/transformers/pit_file/stability_fee/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model PitFileStabilityFeeModel) error + Create(headerID int64, models []PitFileStabilityFeeModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -34,12 +35,38 @@ func NewPitFileStabilityFeeRepository(db *postgres.DB) PitFileStabilityFeeReposi } } -func (repository PitFileStabilityFeeRepository) Create(headerID int64, model PitFileStabilityFeeModel) error { - _, err := repository.db.Exec( - `INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log) +func (repository PitFileStabilityFeeRepository) Create(headerID int64, models []PitFileStabilityFeeModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5)`, - headerID, model.What, model.Data, model.TransactionIndex, model.Raw, - ) + headerID, model.What, model.Data, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository PitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true) return err } @@ -48,8 +75,8 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.pit_file_stability_fee on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR pit_file_stability_fee_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, @@ -57,6 +84,5 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb endingBlockNumber, repository.db.Node.ID, ) - return result, err } diff --git a/pkg/transformers/pit_file/stability_fee/repository_test.go b/pkg/transformers/pit_file/stability_fee/repository_test.go index ab8eb263..ca00d21c 100644 --- a/pkg/transformers/pit_file/stability_fee/repository_test.go +++ b/pkg/transformers/pit_file/stability_fee/repository_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -29,15 +30,24 @@ import ( var _ = Describe("Pit file stability fee repository", func() { Describe("Create", func() { - It("adds a pit file stability fee event", func() { - db := test_config.NewTestDB(core.Node{}) + var ( + db *postgres.DB + pitFileStabilityFeeRepository stability_fee.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db) + pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db) + }) - err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel) + It("adds a pit file stability fee event", func() { + err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}) Expect(err).NotTo(HaveOccurred()) var dbPitFile stability_fee.PitFileStabilityFeeModel @@ -49,30 +59,28 @@ var _ = Describe("Pit file stability fee repository", func() { Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileStabilityFeeModel.Raw)) }) - It("does not duplicate pit file events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + It("marks header as checked for logs", func() { + err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}) + Expect(err).NotTo(HaveOccurred()) - pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db) - err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("does not duplicate pit file events", func() { + err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}) Expect(err).NotTo(HaveOccurred()) - err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel) + err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes pit file if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db) - err = pitFileRepository.Create(headerID, test_data.PitFileStabilityFeeModel) + err = pitFileStabilityFeeRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -85,8 +93,48 @@ var _ = Describe("Pit file stability fee repository", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + pitFileStabilityFeeRepository stability_fee.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + pitFileStabilityFeeRepository = stability_fee.NewPitFileStabilityFeeRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT pit_file_stability_fee_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated pit file event", func() { + It("returns headers that haven't been checked", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) @@ -101,7 +149,7 @@ var _ = Describe("Pit file stability fee repository", func() { Expect(err).NotTo(HaveOccurred()) } pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db) - err := pitFileRepository.Create(headerIDs[1], test_data.PitFileStabilityFeeModel) + err := pitFileRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -112,6 +160,33 @@ var _ = Describe("Pit file stability fee repository", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if pit file stability fee logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + pitFileStabilityFeedBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, pitFileStabilityFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + pitFileStabilityFeeRepository := stability_fee.NewPitFileStabilityFeeRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := pitFileStabilityFeeRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(pitFileStabilityFeedBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -129,7 +204,7 @@ var _ = Describe("Pit file stability fee repository", func() { } pitFileRepository := stability_fee.NewPitFileStabilityFeeRepository(db) pitFileRepositoryTwo := stability_fee.NewPitFileStabilityFeeRepository(dbTwo) - err := pitFileRepository.Create(headerIDs[0], test_data.PitFileStabilityFeeModel) + err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/pit_file/stability_fee/transformer.go b/pkg/transformers/pit_file/stability_fee/transformer.go index 7e424a4f..781c223e 100644 --- a/pkg/transformers/pit_file/stability_fee/transformer.go +++ b/pkg/transformers/pit_file/stability_fee/transformer.go @@ -60,16 +60,20 @@ func (transformer PitFileStabilityFeeTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/pit_file/stability_fee/transformer_test.go b/pkg/transformers/pit_file/stability_fee/transformer_test.go index 8ba7ce10..b2464af7 100644 --- a/pkg/transformers/pit_file/stability_fee/transformer_test.go +++ b/pkg/transformers/pit_file/stability_fee/transformer_test.go @@ -97,6 +97,42 @@ var _ = Describe("Pit file stability fee transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} + mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := stability_fee.PitFileStabilityFeeTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} + mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := stability_fee.PitFileStabilityFeeTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} fetcher := &mocks.MockLogFetcher{} @@ -112,7 +148,7 @@ var _ = Describe("Pit file stability fee transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthPitFileStabilityFeeLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthPitFileStabilityFeeLog})) }) It("returns error if converter returns error", func() { @@ -151,7 +187,7 @@ var _ = Describe("Pit file stability fee transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.PitFileStabilityFeeModel)) + Expect(repository.PassedModels).To(Equal([]stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})) }) It("returns error if repository returns error for create", func() { diff --git a/pkg/transformers/test_data/mocks/bite/converter.go b/pkg/transformers/test_data/mocks/bite/converter.go index 8d902519..690ea408 100644 --- a/pkg/transformers/test_data/mocks/bite/converter.go +++ b/pkg/transformers/test_data/mocks/bite/converter.go @@ -22,24 +22,23 @@ import ( ) type MockBiteConverter struct { - ConverterContract string ConverterAbi string LogsToConvert []types.Log EntitiesToConvert []BiteEntity ConverterError error } -func (mbc *MockBiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) { - mbc.ConverterContract = contractAddress +func (mbc *MockBiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) { mbc.ConverterAbi = contractAbi - mbc.LogsToConvert = append(mbc.LogsToConvert, ethLog) - return test_data.BiteEntity, mbc.ConverterError + mbc.LogsToConvert = append(mbc.LogsToConvert, ethLogs...) + return []BiteEntity{test_data.BiteEntity}, mbc.ConverterError } -func (mbc *MockBiteConverter) ToModel(entity BiteEntity) (BiteModel, error) { - mbc.EntitiesToConvert = append(mbc.EntitiesToConvert, entity) - return test_data.BiteModel, mbc.ConverterError +func (mbc *MockBiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) { + mbc.EntitiesToConvert = append(mbc.EntitiesToConvert, entities...) + return []BiteModel{test_data.BiteModel}, mbc.ConverterError } + func (mbc *MockBiteConverter) SetConverterError(err error) { mbc.ConverterError = err } diff --git a/pkg/transformers/test_data/mocks/bite/repository.go b/pkg/transformers/test_data/mocks/bite/repository.go index 12a7e774..3dea5239 100644 --- a/pkg/transformers/test_data/mocks/bite/repository.go +++ b/pkg/transformers/test_data/mocks/bite/repository.go @@ -15,19 +15,23 @@ package bite import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/bite" ) type MockBiteRepository struct { - createError error - PassedEndingBlockNumber int64 - PassedBiteModel bite.BiteModel - PassedHeaderID int64 - PassedStartingBlockNumber int64 - PassedTransactionIndex uint - missingHeaders []core.Header - missingHeadersErr error + createError error + PassedEndingBlockNumber int64 + PassedBiteModels []bite.BiteModel + PassedHeaderID int64 + PassedStartingBlockNumber int64 + PassedTransactionIndex uint + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error } func (repository *MockBiteRepository) SetCreateError(err error) { @@ -42,14 +46,27 @@ func (repository *MockBiteRepository) SetMissingHeaders(headers []core.Header) { repository.missingHeaders = headers } -func (repository *MockBiteRepository) Create(headerID int64, model bite.BiteModel) error { +func (repository *MockBiteRepository) Create(headerID int64, models []bite.BiteModel) error { repository.PassedHeaderID = headerID - repository.PassedBiteModel = model + repository.PassedBiteModels = models return repository.createError } +func (repository *MockBiteRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockBiteRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } + +func (repository *MockBiteRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockBiteRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/drip_drip/converter.go b/pkg/transformers/test_data/mocks/drip_drip/converter.go index 11c66a48..c6505321 100644 --- a/pkg/transformers/test_data/mocks/drip_drip/converter.go +++ b/pkg/transformers/test_data/mocks/drip_drip/converter.go @@ -21,13 +21,13 @@ import ( ) type MockDripDripConverter struct { - Err error - PassedLog types.Log + Err error + PassedLogs []types.Log } -func (converter *MockDripDripConverter) ToModel(ethLog types.Log) (drip_drip.DripDripModel, error) { - converter.PassedLog = ethLog - return test_data.DripDripModel, converter.Err +func (converter *MockDripDripConverter) ToModels(ethLogs []types.Log) ([]drip_drip.DripDripModel, error) { + converter.PassedLogs = ethLogs + return []drip_drip.DripDripModel{test_data.DripDripModel}, converter.Err } func (converter *MockDripDripConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/drip_drip/repository.go b/pkg/transformers/test_data/mocks/drip_drip/repository.go index c836061b..ca520a13 100644 --- a/pkg/transformers/test_data/mocks/drip_drip/repository.go +++ b/pkg/transformers/test_data/mocks/drip_drip/repository.go @@ -15,32 +15,45 @@ package drip_drip import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" ) type MockDripDripRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel drip_drip.DripDripModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []drip_drip.DripDripModel } -func (repository *MockDripDripRepository) Create(headerID int64, model drip_drip.DripDripModel) error { +func (repository *MockDripDripRepository) Create(headerID int64, models []drip_drip.DripDripModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createErr } +func (repository *MockDripDripRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockDripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockDripDripRepository) SetMarkHeaderCheckedErr(err error) { + repository.markHeaderCheckedErr = err +} + func (repository *MockDripDripRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } @@ -52,3 +65,6 @@ func (repository *MockDripDripRepository) SetMissingHeaders(headers []core.Heade func (repository *MockDripDripRepository) SetCreateError(e error) { repository.createErr = e } +func (repository *MockDripDripRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go b/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go index 13d8a566..85492371 100644 --- a/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go +++ b/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go @@ -22,12 +22,12 @@ import ( type MockDripFileIlkConverter struct { converterErr error - PassedLog types.Log + PassedLogs []types.Log } -func (converter *MockDripFileIlkConverter) ToModel(ethLog types.Log) (ilk.DripFileIlkModel, error) { - converter.PassedLog = ethLog - return test_data.DripFileIlkModel, converter.converterErr +func (converter *MockDripFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.DripFileIlkModel, error) { + converter.PassedLogs = ethLogs + return []ilk.DripFileIlkModel{test_data.DripFileIlkModel}, converter.converterErr } func (converter *MockDripFileIlkConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go b/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go index e08a2e58..b76aa85e 100644 --- a/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go @@ -15,32 +15,45 @@ package ilk import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" ) type MockDripFileIlkRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel ilk.DripFileIlkModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []ilk.DripFileIlkModel } -func (repository *MockDripFileIlkRepository) Create(headerID int64, model ilk.DripFileIlkModel) error { +func (repository *MockDripFileIlkRepository) Create(headerID int64, models []ilk.DripFileIlkModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createErr } +func (repository *MockDripFileIlkRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockDripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockDripFileIlkRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockDripFileIlkRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } @@ -52,3 +65,6 @@ func (repository *MockDripFileIlkRepository) SetMissingHeaders(headers []core.He func (repository *MockDripFileIlkRepository) SetCreateError(e error) { repository.createErr = e } +func (repository *MockDripFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/drip_file/repo/converter.go b/pkg/transformers/test_data/mocks/drip_file/repo/converter.go index 2f9e37f5..0dbe80d0 100644 --- a/pkg/transformers/test_data/mocks/drip_file/repo/converter.go +++ b/pkg/transformers/test_data/mocks/drip_file/repo/converter.go @@ -22,12 +22,12 @@ import ( type MockDripFileRepoConverter struct { converterErr error - PassedLog types.Log + PassedLogs []types.Log } -func (converter *MockDripFileRepoConverter) ToModel(ethLog types.Log) (repo.DripFileRepoModel, error) { - converter.PassedLog = ethLog - return test_data.DripFileRepoModel, converter.converterErr +func (converter *MockDripFileRepoConverter) ToModels(ethLogs []types.Log) ([]repo.DripFileRepoModel, error) { + converter.PassedLogs = ethLogs + return []repo.DripFileRepoModel{test_data.DripFileRepoModel}, converter.converterErr } func (converter *MockDripFileRepoConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/drip_file/repo/repository.go b/pkg/transformers/test_data/mocks/drip_file/repo/repository.go index dcf532b6..be03693a 100644 --- a/pkg/transformers/test_data/mocks/drip_file/repo/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/repo/repository.go @@ -15,32 +15,45 @@ package repo import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" ) type MockDripFileRepoRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel repo.DripFileRepoModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []repo.DripFileRepoModel } -func (repository *MockDripFileRepoRepository) Create(headerID int64, model repo.DripFileRepoModel) error { +func (repository *MockDripFileRepoRepository) Create(headerID int64, models []repo.DripFileRepoModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createErr } +func (repository *MockDripFileRepoRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockDripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockDripFileRepoRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockDripFileRepoRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } @@ -52,3 +65,6 @@ func (repository *MockDripFileRepoRepository) SetMissingHeaders(headers []core.H func (repository *MockDripFileRepoRepository) SetCreateError(e error) { repository.createErr = e } +func (repository *MockDripFileRepoRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/drip_file/vow/converter.go b/pkg/transformers/test_data/mocks/drip_file/vow/converter.go index 95400dd1..7d7adb29 100644 --- a/pkg/transformers/test_data/mocks/drip_file/vow/converter.go +++ b/pkg/transformers/test_data/mocks/drip_file/vow/converter.go @@ -22,12 +22,12 @@ import ( type MockDripFileVowConverter struct { converterErr error - PassedLog types.Log + PassedLogs []types.Log } -func (converter *MockDripFileVowConverter) ToModel(ethLog types.Log) (vow.DripFileVowModel, error) { - converter.PassedLog = ethLog - return test_data.DripFileVowModel, converter.converterErr +func (converter *MockDripFileVowConverter) ToModels(ethLogs []types.Log) ([]vow.DripFileVowModel, error) { + converter.PassedLogs = ethLogs + return []vow.DripFileVowModel{test_data.DripFileVowModel}, converter.converterErr } func (converter *MockDripFileVowConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/drip_file/vow/repository.go b/pkg/transformers/test_data/mocks/drip_file/vow/repository.go index 69404a7d..54e36edd 100644 --- a/pkg/transformers/test_data/mocks/drip_file/vow/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/vow/repository.go @@ -15,32 +15,45 @@ package vow import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" ) type MockDripFileVowRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel vow.DripFileVowModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vow.DripFileVowModel } -func (repository *MockDripFileVowRepository) Create(headerID int64, model vow.DripFileVowModel) error { +func (repository *MockDripFileVowRepository) Create(headerID int64, models []vow.DripFileVowModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createErr } +func (repository *MockDripFileVowRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockDripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockDripFileVowRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockDripFileVowRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } @@ -52,3 +65,6 @@ func (repository *MockDripFileVowRepository) SetMissingHeaders(headers []core.He func (repository *MockDripFileVowRepository) SetCreateError(e error) { repository.createErr = e } +func (repository *MockDripFileVowRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/frob/converter.go b/pkg/transformers/test_data/mocks/frob/converter.go index e69a996b..645307c7 100644 --- a/pkg/transformers/test_data/mocks/frob/converter.go +++ b/pkg/transformers/test_data/mocks/frob/converter.go @@ -22,30 +22,28 @@ import ( ) type MockFrobConverter struct { - PassedContractAddress string - PassedContractABI string - PassedLog types.Log - PassedEntity frob.FrobEntity - toEntityError error - toModelError error + PassedContractABI string + PassedLogs []types.Log + PassedEntities []frob.FrobEntity + toEntityError error + toModelError error } -func (converter *MockFrobConverter) SetToEntityError(err error) { +func (converter *MockFrobConverter) SetToEntitiesError(err error) { converter.toEntityError = err } -func (converter *MockFrobConverter) SetToModelError(err error) { +func (converter *MockFrobConverter) SetToModelsError(err error) { converter.toModelError = err } -func (converter *MockFrobConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (frob.FrobEntity, error) { - converter.PassedContractAddress = contractAddress +func (converter *MockFrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]frob.FrobEntity, error) { converter.PassedContractABI = contractAbi - converter.PassedLog = ethLog - return test_data.FrobEntity, converter.toEntityError + converter.PassedLogs = ethLogs + return []frob.FrobEntity{test_data.FrobEntity}, converter.toEntityError } -func (converter *MockFrobConverter) ToModel(frobEntity frob.FrobEntity) (frob.FrobModel, error) { - converter.PassedEntity = frobEntity - return test_data.FrobModel, converter.toModelError +func (converter *MockFrobConverter) ToModels(frobEntities []frob.FrobEntity) ([]frob.FrobModel, error) { + converter.PassedEntities = frobEntities + return []frob.FrobModel{test_data.FrobModel}, converter.toModelError } diff --git a/pkg/transformers/test_data/mocks/frob/repository.go b/pkg/transformers/test_data/mocks/frob/repository.go index 5358fda6..6cc88c6f 100644 --- a/pkg/transformers/test_data/mocks/frob/repository.go +++ b/pkg/transformers/test_data/mocks/frob/repository.go @@ -15,24 +15,32 @@ package frob import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" ) type MockFrobRepository struct { - createError error - PassedEndingBlockNumber int64 - PassedFrobModel frob.FrobModel - PassedHeaderID int64 - PassedStartingBlockNumber int64 - missingHeaders []core.Header - missingHeadersErr error + createError error + PassedEndingBlockNumber int64 + PassedModels []frob.FrobModel + PassedHeaderID int64 + PassedStartingBlockNumber int64 + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error } func (repository *MockFrobRepository) SetCreateError(err error) { repository.createError = err } +func (repository *MockFrobRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockFrobRepository) SetMissingHeadersErr(err error) { repository.missingHeadersErr = err } @@ -41,14 +49,22 @@ func (repository *MockFrobRepository) SetMissingHeaders(headers []core.Header) { repository.missingHeaders = headers } -func (repository *MockFrobRepository) Create(headerID int64, model frob.FrobModel) error { +func (repository *MockFrobRepository) Create(headerID int64, models []frob.FrobModel) error { repository.PassedHeaderID = headerID - repository.PassedFrobModel = model + repository.PassedModels = models return repository.createError } +func (repository *MockFrobRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockFrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockFrobRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go index 98aa0d79..f4ca9aa7 100644 --- a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go +++ b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go @@ -23,12 +23,12 @@ import ( type MockPitFileDebtCeilingConverter struct { converterErr error - PassedLog types.Log + PassedLogs []types.Log } -func (converter *MockPitFileDebtCeilingConverter) ToModel(ethLog types.Log) (debt_ceiling.PitFileDebtCeilingModel, error) { - converter.PassedLog = ethLog - return test_data.PitFileDebtCeilingModel, converter.converterErr +func (converter *MockPitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]debt_ceiling.PitFileDebtCeilingModel, error) { + converter.PassedLogs = ethLogs + return []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}, converter.converterErr } func (converter *MockPitFileDebtCeilingConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go index 949a56e1..be06442e 100644 --- a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go @@ -15,23 +15,32 @@ package debt_ceiling import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" ) type MockPitFileDebtCeilingRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel debt_ceiling.PitFileDebtCeilingModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []debt_ceiling.PitFileDebtCeilingModel } -func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, model debt_ceiling.PitFileDebtCeilingModel) error { +func (repository *MockPitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + +func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, models []debt_ceiling.PitFileDebtCeilingModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createErr } @@ -41,6 +50,10 @@ func (repository *MockPitFileDebtCeilingRepository) MissingHeaders(startingBlock return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockPitFileDebtCeilingRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockPitFileDebtCeilingRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } @@ -52,3 +65,7 @@ func (repository *MockPitFileDebtCeilingRepository) SetMissingHeaders(headers [] func (repository *MockPitFileDebtCeilingRepository) SetCreateError(e error) { repository.createErr = e } + +func (repository *MockPitFileDebtCeilingRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go b/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go index 47da762a..d7052c42 100644 --- a/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go +++ b/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go @@ -22,7 +22,7 @@ import ( ) type MockPitFileIlkConverter struct { - PassedLog types.Log + PassedLogs []types.Log converterError error } @@ -30,7 +30,7 @@ func (converter *MockPitFileIlkConverter) SetConverterError(err error) { converter.converterError = err } -func (converter *MockPitFileIlkConverter) ToModel(ethLog types.Log) (ilk.PitFileIlkModel, error) { - converter.PassedLog = ethLog - return test_data.PitFileIlkModel, converter.converterError +func (converter *MockPitFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.PitFileIlkModel, error) { + converter.PassedLogs = ethLogs + return []ilk.PitFileIlkModel{test_data.PitFileIlkModel}, converter.converterError } diff --git a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go index 5b75664a..4bb8db36 100644 --- a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go @@ -15,24 +15,32 @@ package ilk import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" ) type MockPitFileIlkRepository struct { - createError error - PassedEndingBlockNumber int64 - PassedModel ilk.PitFileIlkModel - PassedHeaderID int64 - PassedStartingBlockNumber int64 - missingHeaders []core.Header - missingHeadersErr error + createError error + PassedEndingBlockNumber int64 + PassedModels []ilk.PitFileIlkModel + PassedHeaderID int64 + PassedStartingBlockNumber int64 + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error } func (repository *MockPitFileIlkRepository) SetCreateError(err error) { repository.createError = err } +func (repository *MockPitFileIlkRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockPitFileIlkRepository) SetMissingHeadersErr(err error) { repository.missingHeadersErr = err } @@ -41,14 +49,23 @@ func (repository *MockPitFileIlkRepository) SetMissingHeaders(headers []core.Hea repository.missingHeaders = headers } -func (repository *MockPitFileIlkRepository) Create(headerID int64, model ilk.PitFileIlkModel) error { +func (repository *MockPitFileIlkRepository) Create(headerID int64, models []ilk.PitFileIlkModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createError } +func (repository *MockPitFileIlkRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockPitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } + +func (repository *MockPitFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go b/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go index 198cbecb..8539d367 100644 --- a/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go +++ b/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go @@ -23,13 +23,14 @@ import ( type MockPitFileStabilityFeeConverter struct { converterErr error - PassedLog types.Log + PassedLogs []types.Log } -func (converter *MockPitFileStabilityFeeConverter) ToModel(ethLog types.Log) (stability_fee.PitFileStabilityFeeModel, error) { - converter.PassedLog = ethLog - return test_data.PitFileStabilityFeeModel, converter.converterErr +func (converter *MockPitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]stability_fee.PitFileStabilityFeeModel, error) { + converter.PassedLogs = ethLogs + return []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}, converter.converterErr } + func (converter *MockPitFileStabilityFeeConverter) SetConverterError(e error) { converter.converterErr = e } diff --git a/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go b/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go index a05f6eee..907538a8 100644 --- a/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go @@ -15,38 +15,57 @@ package stability_fee import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" ) type MockPitFileStabilityFeeRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel stability_fee.PitFileStabilityFeeModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []stability_fee.PitFileStabilityFeeModel } -func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, model stability_fee.PitFileStabilityFeeModel) error { - repository.PassedModel = model +func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, models []stability_fee.PitFileStabilityFeeModel) error { + repository.PassedModels = models repository.PassedHeaderID = headerID return repository.createErr } +func (repository *MockPitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockPitFileStabilityFeeRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockPitFileStabilityFeeRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockPitFileStabilityFeeRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } + func (repository *MockPitFileStabilityFeeRepository) SetMissingHeaders(headers []core.Header) { repository.missingHeaders = headers } + func (repository *MockPitFileStabilityFeeRepository) SetCreateError(e error) { repository.createErr = e } + +func (repository *MockPitFileStabilityFeeRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/test_data/mocks/vat_init/converter.go b/pkg/transformers/test_data/mocks/vat_init/converter.go index f90f5ac4..3b8b9ccd 100644 --- a/pkg/transformers/test_data/mocks/vat_init/converter.go +++ b/pkg/transformers/test_data/mocks/vat_init/converter.go @@ -23,12 +23,12 @@ import ( type MockVatInitConverter struct { converterErr error - PassedLog types.Log + PassedLogs []types.Log } -func (converter *MockVatInitConverter) ToModel(ethLog types.Log) (vat_init.VatInitModel, error) { - converter.PassedLog = ethLog - return test_data.VatInitModel, converter.converterErr +func (converter *MockVatInitConverter) ToModels(ethLogs []types.Log) ([]vat_init.VatInitModel, error) { + converter.PassedLogs = ethLogs + return []vat_init.VatInitModel{test_data.VatInitModel}, converter.converterErr } func (converter *MockVatInitConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/vat_init/repository.go b/pkg/transformers/test_data/mocks/vat_init/repository.go index a27c5539..7a9978b6 100644 --- a/pkg/transformers/test_data/mocks/vat_init/repository.go +++ b/pkg/transformers/test_data/mocks/vat_init/repository.go @@ -15,32 +15,45 @@ package vat_init import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" ) type MockVatInitRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel vat_init.VatInitModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_init.VatInitModel } -func (repository *MockVatInitRepository) Create(headerID int64, model vat_init.VatInitModel) error { +func (repository *MockVatInitRepository) Create(headerID int64, models []vat_init.VatInitModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createErr } +func (repository *MockVatInitRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockVatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockVatInitRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockVatInitRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } @@ -52,3 +65,7 @@ func (repository *MockVatInitRepository) SetMissingHeaders(headers []core.Header func (repository *MockVatInitRepository) SetCreateError(e error) { repository.createErr = e } + +func (repository *MockVatInitRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/vat_init/converter.go b/pkg/transformers/vat_init/converter.go index 63ecba87..7fc510b5 100644 --- a/pkg/transformers/vat_init/converter.go +++ b/pkg/transformers/vat_init/converter.go @@ -23,23 +23,31 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (VatInitModel, error) + ToModels(ethLogs []types.Log) ([]VatInitModel, error) } type VatInitConverter struct{} -func (VatInitConverter) ToModel(ethLog types.Log) (VatInitModel, error) { - err := verifyLog(ethLog) - if err != nil { - return VatInitModel{}, err +func (VatInitConverter) ToModels(ethLogs []types.Log) ([]VatInitModel, error) { + var models []VatInitModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + raw, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + model := VatInitModel{ + Ilk: ilk, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + models = append(models, model) } - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) - raw, err := json.Marshal(ethLog) - return VatInitModel{ - Ilk: ilk, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/vat_init/converter_test.go b/pkg/transformers/vat_init/converter_test.go index 78b4cb7d..f07625ba 100644 --- a/pkg/transformers/vat_init/converter_test.go +++ b/pkg/transformers/vat_init/converter_test.go @@ -28,7 +28,7 @@ var _ = Describe("Vat init converter", func() { converter := vat_init.VatInitConverter{} badLog := types.Log{} - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -36,9 +36,10 @@ var _ = Describe("Vat init converter", func() { It("converts a log to an model", func() { converter := vat_init.VatInitConverter{} - model, err := converter.ToModel(test_data.EthVatInitLog) + models, err := converter.ToModels([]types.Log{test_data.EthVatInitLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.VatInitModel)) + Expect(len(models)).To(Equal(1)) + Expect(models[0]).To(Equal(test_data.VatInitModel)) }) }) diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index c79861c9..3fc3d56b 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model VatInitModel) error + Create(headerID int64, models []VatInitModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -34,12 +35,31 @@ func NewVatInitRepository(db *postgres.DB) VatInitRepository { } } -func (repository VatInitRepository) Create(headerID int64, model VatInitModel) error { - _, err := repository.db.Exec( - `INSERT into maker.vat_init (header_id, ilk, tx_idx, raw_log) +func (repository VatInitRepository) Create(headerID int64, models []VatInitModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.vat_init (header_id, ilk, tx_idx, raw_log) VALUES($1, $2, $3, $4)`, - headerID, model.Ilk, model.TransactionIndex, model.Raw, - ) + headerID, model.Ilk, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + + return tx.Commit() +} + +func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_init_checked = $2`, headerID, true) return err } @@ -48,8 +68,8 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.vat_init on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_init_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, @@ -57,6 +77,5 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl endingBlockNumber, repository.db.Node.ID, ) - return result, err } diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index 22b9173a..dce264c8 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -21,13 +21,14 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" "github.com/vulcanize/vulcanizedb/test_config" ) -var _ = Describe("", func() { +var _ = Describe("Vat init repository", func() { Describe("Create", func() { It("adds a vat event", func() { db := test_config.NewTestDB(core.Node{}) @@ -37,7 +38,7 @@ var _ = Describe("", func() { Expect(err).NotTo(HaveOccurred()) vatInitRepository := vat_init.NewVatInitRepository(db) - err = vatInitRepository.Create(headerID, test_data.VatInitModel) + err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel}) Expect(err).NotTo(HaveOccurred()) var dbVatInit vat_init.VatInitModel @@ -55,10 +56,10 @@ var _ = Describe("", func() { headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) vatInitRepository := vat_init.NewVatInitRepository(db) - err = vatInitRepository.Create(headerID, test_data.VatInitModel) + err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel}) Expect(err).NotTo(HaveOccurred()) - err = vatInitRepository.Create(headerID, test_data.VatInitModel) + err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -71,7 +72,7 @@ var _ = Describe("", func() { headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) vatInitRepository := vat_init.NewVatInitRepository(db) - err = vatInitRepository.Create(headerID, test_data.VatInitModel) + err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel}) Expect(err).NotTo(HaveOccurred()) _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) @@ -84,8 +85,48 @@ var _ = Describe("", func() { }) }) + Describe("MarkHeaderChecked", func() { + var ( + db *postgres.DB + vatInitRepository vat_init.Repository + err error + headerID int64 + ) + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatInitRepository = vat_init.NewVatInitRepository(db) + }) + + It("creates a row for a new headerID", func() { + err = vatInitRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("updates row when headerID already exists", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) + + err = vatInitRepository.MarkHeaderChecked(headerID) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { - It("returns headers with no associated vat event", func() { + It("returns headers that haven't been checked", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) @@ -100,7 +141,7 @@ var _ = Describe("", func() { Expect(err).NotTo(HaveOccurred()) } vatInitRepository := vat_init.NewVatInitRepository(db) - err := vatInitRepository.Create(headerIDs[1], test_data.VatInitModel) + err := vatInitRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) headers, err := vatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) @@ -111,6 +152,33 @@ var _ = Describe("", func() { Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) }) + It("only treats headers as checked if drip drip logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatInitBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatInitBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + VatInitRepository := vat_init.NewVatInitRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := VatInitRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatInitBlockNumber))) + }) + It("only returns headers associated with the current node", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) @@ -128,7 +196,7 @@ var _ = Describe("", func() { } vatInitRepository := vat_init.NewVatInitRepository(db) vatInitRepositoryTwo := vat_init.NewVatInitRepository(dbTwo) - err := vatInitRepository.Create(headerIDs[0], test_data.VatInitModel) + err := vatInitRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := vatInitRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) diff --git a/pkg/transformers/vat_init/transformer.go b/pkg/transformers/vat_init/transformer.go index 74783f41..2404f67b 100644 --- a/pkg/transformers/vat_init/transformer.go +++ b/pkg/transformers/vat_init/transformer.go @@ -59,16 +59,20 @@ func (transformer VatInitTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/vat_init/transformer_test.go b/pkg/transformers/vat_init/transformer_test.go index c72e5547..95d2acab 100644 --- a/pkg/transformers/vat_init/transformer_test.go +++ b/pkg/transformers/vat_init/transformer_test.go @@ -96,6 +96,42 @@ var _ = Describe("Vat init transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks header checked if no logs returned", func() { + mockConverter := &vat_init_mocks.MockVatInitConverter{} + mockRepository := &vat_init_mocks.MockVatInitRepository{} + headerID := int64(123) + mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_init.VatInitTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_init_mocks.MockVatInitConverter{} + mockRepository := &vat_init_mocks.MockVatInitRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_init.VatInitTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("converts matching logs", func() { converter := &vat_init_mocks.MockVatInitConverter{} fetcher := &mocks.MockLogFetcher{} @@ -111,7 +147,7 @@ var _ = Describe("Vat init transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthVatInitLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatInitLog})) }) It("returns error if converter returns error", func() { @@ -150,7 +186,7 @@ var _ = Describe("Vat init transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.VatInitModel)) + Expect(repository.PassedModels).To(Equal([]vat_init.VatInitModel{test_data.VatInitModel})) }) It("returns error if repository returns error for create", func() {