diff --git a/Supfile b/Supfile index c224ae87..5aef5d17 100644 --- a/Supfile +++ b/Supfile @@ -57,7 +57,8 @@ commands: run: > systemctl daemon-reload && sudo systemctl restart vulcanizedb_light_sync.service && - sudo systemctl restart vulcanizedb_log_sync.service + sudo systemctl restart vulcanizedb_log_sync.service && + sudo systemctl restart vulcanizedb_recheck_sync.service postgraphile: desc: start postgraphile run: systemctl daemon-reload && sudo systemctl restart postgraphile.service diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 73ce9d12..f6681c78 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -22,6 +22,7 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) // backfillMakerLogsCmd represents the backfillMakerLogs command @@ -50,7 +51,7 @@ func backfillMakerLogs() { watcher := shared.NewEventWatcher(db, blockChain) watcher.AddTransformers(transformers.TransformerInitializers()) - err = watcher.Execute() + err = watcher.Execute(constants.HeaderMissing) if err != nil { // TODO Handle watcher error in backfillMakerLogs } diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 240915fd..aa7f9c0f 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -49,6 +49,7 @@ This command expects a light sync to have been run, and the presence of header r } var transformerNames []string +var recheckHeadersArg bool func syncMakerLogs() { ticker := time.NewTicker(pollingInterval) @@ -66,7 +67,11 @@ func syncMakerLogs() { watcher.AddTransformers(initializers) for range ticker.C { - err = watcher.Execute() + if recheckHeadersArg { + err = watcher.Execute(constants.HeaderRecheck) + } else { + err = watcher.Execute(constants.HeaderMissing) + } if err != nil { // TODO Handle watcher errors in ContinuousLogSync } @@ -125,4 +130,5 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer func init() { rootCmd.AddCommand(continuousLogSyncCmd) continuousLogSyncCmd.Flags().StringSliceVar(&transformerNames, "transformers", []string{"all"}, "transformer names to be run during this command") + continuousLogSyncCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.") } diff --git a/db/migrations/00072_change_checked_headers_headers_table_type_int.sql b/db/migrations/00072_change_checked_headers_headers_table_type_int.sql new file mode 100644 index 00000000..af9fa93f --- /dev/null +++ b/db/migrations/00072_change_checked_headers_headers_table_type_int.sql @@ -0,0 +1,175 @@ +-- +goose Up +ALTER TABLE checked_headers + ALTER price_feeds_checked SET DEFAULT null, + ALTER flip_kick_checked SET DEFAULT null, + ALTER frob_checked SET DEFAULT null, + ALTER tend_checked SET DEFAULT null, + ALTER bite_checked SET DEFAULT null, + ALTER dent_checked SET DEFAULT null, + ALTER pit_file_debt_ceiling_checked SET DEFAULT null, + ALTER pit_file_ilk_checked SET DEFAULT null, + ALTER vat_init_checked SET DEFAULT null, + ALTER drip_file_ilk_checked SET DEFAULT null, + ALTER drip_file_repo_checked SET DEFAULT null, + ALTER drip_file_vow_checked SET DEFAULT null, + ALTER deal_checked SET DEFAULT null, + ALTER drip_drip_checked SET DEFAULT null, + ALTER cat_file_chop_lump_checked SET DEFAULT null, + ALTER cat_file_flip_checked SET DEFAULT null, + ALTER cat_file_pit_vow_checked SET DEFAULT null, + ALTER flop_kick_checked SET DEFAULT null, + ALTER vat_move_checked SET DEFAULT null, + ALTER vat_fold_checked SET DEFAULT null, + ALTER vat_heal_checked SET DEFAULT null, + ALTER vat_toll_checked SET DEFAULT null, + ALTER vat_tune_checked SET DEFAULT null, + ALTER vat_grab_checked SET DEFAULT null, + ALTER vat_flux_checked SET DEFAULT null, + ALTER vat_slip_checked SET DEFAULT null, + ALTER vow_flog_checked SET DEFAULT null, + ALTER flap_kick_checked SET DEFAULT null; + +ALTER TABLE checked_headers + ALTER COLUMN price_feeds_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER price_feeds_checked SET DEFAULT 0, + ALTER COLUMN flip_kick_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER flip_kick_checked SET DEFAULT 0, + ALTER COLUMN frob_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER frob_checked SET DEFAULT 0, + ALTER COLUMN tend_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER tend_checked SET DEFAULT 0, + ALTER COLUMN bite_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER bite_checked SET DEFAULT 0, + ALTER COLUMN dent_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER dent_checked SET DEFAULT 0, + ALTER COLUMN pit_file_debt_ceiling_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER pit_file_debt_ceiling_checked SET DEFAULT 0, + ALTER COLUMN pit_file_ilk_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER pit_file_ilk_checked SET DEFAULT 0, + ALTER COLUMN vat_init_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_init_checked SET DEFAULT 0, + ALTER COLUMN drip_file_ilk_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER drip_file_ilk_checked SET DEFAULT 0, + ALTER COLUMN drip_file_repo_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER drip_file_repo_checked SET DEFAULT 0, + ALTER COLUMN drip_file_vow_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER drip_file_vow_checked SET DEFAULT 0, + ALTER COLUMN deal_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER deal_checked SET DEFAULT 0, + ALTER COLUMN drip_drip_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER drip_drip_checked SET DEFAULT 0, + ALTER COLUMN cat_file_chop_lump_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER cat_file_chop_lump_checked SET DEFAULT 0, + ALTER COLUMN cat_file_flip_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER cat_file_flip_checked SET DEFAULT 0, + ALTER COLUMN cat_file_pit_vow_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER cat_file_pit_vow_checked SET DEFAULT 0, + ALTER COLUMN flop_kick_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER flop_kick_checked SET DEFAULT 0, + ALTER COLUMN vat_move_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_move_checked SET DEFAULT 0, + ALTER COLUMN vat_fold_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_fold_checked SET DEFAULT 0, + ALTER COLUMN vat_heal_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_heal_checked SET DEFAULT 0, + ALTER COLUMN vat_toll_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_toll_checked SET DEFAULT 0, + ALTER COLUMN vat_tune_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_tune_checked SET DEFAULT 0, + ALTER COLUMN vat_grab_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_grab_checked SET DEFAULT 0, + ALTER COLUMN vat_flux_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_flux_checked SET DEFAULT 0, + ALTER COLUMN vat_slip_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vat_slip_checked SET DEFAULT 0, + ALTER COLUMN vow_flog_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER vow_flog_checked SET DEFAULT 0, + ALTER COLUMN flap_kick_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END, + ALTER flap_kick_checked SET DEFAULT 0; + +-- +goose Down +ALTER TABLE checked_headers + ALTER price_feeds_checked drop default, + ALTER COLUMN price_feeds_checked SET DATA TYPE boolean USING CASE WHEN price_feeds_checked = 0 THEN FALSE ELSE TRUE END, + ALTER price_feeds_checked SET DEFAULT FALSE, + ALTER flip_kick_checked drop default, + ALTER COLUMN flip_kick_checked SET DATA TYPE boolean USING CASE WHEN flip_kick_checked = 0 THEN FALSE ELSE TRUE END, + ALTER flip_kick_checked SET DEFAULT FALSE, + ALTER frob_checked drop default, + ALTER COLUMN frob_checked SET DATA TYPE boolean USING CASE WHEN frob_checked = 0 THEN FALSE ELSE TRUE END, + ALTER frob_checked SET DEFAULT FALSE, + ALTER tend_checked drop default, + ALTER COLUMN tend_checked SET DATA TYPE boolean USING CASE WHEN tend_checked = 0 THEN FALSE ELSE TRUE END, + ALTER tend_checked SET DEFAULT FALSE, + ALTER bite_checked drop default, + ALTER COLUMN bite_checked SET DATA TYPE boolean USING CASE WHEN bite_checked = 0 THEN FALSE ELSE TRUE END, + ALTER bite_checked SET DEFAULT FALSE, + ALTER dent_checked drop default, + ALTER COLUMN dent_checked SET DATA TYPE boolean USING CASE WHEN dent_checked = 0 THEN FALSE ELSE TRUE END, + ALTER dent_checked SET DEFAULT FALSE, + ALTER pit_file_debt_ceiling_checked drop default, + ALTER COLUMN pit_file_debt_ceiling_checked SET DATA TYPE boolean USING CASE WHEN pit_file_debt_ceiling_checked = 0 THEN FALSE ELSE TRUE END, + ALTER pit_file_debt_ceiling_checked SET DEFAULT FALSE, + ALTER pit_file_ilk_checked drop default, + ALTER COLUMN pit_file_ilk_checked SET DATA TYPE boolean USING CASE WHEN pit_file_ilk_checked = 0 THEN FALSE ELSE TRUE END, + ALTER pit_file_ilk_checked SET DEFAULT FALSE, + ALTER vat_init_checked drop default, + ALTER COLUMN vat_init_checked SET DATA TYPE boolean USING CASE WHEN vat_init_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_init_checked SET DEFAULT FALSE, + ALTER drip_file_ilk_checked drop default, + ALTER COLUMN drip_file_ilk_checked SET DATA TYPE boolean USING CASE WHEN drip_file_ilk_checked = 0 THEN FALSE ELSE TRUE END, + ALTER drip_file_ilk_checked SET DEFAULT FALSE, + ALTER drip_file_repo_checked drop default, + ALTER COLUMN drip_file_repo_checked SET DATA TYPE boolean USING CASE WHEN drip_file_repo_checked = 0 THEN FALSE ELSE TRUE END, + ALTER drip_file_repo_checked SET DEFAULT FALSE, + ALTER drip_file_vow_checked drop default, + ALTER COLUMN drip_file_vow_checked SET DATA TYPE boolean USING CASE WHEN drip_file_vow_checked = 0 THEN FALSE ELSE TRUE END, + ALTER drip_file_vow_checked SET DEFAULT FALSE, + ALTER deal_checked drop default, + ALTER COLUMN deal_checked SET DATA TYPE boolean USING CASE WHEN deal_checked = 0 THEN FALSE ELSE TRUE END, + ALTER deal_checked SET DEFAULT FALSE, + ALTER drip_drip_checked drop default, + ALTER COLUMN drip_drip_checked SET DATA TYPE boolean USING CASE WHEN drip_drip_checked = 0 THEN FALSE ELSE TRUE END, + ALTER drip_drip_checked SET DEFAULT FALSE, + ALTER cat_file_chop_lump_checked drop default, + ALTER COLUMN cat_file_chop_lump_checked SET DATA TYPE boolean USING CASE WHEN cat_file_chop_lump_checked = 0 THEN FALSE ELSE TRUE END, + ALTER cat_file_chop_lump_checked SET DEFAULT FALSE, + ALTER cat_file_flip_checked drop default, + ALTER COLUMN cat_file_flip_checked SET DATA TYPE boolean USING CASE WHEN cat_file_flip_checked = 0 THEN FALSE ELSE TRUE END, + ALTER cat_file_flip_checked SET DEFAULT FALSE, + ALTER cat_file_pit_vow_checked drop default, + ALTER COLUMN cat_file_pit_vow_checked SET DATA TYPE boolean USING CASE WHEN cat_file_pit_vow_checked = 0 THEN FALSE ELSE TRUE END, + ALTER cat_file_pit_vow_checked SET DEFAULT FALSE, + ALTER flop_kick_checked drop default, + ALTER COLUMN flop_kick_checked SET DATA TYPE boolean USING CASE WHEN flop_kick_checked = 0 THEN FALSE ELSE TRUE END, + ALTER flop_kick_checked SET DEFAULT FALSE, + ALTER vat_move_checked drop default, + ALTER COLUMN vat_move_checked SET DATA TYPE boolean USING CASE WHEN vat_move_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_move_checked SET DEFAULT FALSE, + ALTER vat_fold_checked drop default, + ALTER COLUMN vat_fold_checked SET DATA TYPE boolean USING CASE WHEN vat_fold_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_fold_checked SET DEFAULT FALSE, + ALTER vat_heal_checked drop default, + ALTER COLUMN vat_heal_checked SET DATA TYPE boolean USING CASE WHEN vat_heal_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_heal_checked SET DEFAULT FALSE, + ALTER vat_toll_checked drop default, + ALTER COLUMN vat_toll_checked SET DATA TYPE boolean USING CASE WHEN vat_toll_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_toll_checked SET DEFAULT FALSE, + ALTER vat_tune_checked drop default, + ALTER COLUMN vat_tune_checked SET DATA TYPE boolean USING CASE WHEN vat_tune_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_tune_checked SET DEFAULT FALSE, + ALTER vat_grab_checked drop default, + ALTER COLUMN vat_grab_checked SET DATA TYPE boolean USING CASE WHEN vat_grab_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_grab_checked SET DEFAULT FALSE, + ALTER vat_flux_checked drop default, + ALTER COLUMN vat_flux_checked SET DATA TYPE boolean USING CASE WHEN vat_flux_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_flux_checked SET DEFAULT FALSE, + ALTER vat_slip_checked drop default, + ALTER COLUMN vat_slip_checked SET DATA TYPE boolean USING CASE WHEN vat_slip_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vat_slip_checked SET DEFAULT FALSE, + ALTER vow_flog_checked drop default, + ALTER COLUMN vow_flog_checked SET DATA TYPE boolean USING CASE WHEN vow_flog_checked = 0 THEN FALSE ELSE TRUE END, + ALTER vow_flog_checked SET DEFAULT FALSE, + ALTER flap_kick_checked drop default, + ALTER COLUMN flap_kick_checked SET DATA TYPE boolean USING CASE WHEN flap_kick_checked = 0 THEN FALSE ELSE TRUE END, + ALTER flap_kick_checked SET DEFAULT FALSE; diff --git a/db/schema.sql b/db/schema.sql index 2e0a6df6..628fb901 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1357,34 +1357,34 @@ ALTER SEQUENCE public.blocks_id_seq OWNED BY public.blocks.id; CREATE TABLE public.checked_headers ( id integer NOT NULL, header_id integer NOT NULL, - price_feeds_checked boolean DEFAULT false NOT NULL, - flip_kick_checked boolean DEFAULT false NOT NULL, - frob_checked boolean DEFAULT false NOT NULL, - tend_checked boolean DEFAULT false NOT NULL, - bite_checked boolean DEFAULT false NOT NULL, - dent_checked boolean DEFAULT false NOT NULL, - pit_file_debt_ceiling_checked boolean DEFAULT false NOT NULL, - pit_file_ilk_checked boolean DEFAULT false NOT NULL, - vat_init_checked boolean DEFAULT false NOT NULL, - drip_file_ilk_checked boolean DEFAULT false NOT NULL, - drip_file_repo_checked boolean DEFAULT false NOT NULL, - drip_file_vow_checked boolean DEFAULT false NOT NULL, - deal_checked boolean DEFAULT false NOT NULL, - drip_drip_checked boolean DEFAULT false NOT NULL, - cat_file_chop_lump_checked boolean DEFAULT false NOT NULL, - cat_file_flip_checked boolean DEFAULT false NOT NULL, - cat_file_pit_vow_checked boolean DEFAULT false NOT NULL, - flop_kick_checked boolean DEFAULT false NOT NULL, - vat_move_checked boolean DEFAULT false NOT NULL, - vat_fold_checked boolean DEFAULT false NOT NULL, - vat_heal_checked boolean DEFAULT false NOT NULL, - vat_toll_checked boolean DEFAULT false NOT NULL, - vat_tune_checked boolean DEFAULT false NOT NULL, - vat_grab_checked boolean DEFAULT false NOT NULL, - vat_flux_checked boolean DEFAULT false NOT NULL, - vat_slip_checked boolean DEFAULT false NOT NULL, - vow_flog_checked boolean DEFAULT false NOT NULL, - flap_kick_checked boolean DEFAULT false NOT NULL + price_feeds_checked integer DEFAULT 0 NOT NULL, + flip_kick_checked integer DEFAULT 0 NOT NULL, + frob_checked integer DEFAULT 0 NOT NULL, + tend_checked integer DEFAULT 0 NOT NULL, + bite_checked integer DEFAULT 0 NOT NULL, + dent_checked integer DEFAULT 0 NOT NULL, + pit_file_debt_ceiling_checked integer DEFAULT 0 NOT NULL, + pit_file_ilk_checked integer DEFAULT 0 NOT NULL, + vat_init_checked integer DEFAULT 0 NOT NULL, + drip_file_ilk_checked integer DEFAULT 0 NOT NULL, + drip_file_repo_checked integer DEFAULT 0 NOT NULL, + drip_file_vow_checked integer DEFAULT 0 NOT NULL, + deal_checked integer DEFAULT 0 NOT NULL, + drip_drip_checked integer DEFAULT 0 NOT NULL, + cat_file_chop_lump_checked integer DEFAULT 0 NOT NULL, + cat_file_flip_checked integer DEFAULT 0 NOT NULL, + cat_file_pit_vow_checked integer DEFAULT 0 NOT NULL, + flop_kick_checked integer DEFAULT 0 NOT NULL, + vat_move_checked integer DEFAULT 0 NOT NULL, + vat_fold_checked integer DEFAULT 0 NOT NULL, + vat_heal_checked integer DEFAULT 0 NOT NULL, + vat_toll_checked integer DEFAULT 0 NOT NULL, + vat_tune_checked integer DEFAULT 0 NOT NULL, + vat_grab_checked integer DEFAULT 0 NOT NULL, + vat_flux_checked integer DEFAULT 0 NOT NULL, + vat_slip_checked integer DEFAULT 0 NOT NULL, + vow_flog_checked integer DEFAULT 0 NOT NULL, + flap_kick_checked integer DEFAULT 0 NOT NULL ); diff --git a/libraries/shared/event_watcher.go b/libraries/shared/event_watcher.go index d17288be..bdcf0554 100644 --- a/libraries/shared/event_watcher.go +++ b/libraries/shared/event_watcher.go @@ -23,6 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) type EventWatcher struct { @@ -74,7 +75,7 @@ func (watcher *EventWatcher) AddTransformers(initializers []shared.TransformerIn watcher.Chunker.AddConfigs(configs) } -func (watcher *EventWatcher) Execute() error { +func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecution) error { if watcher.Transformers == nil { return fmt.Errorf("No transformers added to watcher") } @@ -83,7 +84,7 @@ func (watcher *EventWatcher) Execute() error { if err != nil { return err } - notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames) + notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders) missingHeaders, err := shared.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL) if err != nil { @@ -107,7 +108,7 @@ func (watcher *EventWatcher) Execute() error { for _, transformer := range watcher.Transformers { transformerName := transformer.GetConfig().TransformerName logChunk := chunkedLogs[transformerName] - err = transformer.Execute(logChunk, header) + err = transformer.Execute(logChunk, header, constants.HeaderMissing) if err != nil { log.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err) return err diff --git a/libraries/shared/event_watcher_test.go b/libraries/shared/event_watcher_test.go index 4a86e5af..485b3682 100644 --- a/libraries/shared/event_watcher_test.go +++ b/libraries/shared/event_watcher_test.go @@ -31,6 +31,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -95,7 +96,7 @@ var _ = Describe("EventWatcher", func() { It("returns an error when run without transformers", func() { watcher := shared.NewEventWatcher(nil, nil) - err := watcher.Execute() + err := watcher.Execute(constants.HeaderMissing) Expect(err).To(MatchError("No transformers added to watcher")) }) @@ -125,7 +126,7 @@ var _ = Describe("EventWatcher", func() { watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) - err := watcher.Execute() + err := watcher.Execute(constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue()) @@ -136,7 +137,7 @@ var _ = Describe("EventWatcher", func() { watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) - err := watcher.Execute() + err := watcher.Execute(constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse()) @@ -167,7 +168,7 @@ var _ = Describe("EventWatcher", func() { watcher.AddTransformers([]shared2.TransformerInitializer{ transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer}) - err := watcher.Execute() + err := watcher.Execute(constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA})) Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB})) @@ -187,7 +188,7 @@ var _ = Describe("EventWatcher", func() { Topic: topic, ContractAddresses: addresses}) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) - err := watcher.Execute() + err := watcher.Execute(constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) fakeHash := common.HexToHash(fakes.FakeHeader.Hash) @@ -203,7 +204,7 @@ var _ = Describe("EventWatcher", func() { mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) - err := watcher.Execute() + err := watcher.Execute(constants.HeaderMissing) Expect(err).To(MatchError(fetcherError)) }) }) diff --git a/pkg/omni/light/repository/header_repository.go b/pkg/omni/light/repository/header_repository.go index c26b3e60..8281bc23 100644 --- a/pkg/omni/light/repository/header_repository.go +++ b/pkg/omni/light/repository/header_repository.go @@ -61,7 +61,7 @@ func (r *headerRepository) AddCheckColumn(id string) error { } pgStr := "ALTER TABLE public.checked_headers ADD COLUMN IF NOT EXISTS " - pgStr = pgStr + id + " BOOLEAN NOT NULL DEFAULT FALSE" + pgStr = pgStr + id + " INTEGER NOT NULL DEFAULT 0" _, err := r.db.Exec(pgStr) if err != nil { return err @@ -80,7 +80,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error { for _, id := range ids { _, ok := r.columns.Get(id) if !ok { - baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " BOOLEAN NOT NULL DEFAULT FALSE," + baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " INTEGER NOT NULL DEFAULT 0," input = append(input, id) } } @@ -100,7 +100,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { _, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`) VALUES ($1, $2) ON CONFLICT (header_id) DO - UPDATE SET `+id+` = $2`, headerID, true) + UPDATE SET `+id+` = checked_headers.`+id+` + 1`, headerID, 1) return err } @@ -112,11 +112,11 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) } pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, " for i := 0; i < len(ids); i++ { - pgStr += "true, " + pgStr += "1, " } pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET " for _, id := range ids { - pgStr += fmt.Sprintf("%s = true, ", id) + pgStr += id + `= checked_headers.` + id + ` + 1, ` } pgStr = pgStr[:len(pgStr)-2] _, err := r.db.Exec(pgStr, headerID) @@ -137,11 +137,11 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [ } pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, " for i := 0; i < len(ids); i++ { - pgStr += "true, " + pgStr += "1, " } pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET " for _, id := range ids { - pgStr += fmt.Sprintf("%s = true, ", id) + pgStr += fmt.Sprintf("%s = checked_headers.%s + 1, ", id, id) } pgStr = pgStr[:len(pgStr)-2] _, err = tx.Exec(pgStr, header.Id) @@ -162,7 +162,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber if endingBlockNumber == -1 { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR ` + id + ` IS FALSE) + WHERE (header_id ISNULL OR checked_headers.` + id + `=0) AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2 ORDER BY headers.block_number` @@ -170,7 +170,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber } else { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR ` + id + ` IS FALSE) + WHERE (header_id ISNULL OR checked_headers.` + id + `=0) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3 @@ -190,7 +190,7 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL` for _, id := range ids { - baseQuery += ` OR ` + id + ` IS FALSE` + baseQuery += ` OR checked_headers.` + id + `= 0` } if endingBlockNumber == -1 { @@ -220,11 +220,11 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id IS NOT NULL` for _, id := range eventIds { - baseQuery += ` AND ` + id + ` IS TRUE` + baseQuery += ` AND ` + id + `!=0` } baseQuery += `) AND (` for _, id := range methodIds { - baseQuery += id + ` IS FALSE AND ` + baseQuery += id + ` =0 AND ` } baseQuery = baseQuery[:len(baseQuery)-5] + `) ` @@ -254,6 +254,6 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) VALUES ($1, $2) ON CONFLICT (header_id) DO - UPDATE SET `+eventID+` = $2`, headerID, true) + UPDATE SET `+eventID+` = checked_headers.`+eventID+` + 1`, headerID, 1) return err } diff --git a/pkg/omni/shared/helpers/test_helpers/database.go b/pkg/omni/shared/helpers/test_helpers/database.go index c2f094d1..4c3c5059 100644 --- a/pkg/omni/shared/helpers/test_helpers/database.go +++ b/pkg/omni/shared/helpers/test_helpers/database.go @@ -257,41 +257,17 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`DELETE FROM receipts`) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DROP TABLE checked_headers`) - Expect(err).NotTo(HaveOccurred()) - - _, err = tx.Exec(`CREATE TABLE checked_headers (id SERIAL PRIMARY KEY, header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`) - Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`ALTER TABLE checked_headers -ADD COLUMN IF NOT EXISTS price_feeds_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS flip_kick_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS frob_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS tend_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS bite_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS dent_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS pit_file_debt_ceiling_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS pit_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_init_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS drip_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS drip_file_repo_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS drip_file_vow_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS deal_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS drip_drip_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS cat_file_chop_lump_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS cat_file_flip_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS cat_file_pit_vow_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS flop_kick_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_move_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_fold_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_heal_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_toll_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_tune_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_grab_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_flux_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vat_slip_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS vow_flog_checked BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS flap_kick_checked BOOLEAN NOT NULL DEFAULT FALSE`) +DROP COLUMN IF EXISTS eventName_contractAddr, +DROP COLUMN IF EXISTS eventName_contractAddr2, +DROP COLUMN IF EXISTS eventName_contractAddr3, +DROP COLUMN IF EXISTS methodName_contractAddr, +DROP COLUMN IF EXISTS methodName_contractAddr2, +DROP COLUMN IF EXISTS methodName_contractAddr3, +DROP COLUMN IF EXISTS transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e, +DROP COLUMN IF EXISTS balanceof_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e, +DROP COLUMN IF EXISTS newowner_0x314159265dd8dbb310642f98f50c066173c1259b, +DROP COLUMN IF EXISTS owner_0x314159265dd8dbb310642f98f50c066173c1259b`) Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) @@ -308,6 +284,9 @@ ADD COLUMN IF NOT EXISTS flap_kick_checked BOOLEAN NOT NULL DEFAULT FALSE`) err = tx.Commit() Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`VACUUM checked_headers`) + Expect(err).NotTo(HaveOccurred()) } func CreateBlock(blockNumber int64, repository repositories.BlockRepository) (blockId int64) { diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 9591bcf8..118831a4 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -18,7 +18,10 @@ package bite import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -49,7 +52,8 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er _, execErr := tx.Exec( `INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)`, + VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, ink = $4, art = $5, iart = $6, tab = $7, nflip = $8, raw_log = $11;`, headerID, biteModel.Ilk, biteModel.Urn, biteModel.Ink, biteModel.Art, biteModel.IArt, biteModel.Tab, biteModel.NFlip, biteModel.LogIndex, biteModel.TransactionIndex, biteModel.Raw, ) if execErr != nil { @@ -76,3 +80,11 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) } + +func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked) +} + +func (repository BiteRepository) RecheckHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked) +} diff --git a/pkg/transformers/cat_file/chop_lump/repository.go b/pkg/transformers/cat_file/chop_lump/repository.go index 52656bc8..e8ebea51 100644 --- a/pkg/transformers/cat_file/chop_lump/repository.go +++ b/pkg/transformers/cat_file/chop_lump/repository.go @@ -18,7 +18,10 @@ package chop_lump import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte _, execErr := tx.Exec( `INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, + VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, data = $4, raw_log = $7;`, headerID, chopLump.Ilk, chopLump.What, chopLump.Data, chopLump.TransactionIndex, chopLump.LogIndex, chopLump.Raw, ) if execErr != nil { @@ -73,6 +77,14 @@ func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) er return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) } +func (repository CatFileChopLumpRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked) +} + +func (repository CatFileChopLumpRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked) +} + func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/cat_file/flip/repository.go b/pkg/transformers/cat_file/flip/repository.go index ac6bc9d4..648effea 100644 --- a/pkg/transformers/cat_file/flip/repository.go +++ b/pkg/transformers/cat_file/flip/repository.go @@ -18,7 +18,10 @@ package flip import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -45,7 +48,8 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac _, execErr := repository.db.Exec( `INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6, $7)`, + VALUES($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, flip = $4, raw_log = $7;`, headerID, flip.Ilk, flip.What, flip.Flip, flip.TransactionIndex, flip.LogIndex, flip.Raw, ) if execErr != nil { @@ -72,6 +76,14 @@ func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) } +func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked) +} + +func (repository CatFileFlipRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked) +} + func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/cat_file/pit_vow/repository.go b/pkg/transformers/cat_file/pit_vow/repository.go index 5e5932a4..bc737c29 100644 --- a/pkg/transformers/cat_file/pit_vow/repository.go +++ b/pkg/transformers/cat_file/pit_vow/repository.go @@ -18,7 +18,10 @@ package pit_vow import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -45,7 +48,8 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf _, execErr := repository.db.Exec( `INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6)`, + VALUES($1, $2, $3, $4, $5, $6) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET what = $2, data = $3, raw_log = $6;`, headerID, vow.What, vow.Data, vow.TransactionIndex, vow.LogIndex, vow.Raw, ) if execErr != nil { @@ -72,6 +76,14 @@ func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) erro return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) } +func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked) +} + +func (repository CatFilePitVowRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked) +} + func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/deal/repository.go b/pkg/transformers/deal/repository.go index 4c2d1451..b0572abd 100644 --- a/pkg/transformers/deal/repository.go +++ b/pkg/transformers/deal/repository.go @@ -18,7 +18,10 @@ package deal import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er _, execErr := tx.Exec( `INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6)`, + VALUES($1, $2, $3, $4, $5, $6) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, contract_address = $3, raw_log = $6;`, headerID, dealModel.BidId, dealModel.ContractAddress, dealModel.LogIndex, dealModel.TransactionIndex, dealModel.Raw, ) if execErr != nil { @@ -73,6 +77,14 @@ func (repository DealRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) } +func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked) +} + +func (repository DealRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked) +} + func (repository *DealRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/dent/repository.go b/pkg/transformers/dent/repository.go index 45fa5593..ceac4e95 100644 --- a/pkg/transformers/dent/repository.go +++ b/pkg/transformers/dent/repository.go @@ -18,7 +18,10 @@ package dent import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -55,25 +58,20 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er _, execErr := tx.Exec( `INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_Id = $2, lot = $3, bid = $4, guy = $5, tic = $6, raw_log = $9;`, headerID, dent.BidId, dent.Lot, dent.Bid, dent.Guy, tic, dent.LogIndex, dent.TransactionIndex, dent.Raw, ) if execErr != nil { - rollbackErr := tx.Rollback() - if rollbackErr != nil { - log.Error("failed to rollback ", rollbackErr) - } + tx.Rollback() return execErr } } - checkHeaderErr := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) - if checkHeaderErr != nil { - rollbackErr := tx.Rollback() - if rollbackErr != nil { - log.Error("failed to rollback ", rollbackErr) - } - return checkHeaderErr + err := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) + if err != nil { + tx.Rollback() + return err } return tx.Commit() } @@ -82,6 +80,14 @@ func (repository DentRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) } +func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked) +} + +func (repository DentRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked) +} + func (repository *DentRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index c7631f01..0c21b060 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -18,7 +18,10 @@ package drip_drip import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -45,7 +48,8 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{} _, execErr := tx.Exec( `INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5)`, + VALUES($1, $2, $3, $4, $5) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk= $2, raw_log = $5;`, headerID, dripDrip.Ilk, dripDrip.LogIndex, dripDrip.TransactionIndex, dripDrip.Raw, ) if execErr != nil { @@ -72,6 +76,14 @@ func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) } +func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked) +} + +func (repository DripDripRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked) +} + func (repository *DripDripRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 136d0e44..dc7eeacd 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -18,7 +18,10 @@ package ilk import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -75,6 +78,14 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) } +func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked) +} + +func (repository DripFileIlkRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked) +} + func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index bc910199..4490af00 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -18,7 +18,10 @@ package repo import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -75,6 +78,14 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) } +func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked) +} + +func (repository DripFileRepoRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked) +} + func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index c4a9ccc4..2f008dc5 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -18,7 +18,10 @@ package vow import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac _, execErr := tx.Exec( `INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6)`, + VALUES($1, $2, $3, $4, $5, $6) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET what = $2, data = $3, raw_log = $6;`, headerID, vow.What, vow.Data, vow.LogIndex, vow.TransactionIndex, vow.Raw, ) if execErr != nil { @@ -74,6 +78,14 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) } +func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked) +} + +func (repository DripFileVowRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked) +} + func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index 3bc8e29b..c22c5a42 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -23,6 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) type LogNoteTransformer struct { @@ -36,7 +37,7 @@ func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) sha return transformer } -func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header) error { +func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header, recheckedHeader constants.TransformerExecution) error { transformerName := transformer.Config.TransformerName // No matching logs, mark the header as checked for this type of logs diff --git a/pkg/transformers/factories/log_note_transformer_test.go b/pkg/transformers/factories/log_note_transformer_test.go index ef0f9524..851472f8 100644 --- a/pkg/transformers/factories/log_note_transformer_test.go +++ b/pkg/transformers/factories/log_note_transformer_test.go @@ -24,6 +24,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "math/rand" @@ -57,14 +58,14 @@ var _ = Describe("LogNoteTransformer", func() { }) It("marks header checked if no logs are provided", func() { - err := transformer.Execute([]types.Log{}, headerOne) + err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { - err := transformer.Execute([]types.Log{}, headerOne) + err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(converter.ToModelsCalledCounter).To(Equal(0)) @@ -72,7 +73,7 @@ var _ = Describe("LogNoteTransformer", func() { }) It("does not call repository.MarkCheckedHeader when there are logs", func() { - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedNotCalled() @@ -81,14 +82,14 @@ var _ = Describe("LogNoteTransformer", func() { It("returns error if marking header checked returns err", func() { repository.SetMarkHeaderCheckedError(fakes.FakeError) - err := transformer.Execute([]types.Log{}, headerOne) + err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts matching logs to models", func() { - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(converter.PassedLogs).To(Equal(logs)) @@ -97,7 +98,7 @@ var _ = Describe("LogNoteTransformer", func() { It("returns error if converter returns error", func() { converter.SetConverterError(fakes.FakeError) - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -105,7 +106,7 @@ var _ = Describe("LogNoteTransformer", func() { It("persists the model", func() { converter.SetReturnModels([]interface{}{model}) - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) @@ -115,7 +116,7 @@ var _ = Describe("LogNoteTransformer", func() { It("returns error if repository returns error for create", func() { repository.SetCreateError(fakes.FakeError) - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/factories/repository.go b/pkg/transformers/factories/repository.go index ed21d955..d953249b 100644 --- a/pkg/transformers/factories/repository.go +++ b/pkg/transformers/factories/repository.go @@ -17,11 +17,14 @@ package factories import ( + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) type Repository interface { Create(headerID int64, models []interface{}) error MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) + RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error) SetDB(db *postgres.DB) } diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index e3307589..cca34a66 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -23,6 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) type Transformer struct { @@ -36,7 +37,7 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transforme return transformer } -func (transformer Transformer) Execute(logs []types.Log, header core.Header) error { +func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error { transformerName := transformer.Config.TransformerName config := transformer.Config diff --git a/pkg/transformers/factories/transformer_test.go b/pkg/transformers/factories/transformer_test.go index 2cb9b21d..3f8c0039 100644 --- a/pkg/transformers/factories/transformer_test.go +++ b/pkg/transformers/factories/transformer_test.go @@ -24,6 +24,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "math/rand" @@ -57,14 +58,14 @@ var _ = Describe("Transformer", func() { }) It("marks header checked if no logs returned", func() { - err := transformer.Execute([]types.Log{}, headerOne) + err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { - err := transformer.Execute([]types.Log{}, headerOne) + err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(converter.ToEntitiesCalledCounter).To(Equal(0)) @@ -73,7 +74,7 @@ var _ = Describe("Transformer", func() { }) It("does not call repository.MarkCheckedHeader when there are logs", func() { - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedNotCalled() @@ -82,14 +83,14 @@ var _ = Describe("Transformer", func() { It("returns error if marking header checked returns err", func() { repository.SetMarkHeaderCheckedError(fakes.FakeError) - err := transformer.Execute([]types.Log{}, headerOne) + err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts an eth log to an entity", func() { - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(converter.ContractAbi).To(Equal(config.ContractAbi)) @@ -99,7 +100,7 @@ var _ = Describe("Transformer", func() { It("returns an error if converter fails", func() { converter.ToEntitiesError = fakes.FakeError - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -108,7 +109,7 @@ var _ = Describe("Transformer", func() { It("converts an entity to a model", func() { converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{})) @@ -118,7 +119,7 @@ var _ = Describe("Transformer", func() { converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} converter.ToModelsError = fakes.FakeError - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) @@ -127,7 +128,7 @@ var _ = Describe("Transformer", func() { It("persists the record", func() { converter.ModelsToReturn = []interface{}{test_data.GenericModel{}} - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) @@ -136,7 +137,7 @@ var _ = Describe("Transformer", func() { It("returns error if persisting the record fails", func() { repository.SetCreateError(fakes.FakeError) - err := transformer.Execute(logs, headerOne) + err := transformer.Execute(logs, headerOne, constants.HeaderMissing) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/flap_kick/repository.go b/pkg/transformers/flap_kick/repository.go index 6ea7f6e0..ea9547dc 100644 --- a/pkg/transformers/flap_kick/repository.go +++ b/pkg/transformers/flap_kick/repository.go @@ -18,7 +18,10 @@ package flap_kick import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -41,7 +44,8 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{ _, execErr := tx.Exec( `INSERT into maker.flap_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) - VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, + VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, gal = $5, "end" = $6, raw_log = $9;`, headerID, flapKickModel.BidId, flapKickModel.Lot, flapKickModel.Bid, flapKickModel.Gal, flapKickModel.End, flapKickModel.TransactionIndex, flapKickModel.LogIndex, flapKickModel.Raw, ) if execErr != nil { @@ -68,6 +72,14 @@ func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) } +func (repository FlapKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked) +} + +func (repository FlapKickRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked) +} + func (repository *FlapKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/flip_kick/repository.go b/pkg/transformers/flip_kick/repository.go index 5197685b..fb18c9f3 100644 --- a/pkg/transformers/flip_kick/repository.go +++ b/pkg/transformers/flip_kick/repository.go @@ -20,6 +20,7 @@ import ( "fmt" log "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -42,7 +43,8 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{} _, execErr := tx.Exec( `INSERT into maker.flip_kick (header_id, bid_id, lot, bid, gal, "end", urn, tab, tx_idx, log_idx, raw_log) - VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11)`, + VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, gal = $5, "end" = $6, urn= $7, tab = $8, raw_log = $11;`, headerID, flipKickModel.BidId, flipKickModel.Lot, flipKickModel.Bid, flipKickModel.Gal, flipKickModel.End, flipKickModel.Urn, flipKickModel.Tab, flipKickModel.TransactionIndex, flipKickModel.LogIndex, flipKickModel.Raw, ) if execErr != nil { @@ -68,6 +70,14 @@ func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) } +func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked) +} + +func (repository FlipKickRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked) +} + func (repository *FlipKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/flop_kick/repository.go b/pkg/transformers/flop_kick/repository.go index d6210330..a71a618b 100644 --- a/pkg/transformers/flop_kick/repository.go +++ b/pkg/transformers/flop_kick/repository.go @@ -18,7 +18,10 @@ package flop_kick import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -42,7 +45,8 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{} _, execErr := tx.Exec( `INSERT into maker.flop_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) - VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, + VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, gal = $5, "end" = $6, raw_log = $9;`, headerID, flopKickModel.BidId, flopKickModel.Lot, flopKickModel.Bid, flopKickModel.Gal, flopKickModel.End, flopKickModel.TransactionIndex, flopKickModel.LogIndex, flopKickModel.Raw, ) if execErr != nil { @@ -70,6 +74,14 @@ func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) } +func (repository FlopKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked) +} + +func (repository FlopKickRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked) +} + func (repository *FlopKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/frob/converter.go b/pkg/transformers/frob/converter.go index 267626af..9d031ca9 100644 --- a/pkg/transformers/frob/converter.go +++ b/pkg/transformers/frob/converter.go @@ -19,8 +19,6 @@ package frob import ( "encoding/json" "fmt" - log "github.com/sirupsen/logrus" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -32,7 +30,6 @@ type FrobConverter struct{} func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) { var entities []interface{} - log.Info("blah") for _, ethLog := range ethLogs { entity := FrobEntity{} address := ethLog.Address diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index 5e336c0c..a8d633c2 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -20,6 +20,7 @@ import ( "fmt" log "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -45,7 +46,8 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er } _, execErr := tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx) - VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)`, + VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET art = $2, dart = $3, dink = $4, iart = $5, ilk = $6, ink = $7, urn = $8, raw_log = $9;`, headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex) if execErr != nil { rollbackErr := tx.Rollback() @@ -70,6 +72,14 @@ func (repository FrobRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) } +func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked) +} + +func (repository FrobRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked) +} + func (repository *FrobRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/integration_tests/bite.go b/pkg/transformers/integration_tests/bite.go index 96d7bc5f..0b273266 100644 --- a/pkg/transformers/integration_tests/bite.go +++ b/pkg/transformers/integration_tests/bite.go @@ -73,7 +73,7 @@ var _ = Describe("Bite Transformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []bite.BiteModel @@ -108,4 +108,71 @@ var _ = Describe("Bite Transformer", func() { Expect(entity.Ilk).To(Equal(expectedEntity.Ilk)) Expect(entity.Ink).To(Equal(expectedEntity.Ink)) }) + + It("rechecks header for bite event", func() { + blockNumber := int64(8956422) + config := testBiteConfig + config.StartingBlockNumber = blockNumber + config.EndingBlockNumber = blockNumber + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + initializer := factories.Transformer{ + Config: config, + Converter: &bite.BiteConverter{}, + Repository: &bite.BiteRepository{}, + } + transformer := initializer.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var biteChecked []int + err = db.Select(&biteChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(biteChecked[0]).To(Equal(2)) + }) + + It("unpacks an event log", func() { + address := common.HexToAddress(test_data.KovanCatContractAddress) + abi, err := geth.ParseAbi(test_data.KovanCatABI) + Expect(err).NotTo(HaveOccurred()) + + contract := bind.NewBoundContract(address, abi, nil, nil, nil) + entity := &bite.BiteEntity{} + + var eventLog = test_data.EthBiteLog + + err = contract.UnpackLog(entity, "Bite", eventLog) + Expect(err).NotTo(HaveOccurred()) + + expectedEntity := test_data.BiteEntity + Expect(entity.Art).To(Equal(expectedEntity.Art)) + Expect(entity.Ilk).To(Equal(expectedEntity.Ilk)) + Expect(entity.Ink).To(Equal(expectedEntity.Ink)) + }) }) diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index 4b84c94d..114dfeaa 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -87,7 +87,7 @@ var _ = Describe("Cat File transformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []chop_lump.CatFileChopLumpModel @@ -108,6 +108,51 @@ var _ = Describe("Cat File transformer", func() { Expect(dbResult[1].LogIndex).To(Equal(uint(4))) }) + It("rechecks header for chop lump event", func() { + // transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014 + chopLumpBlockNumber := int64(8762253) + header, err := persistHeader(db, chopLumpBlockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + config := shared.TransformerConfig{ + TransformerName: constants.CatFileChopLumpLabel, + ContractAddresses: []string{test_data.KovanCatContractAddress}, + ContractAbi: test_data.KovanCatABI, + Topic: test_data.KovanCatFileChopLumpSignature, + StartingBlockNumber: chopLumpBlockNumber, + EndingBlockNumber: chopLumpBlockNumber, + } + + initializer := factories.LogNoteTransformer{ + Config: config, + Converter: &chop_lump.CatFileChopLumpConverter{}, + Repository: &chop_lump.CatFileChopLumpRepository{}, + } + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, chopLumpBlockNumber) + Expect(err).NotTo(HaveOccurred()) + + var catChopLumpChecked []int + err = db.Select(&catChopLumpChecked, `SELECT cat_file_chop_lump_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(catChopLumpChecked[0]).To(Equal(2)) + }) + It("persists a flip event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe flipBlockNumber := int64(8751794) @@ -137,7 +182,7 @@ var _ = Describe("Cat File transformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []flip.CatFileFlipModel @@ -150,6 +195,52 @@ var _ = Describe("Cat File transformer", func() { Expect(dbResult[0].Flip).To(Equal("0x32D496Ad866D110060866B7125981C73642cc509")) }) + It("rechecks a flip event", func() { + // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe + flipBlockNumber := int64(8751794) + header, err := persistHeader(db, flipBlockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + config := shared.TransformerConfig{ + TransformerName: constants.CatFileFlipLabel, + ContractAddresses: []string{test_data.KovanCatContractAddress}, + ContractAbi: test_data.KovanCatABI, + Topic: test_data.KovanCatFileFlipSignature, + StartingBlockNumber: flipBlockNumber, + EndingBlockNumber: flipBlockNumber, + } + + initializer := factories.LogNoteTransformer{ + Config: config, + Converter: &flip.CatFileFlipConverter{}, + Repository: &flip.CatFileFlipRepository{}, + } + + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, flipBlockNumber) + Expect(err).NotTo(HaveOccurred()) + + var catFlipChecked []int + err = db.Select(&catFlipChecked, `SELECT cat_file_flip_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(catFlipChecked[0]).To(Equal(2)) + }) + It("persists a pit vow event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe pitVowBlockNumber := int64(8751794) @@ -178,7 +269,7 @@ var _ = Describe("Cat File transformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []pit_vow.CatFilePitVowModel diff --git a/pkg/transformers/integration_tests/deal.go b/pkg/transformers/integration_tests/deal.go index cdaf26de..8e8a6908 100644 --- a/pkg/transformers/integration_tests/deal.go +++ b/pkg/transformers/integration_tests/deal.go @@ -84,7 +84,7 @@ var _ = Describe("Deal transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel @@ -96,6 +96,34 @@ var _ = Describe("Deal transformer", func() { Expect(dbResult[0].ContractAddress).To(Equal(test_data.KovanFlipperContractAddress)) }) + It("rechecks flip deal event", func() { + // transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58 + flipBlockNumber := int64(8958007) + header, err := persistHeader(db, flipBlockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + initializer.Config.StartingBlockNumber = flipBlockNumber + initializer.Config.EndingBlockNumber = flipBlockNumber + + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, flipBlockNumber) + Expect(err).NotTo(HaveOccurred()) + + var dentChecked []int + err = db.Select(&dentChecked, `SELECT deal_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + }) + It("persists a flop deal log event", func() { //TODO: There are currently no Flop.deal events on Kovan }) @@ -112,7 +140,7 @@ var _ = Describe("Deal transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel diff --git a/pkg/transformers/integration_tests/dent.go b/pkg/transformers/integration_tests/dent.go index 9097da6a..aa12143a 100644 --- a/pkg/transformers/integration_tests/dent.go +++ b/pkg/transformers/integration_tests/dent.go @@ -67,7 +67,7 @@ var _ = Describe("Dent transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer = initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []dent.DentModel @@ -88,6 +88,35 @@ var _ = Describe("Dent transformer", func() { Expect(dbTic).To(Equal(actualTic)) }) + It("rechecks header for flop dent log event", func() { + blockNumber := int64(8955613) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer = initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var dentChecked []int + err = db.Select(&dentChecked, `SELECT dent_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(dentChecked[0]).To(Equal(2)) + }) + It("persists a flip dent log event", func() { //TODO: There are currently no Flip.dent events on Kovan }) diff --git a/pkg/transformers/integration_tests/drip_drip.go b/pkg/transformers/integration_tests/drip_drip.go index 385e6bd7..927ee6eb 100644 --- a/pkg/transformers/integration_tests/drip_drip.go +++ b/pkg/transformers/integration_tests/drip_drip.go @@ -27,6 +27,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -76,7 +77,7 @@ var _ = Describe("DripDrip Transformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResults []drip_drip.DripDripModel @@ -87,4 +88,41 @@ var _ = Describe("DripDrip Transformer", func() { dbResult := dbResults[0] Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) }) + + It("rechecks drip drip event", func() { + blockNumber := int64(8934775) + config.StartingBlockNumber = blockNumber + config.EndingBlockNumber = blockNumber + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + initializer := factories.LogNoteTransformer{ + Config: config, + Converter: &drip_drip.DripDripConverter{}, + Repository: &drip_drip.DripDripRepository{}, + } + transformer := initializer.NewLogNoteTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var dripdripChecked []int + err = db.Select(&dripdripChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + }) }) diff --git a/pkg/transformers/integration_tests/drip_file_vow.go b/pkg/transformers/integration_tests/drip_file_vow.go index 841fc154..459e33eb 100644 --- a/pkg/transformers/integration_tests/drip_file_vow.go +++ b/pkg/transformers/integration_tests/drip_file_vow.go @@ -24,10 +24,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -74,7 +74,7 @@ var _ = Describe("Drip File Vow LogNoteTransformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []vow.DripFileVowModel @@ -85,4 +85,49 @@ var _ = Describe("Drip File Vow LogNoteTransformer", func() { Expect(dbResult[0].What).To(Equal("vow")) Expect(dbResult[0].Data).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1")) }) + + It("rechecks drip file vow event", func() { + blockNumber := int64(8762197) + config := shared.TransformerConfig{ + TransformerName: constants.DripFileVowLabel, + ContractAddresses: []string{test_data.KovanDripContractAddress}, + ContractAbi: test_data.KovanDripABI, + Topic: test_data.KovanDripFileVowSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + initializer := factories.LogNoteTransformer{ + Config: config, + Converter: &vow.DripFileVowConverter{}, + Repository: &vow.DripFileVowRepository{}, + } + transformer := initializer.NewLogNoteTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var dripfilevowChecked []int + err = db.Select(&dripfilevowChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(dripfilevowChecked[0]).To(Equal(2)) + }) }) diff --git a/pkg/transformers/integration_tests/flap_kick.go b/pkg/transformers/integration_tests/flap_kick.go index 0894c4a8..4aecc055 100644 --- a/pkg/transformers/integration_tests/flap_kick.go +++ b/pkg/transformers/integration_tests/flap_kick.go @@ -20,7 +20,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/test_config" "time" @@ -31,6 +30,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) var _ = Describe("FlapKick Transformer", func() { @@ -75,7 +75,7 @@ var _ = Describe("FlapKick Transformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []flap_kick.FlapKickModel @@ -89,4 +89,48 @@ var _ = Describe("FlapKick Transformer", func() { Expect(dbResult[0].Gal).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) }) + + It("rechecks flap kick transformer", func() { + blockNumber := int64(9002933) + config := shared.TransformerConfig{ + TransformerName: constants.FlapKickLabel, + ContractAddresses: []string{test_data.KovanFlapperContractAddress}, + ContractAbi: test_data.KovanFlapperABI, + Topic: test_data.KovanFlapKickSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.Transformer{ + Config: config, + Converter: &flap_kick.FlapKickConverter{}, + Repository: &flap_kick.FlapKickRepository{}, + }.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var flapkickChecked []int + err = db.Select(&flapkickChecked, `SELECT flap_kick_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(flapkickChecked[0]).To(Equal(2)) + }) }) diff --git a/pkg/transformers/integration_tests/flip_kick.go b/pkg/transformers/integration_tests/flip_kick.go index f4e4cd3f..0068db7a 100644 --- a/pkg/transformers/integration_tests/flip_kick.go +++ b/pkg/transformers/integration_tests/flip_kick.go @@ -91,7 +91,7 @@ var _ = Describe("FlipKick Transformer", func() { header) Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []flip_kick.FlipKickModel @@ -105,4 +105,55 @@ var _ = Describe("FlipKick Transformer", func() { Expect(dbResult[0].Gal).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) }) + + It("rechecks flip kick event", func() { + blockNumber := int64(8956476) + config := shared.TransformerConfig{ + TransformerName: constants.FlipKickLabel, + ContractAddresses: []string{test_data.KovanFlipperContractAddress}, + ContractAbi: test_data.KovanFlipperABI, + Topic: test_data.KovanFlipKickSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.Transformer{ + Config: config, + Converter: &flip_kick.FlipKickConverter{}, + Repository: &flip_kick.FlipKickRepository{}, + }.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var flipkickChecked []int + err = db.Select(&flipkickChecked, `SELECT flip_kick_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(flipkickChecked[0]).To(Equal(2)) + }) }) diff --git a/pkg/transformers/integration_tests/flop_kick.go b/pkg/transformers/integration_tests/flop_kick.go index b6ee9ff0..78e26751 100644 --- a/pkg/transformers/integration_tests/flop_kick.go +++ b/pkg/transformers/integration_tests/flop_kick.go @@ -86,7 +86,7 @@ var _ = Describe("FlopKick Transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []flop_kick.Model @@ -102,6 +102,47 @@ var _ = Describe("FlopKick Transformer", func() { Expect(dbResult[0].Lot).To(Equal("115792089237316195423570985008687907853269984665640564039457584007913129639935")) }) + It("rechecks flop kick event", func() { + blockNumber := int64(8672119) + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var dbResult []flop_kick.Model + err = db.Select(&dbResult, `SELECT bid, bid_id, "end", gal, lot FROM maker.flop_kick`) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var flopKickChecked []int + err = db.Select(&flopKickChecked, `SELECT flop_kick_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(flopKickChecked[0]).To(Equal(2)) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].Bid).To(Equal("0")) + Expect(dbResult[0].BidId).To(Equal("1")) + Expect(dbResult[0].End.Equal(time.Unix(1536726768, 0))).To(BeTrue()) + Expect(dbResult[0].Gal).To(Equal("0x9B870D55BaAEa9119dBFa71A92c5E26E79C4726d")) + // this very large number appears to be derived from the data including: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + Expect(dbResult[0].Lot).To(Equal("115792089237316195423570985008687907853269984665640564039457584007913129639935")) + }) + It("fetches and transforms another FlopKick event from Kovan chain", func() { blockNumber := int64(8955611) initializer.Config.StartingBlockNumber = blockNumber @@ -114,7 +155,7 @@ var _ = Describe("FlopKick Transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []flop_kick.Model diff --git a/pkg/transformers/integration_tests/frob.go b/pkg/transformers/integration_tests/frob.go index 64de1ac6..99663e80 100644 --- a/pkg/transformers/integration_tests/frob.go +++ b/pkg/transformers/integration_tests/frob.go @@ -82,7 +82,7 @@ var _ = Describe("Frob Transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []frob.FrobModel @@ -99,6 +99,51 @@ var _ = Describe("Frob Transformer", func() { Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C")) }) + It("rechecks frob event", func() { + blockNumber := int64(8935258) + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var frobChecked []int + err = db.Select(&frobChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(frobChecked[0]).To(Equal(2)) + + var dbResult []frob.FrobModel + err = db.Select(&dbResult, `SELECT art, dart, dink, iart, ilk, ink, urn from maker.frob`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].Art).To(Equal("10000000000000000")) + Expect(dbResult[0].Dart).To(Equal("0")) + Expect(dbResult[0].Dink).To(Equal("10000000000000")) + Expect(dbResult[0].IArt).To(Equal("1495509999999999999992")) + Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + Expect(dbResult[0].Ink).To(Equal("10050100000000000")) + Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C")) + }) + It("unpacks an event log", func() { address := common.HexToAddress(test_data.KovanPitContractAddress) abi, err := geth.ParseAbi(test_data.KovanPitABI) diff --git a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go index 9caecf39..94cf232d 100644 --- a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go +++ b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go @@ -74,7 +74,7 @@ var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() { } transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []debt_ceiling.PitFileDebtCeilingModel @@ -85,4 +85,57 @@ var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() { Expect(dbResult[0].What).To(Equal("Line")) Expect(dbResult[0].Data).To(Equal("10000000.000000000000000000")) }) + + It("rechecks pit file debt ceiling event", func() { + blockNumber := int64(8535578) + config := shared.TransformerConfig{ + TransformerName: constants.PitFileDebtCeilingLabel, + ContractAddresses: []string{test_data.KovanPitContractAddress}, + ContractAbi: test_data.KovanPitABI, + Topic: test_data.KovanPitFileDebtCeilingSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + initializer := factories.LogNoteTransformer{ + Config: config, + Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, + Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, + } + transformer := initializer.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var pitFileDebtCeilingChecked []int + err = db.Select(&pitFileDebtCeilingChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(pitFileDebtCeilingChecked[0]).To(Equal(2)) + + var dbResult []debt_ceiling.PitFileDebtCeilingModel + err = db.Select(&dbResult, `SELECT what, data from maker.pit_file_debt_ceiling`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].What).To(Equal("Line")) + Expect(dbResult[0].Data).To(Equal("10000000.000000000000000000")) + }) }) diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index 85a9be27..5c8f2914 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -79,7 +79,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []ilk.PitFileIlkModel @@ -92,6 +92,45 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648")) }) + It("rechecks pit file ilk event", func() { + blockNumber := int64(9103223) + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var pitFileIlkChecked []int + err = db.Select(&pitFileIlkChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(pitFileIlkChecked[0]).To(Equal(2)) + + var dbResult []ilk.PitFileIlkModel + err = db.Select(&dbResult, `SELECT ilk, what, data from maker.pit_file_ilk`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].Ilk).To(Equal("0x4554480000000000000000000000000000000000000000000000000000000000")) + Expect(dbResult[0].What).To(Equal("spot")) + Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648")) + }) + It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() { blockNumber := int64(8762253) initializer.Config.StartingBlockNumber = blockNumber @@ -105,7 +144,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []ilk.PitFileIlkModel diff --git a/pkg/transformers/integration_tests/price_feeds.go b/pkg/transformers/integration_tests/price_feeds.go index 3fac7d8f..7bb27945 100644 --- a/pkg/transformers/integration_tests/price_feeds.go +++ b/pkg/transformers/integration_tests/price_feeds.go @@ -75,7 +75,7 @@ var _ = Describe("Price feeds transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel @@ -85,6 +85,45 @@ var _ = Describe("Price feeds transformer", func() { Expect(model.MedianizerAddress).To(Equal(addresses[0])) }) + It("rechecks price feed event", func() { + blockNumber := int64(8763054) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + addresses := []string{test_data.KovanPipContractAddress} + initializer.Config.ContractAddresses = addresses + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(addresses), + topics, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var priceFeedChecked []int + err = db.Select(&priceFeedChecked, `SELECT price_feeds_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(priceFeedChecked[0]).To(Equal(2)) + + var model price_feeds.PriceFeedModel + err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(model.UsdValue).To(Equal("207.314891143000011198")) + Expect(model.MedianizerAddress).To(Equal(addresses[0])) + }) + It("persists a MKR/USD price feed event", func() { blockNumber := int64(8763059) header, err := persistHeader(db, blockNumber, blockChain) @@ -101,7 +140,7 @@ var _ = Describe("Price feeds transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel @@ -127,7 +166,7 @@ var _ = Describe("Price feeds transformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index 3c1afc24..d493a030 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -81,7 +81,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel @@ -102,6 +102,52 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(dbTic).To(Equal(actualTic)) }) + It("rechecks tend event", func() { + blockNumber := int64(8935601) + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var tendChecked []int + err = db.Select(&tendChecked, `SELECT tend_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(tendChecked[0]).To(Equal(2)) + + var dbResult []tend.TendModel + err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot from maker.tend where header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].Bid).To(Equal("4000")) + Expect(dbResult[0].BidId).To(Equal("3")) + Expect(dbResult[0].Guy).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) + Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) + + var dbTic int64 + err = db.Get(&dbTic, `SELECT tic FROM maker.tend`) + Expect(err).NotTo(HaveOccurred()) + + actualTic := 1538490276 + constants.TTL + Expect(dbTic).To(Equal(actualTic)) + }) + It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() { blockNumber := int64(8935731) initializer.Config.StartingBlockNumber = blockNumber @@ -114,7 +160,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel @@ -147,7 +193,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel diff --git a/pkg/transformers/integration_tests/vat_flux.go b/pkg/transformers/integration_tests/vat_flux.go index 953b6e11..8e7d91ed 100644 --- a/pkg/transformers/integration_tests/vat_flux.go +++ b/pkg/transformers/integration_tests/vat_flux.go @@ -66,7 +66,7 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { } transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_flux.VatFluxModel @@ -80,4 +80,68 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000")) Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) }) + + It("rechecks vat flux event", func() { + blockNumber := int64(9004474) + config := shared.TransformerConfig{ + TransformerName: constants.VatFluxLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatFluxSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + initializer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_flux.VatFluxConverter{}, + Repository: &vat_flux.VatFluxRepository{}, + } + transformer := initializer.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatFluxChecked []int + err = db.Select(&vatFluxChecked, `SELECT vat_flux_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatFluxChecked[0]).To(Equal(2)) + + var dbResult []vat_flux.VatFluxModel + err = db.Select(&dbResult, `SELECT ilk, src, dst, rad from maker.vat_flux`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + Expect(dbResult[0].Src).To(Equal("0xC0851F73CC8DD5c0765E71980eC7E7Fd1EF74434")) + Expect(dbResult[0].Dst).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) + Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000")) + Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) + }) }) diff --git a/pkg/transformers/integration_tests/vat_fold.go b/pkg/transformers/integration_tests/vat_fold.go index 1b1e8113..8ca669ea 100644 --- a/pkg/transformers/integration_tests/vat_fold.go +++ b/pkg/transformers/integration_tests/vat_fold.go @@ -73,7 +73,7 @@ var _ = Describe("VatFold Transformer", func() { Repository: &vat_fold.VatFoldRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_fold.VatFoldModel @@ -86,4 +86,58 @@ var _ = Describe("VatFold Transformer", func() { Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000")) }) + + It("rechecks vat fold event", func() { + blockNumber := int64(9367233) + config := shared.TransformerConfig{ + TransformerName: constants.VatFoldLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatFoldSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_fold.VatFoldConverter{}, + Repository: &vat_fold.VatFoldRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatFoldChecked []int + err = db.Select(&vatFoldChecked, `SELECT vat_fold_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatFoldChecked[0]).To(Equal(2)) + + var dbResults []vat_fold.VatFoldModel + err = db.Select(&dbResults, `SELECT ilk, urn, rate from maker.vat_fold`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResults)).To(Equal(1)) + dbResult := dbResults[0] + Expect(dbResult.Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) + Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000")) + }) }) diff --git a/pkg/transformers/integration_tests/vat_grab.go b/pkg/transformers/integration_tests/vat_grab.go index caef8338..35061f40 100644 --- a/pkg/transformers/integration_tests/vat_grab.go +++ b/pkg/transformers/integration_tests/vat_grab.go @@ -67,7 +67,7 @@ var _ = Describe("Vat Grab Transformer", func() { Repository: &vat_grab.VatGrabRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_grab.VatGrabModel @@ -87,4 +87,72 @@ var _ = Describe("Vat Grab Transformer", func() { Expect(dbResult[0].Dart).To(Equal(expectedDart.String())) Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) }) + + It("rechecks vat grab event", func() { + blockNumber := int64(8958230) + config := shared.TransformerConfig{ + TransformerName: constants.VatGrabLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatGrabSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_grab.VatGrabConverter{}, + Repository: &vat_grab.VatGrabRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatGrabChecked []int + err = db.Select(&vatGrabChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatGrabChecked[0]).To(Equal(2)) + + var dbResult []vat_grab.VatGrabModel + err = db.Select(&dbResult, `SELECT ilk, urn, v, w, dink, dart from maker.vat_grab`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + Expect(dbResult[0].Urn).To(Equal("0x6a3AE20C315E845B2E398e68EfFe39139eC6060C")) + Expect(dbResult[0].V).To(Equal("0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0")) //cat contract address + Expect(dbResult[0].W).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1")) + expectedDink := new(big.Int) + expectedDink.SetString("115792089237316195423570985008687907853269984665640564039455584007913129639936", 10) + Expect(dbResult[0].Dink).To(Equal(expectedDink.String())) + expectedDart := new(big.Int) + expectedDart.SetString("115792089237316195423570985008687907853269984665640564039441803007913129639936", 10) + Expect(dbResult[0].Dart).To(Equal(expectedDart.String())) + Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) + }) }) diff --git a/pkg/transformers/integration_tests/vat_heal.go b/pkg/transformers/integration_tests/vat_heal.go index 05f7dce3..34c34234 100644 --- a/pkg/transformers/integration_tests/vat_heal.go +++ b/pkg/transformers/integration_tests/vat_heal.go @@ -65,7 +65,7 @@ var _ = Describe("VatHeal Transformer", func() { Repository: &vat_heal.VatHealRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_heal.VatHealModel @@ -78,4 +78,65 @@ var _ = Describe("VatHeal Transformer", func() { Expect(dbResult.V).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) Expect(dbResult.Rad).To(Equal("1000000000000000000000000000")) }) + + It("rechecks vat heal event", func() { + blockNumber := int64(8935578) + config := shared.TransformerConfig{ + TransformerName: constants.VatHealLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatHealSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_heal.VatHealConverter{}, + Repository: &vat_heal.VatHealRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatHealChecked []int + err = db.Select(&vatHealChecked, `SELECT vat_heal_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatHealChecked[0]).To(Equal(2)) + + var dbResults []vat_heal.VatHealModel + err = db.Select(&dbResults, `SELECT urn, v, rad from maker.vat_heal`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResults)).To(Equal(1)) + dbResult := dbResults[0] + Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) + Expect(dbResult.V).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) + Expect(dbResult.Rad).To(Equal("1000000000000000000000000000")) + }) }) diff --git a/pkg/transformers/integration_tests/vat_init.go b/pkg/transformers/integration_tests/vat_init.go index caf390fc..1a9d6db2 100644 --- a/pkg/transformers/integration_tests/vat_init.go +++ b/pkg/transformers/integration_tests/vat_init.go @@ -65,7 +65,7 @@ var _ = Describe("VatInit LogNoteTransformer", func() { Repository: &vat_init.VatInitRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_init.VatInitModel @@ -76,4 +76,64 @@ var _ = Describe("VatInit LogNoteTransformer", func() { dbResult := dbResults[0] Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) }) + + It("rechecks vat init event", func() { + blockNumber := int64(8535561) + config := shared.TransformerConfig{ + TransformerName: constants.VatInitLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatInitSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_init.VatInitConverter{}, + Repository: &vat_init.VatInitRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatInitChecked []int + err = db.Select(&vatInitChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatInitChecked[0]).To(Equal(2)) + + var dbResults []vat_init.VatInitModel + err = db.Select(&dbResults, `SELECT ilk from maker.vat_init`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResults)).To(Equal(1)) + dbResult := dbResults[0] + Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + }) }) diff --git a/pkg/transformers/integration_tests/vat_move.go b/pkg/transformers/integration_tests/vat_move.go index fafab17c..f8e42cb8 100644 --- a/pkg/transformers/integration_tests/vat_move.go +++ b/pkg/transformers/integration_tests/vat_move.go @@ -65,7 +65,7 @@ var _ = Describe("VatMove LogNoteTransformer", func() { Repository: &vat_move.VatMoveRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_move.VatMoveModel @@ -78,4 +78,66 @@ var _ = Describe("VatMove LogNoteTransformer", func() { Expect(dbResult.Dst).To(Equal(common.HexToAddress("0x0000d8b4147eda80fec7122ae16da2479cbd7ffb").String())) Expect(dbResult.Rad).To(Equal("1000000000000000000000000000000000000000000000")) }) + + It("rechecks vat move event", func() { + blockNumber := int64(9004628) + config := shared.TransformerConfig{ + TransformerName: constants.VatMoveLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatMoveSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_move.VatMoveConverter{}, + Repository: &vat_move.VatMoveRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatMoveChecked []int + err = db.Select(&vatMoveChecked, `SELECT vat_move_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatMoveChecked[0]).To(Equal(2)) + + var dbResults []vat_move.VatMoveModel + err = db.Select(&dbResults, `SELECT src, dst, rad from maker.vat_move`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResults)).To(Equal(1)) + dbResult := dbResults[0] + Expect(dbResult.Src).To(Equal(common.HexToAddress("0x8868bad8e74fca4505676d1b5b21ecc23328d132").String())) + Expect(dbResult.Dst).To(Equal(common.HexToAddress("0x0000d8b4147eda80fec7122ae16da2479cbd7ffb").String())) + Expect(dbResult.Rad).To(Equal("1000000000000000000000000000000000000000000000")) + }) }) diff --git a/pkg/transformers/integration_tests/vat_slip.go b/pkg/transformers/integration_tests/vat_slip.go index 0cda5d49..d496fb1a 100644 --- a/pkg/transformers/integration_tests/vat_slip.go +++ b/pkg/transformers/integration_tests/vat_slip.go @@ -57,7 +57,7 @@ var _ = Describe("Vat slip transformer", func() { Repository: &vat_slip.VatSlipRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var headerID int64 @@ -75,4 +75,62 @@ var _ = Describe("Vat slip transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(headerChecked).To(BeTrue()) }) + + It("rechecks vat slip event", func() { + blockNumber := int64(8953655) + config := shared.TransformerConfig{ + TransformerName: constants.VatSlipLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatSlipSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_slip.VatSlipConverter{}, + Repository: &vat_slip.VatSlipRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatSlipChecked []int + err = db.Select(&vatSlipChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatSlipChecked[0]).To(Equal(2)) + + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + var model vat_slip.VatSlipModel + err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(model.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + Expect(model.Guy).To(Equal("0xDA15dCE70ab462E66779f23ee14F21d993789eE3")) + Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000")) + Expect(model.TransactionIndex).To(Equal(uint(0))) + var headerChecked int + err = db.Get(&headerChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(Equal(2)) + }) }) diff --git a/pkg/transformers/integration_tests/vat_tune.go b/pkg/transformers/integration_tests/vat_tune.go index 40c8a23a..da6cd540 100644 --- a/pkg/transformers/integration_tests/vat_tune.go +++ b/pkg/transformers/integration_tests/vat_tune.go @@ -67,7 +67,7 @@ var _ = Describe("VatTune LogNoteTransformer", func() { Repository: &vat_tune.VatTuneRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_tune.VatTuneModel @@ -85,4 +85,71 @@ var _ = Describe("VatTune LogNoteTransformer", func() { Expect(dbResult[0].Dart).To(Equal(expectedDart.String())) Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) }) + + It("transforms VatTune log events", func() { + blockNumber := int64(8761670) + config := shared.TransformerConfig{ + TransformerName: constants.VatTuneLabel, + ContractAddresses: []string{test_data.KovanVatContractAddress}, + ContractAbi: test_data.KovanVatABI, + Topic: test_data.KovanVatTuneSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vat_tune.VatTuneConverter{}, + Repository: &vat_tune.VatTuneRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vatTuneChecked []int + err = db.Select(&vatTuneChecked, `SELECT vat_tune_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vatTuneChecked[0]).To(Equal(2)) + + var dbResult []vat_tune.VatTuneModel + err = db.Select(&dbResult, `SELECT ilk, urn, v, w, dink, dart from maker.vat_tune`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(1)) + Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + Expect(dbResult[0].Urn).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876")) + Expect(dbResult[0].V).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876")) + Expect(dbResult[0].W).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876")) + Expect(dbResult[0].Dink).To(Equal("0")) + expectedDart := new(big.Int) + expectedDart.SetString("115792089237316195423570985008687907853269984665640564039455584007913129639936", 10) + Expect(dbResult[0].Dart).To(Equal(expectedDart.String())) + Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) + }) }) diff --git a/pkg/transformers/integration_tests/vow_flog.go b/pkg/transformers/integration_tests/vow_flog.go index 08d6bfcd..545a7680 100644 --- a/pkg/transformers/integration_tests/vow_flog.go +++ b/pkg/transformers/integration_tests/vow_flog.go @@ -73,7 +73,7 @@ var _ = Describe("VowFlog LogNoteTransformer", func() { Repository: &vow_flog.VowFlogRepository{}, }.NewLogNoteTransformer(db) - err = transformer.Execute(logs, header) + err = transformer.Execute(logs, header, constants.HeaderMissing) Expect(err).NotTo(HaveOccurred()) var dbResult []vow_flog.VowFlogModel @@ -84,4 +84,55 @@ var _ = Describe("VowFlog LogNoteTransformer", func() { Expect(dbResult[0].LogIndex).To(Equal(uint(2))) Expect(dbResult[0].TransactionIndex).To(Equal(uint(2))) }) + + It("rechecks vow flog event", func() { + blockNumber := int64(8946819) + config := shared.TransformerConfig{ + TransformerName: constants.VowFlogLabel, + ContractAddresses: []string{test_data.KovanVowContractAddress}, + ContractAbi: test_data.KovanVowABI, + Topic: test_data.KovanVowFlogSignature, + StartingBlockNumber: blockNumber, + EndingBlockNumber: blockNumber, + } + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + transformer := factories.LogNoteTransformer{ + Config: config, + Converter: &vow_flog.VowFlogConverter{}, + Repository: &vow_flog.VowFlogRepository{}, + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header, constants.HeaderMissing) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header, constants.HeaderRecheck) + Expect(err).NotTo(HaveOccurred()) + + var headerID int64 + err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber) + Expect(err).NotTo(HaveOccurred()) + + var vowFlogChecked []int + err = db.Select(&vowFlogChecked, `SELECT vow_flog_checked FROM public.checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + + Expect(vowFlogChecked[0]).To(Equal(2)) + + var dbResult []vow_flog.VowFlogModel + err = db.Select(&dbResult, `SELECT era, log_idx, tx_idx from maker.vow_flog`) + Expect(err).NotTo(HaveOccurred()) + + Expect(dbResult[0].Era).To(Equal("1538558052")) + Expect(dbResult[0].LogIndex).To(Equal(uint(2))) + Expect(dbResult[0].TransactionIndex).To(Equal(uint(2))) + }) }) diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 118f875c..eff9703a 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -18,7 +18,10 @@ package debt_ceiling import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i _, execErr := tx.Exec( `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, + VALUES($1, $2, $3::NUMERIC, $4, $5, $6) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET what = $2, data = $3, raw_log = $6;`, headerID, pitFileDC.What, pitFileDC.Data, pitFileDC.LogIndex, pitFileDC.TransactionIndex, pitFileDC.Raw, ) @@ -75,6 +79,14 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) } +func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked) +} + +func (repository PitFileDebtCeilingRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked) +} + func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index 0ec9860d..90bd0cfc 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -18,7 +18,10 @@ package ilk import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface _, execErr := tx.Exec( `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, + VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, data = $4, raw_log = $7;`, headerID, pitFileIlk.Ilk, pitFileIlk.What, pitFileIlk.Data, pitFileIlk.LogIndex, pitFileIlk.TransactionIndex, pitFileIlk.Raw, ) if execErr != nil { @@ -73,6 +77,14 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) } +func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked) +} + +func (repository PitFileIlkRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked) +} + func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/price_feeds/repository.go b/pkg/transformers/price_feeds/repository.go index 3ee1fbbd..ed776e73 100644 --- a/pkg/transformers/price_feeds/repository.go +++ b/pkg/transformers/price_feeds/repository.go @@ -18,7 +18,10 @@ package price_feeds import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -43,14 +46,13 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{ return fmt.Errorf("model of type %T, not %T", model, PriceFeedModel{}) } - _, execErr := tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log) - VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw) - if execErr != nil { - rollbackErr := tx.Rollback() - if rollbackErr != nil { - log.Error("failed to rollback ", rollbackErr) - } - return execErr + _, err := tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log) + VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7) + ON CONFLICT (header_id, medianizer_address, tx_idx, log_idx) DO UPDATE SET block_number = $1, usd_value = $4, raw_log = $7;`, + priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw) + if err != nil { + tx.Rollback() + return err } } @@ -69,6 +71,14 @@ func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) } +func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked) +} + +func (repository PriceFeedRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked) +} + func (repository *PriceFeedRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/shared/constants/checked_headers.go b/pkg/transformers/shared/constants/checked_headers.go index 00dae88b..ccbd250e 100644 --- a/pkg/transformers/shared/constants/checked_headers.go +++ b/pkg/transformers/shared/constants/checked_headers.go @@ -1,5 +1,13 @@ package constants +type TransformerExecution bool + +const ( + HeaderRecheck TransformerExecution = true + HeaderMissing TransformerExecution = false + RecheckHeaderCap = "4" +) + var ( BiteChecked = "bite_checked" CatFileChopLumpChecked = "cat_file_chop_lump_checked" diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index 9ee55e8f..834e78ac 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -12,17 +12,17 @@ import ( func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { _, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) - VALUES ($1, $2) + VALUES ($1, $2) ON CONFLICT (header_id) DO - UPDATE SET `+checkedHeadersColumn+` = $2`, headerID, true) + UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1) return err } func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) - VALUES ($1, $2) + VALUES ($1, $2) ON CONFLICT (header_id) DO - UPDATE SET `+checkedHeadersColumn+` = $2`, headerID, true) + UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1) return err } @@ -52,6 +52,31 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D return result, err } +func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) { + var result []core.Header + var query string + var err error + + if endingBlockNumber == -1 { + query = `SELECT headers.id, headers.block_number, headers.hash FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + ` + AND headers.block_number >= $1 + AND headers.eth_node_fingerprint = $2` + err = db.Select(&result, query, startingBlockNumber, db.Node.ID) + } else { + query = `SELECT headers.id, headers.block_number, headers.hash FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + ` + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3` + err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID) + } + + return result, err +} + func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { // Query returns `[]driver.Value`, nullable polymorphic interface var queryResult []driver.Value @@ -79,11 +104,12 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { return columnNames, nil } -// Builds a SQL string that checks if any column value is FALSE, given the column names. +// Builds a SQL string that checks if any column value is 0, given the column names. // Defaults to FALSE when no columns are provided. -// Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)" +// Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)" // [] => "FALSE" -func CreateNotCheckedSQL(boolColumns []string) string { +func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string { + var result bytes.Buffer if len(boolColumns) == 0 { @@ -91,13 +117,24 @@ func CreateNotCheckedSQL(boolColumns []string) string { } result.WriteString("NOT (") + // Loop excluding last column name for _, column := range boolColumns[:len(boolColumns)-1] { - result.WriteString(fmt.Sprintf("%v AND ", column)) + + if recheckHeaders { + result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap)) + } else { + result.WriteString(fmt.Sprintf("%v!=0 AND ", column)) + } } // No trailing "OR" for the last column name - result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1])) + if recheckHeaders { + result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap)) + } else { + result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1])) + + } return result.String() } diff --git a/pkg/transformers/shared/repository_utility_test.go b/pkg/transformers/shared/repository_utility_test.go index b4498afb..1a8f762b 100644 --- a/pkg/transformers/shared/repository_utility_test.go +++ b/pkg/transformers/shared/repository_utility_test.go @@ -17,6 +17,7 @@ package shared_test import ( + "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -25,6 +26,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/test_config" "math/rand" ) @@ -50,7 +52,7 @@ var _ = Describe("Repository utilities", func() { columnNames, err := shared.GetCheckedColumnNames(db) Expect(err).NotTo(HaveOccurred()) - notCheckedSQL = shared.CreateNotCheckedSQL(columnNames) + notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing) startingBlockNumber = rand.Int63() eventSpecificBlockNumber = startingBlockNumber + 1 @@ -118,21 +120,28 @@ var _ = Describe("Repository utilities", func() { Describe("CreateNotCheckedSQL", func() { It("generates a correct SQL string for one column", func() { columns := []string{"columnA"} - expected := "NOT (columnA)" - actual := shared.CreateNotCheckedSQL(columns) + expected := "NOT (columnA!=0)" + actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing) Expect(actual).To(Equal(expected)) }) It("generates a correct SQL string for several columns", func() { columns := []string{"columnA", "columnB"} - expected := "NOT (columnA AND columnB)" - actual := shared.CreateNotCheckedSQL(columns) + expected := "NOT (columnA!=0 AND columnB!=0)" + actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing) Expect(actual).To(Equal(expected)) }) It("defaults to FALSE when there are no columns", func() { expected := "FALSE" - actual := shared.CreateNotCheckedSQL([]string{}) + actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing) + Expect(actual).To(Equal(expected)) + }) + + It("generates a correct SQL string for rechecking headers", func() { + columns := []string{"columnA", "columnB"} + expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap) + actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck) Expect(actual).To(Equal(expected)) }) }) diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 500afec7..7d3c7164 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -22,10 +22,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) type Transformer interface { - Execute(logs []types.Log, header core.Header) error + Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error GetConfig() TransformerConfig } diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 75ff6760..89e3f08e 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -18,7 +18,10 @@ package tend import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -55,7 +58,8 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er _, execErr := tx.Exec( `INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, + VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, guy = $5, tic = $6, raw_log = $9;`, headerID, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tic, tend.LogIndex, tend.TransactionIndex, tend.Raw, ) @@ -83,6 +87,14 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) } +func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked) +} + +func (repository TendRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked) +} + func (repository *TendRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/test_data/mocks/repository.go b/pkg/transformers/test_data/mocks/repository.go index 0a998401..a19e7f86 100644 --- a/pkg/transformers/test_data/mocks/repository.go +++ b/pkg/transformers/test_data/mocks/repository.go @@ -3,6 +3,7 @@ package mocks import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -11,6 +12,9 @@ type MockRepository struct { markHeaderCheckedError error MarkHeaderCheckedPassedHeaderIDs []int64 CreatedHeaderIds []int64 + missingHeaders []core.Header + allHeaders []core.Header + missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 @@ -33,10 +37,34 @@ func (repository *MockRepository) MarkHeaderChecked(headerID int64) error { return repository.markHeaderCheckedError } +func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersError +} + +func (repository *MockRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.allHeaders, nil +} + func (repository *MockRepository) SetDB(db *postgres.DB) { repository.SetDbCalled = true } +func (repository *MockRepository) SetMissingHeadersError(e error) { + repository.missingHeadersError = e +} + +func (repository *MockRepository) SetAllHeaders(headers []core.Header) { + repository.allHeaders = headers +} + +func (repository *MockRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + func (repository *MockRepository) SetMarkHeaderCheckedError(e error) { repository.markHeaderCheckedError = e } diff --git a/pkg/transformers/test_data/mocks/transformer.go b/pkg/transformers/test_data/mocks/transformer.go index 0b054567..452e5794 100644 --- a/pkg/transformers/test_data/mocks/transformer.go +++ b/pkg/transformers/test_data/mocks/transformer.go @@ -5,6 +5,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) type MockTransformer struct { @@ -15,7 +16,7 @@ type MockTransformer struct { config shared.TransformerConfig } -func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { +func (mh *MockTransformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error { if mh.ExecuteError != nil { return mh.ExecuteError } diff --git a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go index d3502760..d12008fa 100644 --- a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go +++ b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go @@ -39,6 +39,7 @@ type CreateBehaviorInputs struct { CheckedHeaderColumnName string LogEventTableName string TestModel interface{} + RecheckTestModel interface{} ModelWithDifferentLogIdx interface{} Repository factories.Repository } @@ -70,11 +71,11 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { err = repository.Create(headerID, []interface{}{logEventModel}) Expect(err).NotTo(HaveOccurred()) - var headerChecked bool + var headerChecked int query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` err = db.Get(&headerChecked, query, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(headerChecked).To(BeTrue()) + Expect(headerChecked).To(Equal(1)) }) It("updates the header to checked if checked headers row already exists", func() { @@ -84,21 +85,11 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { err = repository.Create(headerID, []interface{}{logEventModel}) Expect(err).NotTo(HaveOccurred()) - var headerChecked bool + var headerChecked int query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` err = db.Get(&headerChecked, query, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(headerChecked).To(BeTrue()) - }) - - It("returns an error if inserting duplicate log events", func() { - err = repository.Create(headerID, []interface{}{logEventModel}) - Expect(err).NotTo(HaveOccurred()) - - err = repository.Create(headerID, []interface{}{logEventModel}) - - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + Expect(headerChecked).To(Equal(1)) }) It("allows for multiple log events of the same type in one transaction if they have different log indexes", func() { @@ -147,11 +138,11 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav err = repository.MarkHeaderChecked(headerId) Expect(err).NotTo(HaveOccurred()) - var headerChecked bool + var headerChecked int query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` err = db.Get(&headerChecked, query, headerId) Expect(err).NotTo(HaveOccurred()) - Expect(headerChecked).To(BeTrue()) + Expect(headerChecked).To(Equal(1)) }) It("updates row when headerID already exists", func() { @@ -160,11 +151,11 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav err = repository.MarkHeaderChecked(headerId) Expect(err).NotTo(HaveOccurred()) - var headerChecked bool + var headerChecked int query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` err = db.Get(&headerChecked, query, headerId) Expect(err).NotTo(HaveOccurred()) - Expect(headerChecked).To(BeTrue()) + Expect(headerChecked).To(Equal(1)) }) It("returns an error if upserting a record fails", func() { @@ -173,6 +164,18 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("violates foreign key constraint")) }) + + It("increments header checked", func() { + err = repository.MarkHeaderChecked(headerId) + err = repository.MarkHeaderChecked(headerId) + + Expect(err).NotTo(HaveOccurred()) + var headerChecked int + query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` + err = db.Get(&headerChecked, query, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(Equal(2)) + }) }) } diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index 6b8af24b..5e299ed1 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -18,7 +18,10 @@ package vat_flux import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -45,7 +48,8 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{}) } _, execErr := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log) - VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8)`, + VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, dst = $3, src = $4, rad = $5, raw_log = $8;`, headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw) if execErr != nil { rollbackErr := tx.Rollback() @@ -72,6 +76,14 @@ func (repository VatFluxRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) } +func (repository VatFluxRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatFluxChecked) +} + +func (repository VatFluxRepository) RecheckHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlock, endingBlock, repository.db, constants.VatFluxChecked) +} + func (repository *VatFluxRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index 87aac383..2a199172 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -20,6 +20,7 @@ import ( "fmt" log "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +47,8 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{}) _, execErr := tx.Exec( `INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, + VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, rate = $4, raw_log = $7;`, headerID, vatFold.Ilk, vatFold.Urn, vatFold.Rate, vatFold.LogIndex, vatFold.TransactionIndex, vatFold.Raw, ) if execErr != nil { @@ -74,6 +76,14 @@ func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) } +func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatFoldChecked) +} + +func (repository VatFoldRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatFoldChecked) +} + func (repository *VatFoldRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go index 684426e7..24d1fabb 100644 --- a/pkg/transformers/vat_grab/repository.go +++ b/pkg/transformers/vat_grab/repository.go @@ -4,6 +4,7 @@ import ( "fmt" log "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -30,7 +31,8 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{}) _, execErr := tx.Exec( `INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, + VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, v = $4, w = $5, dink = $6, dart = $7, raw_log = $10;`, headerID, vatGrab.Ilk, vatGrab.Urn, vatGrab.V, vatGrab.W, vatGrab.Dink, vatGrab.Dart, vatGrab.LogIndex, vatGrab.TransactionIndex, vatGrab.Raw, ) if execErr != nil { @@ -56,6 +58,14 @@ func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) } +func (repository VatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatGrabChecked) +} + +func (repository VatGrabRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatGrabChecked) +} + func (repository *VatGrabRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_heal/repository.go b/pkg/transformers/vat_heal/repository.go index 87e14d20..59d567de 100644 --- a/pkg/transformers/vat_heal/repository.go +++ b/pkg/transformers/vat_heal/repository.go @@ -20,6 +20,7 @@ import ( "fmt" log "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -50,7 +51,8 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) } _, execErr := tx.Exec(`INSERT INTO maker.vat_heal (header_id, urn, v, rad, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, + VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) + ON CONFlICT (header_id, tx_idx, log_idx) DO UPDATE SET urn = $2, v = $3, rad = $4, raw_log = $7;`, headerID, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw) if execErr != nil { rollbackErr := tx.Rollback() @@ -72,6 +74,14 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) return tx.Commit() } +func (repository VatHealRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatHealChecked) +} + +func (repository VatHealRepository) RecheckHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlock, endingBlock, repository.db, constants.VatHealChecked) +} + func (repository VatHealRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) } diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index f74ed468..d56c3bcb 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -18,7 +18,10 @@ package vat_init import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) _, execErr := tx.Exec( `INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5)`, + VALUES($1, $2, $3, $4, $5) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, raw_log = $5;`, headerID, vatInit.Ilk, vatInit.LogIndex, vatInit.TransactionIndex, vatInit.Raw, ) if execErr != nil { @@ -74,6 +78,14 @@ func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) } +func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatInitChecked) +} + +func (repository VatInitRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatInitChecked) +} + func (repository *VatInitRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index dc45d4a5..4bd2c26b 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -18,7 +18,10 @@ package vat_move import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) _, execErr := tx.Exec( `INSERT INTO maker.vat_move (header_id, src, dst, rad, log_idx, tx_idx, raw_log) - VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, + VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET src = $2, dst = $3, rad = $4, raw_log = $7;`, headerID, vatMove.Src, vatMove.Dst, vatMove.Rad, vatMove.LogIndex, vatMove.TransactionIndex, vatMove.Raw, ) @@ -71,6 +75,14 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) return tx.Commit() } +func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatMoveChecked) +} + +func (repository VatMoveRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatMoveChecked) +} + func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) } diff --git a/pkg/transformers/vat_slip/repository.go b/pkg/transformers/vat_slip/repository.go index b7e0713f..40f1fb57 100644 --- a/pkg/transformers/vat_slip/repository.go +++ b/pkg/transformers/vat_slip/repository.go @@ -2,7 +2,10 @@ package vat_slip import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -29,7 +32,8 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{}) _, execErr := tx.Exec( `INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, + VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, guy = $3, rad = $4, raw_log = $7;`, headerID, vatSlip.Ilk, vatSlip.Guy, vatSlip.Rad, vatSlip.TransactionIndex, vatSlip.LogIndex, vatSlip.Raw, ) if execErr != nil { @@ -57,6 +61,14 @@ func (repository VatSlipRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) } +func (repository VatSlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatSlipChecked) +} + +func (repository VatSlipRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatSlipChecked) +} + func (repository *VatSlipRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go index 245e4fa3..1af342b4 100644 --- a/pkg/transformers/vat_toll/repository.go +++ b/pkg/transformers/vat_toll/repository.go @@ -2,7 +2,10 @@ package vat_toll import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -56,6 +59,14 @@ func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) } +func (repository VatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTollChecked) +} + +func (repository VatTollRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTollChecked) +} + func (repository *VatTollRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go index 46ea163e..99855bb0 100644 --- a/pkg/transformers/vat_tune/repository.go +++ b/pkg/transformers/vat_tune/repository.go @@ -2,7 +2,10 @@ package vat_tune import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -29,7 +32,8 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) _, execErr := tx.Exec( `INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, + VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, v = $4, w = $5, dink = $6, dart = $7, raw_log = $10;`, headerID, vatTune.Ilk, vatTune.Urn, vatTune.V, vatTune.W, vatTune.Dink, vatTune.Dart, vatTune.TransactionIndex, vatTune.LogIndex, vatTune.Raw, ) if execErr != nil { @@ -56,6 +60,14 @@ func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) } +func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked) +} + +func (repository VatTuneRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked) +} + func (repository *VatTuneRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vow_flog/repository.go b/pkg/transformers/vow_flog/repository.go index 20f06bad..6b1a735d 100644 --- a/pkg/transformers/vow_flog/repository.go +++ b/pkg/transformers/vow_flog/repository.go @@ -18,7 +18,10 @@ package vow_flog import ( "fmt" + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -46,7 +49,8 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{}) _, execErr := tx.Exec( `INSERT into maker.vow_flog (header_id, era, log_idx, tx_idx, raw_log) - VALUES($1, $2::NUMERIC, $3, $4, $5)`, + VALUES($1, $2::NUMERIC, $3, $4, $5) + ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET era = $2, raw_log = $5;`, headerID, flog.Era, flog.LogIndex, flog.TransactionIndex, flog.Raw, ) @@ -75,6 +79,14 @@ func (repository VowFlogRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) } +func (repository VowFlogRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VowFlogChecked) +} + +func (repository VowFlogRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VowFlogChecked) +} + func (repository *VowFlogRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/test_config/test_config.go b/test_config/test_config.go index 726c438a..e11ddea3 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -55,6 +55,7 @@ func setTestConfig() { hn := TestConfig.GetString("database.hostname") port := TestConfig.GetInt("database.port") name := TestConfig.GetString("database.name") + DBConfig = config.Database{ Hostname: hn, Name: name,