cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

* update transformer to able to recheck headers (#4)

Browse Source 
* update transformer to able to recheck headers
* put cap on rechecking header
* integration test for recheck headers
* use enum for recheck headers param; make recheck cap configurable
* update integration tests with new test config
* update omni pkg with new recheck header column type
* update migration with new migration tool and final tweaks needed to accommodate changes in omni pkg
This commit is contained in:
Takayuki Goto 2019-02-08 10:35:46 -06:00 committed by GitHub
parent 219f099b0c
commit 0a024d429d
74 changed files with 1994 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Supfile
View File

@ -57,7 +57,8 @@ commands:
run: > run: >
systemctl daemon-reload && systemctl daemon-reload &&
sudo systemctl restart vulcanizedb_light_sync.service && sudo systemctl restart vulcanizedb_light_sync.service &&
sudo systemctl restart vulcanizedb_log_sync.service sudo systemctl restart vulcanizedb_log_sync.service &&
sudo systemctl restart vulcanizedb_recheck_sync.service
postgraphile: postgraphile:
desc: start postgraphile desc: start postgraphile
run: systemctl daemon-reload && sudo systemctl restart postgraphile.service run: systemctl daemon-reload && sudo systemctl restart postgraphile.service

3
cmd/backfillMakerLogs.go
View File

@ -22,6 +22,7 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/pkg/transformers"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
// backfillMakerLogsCmd represents the backfillMakerLogs command // backfillMakerLogsCmd represents the backfillMakerLogs command
@ -50,7 +51,7 @@ func backfillMakerLogs() {
watcher := shared.NewEventWatcher(db, blockChain) watcher := shared.NewEventWatcher(db, blockChain)
watcher.AddTransformers(transformers.TransformerInitializers()) watcher.AddTransformers(transformers.TransformerInitializers())
err = watcher.Execute() err = watcher.Execute(constants.HeaderMissing)
if err != nil { if err != nil {
// TODO Handle watcher error in backfillMakerLogs // TODO Handle watcher error in backfillMakerLogs
} }

8
cmd/continuousLogSync.go
View File

@ -49,6 +49,7 @@ This command expects a light sync to have been run, and the presence of header r
} }
var transformerNames []string var transformerNames []string
var recheckHeadersArg bool
func syncMakerLogs() { func syncMakerLogs() {
ticker := time.NewTicker(pollingInterval) ticker := time.NewTicker(pollingInterval)
@ -66,7 +67,11 @@ func syncMakerLogs() {
watcher.AddTransformers(initializers) watcher.AddTransformers(initializers)
for range ticker.C { for range ticker.C {
err = watcher.Execute() if recheckHeadersArg {
err = watcher.Execute(constants.HeaderRecheck)
} else {
err = watcher.Execute(constants.HeaderMissing)
}
if err != nil { if err != nil {
// TODO Handle watcher errors in ContinuousLogSync // TODO Handle watcher errors in ContinuousLogSync
} }
@ -125,4 +130,5 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer
func init() { func init() {
rootCmd.AddCommand(continuousLogSyncCmd) rootCmd.AddCommand(continuousLogSyncCmd)
continuousLogSyncCmd.Flags().StringSliceVar(&transformerNames, "transformers", []string{"all"}, "transformer names to be run during this command") continuousLogSyncCmd.Flags().StringSliceVar(&transformerNames, "transformers", []string{"all"}, "transformer names to be run during this command")
continuousLogSyncCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
} }

175
db/migrations/00072_change_checked_headers_headers_table_type_int.sql Normal file
View File

@ -0,0 +1,175 @@
-- +goose Up
ALTER TABLE checked_headers
ALTER price_feeds_checked SET DEFAULT null,
ALTER flip_kick_checked SET DEFAULT null,
ALTER frob_checked SET DEFAULT null,
ALTER tend_checked SET DEFAULT null,
ALTER bite_checked SET DEFAULT null,
ALTER dent_checked SET DEFAULT null,
ALTER pit_file_debt_ceiling_checked SET DEFAULT null,
ALTER pit_file_ilk_checked SET DEFAULT null,
ALTER vat_init_checked SET DEFAULT null,
ALTER drip_file_ilk_checked SET DEFAULT null,
ALTER drip_file_repo_checked SET DEFAULT null,
ALTER drip_file_vow_checked SET DEFAULT null,
ALTER deal_checked SET DEFAULT null,
ALTER drip_drip_checked SET DEFAULT null,
ALTER cat_file_chop_lump_checked SET DEFAULT null,
ALTER cat_file_flip_checked SET DEFAULT null,
ALTER cat_file_pit_vow_checked SET DEFAULT null,
ALTER flop_kick_checked SET DEFAULT null,
ALTER vat_move_checked SET DEFAULT null,
ALTER vat_fold_checked SET DEFAULT null,
ALTER vat_heal_checked SET DEFAULT null,
ALTER vat_toll_checked SET DEFAULT null,
ALTER vat_tune_checked SET DEFAULT null,
ALTER vat_grab_checked SET DEFAULT null,
ALTER vat_flux_checked SET DEFAULT null,
ALTER vat_slip_checked SET DEFAULT null,
ALTER vow_flog_checked SET DEFAULT null,
ALTER flap_kick_checked SET DEFAULT null;
ALTER TABLE checked_headers
ALTER COLUMN price_feeds_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER price_feeds_checked SET DEFAULT 0,
ALTER COLUMN flip_kick_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER flip_kick_checked SET DEFAULT 0,
ALTER COLUMN frob_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER frob_checked SET DEFAULT 0,
ALTER COLUMN tend_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER tend_checked SET DEFAULT 0,
ALTER COLUMN bite_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER bite_checked SET DEFAULT 0,
ALTER COLUMN dent_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER dent_checked SET DEFAULT 0,
ALTER COLUMN pit_file_debt_ceiling_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER pit_file_debt_ceiling_checked SET DEFAULT 0,
ALTER COLUMN pit_file_ilk_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER pit_file_ilk_checked SET DEFAULT 0,
ALTER COLUMN vat_init_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_init_checked SET DEFAULT 0,
ALTER COLUMN drip_file_ilk_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER drip_file_ilk_checked SET DEFAULT 0,
ALTER COLUMN drip_file_repo_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER drip_file_repo_checked SET DEFAULT 0,
ALTER COLUMN drip_file_vow_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER drip_file_vow_checked SET DEFAULT 0,
ALTER COLUMN deal_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER deal_checked SET DEFAULT 0,
ALTER COLUMN drip_drip_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER drip_drip_checked SET DEFAULT 0,
ALTER COLUMN cat_file_chop_lump_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER cat_file_chop_lump_checked SET DEFAULT 0,
ALTER COLUMN cat_file_flip_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER cat_file_flip_checked SET DEFAULT 0,
ALTER COLUMN cat_file_pit_vow_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER cat_file_pit_vow_checked SET DEFAULT 0,
ALTER COLUMN flop_kick_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER flop_kick_checked SET DEFAULT 0,
ALTER COLUMN vat_move_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_move_checked SET DEFAULT 0,
ALTER COLUMN vat_fold_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_fold_checked SET DEFAULT 0,
ALTER COLUMN vat_heal_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_heal_checked SET DEFAULT 0,
ALTER COLUMN vat_toll_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_toll_checked SET DEFAULT 0,
ALTER COLUMN vat_tune_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_tune_checked SET DEFAULT 0,
ALTER COLUMN vat_grab_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_grab_checked SET DEFAULT 0,
ALTER COLUMN vat_flux_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_flux_checked SET DEFAULT 0,
ALTER COLUMN vat_slip_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vat_slip_checked SET DEFAULT 0,
ALTER COLUMN vow_flog_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER vow_flog_checked SET DEFAULT 0,
ALTER COLUMN flap_kick_checked TYPE integer USING CASE WHEN false THEN 0 ELSE 1 END,
ALTER flap_kick_checked SET DEFAULT 0;
-- +goose Down
ALTER TABLE checked_headers
ALTER price_feeds_checked drop default,
ALTER COLUMN price_feeds_checked SET DATA TYPE boolean USING CASE WHEN price_feeds_checked = 0 THEN FALSE ELSE TRUE END,
ALTER price_feeds_checked SET DEFAULT FALSE,
ALTER flip_kick_checked drop default,
ALTER COLUMN flip_kick_checked SET DATA TYPE boolean USING CASE WHEN flip_kick_checked = 0 THEN FALSE ELSE TRUE END,
ALTER flip_kick_checked SET DEFAULT FALSE,
ALTER frob_checked drop default,
ALTER COLUMN frob_checked SET DATA TYPE boolean USING CASE WHEN frob_checked = 0 THEN FALSE ELSE TRUE END,
ALTER frob_checked SET DEFAULT FALSE,
ALTER tend_checked drop default,
ALTER COLUMN tend_checked SET DATA TYPE boolean USING CASE WHEN tend_checked = 0 THEN FALSE ELSE TRUE END,
ALTER tend_checked SET DEFAULT FALSE,
ALTER bite_checked drop default,
ALTER COLUMN bite_checked SET DATA TYPE boolean USING CASE WHEN bite_checked = 0 THEN FALSE ELSE TRUE END,
ALTER bite_checked SET DEFAULT FALSE,
ALTER dent_checked drop default,
ALTER COLUMN dent_checked SET DATA TYPE boolean USING CASE WHEN dent_checked = 0 THEN FALSE ELSE TRUE END,
ALTER dent_checked SET DEFAULT FALSE,
ALTER pit_file_debt_ceiling_checked drop default,
ALTER COLUMN pit_file_debt_ceiling_checked SET DATA TYPE boolean USING CASE WHEN pit_file_debt_ceiling_checked = 0 THEN FALSE ELSE TRUE END,
ALTER pit_file_debt_ceiling_checked SET DEFAULT FALSE,
ALTER pit_file_ilk_checked drop default,
ALTER COLUMN pit_file_ilk_checked SET DATA TYPE boolean USING CASE WHEN pit_file_ilk_checked = 0 THEN FALSE ELSE TRUE END,
ALTER pit_file_ilk_checked SET DEFAULT FALSE,
ALTER vat_init_checked drop default,
ALTER COLUMN vat_init_checked SET DATA TYPE boolean USING CASE WHEN vat_init_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_init_checked SET DEFAULT FALSE,
ALTER drip_file_ilk_checked drop default,
ALTER COLUMN drip_file_ilk_checked SET DATA TYPE boolean USING CASE WHEN drip_file_ilk_checked = 0 THEN FALSE ELSE TRUE END,
ALTER drip_file_ilk_checked SET DEFAULT FALSE,
ALTER drip_file_repo_checked drop default,
ALTER COLUMN drip_file_repo_checked SET DATA TYPE boolean USING CASE WHEN drip_file_repo_checked = 0 THEN FALSE ELSE TRUE END,
ALTER drip_file_repo_checked SET DEFAULT FALSE,
ALTER drip_file_vow_checked drop default,
ALTER COLUMN drip_file_vow_checked SET DATA TYPE boolean USING CASE WHEN drip_file_vow_checked = 0 THEN FALSE ELSE TRUE END,
ALTER drip_file_vow_checked SET DEFAULT FALSE,
ALTER deal_checked drop default,
ALTER COLUMN deal_checked SET DATA TYPE boolean USING CASE WHEN deal_checked = 0 THEN FALSE ELSE TRUE END,
ALTER deal_checked SET DEFAULT FALSE,
ALTER drip_drip_checked drop default,
ALTER COLUMN drip_drip_checked SET DATA TYPE boolean USING CASE WHEN drip_drip_checked = 0 THEN FALSE ELSE TRUE END,
ALTER drip_drip_checked SET DEFAULT FALSE,
ALTER cat_file_chop_lump_checked drop default,
ALTER COLUMN cat_file_chop_lump_checked SET DATA TYPE boolean USING CASE WHEN cat_file_chop_lump_checked = 0 THEN FALSE ELSE TRUE END,
ALTER cat_file_chop_lump_checked SET DEFAULT FALSE,
ALTER cat_file_flip_checked drop default,
ALTER COLUMN cat_file_flip_checked SET DATA TYPE boolean USING CASE WHEN cat_file_flip_checked = 0 THEN FALSE ELSE TRUE END,
ALTER cat_file_flip_checked SET DEFAULT FALSE,
ALTER cat_file_pit_vow_checked drop default,
ALTER COLUMN cat_file_pit_vow_checked SET DATA TYPE boolean USING CASE WHEN cat_file_pit_vow_checked = 0 THEN FALSE ELSE TRUE END,
ALTER cat_file_pit_vow_checked SET DEFAULT FALSE,
ALTER flop_kick_checked drop default,
ALTER COLUMN flop_kick_checked SET DATA TYPE boolean USING CASE WHEN flop_kick_checked = 0 THEN FALSE ELSE TRUE END,
ALTER flop_kick_checked SET DEFAULT FALSE,
ALTER vat_move_checked drop default,
ALTER COLUMN vat_move_checked SET DATA TYPE boolean USING CASE WHEN vat_move_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_move_checked SET DEFAULT FALSE,
ALTER vat_fold_checked drop default,
ALTER COLUMN vat_fold_checked SET DATA TYPE boolean USING CASE WHEN vat_fold_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_fold_checked SET DEFAULT FALSE,
ALTER vat_heal_checked drop default,
ALTER COLUMN vat_heal_checked SET DATA TYPE boolean USING CASE WHEN vat_heal_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_heal_checked SET DEFAULT FALSE,
ALTER vat_toll_checked drop default,
ALTER COLUMN vat_toll_checked SET DATA TYPE boolean USING CASE WHEN vat_toll_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_toll_checked SET DEFAULT FALSE,
ALTER vat_tune_checked drop default,
ALTER COLUMN vat_tune_checked SET DATA TYPE boolean USING CASE WHEN vat_tune_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_tune_checked SET DEFAULT FALSE,
ALTER vat_grab_checked drop default,
ALTER COLUMN vat_grab_checked SET DATA TYPE boolean USING CASE WHEN vat_grab_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_grab_checked SET DEFAULT FALSE,
ALTER vat_flux_checked drop default,
ALTER COLUMN vat_flux_checked SET DATA TYPE boolean USING CASE WHEN vat_flux_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_flux_checked SET DEFAULT FALSE,
ALTER vat_slip_checked drop default,
ALTER COLUMN vat_slip_checked SET DATA TYPE boolean USING CASE WHEN vat_slip_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vat_slip_checked SET DEFAULT FALSE,
ALTER vow_flog_checked drop default,
ALTER COLUMN vow_flog_checked SET DATA TYPE boolean USING CASE WHEN vow_flog_checked = 0 THEN FALSE ELSE TRUE END,
ALTER vow_flog_checked SET DEFAULT FALSE,
ALTER flap_kick_checked drop default,
ALTER COLUMN flap_kick_checked SET DATA TYPE boolean USING CASE WHEN flap_kick_checked = 0 THEN FALSE ELSE TRUE END,
ALTER flap_kick_checked SET DEFAULT FALSE;

56
db/schema.sql
View File

@ -1357,34 +1357,34 @@ ALTER SEQUENCE public.blocks_id_seq OWNED BY public.blocks.id;
CREATE TABLE public.checked_headers ( CREATE TABLE public.checked_headers (
id integer NOT NULL, id integer NOT NULL,
header_id integer NOT NULL, header_id integer NOT NULL,
price_feeds_checked boolean DEFAULT false NOT NULL, price_feeds_checked integer DEFAULT 0 NOT NULL,
flip_kick_checked boolean DEFAULT false NOT NULL, flip_kick_checked integer DEFAULT 0 NOT NULL,
frob_checked boolean DEFAULT false NOT NULL, frob_checked integer DEFAULT 0 NOT NULL,
tend_checked boolean DEFAULT false NOT NULL, tend_checked integer DEFAULT 0 NOT NULL,
bite_checked boolean DEFAULT false NOT NULL, bite_checked integer DEFAULT 0 NOT NULL,
dent_checked boolean DEFAULT false NOT NULL, dent_checked integer DEFAULT 0 NOT NULL,
pit_file_debt_ceiling_checked boolean DEFAULT false NOT NULL, pit_file_debt_ceiling_checked integer DEFAULT 0 NOT NULL,
pit_file_ilk_checked boolean DEFAULT false NOT NULL, pit_file_ilk_checked integer DEFAULT 0 NOT NULL,
vat_init_checked boolean DEFAULT false NOT NULL, vat_init_checked integer DEFAULT 0 NOT NULL,
drip_file_ilk_checked boolean DEFAULT false NOT NULL, drip_file_ilk_checked integer DEFAULT 0 NOT NULL,
drip_file_repo_checked boolean DEFAULT false NOT NULL, drip_file_repo_checked integer DEFAULT 0 NOT NULL,
drip_file_vow_checked boolean DEFAULT false NOT NULL, drip_file_vow_checked integer DEFAULT 0 NOT NULL,
deal_checked boolean DEFAULT false NOT NULL, deal_checked integer DEFAULT 0 NOT NULL,
drip_drip_checked boolean DEFAULT false NOT NULL, drip_drip_checked integer DEFAULT 0 NOT NULL,
cat_file_chop_lump_checked boolean DEFAULT false NOT NULL, cat_file_chop_lump_checked integer DEFAULT 0 NOT NULL,
cat_file_flip_checked boolean DEFAULT false NOT NULL, cat_file_flip_checked integer DEFAULT 0 NOT NULL,
cat_file_pit_vow_checked boolean DEFAULT false NOT NULL, cat_file_pit_vow_checked integer DEFAULT 0 NOT NULL,
flop_kick_checked boolean DEFAULT false NOT NULL, flop_kick_checked integer DEFAULT 0 NOT NULL,
vat_move_checked boolean DEFAULT false NOT NULL, vat_move_checked integer DEFAULT 0 NOT NULL,
vat_fold_checked boolean DEFAULT false NOT NULL, vat_fold_checked integer DEFAULT 0 NOT NULL,
vat_heal_checked boolean DEFAULT false NOT NULL, vat_heal_checked integer DEFAULT 0 NOT NULL,
vat_toll_checked boolean DEFAULT false NOT NULL, vat_toll_checked integer DEFAULT 0 NOT NULL,
vat_tune_checked boolean DEFAULT false NOT NULL, vat_tune_checked integer DEFAULT 0 NOT NULL,
vat_grab_checked boolean DEFAULT false NOT NULL, vat_grab_checked integer DEFAULT 0 NOT NULL,
vat_flux_checked boolean DEFAULT false NOT NULL, vat_flux_checked integer DEFAULT 0 NOT NULL,
vat_slip_checked boolean DEFAULT false NOT NULL, vat_slip_checked integer DEFAULT 0 NOT NULL,
vow_flog_checked boolean DEFAULT false NOT NULL, vow_flog_checked integer DEFAULT 0 NOT NULL,
flap_kick_checked boolean DEFAULT false NOT NULL flap_kick_checked integer DEFAULT 0 NOT NULL
); );

7
libraries/shared/event_watcher.go
View File

@ -23,6 +23,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
type EventWatcher struct { type EventWatcher struct {
@ -74,7 +75,7 @@ func (watcher *EventWatcher) AddTransformers(initializers []shared.TransformerIn
watcher.Chunker.AddConfigs(configs) watcher.Chunker.AddConfigs(configs)
} }
func (watcher *EventWatcher) Execute() error { func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecution) error {
if watcher.Transformers == nil { if watcher.Transformers == nil {
return fmt.Errorf("No transformers added to watcher") return fmt.Errorf("No transformers added to watcher")
} }
@ -83,7 +84,7 @@ func (watcher *EventWatcher) Execute() error {
if err != nil { if err != nil {
return err return err
} }
notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames) notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
missingHeaders, err := shared.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL) missingHeaders, err := shared.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
if err != nil { if err != nil {
@ -107,7 +108,7 @@ func (watcher *EventWatcher) Execute() error {
for _, transformer := range watcher.Transformers { for _, transformer := range watcher.Transformers {
transformerName := transformer.GetConfig().TransformerName transformerName := transformer.GetConfig().TransformerName
logChunk := chunkedLogs[transformerName] logChunk := chunkedLogs[transformerName]
err = transformer.Execute(logChunk, header) err = transformer.Execute(logChunk, header, constants.HeaderMissing)
if err != nil { if err != nil {
log.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err) log.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
return err return err

13
libraries/shared/event_watcher_test.go
View File

@ -31,6 +31,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/fakes"
shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/test_config"
) )
@ -95,7 +96,7 @@ var _ = Describe("EventWatcher", func() {
It("returns an error when run without transformers", func() { It("returns an error when run without transformers", func() {
watcher := shared.NewEventWatcher(nil, nil) watcher := shared.NewEventWatcher(nil, nil)
err := watcher.Execute() err := watcher.Execute(constants.HeaderMissing)
Expect(err).To(MatchError("No transformers added to watcher")) Expect(err).To(MatchError("No transformers added to watcher"))
}) })
@ -125,7 +126,7 @@ var _ = Describe("EventWatcher", func() {
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute() err := watcher.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue()) Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
@ -136,7 +137,7 @@ var _ = Describe("EventWatcher", func() {
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) repository.SetMissingHeaders([]core.Header{fakes.FakeHeader})
err := watcher.Execute() err := watcher.Execute(constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse()) Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse())
@ -167,7 +168,7 @@ var _ = Describe("EventWatcher", func() {
watcher.AddTransformers([]shared2.TransformerInitializer{ watcher.AddTransformers([]shared2.TransformerInitializer{
transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer}) transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer})
err := watcher.Execute() err := watcher.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA})) Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA}))
Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB})) Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB}))
@ -187,7 +188,7 @@ var _ = Describe("EventWatcher", func() {
Topic: topic, ContractAddresses: addresses}) Topic: topic, ContractAddresses: addresses})
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
err := watcher.Execute() err := watcher.Execute(constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
fakeHash := common.HexToHash(fakes.FakeHeader.Hash) fakeHash := common.HexToHash(fakes.FakeHeader.Hash)
@ -203,7 +204,7 @@ var _ = Describe("EventWatcher", func() {
mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError) mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError)
watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
err := watcher.Execute() err := watcher.Execute(constants.HeaderMissing)
Expect(err).To(MatchError(fetcherError)) Expect(err).To(MatchError(fetcherError))
}) })
}) })

26
pkg/omni/light/repository/header_repository.go
View File

@ -61,7 +61,7 @@ func (r *headerRepository) AddCheckColumn(id string) error {
} }
pgStr := "ALTER TABLE public.checked_headers ADD COLUMN IF NOT EXISTS " pgStr := "ALTER TABLE public.checked_headers ADD COLUMN IF NOT EXISTS "
pgStr = pgStr + id + " BOOLEAN NOT NULL DEFAULT FALSE" pgStr = pgStr + id + " INTEGER NOT NULL DEFAULT 0"
_, err := r.db.Exec(pgStr) _, err := r.db.Exec(pgStr)
if err != nil { if err != nil {
return err return err
@ -80,7 +80,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error {
for _, id := range ids { for _, id := range ids {
_, ok := r.columns.Get(id) _, ok := r.columns.Get(id)
if !ok { if !ok {
baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " BOOLEAN NOT NULL DEFAULT FALSE," baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " INTEGER NOT NULL DEFAULT 0,"
input = append(input, id) input = append(input, id)
} }
} }
@ -100,7 +100,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`) _, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
VALUES ($1, $2) VALUES ($1, $2)
ON CONFLICT (header_id) DO ON CONFLICT (header_id) DO
UPDATE SET `+id+` = $2`, headerID, true) UPDATE SET `+id+` = checked_headers.`+id+` + 1`, headerID, 1)
return err return err
} }
@ -112,11 +112,11 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
} }
pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, " pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
for i := 0; i < len(ids); i++ { for i := 0; i < len(ids); i++ {
pgStr += "true, " pgStr += "1, "
} }
pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET " pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
for _, id := range ids { for _, id := range ids {
pgStr += fmt.Sprintf("%s = true, ", id) pgStr += id + `= checked_headers.` + id + ` + 1, `
} }
pgStr = pgStr[:len(pgStr)-2] pgStr = pgStr[:len(pgStr)-2]
_, err := r.db.Exec(pgStr, headerID) _, err := r.db.Exec(pgStr, headerID)
@ -137,11 +137,11 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [
} }
pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, " pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
for i := 0; i < len(ids); i++ { for i := 0; i < len(ids); i++ {
pgStr += "true, " pgStr += "1, "
} }
pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET " pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
for _, id := range ids { for _, id := range ids {
pgStr += fmt.Sprintf("%s = true, ", id) pgStr += fmt.Sprintf("%s = checked_headers.%s + 1, ", id, id)
} }
pgStr = pgStr[:len(pgStr)-2] pgStr = pgStr[:len(pgStr)-2]
_, err = tx.Exec(pgStr, header.Id) _, err = tx.Exec(pgStr, header.Id)
@ -162,7 +162,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
if endingBlockNumber == -1 { if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + id + ` IS FALSE) WHERE (header_id ISNULL OR checked_headers.` + id + `=0)
AND headers.block_number >= $1 AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2 AND headers.eth_node_fingerprint = $2
ORDER BY headers.block_number` ORDER BY headers.block_number`
@ -170,7 +170,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
} else { } else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + id + ` IS FALSE) WHERE (header_id ISNULL OR checked_headers.` + id + `=0)
AND headers.block_number >= $1 AND headers.block_number >= $1
AND headers.block_number <= $2 AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3 AND headers.eth_node_fingerprint = $3
@ -190,7 +190,7 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL` WHERE (header_id ISNULL`
for _, id := range ids { for _, id := range ids {
baseQuery += ` OR ` + id + ` IS FALSE` baseQuery += ` OR checked_headers.` + id + `= 0`
} }
if endingBlockNumber == -1 { if endingBlockNumber == -1 {
@ -220,11 +220,11 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id IS NOT NULL` WHERE (header_id IS NOT NULL`
for _, id := range eventIds { for _, id := range eventIds {
baseQuery += ` AND ` + id + ` IS TRUE` baseQuery += ` AND ` + id + `!=0`
} }
baseQuery += `) AND (` baseQuery += `) AND (`
for _, id := range methodIds { for _, id := range methodIds {
baseQuery += id + ` IS FALSE AND ` baseQuery += id + ` =0 AND `
} }
baseQuery = baseQuery[:len(baseQuery)-5] + `) ` baseQuery = baseQuery[:len(baseQuery)-5] + `) `
@ -254,6 +254,6 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string)
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
VALUES ($1, $2) VALUES ($1, $2)
ON CONFLICT (header_id) DO ON CONFLICT (header_id) DO
UPDATE SET `+eventID+` = $2`, headerID, true) UPDATE SET `+eventID+` = checked_headers.`+eventID+` + 1`, headerID, 1)
return err return err
} }

47
pkg/omni/shared/helpers/test_helpers/database.go
View File

@ -257,41 +257,17 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`DELETE FROM receipts`) _, err = tx.Exec(`DELETE FROM receipts`)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP TABLE checked_headers`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`CREATE TABLE checked_headers (id SERIAL PRIMARY KEY, header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`ALTER TABLE checked_headers _, err = tx.Exec(`ALTER TABLE checked_headers
ADD COLUMN IF NOT EXISTS price_feeds_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS eventName_contractAddr,
ADD COLUMN IF NOT EXISTS flip_kick_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS eventName_contractAddr2,
ADD COLUMN IF NOT EXISTS frob_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS eventName_contractAddr3,
ADD COLUMN IF NOT EXISTS tend_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS methodName_contractAddr,
ADD COLUMN IF NOT EXISTS bite_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS methodName_contractAddr2,
ADD COLUMN IF NOT EXISTS dent_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS methodName_contractAddr3,
ADD COLUMN IF NOT EXISTS pit_file_debt_ceiling_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e,
ADD COLUMN IF NOT EXISTS pit_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS balanceof_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e,
ADD COLUMN IF NOT EXISTS vat_init_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS newowner_0x314159265dd8dbb310642f98f50c066173c1259b,
ADD COLUMN IF NOT EXISTS drip_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN IF EXISTS owner_0x314159265dd8dbb310642f98f50c066173c1259b`)
ADD COLUMN IF NOT EXISTS drip_file_repo_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS drip_file_vow_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS deal_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS drip_drip_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS cat_file_chop_lump_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS cat_file_flip_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS cat_file_pit_vow_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS flop_kick_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_move_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_fold_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_heal_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_toll_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_tune_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_grab_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_flux_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_slip_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vow_flog_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS flap_kick_checked BOOLEAN NOT NULL DEFAULT FALSE`)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) _, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
@ -308,6 +284,9 @@ ADD COLUMN IF NOT EXISTS flap_kick_checked BOOLEAN NOT NULL DEFAULT FALSE`)
err = tx.Commit() err = tx.Commit()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`VACUUM checked_headers`)
Expect(err).NotTo(HaveOccurred())
} }
func CreateBlock(blockNumber int64, repository repositories.BlockRepository) (blockId int64) { func CreateBlock(blockNumber int64, repository repositories.BlockRepository) (blockId int64) {

14
pkg/transformers/bite/repository.go
View File

@ -18,7 +18,10 @@ package bite
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -49,7 +52,8 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log) `INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)`, VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, ink = $4, art = $5, iart = $6, tab = $7, nflip = $8, raw_log = $11;`,
headerID, biteModel.Ilk, biteModel.Urn, biteModel.Ink, biteModel.Art, biteModel.IArt, biteModel.Tab, biteModel.NFlip, biteModel.LogIndex, biteModel.TransactionIndex, biteModel.Raw, headerID, biteModel.Ilk, biteModel.Urn, biteModel.Ink, biteModel.Art, biteModel.IArt, biteModel.Tab, biteModel.NFlip, biteModel.LogIndex, biteModel.TransactionIndex, biteModel.Raw,
) )
if execErr != nil { if execErr != nil {
@ -76,3 +80,11 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er
func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked)
} }
func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked)
}
func (repository BiteRepository) RecheckHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked)
}

14
pkg/transformers/cat_file/chop_lump/repository.go
View File

@ -18,7 +18,10 @@ package chop_lump
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log) `INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, data = $4, raw_log = $7;`,
headerID, chopLump.Ilk, chopLump.What, chopLump.Data, chopLump.TransactionIndex, chopLump.LogIndex, chopLump.Raw, headerID, chopLump.Ilk, chopLump.What, chopLump.Data, chopLump.TransactionIndex, chopLump.LogIndex, chopLump.Raw,
) )
if execErr != nil { if execErr != nil {
@ -73,6 +77,14 @@ func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) er
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked)
} }
func (repository CatFileChopLumpRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked)
}
func (repository CatFileChopLumpRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked)
}
func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) { func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/cat_file/flip/repository.go
View File

@ -18,7 +18,10 @@ package flip
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -45,7 +48,8 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac
_, execErr := repository.db.Exec( _, execErr := repository.db.Exec(
`INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log) `INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7)`, VALUES($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, flip = $4, raw_log = $7;`,
headerID, flip.Ilk, flip.What, flip.Flip, flip.TransactionIndex, flip.LogIndex, flip.Raw, headerID, flip.Ilk, flip.What, flip.Flip, flip.TransactionIndex, flip.LogIndex, flip.Raw,
) )
if execErr != nil { if execErr != nil {
@ -72,6 +76,14 @@ func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked)
} }
func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked)
}
func (repository CatFileFlipRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked)
}
func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) { func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/cat_file/pit_vow/repository.go
View File

@ -18,7 +18,10 @@ package pit_vow
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -45,7 +48,8 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf
_, execErr := repository.db.Exec( _, execErr := repository.db.Exec(
`INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log) `INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)`, VALUES($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET what = $2, data = $3, raw_log = $6;`,
headerID, vow.What, vow.Data, vow.TransactionIndex, vow.LogIndex, vow.Raw, headerID, vow.What, vow.Data, vow.TransactionIndex, vow.LogIndex, vow.Raw,
) )
if execErr != nil { if execErr != nil {
@ -72,6 +76,14 @@ func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) erro
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked)
} }
func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked)
}
func (repository CatFilePitVowRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked)
}
func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) { func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/deal/repository.go
View File

@ -18,7 +18,10 @@ package deal
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log) `INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)`, VALUES($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, contract_address = $3, raw_log = $6;`,
headerID, dealModel.BidId, dealModel.ContractAddress, dealModel.LogIndex, dealModel.TransactionIndex, dealModel.Raw, headerID, dealModel.BidId, dealModel.ContractAddress, dealModel.LogIndex, dealModel.TransactionIndex, dealModel.Raw,
) )
if execErr != nil { if execErr != nil {
@ -73,6 +77,14 @@ func (repository DealRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked)
} }
func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked)
}
func (repository DealRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked)
}
func (repository *DealRepository) SetDB(db *postgres.DB) { func (repository *DealRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

30
pkg/transformers/dent/repository.go
View File

@ -18,7 +18,10 @@ package dent
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -55,25 +58,20 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) `INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`, VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_Id = $2, lot = $3, bid = $4, guy = $5, tic = $6, raw_log = $9;`,
headerID, dent.BidId, dent.Lot, dent.Bid, dent.Guy, tic, dent.LogIndex, dent.TransactionIndex, dent.Raw, headerID, dent.BidId, dent.Lot, dent.Bid, dent.Guy, tic, dent.LogIndex, dent.TransactionIndex, dent.Raw,
) )
if execErr != nil { if execErr != nil {
rollbackErr := tx.Rollback() tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return execErr return execErr
} }
} }
checkHeaderErr := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) err := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked)
if checkHeaderErr != nil { if err != nil {
rollbackErr := tx.Rollback() tx.Rollback()
if rollbackErr != nil { return err
log.Error("failed to rollback ", rollbackErr)
}
return checkHeaderErr
} }
return tx.Commit() return tx.Commit()
} }
@ -82,6 +80,14 @@ func (repository DentRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked)
} }
func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked)
}
func (repository DentRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked)
}
func (repository *DentRepository) SetDB(db *postgres.DB) { func (repository *DentRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/drip_drip/repository.go
View File

@ -18,7 +18,10 @@ package drip_drip
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -45,7 +48,8 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{}
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log) `INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5)`, VALUES($1, $2, $3, $4, $5)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk= $2, raw_log = $5;`,
headerID, dripDrip.Ilk, dripDrip.LogIndex, dripDrip.TransactionIndex, dripDrip.Raw, headerID, dripDrip.Ilk, dripDrip.LogIndex, dripDrip.TransactionIndex, dripDrip.Raw,
) )
if execErr != nil { if execErr != nil {
@ -72,6 +76,14 @@ func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked)
} }
func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked)
}
func (repository DripDripRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked)
}
func (repository *DripDripRepository) SetDB(db *postgres.DB) { func (repository *DripDripRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

11
pkg/transformers/drip_file/ilk/repository.go
View File

@ -18,7 +18,10 @@ package ilk
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -75,6 +78,14 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked)
} }
func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked)
}
func (repository DripFileIlkRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked)
}
func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

11
pkg/transformers/drip_file/repo/repository.go
View File

@ -18,7 +18,10 @@ package repo
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -75,6 +78,14 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked)
} }
func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked)
}
func (repository DripFileRepoRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked)
}
func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/drip_file/vow/repository.go
View File

@ -18,7 +18,10 @@ package vow
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log) `INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)`, VALUES($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET what = $2, data = $3, raw_log = $6;`,
headerID, vow.What, vow.Data, vow.LogIndex, vow.TransactionIndex, vow.Raw, headerID, vow.What, vow.Data, vow.LogIndex, vow.TransactionIndex, vow.Raw,
) )
if execErr != nil { if execErr != nil {
@ -74,6 +78,14 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked)
} }
func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked)
}
func (repository DripFileVowRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked)
}
func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { func (repository *DripFileVowRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

3
pkg/transformers/factories/log_note_transformer.go
View File

@ -23,6 +23,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
type LogNoteTransformer struct { type LogNoteTransformer struct {
@ -36,7 +37,7 @@ func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) sha
return transformer return transformer
} }
func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header) error { func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header, recheckedHeader constants.TransformerExecution) error {
transformerName := transformer.Config.TransformerName transformerName := transformer.Config.TransformerName
// No matching logs, mark the header as checked for this type of logs // No matching logs, mark the header as checked for this type of logs

17
pkg/transformers/factories/log_note_transformer_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"math/rand" "math/rand"
@ -57,14 +58,14 @@ var _ = Describe("LogNoteTransformer", func() {
}) })
It("marks header checked if no logs are provided", func() { It("marks header checked if no logs are provided", func() {
err := transformer.Execute([]types.Log{}, headerOne) err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
}) })
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
err := transformer.Execute([]types.Log{}, headerOne) err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.ToModelsCalledCounter).To(Equal(0)) Expect(converter.ToModelsCalledCounter).To(Equal(0))
@ -72,7 +73,7 @@ var _ = Describe("LogNoteTransformer", func() {
}) })
It("does not call repository.MarkCheckedHeader when there are logs", func() { It("does not call repository.MarkCheckedHeader when there are logs", func() {
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled() repository.AssertMarkHeaderCheckedNotCalled()
@ -81,14 +82,14 @@ var _ = Describe("LogNoteTransformer", func() {
It("returns error if marking header checked returns err", func() { It("returns error if marking header checked returns err", func() {
repository.SetMarkHeaderCheckedError(fakes.FakeError) repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute([]types.Log{}, headerOne) err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
}) })
It("converts matching logs to models", func() { It("converts matching logs to models", func() {
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLogs).To(Equal(logs)) Expect(converter.PassedLogs).To(Equal(logs))
@ -97,7 +98,7 @@ var _ = Describe("LogNoteTransformer", func() {
It("returns error if converter returns error", func() { It("returns error if converter returns error", func() {
converter.SetConverterError(fakes.FakeError) converter.SetConverterError(fakes.FakeError)
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
@ -105,7 +106,7 @@ var _ = Describe("LogNoteTransformer", func() {
It("persists the model", func() { It("persists the model", func() {
converter.SetReturnModels([]interface{}{model}) converter.SetReturnModels([]interface{}{model})
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@ -115,7 +116,7 @@ var _ = Describe("LogNoteTransformer", func() {
It("returns error if repository returns error for create", func() { It("returns error if repository returns error for create", func() {
repository.SetCreateError(fakes.FakeError) repository.SetCreateError(fakes.FakeError)
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))

3
pkg/transformers/factories/repository.go
View File

@ -17,11 +17,14 @@
package factories package factories
import ( import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
type Repository interface { type Repository interface {
Create(headerID int64, models []interface{}) error Create(headerID int64, models []interface{}) error
MarkHeaderChecked(headerID int64) error MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
SetDB(db *postgres.DB) SetDB(db *postgres.DB)
} }

3
pkg/transformers/factories/transformer.go
View File

@ -23,6 +23,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
type Transformer struct { type Transformer struct {
@ -36,7 +37,7 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transforme
return transformer return transformer
} }
func (transformer Transformer) Execute(logs []types.Log, header core.Header) error { func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
transformerName := transformer.Config.TransformerName transformerName := transformer.Config.TransformerName
config := transformer.Config config := transformer.Config

21
pkg/transformers/factories/transformer_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"math/rand" "math/rand"
@ -57,14 +58,14 @@ var _ = Describe("Transformer", func() {
}) })
It("marks header checked if no logs returned", func() { It("marks header checked if no logs returned", func() {
err := transformer.Execute([]types.Log{}, headerOne) err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
}) })
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
err := transformer.Execute([]types.Log{}, headerOne) err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.ToEntitiesCalledCounter).To(Equal(0)) Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
@ -73,7 +74,7 @@ var _ = Describe("Transformer", func() {
}) })
It("does not call repository.MarkCheckedHeader when there are logs", func() { It("does not call repository.MarkCheckedHeader when there are logs", func() {
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled() repository.AssertMarkHeaderCheckedNotCalled()
@ -82,14 +83,14 @@ var _ = Describe("Transformer", func() {
It("returns error if marking header checked returns err", func() { It("returns error if marking header checked returns err", func() {
repository.SetMarkHeaderCheckedError(fakes.FakeError) repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := transformer.Execute([]types.Log{}, headerOne) err := transformer.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
}) })
It("converts an eth log to an entity", func() { It("converts an eth log to an entity", func() {
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.ContractAbi).To(Equal(config.ContractAbi)) Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
@ -99,7 +100,7 @@ var _ = Describe("Transformer", func() {
It("returns an error if converter fails", func() { It("returns an error if converter fails", func() {
converter.ToEntitiesError = fakes.FakeError converter.ToEntitiesError = fakes.FakeError
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
@ -108,7 +109,7 @@ var _ = Describe("Transformer", func() {
It("converts an entity to a model", func() { It("converts an entity to a model", func() {
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{})) Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
@ -118,7 +119,7 @@ var _ = Describe("Transformer", func() {
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
converter.ToModelsError = fakes.FakeError converter.ToModelsError = fakes.FakeError
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))
@ -127,7 +128,7 @@ var _ = Describe("Transformer", func() {
It("persists the record", func() { It("persists the record", func() {
converter.ModelsToReturn = []interface{}{test_data.GenericModel{}} converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@ -136,7 +137,7 @@ var _ = Describe("Transformer", func() {
It("returns error if persisting the record fails", func() { It("returns error if persisting the record fails", func() {
repository.SetCreateError(fakes.FakeError) repository.SetCreateError(fakes.FakeError)
err := transformer.Execute(logs, headerOne) err := transformer.Execute(logs, headerOne, constants.HeaderMissing)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))

14
pkg/transformers/flap_kick/repository.go
View File

@ -18,7 +18,10 @@ package flap_kick
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -41,7 +44,8 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.flap_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) `INSERT into maker.flap_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, gal = $5, "end" = $6, raw_log = $9;`,
headerID, flapKickModel.BidId, flapKickModel.Lot, flapKickModel.Bid, flapKickModel.Gal, flapKickModel.End, flapKickModel.TransactionIndex, flapKickModel.LogIndex, flapKickModel.Raw, headerID, flapKickModel.BidId, flapKickModel.Lot, flapKickModel.Bid, flapKickModel.Gal, flapKickModel.End, flapKickModel.TransactionIndex, flapKickModel.LogIndex, flapKickModel.Raw,
) )
if execErr != nil { if execErr != nil {
@ -68,6 +72,14 @@ func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked)
} }
func (repository FlapKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked)
}
func (repository FlapKickRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked)
}
func (repository *FlapKickRepository) SetDB(db *postgres.DB) { func (repository *FlapKickRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

12
pkg/transformers/flip_kick/repository.go
View File

@ -20,6 +20,7 @@ import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -42,7 +43,8 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{}
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.flip_kick (header_id, bid_id, lot, bid, gal, "end", urn, tab, tx_idx, log_idx, raw_log) `INSERT into maker.flip_kick (header_id, bid_id, lot, bid, gal, "end", urn, tab, tx_idx, log_idx, raw_log)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, gal = $5, "end" = $6, urn= $7, tab = $8, raw_log = $11;`,
headerID, flipKickModel.BidId, flipKickModel.Lot, flipKickModel.Bid, flipKickModel.Gal, flipKickModel.End, flipKickModel.Urn, flipKickModel.Tab, flipKickModel.TransactionIndex, flipKickModel.LogIndex, flipKickModel.Raw, headerID, flipKickModel.BidId, flipKickModel.Lot, flipKickModel.Bid, flipKickModel.Gal, flipKickModel.End, flipKickModel.Urn, flipKickModel.Tab, flipKickModel.TransactionIndex, flipKickModel.LogIndex, flipKickModel.Raw,
) )
if execErr != nil { if execErr != nil {
@ -68,6 +70,14 @@ func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked)
} }
func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked)
}
func (repository FlipKickRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked)
}
func (repository *FlipKickRepository) SetDB(db *postgres.DB) { func (repository *FlipKickRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/flop_kick/repository.go
View File

@ -18,7 +18,10 @@ package flop_kick
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -42,7 +45,8 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{}
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.flop_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) `INSERT into maker.flop_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, gal = $5, "end" = $6, raw_log = $9;`,
headerID, flopKickModel.BidId, flopKickModel.Lot, flopKickModel.Bid, flopKickModel.Gal, flopKickModel.End, flopKickModel.TransactionIndex, flopKickModel.LogIndex, flopKickModel.Raw, headerID, flopKickModel.BidId, flopKickModel.Lot, flopKickModel.Bid, flopKickModel.Gal, flopKickModel.End, flopKickModel.TransactionIndex, flopKickModel.LogIndex, flopKickModel.Raw,
) )
if execErr != nil { if execErr != nil {
@ -70,6 +74,14 @@ func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked)
} }
func (repository FlopKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked)
}
func (repository FlopKickRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked)
}
func (repository *FlopKickRepository) SetDB(db *postgres.DB) { func (repository *FlopKickRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

3
pkg/transformers/frob/converter.go
View File

@ -19,8 +19,6 @@ package frob
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
log "github.com/sirupsen/logrus"
"github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
@ -32,7 +30,6 @@ type FrobConverter struct{}
func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) { func (FrobConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
var entities []interface{} var entities []interface{}
log.Info("blah")
for _, ethLog := range ethLogs { for _, ethLog := range ethLogs {
entity := FrobEntity{} entity := FrobEntity{}
address := ethLog.Address address := ethLog.Address

12
pkg/transformers/frob/repository.go
View File

@ -20,6 +20,7 @@ import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -45,7 +46,8 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er
} }
_, execErr := tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx) _, execErr := tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET art = $2, dart = $3, dink = $4, iart = $5, ilk = $6, ink = $7, urn = $8, raw_log = $9;`,
headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex) headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex)
if execErr != nil { if execErr != nil {
rollbackErr := tx.Rollback() rollbackErr := tx.Rollback()
@ -70,6 +72,14 @@ func (repository FrobRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked)
} }
func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked)
}
func (repository FrobRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked)
}
func (repository *FrobRepository) SetDB(db *postgres.DB) { func (repository *FrobRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

69
pkg/transformers/integration_tests/bite.go
View File

@ -73,7 +73,7 @@ var _ = Describe("Bite Transformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []bite.BiteModel var dbResult []bite.BiteModel
@ -108,4 +108,71 @@ var _ = Describe("Bite Transformer", func() {
Expect(entity.Ilk).To(Equal(expectedEntity.Ilk)) Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
Expect(entity.Ink).To(Equal(expectedEntity.Ink)) Expect(entity.Ink).To(Equal(expectedEntity.Ink))
}) })
It("rechecks header for bite event", func() {
blockNumber := int64(8956422)
config := testBiteConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.Transformer{
Config: config,
Converter: &bite.BiteConverter{},
Repository: &bite.BiteRepository{},
}
transformer := initializer.NewTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var biteChecked []int
err = db.Select(&biteChecked, `SELECT bite_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(biteChecked[0]).To(Equal(2))
})
It("unpacks an event log", func() {
address := common.HexToAddress(test_data.KovanCatContractAddress)
abi, err := geth.ParseAbi(test_data.KovanCatABI)
Expect(err).NotTo(HaveOccurred())
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
entity := &bite.BiteEntity{}
var eventLog = test_data.EthBiteLog
err = contract.UnpackLog(entity, "Bite", eventLog)
Expect(err).NotTo(HaveOccurred())
expectedEntity := test_data.BiteEntity
Expect(entity.Art).To(Equal(expectedEntity.Art))
Expect(entity.Ilk).To(Equal(expectedEntity.Ilk))
Expect(entity.Ink).To(Equal(expectedEntity.Ink))
})
}) })

97
pkg/transformers/integration_tests/cat_file.go
View File

@ -87,7 +87,7 @@ var _ = Describe("Cat File transformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []chop_lump.CatFileChopLumpModel var dbResult []chop_lump.CatFileChopLumpModel
@ -108,6 +108,51 @@ var _ = Describe("Cat File transformer", func() {
Expect(dbResult[1].LogIndex).To(Equal(uint(4))) Expect(dbResult[1].LogIndex).To(Equal(uint(4)))
}) })
It("rechecks header for chop lump event", func() {
// transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014
chopLumpBlockNumber := int64(8762253)
header, err := persistHeader(db, chopLumpBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := shared.TransformerConfig{
TransformerName: constants.CatFileChopLumpLabel,
ContractAddresses: []string{test_data.KovanCatContractAddress},
ContractAbi: test_data.KovanCatABI,
Topic: test_data.KovanCatFileChopLumpSignature,
StartingBlockNumber: chopLumpBlockNumber,
EndingBlockNumber: chopLumpBlockNumber,
}
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &chop_lump.CatFileChopLumpConverter{},
Repository: &chop_lump.CatFileChopLumpRepository{},
}
transformer := initializer.NewLogNoteTransformer(db)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, chopLumpBlockNumber)
Expect(err).NotTo(HaveOccurred())
var catChopLumpChecked []int
err = db.Select(&catChopLumpChecked, `SELECT cat_file_chop_lump_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(catChopLumpChecked[0]).To(Equal(2))
})
It("persists a flip event", func() { It("persists a flip event", func() {
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
flipBlockNumber := int64(8751794) flipBlockNumber := int64(8751794)
@ -137,7 +182,7 @@ var _ = Describe("Cat File transformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flip.CatFileFlipModel var dbResult []flip.CatFileFlipModel
@ -150,6 +195,52 @@ var _ = Describe("Cat File transformer", func() {
Expect(dbResult[0].Flip).To(Equal("0x32D496Ad866D110060866B7125981C73642cc509")) Expect(dbResult[0].Flip).To(Equal("0x32D496Ad866D110060866B7125981C73642cc509"))
}) })
It("rechecks a flip event", func() {
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
flipBlockNumber := int64(8751794)
header, err := persistHeader(db, flipBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
config := shared.TransformerConfig{
TransformerName: constants.CatFileFlipLabel,
ContractAddresses: []string{test_data.KovanCatContractAddress},
ContractAbi: test_data.KovanCatABI,
Topic: test_data.KovanCatFileFlipSignature,
StartingBlockNumber: flipBlockNumber,
EndingBlockNumber: flipBlockNumber,
}
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &flip.CatFileFlipConverter{},
Repository: &flip.CatFileFlipRepository{},
}
transformer := initializer.NewLogNoteTransformer(db)
logs, err := fetcher.FetchLogs(
[]common.Address{common.HexToAddress(config.ContractAddresses[0])},
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, flipBlockNumber)
Expect(err).NotTo(HaveOccurred())
var catFlipChecked []int
err = db.Select(&catFlipChecked, `SELECT cat_file_flip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(catFlipChecked[0]).To(Equal(2))
})
It("persists a pit vow event", func() { It("persists a pit vow event", func() {
// transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe
pitVowBlockNumber := int64(8751794) pitVowBlockNumber := int64(8751794)
@ -178,7 +269,7 @@ var _ = Describe("Cat File transformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []pit_vow.CatFilePitVowModel var dbResult []pit_vow.CatFilePitVowModel

32
pkg/transformers/integration_tests/deal.go
View File

@ -84,7 +84,7 @@ var _ = Describe("Deal transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []deal.DealModel var dbResult []deal.DealModel
@ -96,6 +96,34 @@ var _ = Describe("Deal transformer", func() {
Expect(dbResult[0].ContractAddress).To(Equal(test_data.KovanFlipperContractAddress)) Expect(dbResult[0].ContractAddress).To(Equal(test_data.KovanFlipperContractAddress))
}) })
It("rechecks flip deal event", func() {
// transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58
flipBlockNumber := int64(8958007)
header, err := persistHeader(db, flipBlockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer.Config.StartingBlockNumber = flipBlockNumber
initializer.Config.EndingBlockNumber = flipBlockNumber
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, flipBlockNumber)
Expect(err).NotTo(HaveOccurred())
var dentChecked []int
err = db.Select(&dentChecked, `SELECT deal_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
})
It("persists a flop deal log event", func() { It("persists a flop deal log event", func() {
//TODO: There are currently no Flop.deal events on Kovan //TODO: There are currently no Flop.deal events on Kovan
}) })
@ -112,7 +140,7 @@ var _ = Describe("Deal transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []deal.DealModel var dbResult []deal.DealModel

31
pkg/transformers/integration_tests/dent.go
View File

@ -67,7 +67,7 @@ var _ = Describe("Dent transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer = initializer.NewLogNoteTransformer(db) transformer = initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []dent.DentModel var dbResult []dent.DentModel
@ -88,6 +88,35 @@ var _ = Describe("Dent transformer", func() {
Expect(dbTic).To(Equal(actualTic)) Expect(dbTic).To(Equal(actualTic))
}) })
It("rechecks header for flop dent log event", func() {
blockNumber := int64(8955613)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer = initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var dentChecked []int
err = db.Select(&dentChecked, `SELECT dent_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dentChecked[0]).To(Equal(2))
})
It("persists a flip dent log event", func() { It("persists a flip dent log event", func() {
//TODO: There are currently no Flip.dent events on Kovan //TODO: There are currently no Flip.dent events on Kovan
}) })

40
pkg/transformers/integration_tests/drip_drip.go
View File

@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/test_config"
) )
@ -76,7 +77,7 @@ var _ = Describe("DripDrip Transformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []drip_drip.DripDripModel var dbResults []drip_drip.DripDripModel
@ -87,4 +88,41 @@ var _ = Describe("DripDrip Transformer", func() {
dbResult := dbResults[0] dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
}) })
It("rechecks drip drip event", func() {
blockNumber := int64(8934775)
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &drip_drip.DripDripConverter{},
Repository: &drip_drip.DripDripRepository{},
}
transformer := initializer.NewLogNoteTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var dripdripChecked []int
err = db.Select(&dripdripChecked, `SELECT drip_drip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
})
}) })

49
pkg/transformers/integration_tests/drip_file_vow.go
View File

@ -24,10 +24,10 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/test_config"
) )
@ -74,7 +74,7 @@ var _ = Describe("Drip File Vow LogNoteTransformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vow.DripFileVowModel var dbResult []vow.DripFileVowModel
@ -85,4 +85,49 @@ var _ = Describe("Drip File Vow LogNoteTransformer", func() {
Expect(dbResult[0].What).To(Equal("vow")) Expect(dbResult[0].What).To(Equal("vow"))
Expect(dbResult[0].Data).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1")) Expect(dbResult[0].Data).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1"))
}) })
It("rechecks drip file vow event", func() {
blockNumber := int64(8762197)
config := shared.TransformerConfig{
TransformerName: constants.DripFileVowLabel,
ContractAddresses: []string{test_data.KovanDripContractAddress},
ContractAbi: test_data.KovanDripABI,
Topic: test_data.KovanDripFileVowSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &vow.DripFileVowConverter{},
Repository: &vow.DripFileVowRepository{},
}
transformer := initializer.NewLogNoteTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var dripfilevowChecked []int
err = db.Select(&dripfilevowChecked, `SELECT drip_file_vow_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dripfilevowChecked[0]).To(Equal(2))
})
}) })

48
pkg/transformers/integration_tests/flap_kick.go
View File

@ -20,7 +20,6 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/test_config"
"time" "time"
@ -31,6 +30,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
var _ = Describe("FlapKick Transformer", func() { var _ = Describe("FlapKick Transformer", func() {
@ -75,7 +75,7 @@ var _ = Describe("FlapKick Transformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flap_kick.FlapKickModel var dbResult []flap_kick.FlapKickModel
@ -89,4 +89,48 @@ var _ = Describe("FlapKick Transformer", func() {
Expect(dbResult[0].Gal).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(dbResult[0].Gal).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000"))
}) })
It("rechecks flap kick transformer", func() {
blockNumber := int64(9002933)
config := shared.TransformerConfig{
TransformerName: constants.FlapKickLabel,
ContractAddresses: []string{test_data.KovanFlapperContractAddress},
ContractAbi: test_data.KovanFlapperABI,
Topic: test_data.KovanFlapKickSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
transformer := factories.Transformer{
Config: config,
Converter: &flap_kick.FlapKickConverter{},
Repository: &flap_kick.FlapKickRepository{},
}.NewTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var flapkickChecked []int
err = db.Select(&flapkickChecked, `SELECT flap_kick_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(flapkickChecked[0]).To(Equal(2))
})
}) })

53
pkg/transformers/integration_tests/flip_kick.go
View File

@ -91,7 +91,7 @@ var _ = Describe("FlipKick Transformer", func() {
header) header)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flip_kick.FlipKickModel var dbResult []flip_kick.FlipKickModel
@ -105,4 +105,55 @@ var _ = Describe("FlipKick Transformer", func() {
Expect(dbResult[0].Gal).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1")) Expect(dbResult[0].Gal).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1"))
Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000"))
}) })
It("rechecks flip kick event", func() {
blockNumber := int64(8956476)
config := shared.TransformerConfig{
TransformerName: constants.FlipKickLabel,
ContractAddresses: []string{test_data.KovanFlipperContractAddress},
ContractAbi: test_data.KovanFlipperABI,
Topic: test_data.KovanFlipKickSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
transformer := factories.Transformer{
Config: config,
Converter: &flip_kick.FlipKickConverter{},
Repository: &flip_kick.FlipKickRepository{},
}.NewTransformer(db)
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var flipkickChecked []int
err = db.Select(&flipkickChecked, `SELECT flip_kick_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(flipkickChecked[0]).To(Equal(2))
})
}) })

45
pkg/transformers/integration_tests/flop_kick.go
View File

@ -86,7 +86,7 @@ var _ = Describe("FlopKick Transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db) transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flop_kick.Model var dbResult []flop_kick.Model
@ -102,6 +102,47 @@ var _ = Describe("FlopKick Transformer", func() {
Expect(dbResult[0].Lot).To(Equal("115792089237316195423570985008687907853269984665640564039457584007913129639935")) Expect(dbResult[0].Lot).To(Equal("115792089237316195423570985008687907853269984665640564039457584007913129639935"))
}) })
It("rechecks flop kick event", func() {
blockNumber := int64(8672119)
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var dbResult []flop_kick.Model
err = db.Select(&dbResult, `SELECT bid, bid_id, "end", gal, lot FROM maker.flop_kick`)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var flopKickChecked []int
err = db.Select(&flopKickChecked, `SELECT flop_kick_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(flopKickChecked[0]).To(Equal(2))
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Bid).To(Equal("0"))
Expect(dbResult[0].BidId).To(Equal("1"))
Expect(dbResult[0].End.Equal(time.Unix(1536726768, 0))).To(BeTrue())
Expect(dbResult[0].Gal).To(Equal("0x9B870D55BaAEa9119dBFa71A92c5E26E79C4726d"))
// this very large number appears to be derived from the data including: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
Expect(dbResult[0].Lot).To(Equal("115792089237316195423570985008687907853269984665640564039457584007913129639935"))
})
It("fetches and transforms another FlopKick event from Kovan chain", func() { It("fetches and transforms another FlopKick event from Kovan chain", func() {
blockNumber := int64(8955611) blockNumber := int64(8955611)
initializer.Config.StartingBlockNumber = blockNumber initializer.Config.StartingBlockNumber = blockNumber
@ -114,7 +155,7 @@ var _ = Describe("FlopKick Transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db) transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []flop_kick.Model var dbResult []flop_kick.Model

47
pkg/transformers/integration_tests/frob.go
View File

@ -82,7 +82,7 @@ var _ = Describe("Frob Transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db) transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []frob.FrobModel var dbResult []frob.FrobModel
@ -99,6 +99,51 @@ var _ = Describe("Frob Transformer", func() {
Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C")) Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C"))
}) })
It("rechecks frob event", func() {
blockNumber := int64(8935258)
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var frobChecked []int
err = db.Select(&frobChecked, `SELECT frob_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(frobChecked[0]).To(Equal(2))
var dbResult []frob.FrobModel
err = db.Select(&dbResult, `SELECT art, dart, dink, iart, ilk, ink, urn from maker.frob`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Art).To(Equal("10000000000000000"))
Expect(dbResult[0].Dart).To(Equal("0"))
Expect(dbResult[0].Dink).To(Equal("10000000000000"))
Expect(dbResult[0].IArt).To(Equal("1495509999999999999992"))
Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Ink).To(Equal("10050100000000000"))
Expect(dbResult[0].Urn).To(Equal("0xc8E093e5f3F9B5Aa6A6b33ea45960b93C161430C"))
})
It("unpacks an event log", func() { It("unpacks an event log", func() {
address := common.HexToAddress(test_data.KovanPitContractAddress) address := common.HexToAddress(test_data.KovanPitContractAddress)
abi, err := geth.ParseAbi(test_data.KovanPitABI) abi, err := geth.ParseAbi(test_data.KovanPitABI)

55
pkg/transformers/integration_tests/pit_file_debt_ceiling.go
View File

@ -74,7 +74,7 @@ var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() {
} }
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []debt_ceiling.PitFileDebtCeilingModel var dbResult []debt_ceiling.PitFileDebtCeilingModel
@ -85,4 +85,57 @@ var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() {
Expect(dbResult[0].What).To(Equal("Line")) Expect(dbResult[0].What).To(Equal("Line"))
Expect(dbResult[0].Data).To(Equal("10000000.000000000000000000")) Expect(dbResult[0].Data).To(Equal("10000000.000000000000000000"))
}) })
It("rechecks pit file debt ceiling event", func() {
blockNumber := int64(8535578)
config := shared.TransformerConfig{
TransformerName: constants.PitFileDebtCeilingLabel,
ContractAddresses: []string{test_data.KovanPitContractAddress},
ContractAbi: test_data.KovanPitABI,
Topic: test_data.KovanPitFileDebtCeilingSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &debt_ceiling.PitFileDebtCeilingConverter{},
Repository: &debt_ceiling.PitFileDebtCeilingRepository{},
}
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var pitFileDebtCeilingChecked []int
err = db.Select(&pitFileDebtCeilingChecked, `SELECT pit_file_debt_ceiling_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(pitFileDebtCeilingChecked[0]).To(Equal(2))
var dbResult []debt_ceiling.PitFileDebtCeilingModel
err = db.Select(&dbResult, `SELECT what, data from maker.pit_file_debt_ceiling`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].What).To(Equal("Line"))
Expect(dbResult[0].Data).To(Equal("10000000.000000000000000000"))
})
}) })

43
pkg/transformers/integration_tests/pit_file_ilk.go
View File

@ -79,7 +79,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []ilk.PitFileIlkModel var dbResult []ilk.PitFileIlkModel
@ -92,6 +92,45 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648")) Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648"))
}) })
It("rechecks pit file ilk event", func() {
blockNumber := int64(9103223)
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var pitFileIlkChecked []int
err = db.Select(&pitFileIlkChecked, `SELECT pit_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(pitFileIlkChecked[0]).To(Equal(2))
var dbResult []ilk.PitFileIlkModel
err = db.Select(&dbResult, `SELECT ilk, what, data from maker.pit_file_ilk`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("0x4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].What).To(Equal("spot"))
Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648"))
})
It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() { It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() {
blockNumber := int64(8762253) blockNumber := int64(8762253)
initializer.Config.StartingBlockNumber = blockNumber initializer.Config.StartingBlockNumber = blockNumber
@ -105,7 +144,7 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []ilk.PitFileIlkModel var dbResult []ilk.PitFileIlkModel

45
pkg/transformers/integration_tests/price_feeds.go
View File

@ -75,7 +75,7 @@ var _ = Describe("Price feeds transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel var model price_feeds.PriceFeedModel
@ -85,6 +85,45 @@ var _ = Describe("Price feeds transformer", func() {
Expect(model.MedianizerAddress).To(Equal(addresses[0])) Expect(model.MedianizerAddress).To(Equal(addresses[0]))
}) })
It("rechecks price feed event", func() {
blockNumber := int64(8763054)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
addresses := []string{test_data.KovanPipContractAddress}
initializer.Config.ContractAddresses = addresses
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(addresses),
topics,
header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var priceFeedChecked []int
err = db.Select(&priceFeedChecked, `SELECT price_feeds_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(priceFeedChecked[0]).To(Equal(2))
var model price_feeds.PriceFeedModel
err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(model.UsdValue).To(Equal("207.314891143000011198"))
Expect(model.MedianizerAddress).To(Equal(addresses[0]))
})
It("persists a MKR/USD price feed event", func() { It("persists a MKR/USD price feed event", func() {
blockNumber := int64(8763059) blockNumber := int64(8763059)
header, err := persistHeader(db, blockNumber, blockChain) header, err := persistHeader(db, blockNumber, blockChain)
@ -101,7 +140,7 @@ var _ = Describe("Price feeds transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel var model price_feeds.PriceFeedModel
@ -127,7 +166,7 @@ var _ = Describe("Price feeds transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var model price_feeds.PriceFeedModel var model price_feeds.PriceFeedModel

52
pkg/transformers/integration_tests/tend.go
View File

@ -81,7 +81,7 @@ var _ = Describe("Tend LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel var dbResult []tend.TendModel
@ -102,6 +102,52 @@ var _ = Describe("Tend LogNoteTransformer", func() {
Expect(dbTic).To(Equal(actualTic)) Expect(dbTic).To(Equal(actualTic))
}) })
It("rechecks tend event", func() {
blockNumber := int64(8935601)
initializer.Config.StartingBlockNumber = blockNumber
initializer.Config.EndingBlockNumber = blockNumber
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
logs, err := fetcher.FetchLogs(addresses, topics, header)
Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var tendChecked []int
err = db.Select(&tendChecked, `SELECT tend_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(tendChecked[0]).To(Equal(2))
var dbResult []tend.TendModel
err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot from maker.tend where header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Bid).To(Equal("4000"))
Expect(dbResult[0].BidId).To(Equal("3"))
Expect(dbResult[0].Guy).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(dbResult[0].Lot).To(Equal("1000000000000000000"))
var dbTic int64
err = db.Get(&dbTic, `SELECT tic FROM maker.tend`)
Expect(err).NotTo(HaveOccurred())
actualTic := 1538490276 + constants.TTL
Expect(dbTic).To(Equal(actualTic))
})
It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() { It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() {
blockNumber := int64(8935731) blockNumber := int64(8935731)
initializer.Config.StartingBlockNumber = blockNumber initializer.Config.StartingBlockNumber = blockNumber
@ -114,7 +160,7 @@ var _ = Describe("Tend LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel var dbResult []tend.TendModel
@ -147,7 +193,7 @@ var _ = Describe("Tend LogNoteTransformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []tend.TendModel var dbResult []tend.TendModel

66
pkg/transformers/integration_tests/vat_flux.go
View File

@ -66,7 +66,7 @@ var _ = Describe("VatFlux LogNoteTransformer", func() {
} }
transformer := initializer.NewLogNoteTransformer(db) transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vat_flux.VatFluxModel var dbResult []vat_flux.VatFluxModel
@ -80,4 +80,68 @@ var _ = Describe("VatFlux LogNoteTransformer", func() {
Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000")) Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000"))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) Expect(dbResult[0].TransactionIndex).To(Equal(uint(0)))
}) })
It("rechecks vat flux event", func() {
blockNumber := int64(9004474)
config := shared.TransformerConfig{
TransformerName: constants.VatFluxLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatFluxSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_flux.VatFluxConverter{},
Repository: &vat_flux.VatFluxRepository{},
}
transformer := initializer.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatFluxChecked []int
err = db.Select(&vatFluxChecked, `SELECT vat_flux_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatFluxChecked[0]).To(Equal(2))
var dbResult []vat_flux.VatFluxModel
err = db.Select(&dbResult, `SELECT ilk, src, dst, rad from maker.vat_flux`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Src).To(Equal("0xC0851F73CC8DD5c0765E71980eC7E7Fd1EF74434"))
Expect(dbResult[0].Dst).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000"))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(0)))
})
}) })

56
pkg/transformers/integration_tests/vat_fold.go
View File

@ -73,7 +73,7 @@ var _ = Describe("VatFold Transformer", func() {
Repository: &vat_fold.VatFoldRepository{}, Repository: &vat_fold.VatFoldRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_fold.VatFoldModel var dbResults []vat_fold.VatFoldModel
@ -86,4 +86,58 @@ var _ = Describe("VatFold Transformer", func() {
Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000")) Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000"))
}) })
It("rechecks vat fold event", func() {
blockNumber := int64(9367233)
config := shared.TransformerConfig{
TransformerName: constants.VatFoldLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatFoldSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_fold.VatFoldConverter{},
Repository: &vat_fold.VatFoldRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatFoldChecked []int
err = db.Select(&vatFoldChecked, `SELECT vat_fold_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatFoldChecked[0]).To(Equal(2))
var dbResults []vat_fold.VatFoldModel
err = db.Select(&dbResults, `SELECT ilk, urn, rate from maker.vat_fold`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResults)).To(Equal(1))
dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000"))
})
}) })

70
pkg/transformers/integration_tests/vat_grab.go
View File

@ -67,7 +67,7 @@ var _ = Describe("Vat Grab Transformer", func() {
Repository: &vat_grab.VatGrabRepository{}, Repository: &vat_grab.VatGrabRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vat_grab.VatGrabModel var dbResult []vat_grab.VatGrabModel
@ -87,4 +87,72 @@ var _ = Describe("Vat Grab Transformer", func() {
Expect(dbResult[0].Dart).To(Equal(expectedDart.String())) Expect(dbResult[0].Dart).To(Equal(expectedDart.String()))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) Expect(dbResult[0].TransactionIndex).To(Equal(uint(0)))
}) })
It("rechecks vat grab event", func() {
blockNumber := int64(8958230)
config := shared.TransformerConfig{
TransformerName: constants.VatGrabLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatGrabSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_grab.VatGrabConverter{},
Repository: &vat_grab.VatGrabRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatGrabChecked []int
err = db.Select(&vatGrabChecked, `SELECT vat_grab_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatGrabChecked[0]).To(Equal(2))
var dbResult []vat_grab.VatGrabModel
err = db.Select(&dbResult, `SELECT ilk, urn, v, w, dink, dart from maker.vat_grab`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Urn).To(Equal("0x6a3AE20C315E845B2E398e68EfFe39139eC6060C"))
Expect(dbResult[0].V).To(Equal("0x2F34f22a00eE4b7a8F8BBC4eAee1658774c624e0")) //cat contract address
Expect(dbResult[0].W).To(Equal("0x3728e9777B2a0a611ee0F89e00E01044ce4736d1"))
expectedDink := new(big.Int)
expectedDink.SetString("115792089237316195423570985008687907853269984665640564039455584007913129639936", 10)
Expect(dbResult[0].Dink).To(Equal(expectedDink.String()))
expectedDart := new(big.Int)
expectedDart.SetString("115792089237316195423570985008687907853269984665640564039441803007913129639936", 10)
Expect(dbResult[0].Dart).To(Equal(expectedDart.String()))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(0)))
})
}) })

63
pkg/transformers/integration_tests/vat_heal.go
View File

@ -65,7 +65,7 @@ var _ = Describe("VatHeal Transformer", func() {
Repository: &vat_heal.VatHealRepository{}, Repository: &vat_heal.VatHealRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_heal.VatHealModel var dbResults []vat_heal.VatHealModel
@ -78,4 +78,65 @@ var _ = Describe("VatHeal Transformer", func() {
Expect(dbResult.V).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String())) Expect(dbResult.V).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
Expect(dbResult.Rad).To(Equal("1000000000000000000000000000")) Expect(dbResult.Rad).To(Equal("1000000000000000000000000000"))
}) })
It("rechecks vat heal event", func() {
blockNumber := int64(8935578)
config := shared.TransformerConfig{
TransformerName: constants.VatHealLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatHealSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_heal.VatHealConverter{},
Repository: &vat_heal.VatHealRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatHealChecked []int
err = db.Select(&vatHealChecked, `SELECT vat_heal_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatHealChecked[0]).To(Equal(2))
var dbResults []vat_heal.VatHealModel
err = db.Select(&dbResults, `SELECT urn, v, rad from maker.vat_heal`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResults)).To(Equal(1))
dbResult := dbResults[0]
Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
Expect(dbResult.V).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
Expect(dbResult.Rad).To(Equal("1000000000000000000000000000"))
})
}) })

62
pkg/transformers/integration_tests/vat_init.go
View File

@ -65,7 +65,7 @@ var _ = Describe("VatInit LogNoteTransformer", func() {
Repository: &vat_init.VatInitRepository{}, Repository: &vat_init.VatInitRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_init.VatInitModel var dbResults []vat_init.VatInitModel
@ -76,4 +76,64 @@ var _ = Describe("VatInit LogNoteTransformer", func() {
dbResult := dbResults[0] dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
}) })
It("rechecks vat init event", func() {
blockNumber := int64(8535561)
config := shared.TransformerConfig{
TransformerName: constants.VatInitLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatInitSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_init.VatInitConverter{},
Repository: &vat_init.VatInitRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatInitChecked []int
err = db.Select(&vatInitChecked, `SELECT vat_init_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatInitChecked[0]).To(Equal(2))
var dbResults []vat_init.VatInitModel
err = db.Select(&dbResults, `SELECT ilk from maker.vat_init`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResults)).To(Equal(1))
dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
})
}) })

64
pkg/transformers/integration_tests/vat_move.go
View File

@ -65,7 +65,7 @@ var _ = Describe("VatMove LogNoteTransformer", func() {
Repository: &vat_move.VatMoveRepository{}, Repository: &vat_move.VatMoveRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResults []vat_move.VatMoveModel var dbResults []vat_move.VatMoveModel
@ -78,4 +78,66 @@ var _ = Describe("VatMove LogNoteTransformer", func() {
Expect(dbResult.Dst).To(Equal(common.HexToAddress("0x0000d8b4147eda80fec7122ae16da2479cbd7ffb").String())) Expect(dbResult.Dst).To(Equal(common.HexToAddress("0x0000d8b4147eda80fec7122ae16da2479cbd7ffb").String()))
Expect(dbResult.Rad).To(Equal("1000000000000000000000000000000000000000000000")) Expect(dbResult.Rad).To(Equal("1000000000000000000000000000000000000000000000"))
}) })
It("rechecks vat move event", func() {
blockNumber := int64(9004628)
config := shared.TransformerConfig{
TransformerName: constants.VatMoveLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatMoveSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_move.VatMoveConverter{},
Repository: &vat_move.VatMoveRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatMoveChecked []int
err = db.Select(&vatMoveChecked, `SELECT vat_move_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatMoveChecked[0]).To(Equal(2))
var dbResults []vat_move.VatMoveModel
err = db.Select(&dbResults, `SELECT src, dst, rad from maker.vat_move`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResults)).To(Equal(1))
dbResult := dbResults[0]
Expect(dbResult.Src).To(Equal(common.HexToAddress("0x8868bad8e74fca4505676d1b5b21ecc23328d132").String()))
Expect(dbResult.Dst).To(Equal(common.HexToAddress("0x0000d8b4147eda80fec7122ae16da2479cbd7ffb").String()))
Expect(dbResult.Rad).To(Equal("1000000000000000000000000000000000000000000000"))
})
}) })

60
pkg/transformers/integration_tests/vat_slip.go
View File

@ -57,7 +57,7 @@ var _ = Describe("Vat slip transformer", func() {
Repository: &vat_slip.VatSlipRepository{}, Repository: &vat_slip.VatSlipRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var headerID int64 var headerID int64
@ -75,4 +75,62 @@ var _ = Describe("Vat slip transformer", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue()) Expect(headerChecked).To(BeTrue())
}) })
It("rechecks vat slip event", func() {
blockNumber := int64(8953655)
config := shared.TransformerConfig{
TransformerName: constants.VatSlipLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatSlipSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_slip.VatSlipConverter{},
Repository: &vat_slip.VatSlipRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatSlipChecked []int
err = db.Select(&vatSlipChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatSlipChecked[0]).To(Equal(2))
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var model vat_slip.VatSlipModel
err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(model.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(model.Guy).To(Equal("0xDA15dCE70ab462E66779f23ee14F21d993789eE3"))
Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000"))
Expect(model.TransactionIndex).To(Equal(uint(0)))
var headerChecked int
err = db.Get(&headerChecked, `SELECT vat_slip_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(Equal(2))
})
}) })

69
pkg/transformers/integration_tests/vat_tune.go
View File

@ -67,7 +67,7 @@ var _ = Describe("VatTune LogNoteTransformer", func() {
Repository: &vat_tune.VatTuneRepository{}, Repository: &vat_tune.VatTuneRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vat_tune.VatTuneModel var dbResult []vat_tune.VatTuneModel
@ -85,4 +85,71 @@ var _ = Describe("VatTune LogNoteTransformer", func() {
Expect(dbResult[0].Dart).To(Equal(expectedDart.String())) Expect(dbResult[0].Dart).To(Equal(expectedDart.String()))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(0))) Expect(dbResult[0].TransactionIndex).To(Equal(uint(0)))
}) })
It("transforms VatTune log events", func() {
blockNumber := int64(8761670)
config := shared.TransformerConfig{
TransformerName: constants.VatTuneLabel,
ContractAddresses: []string{test_data.KovanVatContractAddress},
ContractAbi: test_data.KovanVatABI,
Topic: test_data.KovanVatTuneSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vat_tune.VatTuneConverter{},
Repository: &vat_tune.VatTuneRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vatTuneChecked []int
err = db.Select(&vatTuneChecked, `SELECT vat_tune_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vatTuneChecked[0]).To(Equal(2))
var dbResult []vat_tune.VatTuneModel
err = db.Select(&dbResult, `SELECT ilk, urn, v, w, dink, dart from maker.vat_tune`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000"))
Expect(dbResult[0].Urn).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))
Expect(dbResult[0].V).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))
Expect(dbResult[0].W).To(Equal("0x4F26FfBe5F04ED43630fdC30A87638d53D0b0876"))
Expect(dbResult[0].Dink).To(Equal("0"))
expectedDart := new(big.Int)
expectedDart.SetString("115792089237316195423570985008687907853269984665640564039455584007913129639936", 10)
Expect(dbResult[0].Dart).To(Equal(expectedDart.String()))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(0)))
})
}) })

53
pkg/transformers/integration_tests/vow_flog.go
View File

@ -73,7 +73,7 @@ var _ = Describe("VowFlog LogNoteTransformer", func() {
Repository: &vow_flog.VowFlogRepository{}, Repository: &vow_flog.VowFlogRepository{},
}.NewLogNoteTransformer(db) }.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header) err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var dbResult []vow_flog.VowFlogModel var dbResult []vow_flog.VowFlogModel
@ -84,4 +84,55 @@ var _ = Describe("VowFlog LogNoteTransformer", func() {
Expect(dbResult[0].LogIndex).To(Equal(uint(2))) Expect(dbResult[0].LogIndex).To(Equal(uint(2)))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(2))) Expect(dbResult[0].TransactionIndex).To(Equal(uint(2)))
}) })
It("rechecks vow flog event", func() {
blockNumber := int64(8946819)
config := shared.TransformerConfig{
TransformerName: constants.VowFlogLabel,
ContractAddresses: []string{test_data.KovanVowContractAddress},
ContractAbi: test_data.KovanVowABI,
Topic: test_data.KovanVowFlogSignature,
StartingBlockNumber: blockNumber,
EndingBlockNumber: blockNumber,
}
header, err := persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
fetcher := shared.NewFetcher(blockChain)
logs, err := fetcher.FetchLogs(
shared.HexStringsToAddresses(config.ContractAddresses),
[]common.Hash{common.HexToHash(config.Topic)},
header)
Expect(err).NotTo(HaveOccurred())
transformer := factories.LogNoteTransformer{
Config: config,
Converter: &vow_flog.VowFlogConverter{},
Repository: &vow_flog.VowFlogRepository{},
}.NewLogNoteTransformer(db)
err = transformer.Execute(logs, header, constants.HeaderMissing)
Expect(err).NotTo(HaveOccurred())
err = transformer.Execute(logs, header, constants.HeaderRecheck)
Expect(err).NotTo(HaveOccurred())
var headerID int64
err = db.Get(&headerID, `SELECT id FROM public.headers WHERE block_number = $1`, blockNumber)
Expect(err).NotTo(HaveOccurred())
var vowFlogChecked []int
err = db.Select(&vowFlogChecked, `SELECT vow_flog_checked FROM public.checked_headers WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(vowFlogChecked[0]).To(Equal(2))
var dbResult []vow_flog.VowFlogModel
err = db.Select(&dbResult, `SELECT era, log_idx, tx_idx from maker.vow_flog`)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult[0].Era).To(Equal("1538558052"))
Expect(dbResult[0].LogIndex).To(Equal(uint(2)))
Expect(dbResult[0].TransactionIndex).To(Equal(uint(2)))
})
}) })

14
pkg/transformers/pit_file/debt_ceiling/repository.go
View File

@ -18,7 +18,10 @@ package debt_ceiling
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, log_idx, tx_idx, raw_log) `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, VALUES($1, $2, $3::NUMERIC, $4, $5, $6)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET what = $2, data = $3, raw_log = $6;`,
headerID, pitFileDC.What, pitFileDC.Data, pitFileDC.LogIndex, pitFileDC.TransactionIndex, pitFileDC.Raw, headerID, pitFileDC.What, pitFileDC.Data, pitFileDC.LogIndex, pitFileDC.TransactionIndex, pitFileDC.Raw,
) )
@ -75,6 +79,14 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64)
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked)
} }
func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked)
}
func (repository PitFileDebtCeilingRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked)
}
func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/pit_file/ilk/repository.go
View File

@ -18,7 +18,10 @@ package ilk
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log) `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, data = $4, raw_log = $7;`,
headerID, pitFileIlk.Ilk, pitFileIlk.What, pitFileIlk.Data, pitFileIlk.LogIndex, pitFileIlk.TransactionIndex, pitFileIlk.Raw, headerID, pitFileIlk.Ilk, pitFileIlk.What, pitFileIlk.Data, pitFileIlk.LogIndex, pitFileIlk.TransactionIndex, pitFileIlk.Raw,
) )
if execErr != nil { if execErr != nil {
@ -73,6 +77,14 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked)
} }
func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked)
}
func (repository PitFileIlkRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked)
}
func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

26
pkg/transformers/price_feeds/repository.go
View File

@ -18,7 +18,10 @@ package price_feeds
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -43,14 +46,13 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{
return fmt.Errorf("model of type %T, not %T", model, PriceFeedModel{}) return fmt.Errorf("model of type %T, not %T", model, PriceFeedModel{})
} }
_, execErr := tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log) _, err := tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log)
VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw) VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)
if execErr != nil { ON CONFLICT (header_id, medianizer_address, tx_idx, log_idx) DO UPDATE SET block_number = $1, usd_value = $4, raw_log = $7;`,
rollbackErr := tx.Rollback() priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw)
if rollbackErr != nil { if err != nil {
log.Error("failed to rollback ", rollbackErr) tx.Rollback()
} return err
return execErr
} }
} }
@ -69,6 +71,14 @@ func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked)
} }
func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked)
}
func (repository PriceFeedRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked)
}
func (repository *PriceFeedRepository) SetDB(db *postgres.DB) { func (repository *PriceFeedRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

8
pkg/transformers/shared/constants/checked_headers.go
View File

@ -1,5 +1,13 @@
package constants package constants
type TransformerExecution bool
const (
HeaderRecheck TransformerExecution = true
HeaderMissing TransformerExecution = false
RecheckHeaderCap = "4"
)
var ( var (
BiteChecked = "bite_checked" BiteChecked = "bite_checked"
CatFileChopLumpChecked = "cat_file_chop_lump_checked" CatFileChopLumpChecked = "cat_file_chop_lump_checked"

55
pkg/transformers/shared/repository.go
View File

@ -12,17 +12,17 @@ import (
func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) _, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
VALUES ($1, $2) VALUES ($1, $2)
ON CONFLICT (header_id) DO ON CONFLICT (header_id) DO
UPDATE SET `+checkedHeadersColumn+` = $2`, headerID, true) UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
return err return err
} }
func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error { func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
VALUES ($1, $2) VALUES ($1, $2)
ON CONFLICT (header_id) DO ON CONFLICT (header_id) DO
UPDATE SET `+checkedHeadersColumn+` = $2`, headerID, true) UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
return err return err
} }
@ -52,6 +52,31 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
return result, err return result, err
} }
func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
var result []core.Header
var query string
var err error
if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2`
err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
} else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`
err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
}
return result, err
}
func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
// Query returns `[]driver.Value`, nullable polymorphic interface // Query returns `[]driver.Value`, nullable polymorphic interface
var queryResult []driver.Value var queryResult []driver.Value
@ -79,11 +104,12 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
return columnNames, nil return columnNames, nil
} }
// Builds a SQL string that checks if any column value is FALSE, given the column names. // Builds a SQL string that checks if any column value is 0, given the column names.
// Defaults to FALSE when no columns are provided. // Defaults to FALSE when no columns are provided.
// Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)" // Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
// [] => "FALSE" // [] => "FALSE"
func CreateNotCheckedSQL(boolColumns []string) string { func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
var result bytes.Buffer var result bytes.Buffer
if len(boolColumns) == 0 { if len(boolColumns) == 0 {
@ -91,13 +117,24 @@ func CreateNotCheckedSQL(boolColumns []string) string {
} }
result.WriteString("NOT (") result.WriteString("NOT (")
// Loop excluding last column name // Loop excluding last column name
for _, column := range boolColumns[:len(boolColumns)-1] { for _, column := range boolColumns[:len(boolColumns)-1] {
result.WriteString(fmt.Sprintf("%v AND ", column))
if recheckHeaders {
result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap))
} else {
result.WriteString(fmt.Sprintf("%v!=0 AND ", column))
}
} }
// No trailing "OR" for the last column name // No trailing "OR" for the last column name
result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1])) if recheckHeaders {
result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
} else {
result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1]))
}
return result.String() return result.String()
} }

21
pkg/transformers/shared/repository_utility_test.go
View File

@ -17,6 +17,7 @@
package shared_test package shared_test
import ( import (
"fmt"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
@ -25,6 +26,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/test_config"
"math/rand" "math/rand"
) )
@ -50,7 +52,7 @@ var _ = Describe("Repository utilities", func() {
columnNames, err := shared.GetCheckedColumnNames(db) columnNames, err := shared.GetCheckedColumnNames(db)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
notCheckedSQL = shared.CreateNotCheckedSQL(columnNames) notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing)
startingBlockNumber = rand.Int63() startingBlockNumber = rand.Int63()
eventSpecificBlockNumber = startingBlockNumber + 1 eventSpecificBlockNumber = startingBlockNumber + 1
@ -118,21 +120,28 @@ var _ = Describe("Repository utilities", func() {
Describe("CreateNotCheckedSQL", func() { Describe("CreateNotCheckedSQL", func() {
It("generates a correct SQL string for one column", func() { It("generates a correct SQL string for one column", func() {
columns := []string{"columnA"} columns := []string{"columnA"}
expected := "NOT (columnA)" expected := "NOT (columnA!=0)"
actual := shared.CreateNotCheckedSQL(columns) actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
Expect(actual).To(Equal(expected)) Expect(actual).To(Equal(expected))
}) })
It("generates a correct SQL string for several columns", func() { It("generates a correct SQL string for several columns", func() {
columns := []string{"columnA", "columnB"} columns := []string{"columnA", "columnB"}
expected := "NOT (columnA AND columnB)" expected := "NOT (columnA!=0 AND columnB!=0)"
actual := shared.CreateNotCheckedSQL(columns) actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
Expect(actual).To(Equal(expected)) Expect(actual).To(Equal(expected))
}) })
It("defaults to FALSE when there are no columns", func() { It("defaults to FALSE when there are no columns", func() {
expected := "FALSE" expected := "FALSE"
actual := shared.CreateNotCheckedSQL([]string{}) actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing)
Expect(actual).To(Equal(expected))
})
It("generates a correct SQL string for rechecking headers", func() {
columns := []string{"columnA", "columnB"}
expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck)
Expect(actual).To(Equal(expected)) Expect(actual).To(Equal(expected))
}) })
}) })

3
pkg/transformers/shared/transformer.go
View File

@ -22,10 +22,11 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
type Transformer interface { type Transformer interface {
Execute(logs []types.Log, header core.Header) error Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error
GetConfig() TransformerConfig GetConfig() TransformerConfig
} }

14
pkg/transformers/tend/repository.go
View File

@ -18,7 +18,10 @@ package tend
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -55,7 +58,8 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) `INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, lot = $3, bid = $4, guy = $5, tic = $6, raw_log = $9;`,
headerID, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tic, tend.LogIndex, tend.TransactionIndex, tend.Raw, headerID, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tic, tend.LogIndex, tend.TransactionIndex, tend.Raw,
) )
@ -83,6 +87,14 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked)
} }
func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked)
}
func (repository TendRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked)
}
func (repository *TendRepository) SetDB(db *postgres.DB) { func (repository *TendRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

28
pkg/transformers/test_data/mocks/repository.go
View File

@ -3,6 +3,7 @@ package mocks
import ( import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
@ -11,6 +12,9 @@ type MockRepository struct {
markHeaderCheckedError error markHeaderCheckedError error
MarkHeaderCheckedPassedHeaderIDs []int64 MarkHeaderCheckedPassedHeaderIDs []int64
CreatedHeaderIds []int64 CreatedHeaderIds []int64
missingHeaders []core.Header
allHeaders []core.Header
missingHeadersError error
PassedStartingBlockNumber int64 PassedStartingBlockNumber int64
PassedEndingBlockNumber int64 PassedEndingBlockNumber int64
PassedHeaderID int64 PassedHeaderID int64
@ -33,10 +37,34 @@ func (repository *MockRepository) MarkHeaderChecked(headerID int64) error {
return repository.markHeaderCheckedError return repository.markHeaderCheckedError
} }
func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.missingHeaders, repository.missingHeadersError
}
func (repository *MockRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
repository.PassedStartingBlockNumber = startingBlockNumber
repository.PassedEndingBlockNumber = endingBlockNumber
return repository.allHeaders, nil
}
func (repository *MockRepository) SetDB(db *postgres.DB) { func (repository *MockRepository) SetDB(db *postgres.DB) {
repository.SetDbCalled = true repository.SetDbCalled = true
} }
func (repository *MockRepository) SetMissingHeadersError(e error) {
repository.missingHeadersError = e
}
func (repository *MockRepository) SetAllHeaders(headers []core.Header) {
repository.allHeaders = headers
}
func (repository *MockRepository) SetMissingHeaders(headers []core.Header) {
repository.missingHeaders = headers
}
func (repository *MockRepository) SetMarkHeaderCheckedError(e error) { func (repository *MockRepository) SetMarkHeaderCheckedError(e error) {
repository.markHeaderCheckedError = e repository.markHeaderCheckedError = e
} }

3
pkg/transformers/test_data/mocks/transformer.go
View File

@ -5,6 +5,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
type MockTransformer struct { type MockTransformer struct {
@ -15,7 +16,7 @@ type MockTransformer struct {
config shared.TransformerConfig config shared.TransformerConfig
} }
func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { func (mh *MockTransformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
if mh.ExecuteError != nil { if mh.ExecuteError != nil {
return mh.ExecuteError return mh.ExecuteError
} }

39
pkg/transformers/test_data/shared_behaviors/repository_behaviors.go
View File

@ -39,6 +39,7 @@ type CreateBehaviorInputs struct {
CheckedHeaderColumnName string CheckedHeaderColumnName string
LogEventTableName string LogEventTableName string
TestModel interface{} TestModel interface{}
RecheckTestModel interface{}
ModelWithDifferentLogIdx interface{} ModelWithDifferentLogIdx interface{}
Repository factories.Repository Repository factories.Repository
} }
@ -70,11 +71,11 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
err = repository.Create(headerID, []interface{}{logEventModel}) err = repository.Create(headerID, []interface{}{logEventModel})
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var headerChecked bool var headerChecked int
query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1`
err = db.Get(&headerChecked, query, headerID) err = db.Get(&headerChecked, query, headerID)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue()) Expect(headerChecked).To(Equal(1))
}) })
It("updates the header to checked if checked headers row already exists", func() { It("updates the header to checked if checked headers row already exists", func() {
@ -84,21 +85,11 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
err = repository.Create(headerID, []interface{}{logEventModel}) err = repository.Create(headerID, []interface{}{logEventModel})
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var headerChecked bool var headerChecked int
query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1`
err = db.Get(&headerChecked, query, headerID) err = db.Get(&headerChecked, query, headerID)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue()) Expect(headerChecked).To(Equal(1))
})
It("returns an error if inserting duplicate log events", func() {
err = repository.Create(headerID, []interface{}{logEventModel})
Expect(err).NotTo(HaveOccurred())
err = repository.Create(headerID, []interface{}{logEventModel})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
}) })
It("allows for multiple log events of the same type in one transaction if they have different log indexes", func() { It("allows for multiple log events of the same type in one transaction if they have different log indexes", func() {
@ -147,11 +138,11 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav
err = repository.MarkHeaderChecked(headerId) err = repository.MarkHeaderChecked(headerId)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var headerChecked bool var headerChecked int
query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1`
err = db.Get(&headerChecked, query, headerId) err = db.Get(&headerChecked, query, headerId)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue()) Expect(headerChecked).To(Equal(1))
}) })
It("updates row when headerID already exists", func() { It("updates row when headerID already exists", func() {
@ -160,11 +151,11 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav
err = repository.MarkHeaderChecked(headerId) err = repository.MarkHeaderChecked(headerId)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var headerChecked bool var headerChecked int
query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1` query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1`
err = db.Get(&headerChecked, query, headerId) err = db.Get(&headerChecked, query, headerId)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(BeTrue()) Expect(headerChecked).To(Equal(1))
}) })
It("returns an error if upserting a record fails", func() { It("returns an error if upserting a record fails", func() {
@ -173,6 +164,18 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("violates foreign key constraint")) Expect(err.Error()).To(ContainSubstring("violates foreign key constraint"))
}) })
It("increments header checked", func() {
err = repository.MarkHeaderChecked(headerId)
err = repository.MarkHeaderChecked(headerId)
Expect(err).NotTo(HaveOccurred())
var headerChecked int
query := `SELECT ` + checkedHeaderColumn + ` FROM public.checked_headers WHERE header_id = $1`
err = db.Get(&headerChecked, query, headerId)
Expect(err).NotTo(HaveOccurred())
Expect(headerChecked).To(Equal(2))
})
}) })
} }

14
pkg/transformers/vat_flux/repository.go
View File

@ -18,7 +18,10 @@ package vat_flux
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -45,7 +48,8 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{})
} }
_, execErr := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log) _, execErr := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8)`, VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, dst = $3, src = $4, rad = $5, raw_log = $8;`,
headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw) headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw)
if execErr != nil { if execErr != nil {
rollbackErr := tx.Rollback() rollbackErr := tx.Rollback()
@ -72,6 +76,14 @@ func (repository VatFluxRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked)
} }
func (repository VatFluxRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatFluxChecked)
}
func (repository VatFluxRepository) RecheckHeaders(startingBlock, endingBlock int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlock, endingBlock, repository.db, constants.VatFluxChecked)
}
func (repository *VatFluxRepository) SetDB(db *postgres.DB) { func (repository *VatFluxRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

12
pkg/transformers/vat_fold/repository.go
View File

@ -20,6 +20,7 @@ import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +47,8 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{})
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log) `INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, rate = $4, raw_log = $7;`,
headerID, vatFold.Ilk, vatFold.Urn, vatFold.Rate, vatFold.LogIndex, vatFold.TransactionIndex, vatFold.Raw, headerID, vatFold.Ilk, vatFold.Urn, vatFold.Rate, vatFold.LogIndex, vatFold.TransactionIndex, vatFold.Raw,
) )
if execErr != nil { if execErr != nil {
@ -74,6 +76,14 @@ func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked)
} }
func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatFoldChecked)
}
func (repository VatFoldRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatFoldChecked)
}
func (repository *VatFoldRepository) SetDB(db *postgres.DB) { func (repository *VatFoldRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

12
pkg/transformers/vat_grab/repository.go
View File

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -30,7 +31,8 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{})
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log) `INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, v = $4, w = $5, dink = $6, dart = $7, raw_log = $10;`,
headerID, vatGrab.Ilk, vatGrab.Urn, vatGrab.V, vatGrab.W, vatGrab.Dink, vatGrab.Dart, vatGrab.LogIndex, vatGrab.TransactionIndex, vatGrab.Raw, headerID, vatGrab.Ilk, vatGrab.Urn, vatGrab.V, vatGrab.W, vatGrab.Dink, vatGrab.Dart, vatGrab.LogIndex, vatGrab.TransactionIndex, vatGrab.Raw,
) )
if execErr != nil { if execErr != nil {
@ -56,6 +58,14 @@ func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked)
} }
func (repository VatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatGrabChecked)
}
func (repository VatGrabRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatGrabChecked)
}
func (repository *VatGrabRepository) SetDB(db *postgres.DB) { func (repository *VatGrabRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

12
pkg/transformers/vat_heal/repository.go
View File

@ -20,6 +20,7 @@ import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -50,7 +51,8 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{})
} }
_, execErr := tx.Exec(`INSERT INTO maker.vat_heal (header_id, urn, v, rad, log_idx, tx_idx, raw_log) _, execErr := tx.Exec(`INSERT INTO maker.vat_heal (header_id, urn, v, rad, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)
ON CONFlICT (header_id, tx_idx, log_idx) DO UPDATE SET urn = $2, v = $3, rad = $4, raw_log = $7;`,
headerID, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw) headerID, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw)
if execErr != nil { if execErr != nil {
rollbackErr := tx.Rollback() rollbackErr := tx.Rollback()
@ -72,6 +74,14 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{})
return tx.Commit() return tx.Commit()
} }
func (repository VatHealRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatHealChecked)
}
func (repository VatHealRepository) RecheckHeaders(startingBlock, endingBlock int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlock, endingBlock, repository.db, constants.VatHealChecked)
}
func (repository VatHealRepository) MarkHeaderChecked(headerId int64) error { func (repository VatHealRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) return shared.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked)
} }

14
pkg/transformers/vat_init/repository.go
View File

@ -18,7 +18,10 @@ package vat_init
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{})
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log) `INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5)`, VALUES($1, $2, $3, $4, $5)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, raw_log = $5;`,
headerID, vatInit.Ilk, vatInit.LogIndex, vatInit.TransactionIndex, vatInit.Raw, headerID, vatInit.Ilk, vatInit.LogIndex, vatInit.TransactionIndex, vatInit.Raw,
) )
if execErr != nil { if execErr != nil {
@ -74,6 +78,14 @@ func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked)
} }
func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatInitChecked)
}
func (repository VatInitRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatInitChecked)
}
func (repository *VatInitRepository) SetDB(db *postgres.DB) { func (repository *VatInitRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/vat_move/repository.go
View File

@ -18,7 +18,10 @@ package vat_move
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{})
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT INTO maker.vat_move (header_id, src, dst, rad, log_idx, tx_idx, raw_log) `INSERT INTO maker.vat_move (header_id, src, dst, rad, log_idx, tx_idx, raw_log)
VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET src = $2, dst = $3, rad = $4, raw_log = $7;`,
headerID, vatMove.Src, vatMove.Dst, vatMove.Rad, vatMove.LogIndex, vatMove.TransactionIndex, vatMove.Raw, headerID, vatMove.Src, vatMove.Dst, vatMove.Rad, vatMove.LogIndex, vatMove.TransactionIndex, vatMove.Raw,
) )
@ -71,6 +75,14 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{})
return tx.Commit() return tx.Commit()
} }
func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatMoveChecked)
}
func (repository VatMoveRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatMoveChecked)
}
func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked)
} }

14
pkg/transformers/vat_slip/repository.go
View File

@ -2,7 +2,10 @@ package vat_slip
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -29,7 +32,8 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{})
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log) `INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, guy = $3, rad = $4, raw_log = $7;`,
headerID, vatSlip.Ilk, vatSlip.Guy, vatSlip.Rad, vatSlip.TransactionIndex, vatSlip.LogIndex, vatSlip.Raw, headerID, vatSlip.Ilk, vatSlip.Guy, vatSlip.Rad, vatSlip.TransactionIndex, vatSlip.LogIndex, vatSlip.Raw,
) )
if execErr != nil { if execErr != nil {
@ -57,6 +61,14 @@ func (repository VatSlipRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked)
} }
func (repository VatSlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatSlipChecked)
}
func (repository VatSlipRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatSlipChecked)
}
func (repository *VatSlipRepository) SetDB(db *postgres.DB) { func (repository *VatSlipRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

11
pkg/transformers/vat_toll/repository.go
View File

@ -2,7 +2,10 @@ package vat_toll
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -56,6 +59,14 @@ func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked)
} }
func (repository VatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTollChecked)
}
func (repository VatTollRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTollChecked)
}
func (repository *VatTollRepository) SetDB(db *postgres.DB) { func (repository *VatTollRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/vat_tune/repository.go
View File

@ -2,7 +2,10 @@ package vat_tune
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -29,7 +32,8 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{})
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log) `INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, v = $4, w = $5, dink = $6, dart = $7, raw_log = $10;`,
headerID, vatTune.Ilk, vatTune.Urn, vatTune.V, vatTune.W, vatTune.Dink, vatTune.Dart, vatTune.TransactionIndex, vatTune.LogIndex, vatTune.Raw, headerID, vatTune.Ilk, vatTune.Urn, vatTune.V, vatTune.W, vatTune.Dink, vatTune.Dart, vatTune.TransactionIndex, vatTune.LogIndex, vatTune.Raw,
) )
if execErr != nil { if execErr != nil {
@ -56,6 +60,14 @@ func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked)
} }
func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked)
}
func (repository VatTuneRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked)
}
func (repository *VatTuneRepository) SetDB(db *postgres.DB) { func (repository *VatTuneRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

14
pkg/transformers/vow_flog/repository.go
View File

@ -18,7 +18,10 @@ package vow_flog
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
@ -46,7 +49,8 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{})
_, execErr := tx.Exec( _, execErr := tx.Exec(
`INSERT into maker.vow_flog (header_id, era, log_idx, tx_idx, raw_log) `INSERT into maker.vow_flog (header_id, era, log_idx, tx_idx, raw_log)
VALUES($1, $2::NUMERIC, $3, $4, $5)`, VALUES($1, $2::NUMERIC, $3, $4, $5)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET era = $2, raw_log = $5;`,
headerID, flog.Era, flog.LogIndex, flog.TransactionIndex, flog.Raw, headerID, flog.Era, flog.LogIndex, flog.TransactionIndex, flog.Raw,
) )
@ -75,6 +79,14 @@ func (repository VowFlogRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked)
} }
func (repository VowFlogRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VowFlogChecked)
}
func (repository VowFlogRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VowFlogChecked)
}
func (repository *VowFlogRepository) SetDB(db *postgres.DB) { func (repository *VowFlogRepository) SetDB(db *postgres.DB) {
repository.db = db repository.db = db
} }

1
test_config/test_config.go
View File

@ -55,6 +55,7 @@ func setTestConfig() {
hn := TestConfig.GetString("database.hostname") hn := TestConfig.GetString("database.hostname")
port := TestConfig.GetInt("database.port") port := TestConfig.GetInt("database.port")
name := TestConfig.GetString("database.name") name := TestConfig.GetString("database.name")
DBConfig = config.Database{ DBConfig = config.Database{
Hostname: hn, Hostname: hn,
Name: name, Name: name,