deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

* update transformer to able to recheck headers (#4)

Browse Source 
* update transformer to able to recheck headers
* put cap on rechecking header
* integration test for recheck headers
* use enum for recheck headers param; make recheck cap configurable
* update integration tests with new test config
* update omni pkg with new recheck header column type
* update migration with new migration tool and final tweaks needed to accommodate changes in omni pkg
This commit is contained in:
Takayuki Goto
2019-02-08 10:35:46 -06:00
committed by GitHub
parent 219f099b0c
commit 0a024d429d
74 changed files with 1994 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/omni/light/repository/header_repository.go
+13 -13
View File
@@ -61,7 +61,7 @@ func (r *headerRepository) AddCheckColumn(id string) error {
}
pgStr := "ALTER TABLE public.checked_headers ADD COLUMN IF NOT EXISTS "
pgStr = pgStr + id + " BOOLEAN NOT NULL DEFAULT FALSE"
pgStr = pgStr + id + " INTEGER NOT NULL DEFAULT 0"
_, err := r.db.Exec(pgStr)
if err != nil {
return err
@@ -80,7 +80,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error {
for _, id := range ids {
_, ok := r.columns.Get(id)
if !ok {
baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " BOOLEAN NOT NULL DEFAULT FALSE,"
baseQuery += " ADD COLUMN IF NOT EXISTS " + id + " INTEGER NOT NULL DEFAULT 0,"
input = append(input, id)
}
}
@@ -100,7 +100,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET `+id+` = $2`, headerID, true)
UPDATE SET `+id+` = checked_headers.`+id+` + 1`, headerID, 1)
return err
}
@@ -112,11 +112,11 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
}
pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
for i := 0; i < len(ids); i++ {
pgStr += "true, "
pgStr += "1, "
}
pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
for _, id := range ids {
pgStr += fmt.Sprintf("%s = true, ", id)
pgStr += id + `= checked_headers.` + id + ` + 1, `
}
pgStr = pgStr[:len(pgStr)-2]
_, err := r.db.Exec(pgStr, headerID)
@@ -137,11 +137,11 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [
}
pgStr = pgStr[:len(pgStr)-2] + ") VALUES ($1, "
for i := 0; i < len(ids); i++ {
pgStr += "true, "
pgStr += "1, "
}
pgStr = pgStr[:len(pgStr)-2] + ") ON CONFLICT (header_id) DO UPDATE SET "
for _, id := range ids {
pgStr += fmt.Sprintf("%s = true, ", id)
pgStr += fmt.Sprintf("%s = checked_headers.%s + 1, ", id, id)
}
pgStr = pgStr[:len(pgStr)-2]
_, err = tx.Exec(pgStr, header.Id)
@@ -162,7 +162,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + id + ` IS FALSE)
WHERE (header_id ISNULL OR checked_headers.` + id + `=0)
AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2
ORDER BY headers.block_number`
@@ -170,7 +170,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
} else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + id + ` IS FALSE)
WHERE (header_id ISNULL OR checked_headers.` + id + `=0)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3
@@ -190,7 +190,7 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL`
for _, id := range ids {
baseQuery += ` OR ` + id + ` IS FALSE`
baseQuery += ` OR checked_headers.` + id + `= 0`
}
if endingBlockNumber == -1 {
@@ -220,11 +220,11 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id IS NOT NULL`
for _, id := range eventIds {
baseQuery += ` AND ` + id + ` IS TRUE`
baseQuery += ` AND ` + id + `!=0`
}
baseQuery += `) AND (`
for _, id := range methodIds {
baseQuery += id + ` IS FALSE AND `
baseQuery += id + ` =0 AND `
}
baseQuery = baseQuery[:len(baseQuery)-5] + `) `
@@ -254,6 +254,6 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string)
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET `+eventID+` = $2`, headerID, true)
UPDATE SET `+eventID+` = checked_headers.`+eventID+` + 1`, headerID, 1)
return err
}
pkg/omni/shared/helpers/test_helpers/database.go
+13 -34
View File
@@ -257,41 +257,17 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`DELETE FROM receipts`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP TABLE checked_headers`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`CREATE TABLE checked_headers (id SERIAL PRIMARY KEY, header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`ALTER TABLE checked_headers
ADD COLUMN IF NOT EXISTS price_feeds_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS flip_kick_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS frob_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS tend_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS bite_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS dent_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS pit_file_debt_ceiling_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS pit_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_init_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS drip_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS drip_file_repo_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS drip_file_vow_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS deal_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS drip_drip_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS cat_file_chop_lump_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS cat_file_flip_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS cat_file_pit_vow_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS flop_kick_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_move_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_fold_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_heal_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_toll_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_tune_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_grab_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_flux_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vat_slip_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS vow_flog_checked BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS flap_kick_checked BOOLEAN NOT NULL DEFAULT FALSE`)
DROP COLUMN IF EXISTS eventName_contractAddr,
DROP COLUMN IF EXISTS eventName_contractAddr2,
DROP COLUMN IF EXISTS eventName_contractAddr3,
DROP COLUMN IF EXISTS methodName_contractAddr,
DROP COLUMN IF EXISTS methodName_contractAddr2,
DROP COLUMN IF EXISTS methodName_contractAddr3,
DROP COLUMN IF EXISTS transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e,
DROP COLUMN IF EXISTS balanceof_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e,
DROP COLUMN IF EXISTS newowner_0x314159265dd8dbb310642f98f50c066173c1259b,
DROP COLUMN IF EXISTS owner_0x314159265dd8dbb310642f98f50c066173c1259b`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
@@ -308,6 +284,9 @@ ADD COLUMN IF NOT EXISTS flap_kick_checked BOOLEAN NOT NULL DEFAULT FALSE`)
err = tx.Commit()
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`VACUUM checked_headers`)
Expect(err).NotTo(HaveOccurred())
}
func CreateBlock(blockNumber int64, repository repositories.BlockRepository) (blockId int64) {