From 33b293f2bba6cac08c963a55a51ef75b981582d8 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Mon, 9 May 2022 17:22:19 +0530 Subject: [PATCH 1/2] Fix migrations naming --- README.md | 2 +- ...hile_triggers.sql => 00015_create_postgraphile_triggers.sql} | 0 ...e_db_version_table.sql => 00016_create_db_version_table.sql} | 0 ...{00018_update_db_version.sql => 00017_update_db_version.sql} | 0 ...vert_to_hypertables.sql => 00018_convert_to_hypertables.sql} | 0 .../{00020_add_data_nodes.sql => 00019_add_data_nodes.sql} | 0 ...rtables.sql => 00020_convert_to_distributed_hypertables.sql} | 0 ...{00022_attach_data_nodes.sql => 00021_attach_data_nodes.sql} | 0 ...e_stored_functions.sql => 00022_create_stored_functions.sql} | 0 ..._replication_factor.sql => 00023_set_replication_factor.sql} | 0 scripts/startup_script.sh | 2 +- 11 files changed, 2 insertions(+), 2 deletions(-) rename db/migrations/{00016_create_postgraphile_triggers.sql => 00015_create_postgraphile_triggers.sql} (100%) rename db/migrations/{00017_create_db_version_table.sql => 00016_create_db_version_table.sql} (100%) rename db/migrations/{00018_update_db_version.sql => 00017_update_db_version.sql} (100%) rename db/migrations/{00019_convert_to_hypertables.sql => 00018_convert_to_hypertables.sql} (100%) rename db/migrations/{00020_add_data_nodes.sql => 00019_add_data_nodes.sql} (100%) rename db/migrations/{00021_convert_to_distributed_hypertables.sql => 00020_convert_to_distributed_hypertables.sql} (100%) rename db/migrations/{00022_attach_data_nodes.sql => 00021_attach_data_nodes.sql} (100%) rename db/migrations/{00023_create_stored_functions.sql => 00022_create_stored_functions.sql} (100%) rename db/migrations/{00024_set_replication_factor.sql => 00023_set_replication_factor.sql} (100%) diff --git a/README.md b/README.md index c456ca1..64c4e22 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Schemas and utils for IPLD ETH Postgres database * Edit [startup_script.sh](./scripts/startup_script.sh) to change the number of migrations to be run: ```bash - ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 23 + ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 22 ``` * In another `ipld-eth-db` terminal window, build an image `migrations-test` using [Dockerfile](./db/Dockerfile): diff --git a/db/migrations/00016_create_postgraphile_triggers.sql b/db/migrations/00015_create_postgraphile_triggers.sql similarity index 100% rename from db/migrations/00016_create_postgraphile_triggers.sql rename to db/migrations/00015_create_postgraphile_triggers.sql diff --git a/db/migrations/00017_create_db_version_table.sql b/db/migrations/00016_create_db_version_table.sql similarity index 100% rename from db/migrations/00017_create_db_version_table.sql rename to db/migrations/00016_create_db_version_table.sql diff --git a/db/migrations/00018_update_db_version.sql b/db/migrations/00017_update_db_version.sql similarity index 100% rename from db/migrations/00018_update_db_version.sql rename to db/migrations/00017_update_db_version.sql diff --git a/db/migrations/00019_convert_to_hypertables.sql b/db/migrations/00018_convert_to_hypertables.sql similarity index 100% rename from db/migrations/00019_convert_to_hypertables.sql rename to db/migrations/00018_convert_to_hypertables.sql diff --git a/db/migrations/00020_add_data_nodes.sql b/db/migrations/00019_add_data_nodes.sql similarity index 100% rename from db/migrations/00020_add_data_nodes.sql rename to db/migrations/00019_add_data_nodes.sql diff --git a/db/migrations/00021_convert_to_distributed_hypertables.sql b/db/migrations/00020_convert_to_distributed_hypertables.sql similarity index 100% rename from db/migrations/00021_convert_to_distributed_hypertables.sql rename to db/migrations/00020_convert_to_distributed_hypertables.sql diff --git a/db/migrations/00022_attach_data_nodes.sql b/db/migrations/00021_attach_data_nodes.sql similarity index 100% rename from db/migrations/00022_attach_data_nodes.sql rename to db/migrations/00021_attach_data_nodes.sql diff --git a/db/migrations/00023_create_stored_functions.sql b/db/migrations/00022_create_stored_functions.sql similarity index 100% rename from db/migrations/00023_create_stored_functions.sql rename to db/migrations/00022_create_stored_functions.sql diff --git a/db/migrations/00024_set_replication_factor.sql b/db/migrations/00023_set_replication_factor.sql similarity index 100% rename from db/migrations/00024_set_replication_factor.sql rename to db/migrations/00023_set_replication_factor.sql diff --git a/scripts/startup_script.sh b/scripts/startup_script.sh index 2886c64..b07f2f4 100755 --- a/scripts/startup_script.sh +++ b/scripts/startup_script.sh @@ -8,7 +8,7 @@ VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME # Run the DB migrations echo "Connecting with: $VDB_PG_CONNECT" echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 23 +./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 22 # If the db migrations ran without err if [[ $? -eq 0 ]]; then From 0ccb54c770867d5e6b0a5ff863e657707a003c40 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Mon, 9 May 2022 19:04:32 +0530 Subject: [PATCH 2/2] Skip migration to create hypertables --- README.md | 2 +- ...ata_nodes.sql => 00018_add_data_nodes.sql} | 0 .../00018_convert_to_hypertables.sql | 67 ------------------- ...19_convert_to_distributed_hypertables.sql} | 26 +++---- ..._nodes.sql => 00020_attach_data_nodes.sql} | 0 ....sql => 00021_create_stored_functions.sql} | 0 ...r.sql => 00022_set_replication_factor.sql} | 0 scripts/startup_script.sh | 2 +- 8 files changed, 11 insertions(+), 86 deletions(-) rename db/migrations/{00019_add_data_nodes.sql => 00018_add_data_nodes.sql} (100%) delete mode 100644 db/migrations/00018_convert_to_hypertables.sql rename db/migrations/{00020_convert_to_distributed_hypertables.sql => 00019_convert_to_distributed_hypertables.sql} (83%) rename db/migrations/{00021_attach_data_nodes.sql => 00020_attach_data_nodes.sql} (100%) rename db/migrations/{00022_create_stored_functions.sql => 00021_create_stored_functions.sql} (100%) rename db/migrations/{00023_set_replication_factor.sql => 00022_set_replication_factor.sql} (100%) diff --git a/README.md b/README.md index 64c4e22..e542627 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Schemas and utils for IPLD ETH Postgres database * Edit [startup_script.sh](./scripts/startup_script.sh) to change the number of migrations to be run: ```bash - ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 22 + ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 21 ``` * In another `ipld-eth-db` terminal window, build an image `migrations-test` using [Dockerfile](./db/Dockerfile): diff --git a/db/migrations/00019_add_data_nodes.sql b/db/migrations/00018_add_data_nodes.sql similarity index 100% rename from db/migrations/00019_add_data_nodes.sql rename to db/migrations/00018_add_data_nodes.sql diff --git a/db/migrations/00018_convert_to_hypertables.sql b/db/migrations/00018_convert_to_hypertables.sql deleted file mode 100644 index 026472d..0000000 --- a/db/migrations/00018_convert_to_hypertables.sql +++ /dev/null @@ -1,67 +0,0 @@ --- +goose Up -SELECT create_hypertable('public.blocks', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.header_cids', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.uncle_cids', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.transaction_cids', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.receipt_cids', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.state_cids', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.storage_cids', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.state_accounts', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.access_list_elements', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.log_cids', 'block_number', migrate_data => true, chunk_time_interval => 32768); - --- update version -INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0-h') - ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0-h', NOW()); - --- +goose Down -INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0') - ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0', NOW()); --- reversing conversion to hypertable requires migrating all data from every chunk back to a single table --- create new regular tables -CREATE TABLE eth.log_cids_i (LIKE eth.log_cids INCLUDING ALL); -CREATE TABLE eth.access_list_elements_i (LIKE eth.access_list_elements INCLUDING ALL); -CREATE TABLE eth.state_accounts_i (LIKE eth.state_accounts INCLUDING ALL); -CREATE TABLE eth.storage_cids_i (LIKE eth.storage_cids INCLUDING ALL); -CREATE TABLE eth.state_cids_i (LIKE eth.state_cids INCLUDING ALL); -CREATE TABLE eth.receipt_cids_i (LIKE eth.receipt_cids INCLUDING ALL); -CREATE TABLE eth.transaction_cids_i (LIKE eth.transaction_cids INCLUDING ALL); -CREATE TABLE eth.uncle_cids_i (LIKE eth.uncle_cids INCLUDING ALL); -CREATE TABLE eth.header_cids_i (LIKE eth.header_cids INCLUDING ALL); -CREATE TABLE public.blocks_i (LIKE public.blocks INCLUDING ALL); - --- migrate data -INSERT INTO eth.log_cids_i (SELECT * FROM eth.log_cids); -INSERT INTO eth.access_list_elements_i (SELECT * FROM eth.access_list_elements); -INSERT INTO eth.state_accounts_i (SELECT * FROM eth.state_accounts); -INSERT INTO eth.storage_cids_i (SELECT * FROM eth.storage_cids); -INSERT INTO eth.state_cids_i (SELECT * FROM eth.state_cids); -INSERT INTO eth.receipt_cids_i (SELECT * FROM eth.receipt_cids); -INSERT INTO eth.transaction_cids_i (SELECT * FROM eth.transaction_cids); -INSERT INTO eth.uncle_cids_i (SELECT * FROM eth.uncle_cids); -INSERT INTO eth.header_cids_i (SELECT * FROM eth.header_cids); -INSERT INTO public.blocks_i (SELECT * FROM public.blocks); - --- drops hypertables -DROP TABLE eth.log_cids; -DROP TABLE eth.access_list_elements; -DROP TABLE eth.state_accounts; -DROP TABLE eth.storage_cids; -DROP TABLE eth.state_cids; -DROP TABLE eth.receipt_cids; -DROP TABLE eth.transaction_cids; -DROP TABLE eth.uncle_cids; -DROP TABLE eth.header_cids; -DROP TABLE public.blocks; - --- rename new tables -ALTER TABLE eth.log_cids_i RENAME TO log_cids; -ALTER TABLE eth.access_list_elements_i RENAME TO access_list_elements; -ALTER TABLE eth.state_accounts_i RENAME TO state_accounts; -ALTER TABLE eth.storage_cids_i RENAME TO storage_cids; -ALTER TABLE eth.state_cids_i RENAME TO state_cids; -ALTER TABLE eth.receipt_cids_i RENAME TO receipt_cids; -ALTER TABLE eth.transaction_cids_i RENAME TO transaction_cids; -ALTER TABLE eth.uncle_cids_i RENAME TO uncle_cids; -ALTER TABLE eth.header_cids_i RENAME TO header_cids; -ALTER TABLE public.blocks_i RENAME TO blocks; diff --git a/db/migrations/00020_convert_to_distributed_hypertables.sql b/db/migrations/00019_convert_to_distributed_hypertables.sql similarity index 83% rename from db/migrations/00020_convert_to_distributed_hypertables.sql rename to db/migrations/00019_convert_to_distributed_hypertables.sql index f861678..e755510 100644 --- a/db/migrations/00020_convert_to_distributed_hypertables.sql +++ b/db/migrations/00019_convert_to_distributed_hypertables.sql @@ -1,4 +1,8 @@ -- +goose Up +-- creating distributed hypertables from duplicate tables for now as we are getting the following error (while running geth +-- unit tests) if regular tables are directly converted to distributed hypertables +-- error: "cannot PREPARE a transaction that has executed LISTEN, UNLISTEN, or NOTIFY" + -- create new regular tables CREATE TABLE eth.log_cids_i (LIKE eth.log_cids INCLUDING ALL); CREATE TABLE eth.access_list_elements_i (LIKE eth.access_list_elements INCLUDING ALL); @@ -35,7 +39,7 @@ INSERT INTO eth.uncle_cids_i (SELECT * FROM eth.uncle_cids); INSERT INTO eth.header_cids_i (SELECT * FROM eth.header_cids); INSERT INTO public.blocks_i (SELECT * FROM public.blocks); --- drops hypertables +-- drop tables DROP TABLE eth.log_cids; DROP TABLE eth.access_list_elements; DROP TABLE eth.state_accounts; @@ -74,9 +78,9 @@ INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.00-dh') ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0-dh', NOW()); -- +goose Down -INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0-h') - ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0-h', NOW()); --- reversing conversion to hypertable requires migrating all data from every chunk back to a single table +INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0') + ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0', NOW()); +-- reversing conversion to hypertables requires migrating all data from every chunk back to a single table -- create new regular tables CREATE TABLE eth.log_cids_i (LIKE eth.log_cids INCLUDING ALL); CREATE TABLE eth.access_list_elements_i (LIKE eth.access_list_elements INCLUDING ALL); @@ -89,18 +93,6 @@ CREATE TABLE eth.uncle_cids_i (LIKE eth.uncle_cids INCLUDING ALL); CREATE TABLE eth.header_cids_i (LIKE eth.header_cids INCLUDING ALL); CREATE TABLE public.blocks_i (LIKE public.blocks INCLUDING ALL); --- turn them into hypertables -SELECT create_hypertable('public.blocks_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.header_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.uncle_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.transaction_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.receipt_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.state_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.storage_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.state_accounts_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.access_list_elements_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); -SELECT create_hypertable('eth.log_cids_i', 'block_number', migrate_data => true, chunk_time_interval => 32768); - -- migrate data INSERT INTO eth.log_cids_i (SELECT * FROM eth.log_cids); INSERT INTO eth.access_list_elements_i (SELECT * FROM eth.access_list_elements); @@ -113,7 +105,7 @@ INSERT INTO eth.uncle_cids_i (SELECT * FROM eth.uncle_cids); INSERT INTO eth.header_cids_i (SELECT * FROM eth.header_cids); INSERT INTO public.blocks_i (SELECT * FROM public.blocks); --- drops distributed hypertables +-- drop distributed hypertables DROP TABLE eth.log_cids; DROP TABLE eth.access_list_elements; DROP TABLE eth.state_accounts; diff --git a/db/migrations/00021_attach_data_nodes.sql b/db/migrations/00020_attach_data_nodes.sql similarity index 100% rename from db/migrations/00021_attach_data_nodes.sql rename to db/migrations/00020_attach_data_nodes.sql diff --git a/db/migrations/00022_create_stored_functions.sql b/db/migrations/00021_create_stored_functions.sql similarity index 100% rename from db/migrations/00022_create_stored_functions.sql rename to db/migrations/00021_create_stored_functions.sql diff --git a/db/migrations/00023_set_replication_factor.sql b/db/migrations/00022_set_replication_factor.sql similarity index 100% rename from db/migrations/00023_set_replication_factor.sql rename to db/migrations/00022_set_replication_factor.sql diff --git a/scripts/startup_script.sh b/scripts/startup_script.sh index b07f2f4..514a572 100755 --- a/scripts/startup_script.sh +++ b/scripts/startup_script.sh @@ -8,7 +8,7 @@ VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME # Run the DB migrations echo "Connecting with: $VDB_PG_CONNECT" echo "Running database migrations" -./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 22 +./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 21 # If the db migrations ran without err if [[ $? -eq 0 ]]; then