diff --git a/lib/harmony/harmonydb/sql/20230719-harmony.sql b/lib/harmony/harmonydb/sql/20230719-harmony.sql deleted file mode 100644 index e7b1795c5..000000000 --- a/lib/harmony/harmonydb/sql/20230719-harmony.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* For HarmonyTask base implementation. */ - -CREATE TABLE harmony_machines ( - id SERIAL PRIMARY KEY NOT NULL, - last_contact TIMESTAMP NOT NULL DEFAULT current_timestamp, - host_and_port varchar(300) NOT NULL, - cpu INTEGER NOT NULL, - ram BIGINT NOT NULL, - gpu FLOAT NOT NULL -); - -CREATE TABLE harmony_task ( - id SERIAL PRIMARY KEY NOT NULL, - initiated_by INTEGER, - update_time TIMESTAMP NOT NULL DEFAULT current_timestamp, - posted_time TIMESTAMP NOT NULL, - owner_id INTEGER REFERENCES harmony_machines (id) ON DELETE SET NULL, - added_by INTEGER NOT NULL, - previous_task INTEGER, - name varchar(16) NOT NULL -); -COMMENT ON COLUMN harmony_task.initiated_by IS 'The task ID whose completion occasioned this task.'; -COMMENT ON COLUMN harmony_task.owner_id IS 'The foreign key to harmony_machines.'; -COMMENT ON COLUMN harmony_task.name IS 'The name of the task type.'; -COMMENT ON COLUMN harmony_task.owner_id IS 'may be null if between owners or not yet taken'; -COMMENT ON COLUMN harmony_task.update_time IS 'When it was last modified. not a heartbeat'; - -CREATE TABLE harmony_task_history ( - id SERIAL PRIMARY KEY NOT NULL, - task_id INTEGER NOT NULL, - name VARCHAR(16) NOT NULL, - posted TIMESTAMP NOT NULL, - work_start TIMESTAMP NOT NULL, - work_end TIMESTAMP NOT NULL, - result BOOLEAN NOT NULL, - err varchar -); -COMMENT ON COLUMN harmony_task_history.result IS 'Use to detemine if this was a successful run.'; - -CREATE TABLE harmony_task_follow ( - id SERIAL PRIMARY KEY NOT NULL, - owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE, - to_type VARCHAR(16) NOT NULL, - from_type VARCHAR(16) NOT NULL -); - -CREATE TABLE harmony_task_impl ( - id SERIAL PRIMARY KEY NOT NULL, - owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE, - name VARCHAR(16) NOT NULL -); \ No newline at end of file diff --git a/lib/harmony/harmonydb/sql/20230823-wdpost.sql b/lib/harmony/harmonydb/sql/20230823-wdpost.sql deleted file mode 100644 index c6f993d76..000000000 --- a/lib/harmony/harmonydb/sql/20230823-wdpost.sql +++ /dev/null @@ -1,48 +0,0 @@ -create table wdpost_partition_tasks -( - task_id bigint not null - constraint wdpost_partition_tasks_pk - primary key, - sp_id bigint not null, - proving_period_start bigint not null, - deadline_index bigint not null, - partition_index bigint not null, - constraint wdpost_partition_tasks_identity_key - unique (sp_id, proving_period_start, deadline_index, partition_index) -); - -comment on column wdpost_partition_tasks.task_id is 'harmonytask task ID'; -comment on column wdpost_partition_tasks.sp_id is 'storage provider ID'; -comment on column wdpost_partition_tasks.proving_period_start is 'proving period start'; -comment on column wdpost_partition_tasks.deadline_index is 'deadline index within the proving period'; -comment on column wdpost_partition_tasks.partition_index is 'partition index within the deadline'; - -create table wdpost_proofs -( - sp_id bigint not null, - proving_period_start bigint not null, - deadline bigint not null, - partition bigint not null, - submit_at_epoch bigint not null, - submit_by_epoch bigint not null, - proof_params bytea, - - submit_task_id bigint, - message_cid text, - - constraint wdpost_proofs_identity_key - unique (sp_id, proving_period_start, deadline, partition) -); - -create table wdpost_recovery_tasks -( - task_id bigint not null - constraint wdpost_recovery_tasks_pk - primary key, - sp_id bigint not null, - proving_period_start bigint not null, - deadline_index bigint not null, - partition_index bigint not null, - constraint wdpost_recovery_tasks_identity_key - unique (sp_id, proving_period_start, deadline_index, partition_index) -); \ No newline at end of file diff --git a/lib/harmony/harmonydb/sql/20230919-config.sql b/lib/harmony/harmonydb/sql/20230919-config.sql deleted file mode 100644 index 84699a0d5..000000000 --- a/lib/harmony/harmonydb/sql/20230919-config.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE harmony_config ( - id SERIAL PRIMARY KEY NOT NULL, - title VARCHAR(300) UNIQUE NOT NULL, - config TEXT NOT NULL -); \ No newline at end of file diff --git a/lib/harmony/harmonydb/sql/20231103-chain_sends.sql b/lib/harmony/harmonydb/sql/20231103-chain_sends.sql deleted file mode 100644 index e70cf3738..000000000 --- a/lib/harmony/harmonydb/sql/20231103-chain_sends.sql +++ /dev/null @@ -1,55 +0,0 @@ -create table message_sends -( - from_key text not null, - to_addr text not null, - send_reason text not null, - send_task_id bigint not null, - - unsigned_data bytea not null, - unsigned_cid text not null, - - nonce bigint, - signed_data bytea, - signed_json jsonb, - signed_cid text, - - send_time timestamp default null, - send_success boolean default null, - send_error text, - - constraint message_sends_pk - primary key (send_task_id, from_key) -); - -comment on column message_sends.from_key is 'text f[1/3/4]... address'; -comment on column message_sends.to_addr is 'text f[0/1/2/3/4]... address'; -comment on column message_sends.send_reason is 'optional description of send reason'; -comment on column message_sends.send_task_id is 'harmony task id of the send task'; - -comment on column message_sends.unsigned_data is 'unsigned message data'; -comment on column message_sends.unsigned_cid is 'unsigned message cid'; - -comment on column message_sends.nonce is 'assigned message nonce, set while the send task is executing'; -comment on column message_sends.signed_data is 'signed message data, set while the send task is executing'; -comment on column message_sends.signed_cid is 'signed message cid, set while the send task is executing'; - -comment on column message_sends.send_time is 'time when the send task was executed, set after pushing the message to the network'; -comment on column message_sends.send_success is 'whether this message was broadcasted to the network already, null if not yet attempted, true if successful, false if failed'; -comment on column message_sends.send_error is 'error message if send_success is false'; - -create unique index message_sends_success_index - on message_sends (from_key, nonce) - where send_success is not false; - -comment on index message_sends_success_index is -'message_sends_success_index enforces sender/nonce uniqueness, it is a conditional index that only indexes rows where send_success is not false. This allows us to have multiple rows with the same sender/nonce, as long as only one of them was successfully broadcasted (true) to the network or is in the process of being broadcasted (null).'; - -create table message_send_locks -( - from_key text not null, - task_id bigint not null, - claimed_at timestamp not null, - - constraint message_send_locks_pk - primary key (from_key) -); diff --git a/lib/harmony/harmonydb/sql/20231110-mining_tasks.sql b/lib/harmony/harmonydb/sql/20231110-mining_tasks.sql deleted file mode 100644 index 15b478f4d..000000000 --- a/lib/harmony/harmonydb/sql/20231110-mining_tasks.sql +++ /dev/null @@ -1,39 +0,0 @@ -create table mining_tasks -( - task_id bigint not null - constraint mining_tasks_pk - primary key, - sp_id bigint not null, - epoch bigint not null, - base_compute_time timestamp not null, - - won bool not null default false, - mined_cid text, - mined_header jsonb, - mined_at timestamp, - - submitted_at timestamp, - - constraint mining_tasks_sp_epoch - unique (sp_id, epoch) -); - -create table mining_base_block -( - id bigserial not null - constraint mining_base_block_pk - primary key, - task_id bigint not null - constraint mining_base_block_mining_tasks_task_id_fk - references mining_tasks - on delete cascade, - sp_id bigint, - block_cid text not null, - - no_win bool not null default false, - - constraint mining_base_block_pk2 - unique (sp_id, task_id, block_cid) -); - -CREATE UNIQUE INDEX mining_base_block_cid_k ON mining_base_block (sp_id, block_cid) WHERE no_win = false; diff --git a/lib/harmony/harmonydb/sql/20231113-harmony_taskhistory_oops.sql b/lib/harmony/harmonydb/sql/20231113-harmony_taskhistory_oops.sql deleted file mode 100644 index 7a71d98ae..000000000 --- a/lib/harmony/harmonydb/sql/20231113-harmony_taskhistory_oops.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE harmony_task_history ADD COLUMN completed_by_host_and_port varchar(300) NOT NULL; \ No newline at end of file diff --git a/lib/harmony/harmonydb/sql/20231120-testing1.sql b/lib/harmony/harmonydb/sql/20231120-testing1.sql deleted file mode 100644 index 71daaef69..000000000 --- a/lib/harmony/harmonydb/sql/20231120-testing1.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE harmony_test ( - task_id bigint - constraint harmony_test_pk - primary key, - options text, - result text -); -ALTER TABLE wdpost_proofs ADD COLUMN test_task_id bigint; \ No newline at end of file diff --git a/lib/harmony/harmonydb/sql/20231217-sdr-pipeline.sql b/lib/harmony/harmonydb/sql/20231217-sdr-pipeline.sql deleted file mode 100644 index b7ef2a0b5..000000000 --- a/lib/harmony/harmonydb/sql/20231217-sdr-pipeline.sql +++ /dev/null @@ -1,138 +0,0 @@ --- NOTE: task_ids can be the same between different task types and between different sectors --- e.g. SN-supraseal doing 128 sdr/TreeC/TreeR with the same task_id - -create table sectors_sdr_pipeline ( - sp_id bigint not null, - sector_number bigint not null, - - -- at request time - create_time timestamp not null default current_timestamp, - reg_seal_proof int not null, - - -- sdr - ticket_epoch bigint, - ticket_value bytea, - - task_id_sdr bigint, - after_sdr bool not null default false, - - -- tree D - tree_d_cid text, -- commd from treeD compute, should match comm_d_cid - - task_id_tree_d bigint, - after_tree_d bool not null default false, - - -- tree C - task_id_tree_c bigint, - after_tree_c bool not null default false, - - -- tree R - tree_r_cid text, -- commr from treeR compute - - task_id_tree_r bigint, - after_tree_r bool not null default false, - - -- precommit message sending - precommit_msg_cid text, - - task_id_precommit_msg bigint, - after_precommit_msg bool not null default false, - - -- precommit message wait - seed_epoch bigint, - precommit_msg_tsk bytea, - after_precommit_msg_success bool not null default false, - - -- seed - seed_value bytea, - - -- Commit (PoRep snark) - task_id_porep bigint, - porep_proof bytea, - after_porep bool not null default false, - - -- Finalize (trim cache) - task_id_finalize bigint, - after_finalize bool not null default false, - - -- MoveStorage (move data to storage) - task_id_move_storage bigint, - after_move_storage bool not null default false, - - -- Commit message sending - commit_msg_cid text, - - task_id_commit_msg bigint, - after_commit_msg bool not null default false, - - -- Commit message wait - commit_msg_tsk bytea, - after_commit_msg_success bool not null default false, - - -- Failure handling - failed bool not null default false, - failed_at timestamp, - failed_reason varchar(20) not null default '', - failed_reason_msg text not null default '', - - -- foreign key - -- NOTE: Following keys were dropped in 20240507-sdr-pipeline-fk-drop.sql - foreign key (task_id_sdr) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_tree_d) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_tree_c) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_tree_r) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_precommit_msg) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_porep) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_finalize) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_move_storage) references harmony_task (id) on delete set null, -- dropped - foreign key (task_id_commit_msg) references harmony_task (id) on delete set null, -- dropped - - -- constraints - primary key (sp_id, sector_number) -); - -create table sectors_sdr_initial_pieces ( - sp_id bigint not null, - sector_number bigint not null, - - piece_index bigint not null, - piece_cid text not null, - piece_size bigint not null, -- padded size - - -- data source - data_url text not null, - data_headers jsonb not null default '{}', - data_raw_size bigint not null, - data_delete_on_finalize bool not null, - - -- deal info - f05_publish_cid text, - f05_deal_id bigint, - f05_deal_proposal jsonb, - f05_deal_start_epoch bigint, - f05_deal_end_epoch bigint, - - -- ddo deal info - -- added in 20240402-sdr-pipeline-ddo-deal-info.sql - -- direct_start_epoch bigint, - -- direct_end_epoch bigint, - -- direct_piece_activation_manifest jsonb, - - -- created_at added in 20240508-open-deal-sectors.sql - -- created_at timestamp, - - -- open_sector_pieces table is a copy of this - -- all alters should happen on both tables except constraints - - -- foreign key - foreign key (sp_id, sector_number) references sectors_sdr_pipeline (sp_id, sector_number) on delete cascade, - - primary key (sp_id, sector_number, piece_index) -); - -comment on column sectors_sdr_initial_pieces.piece_size is 'padded size of the piece'; - -create table sectors_allocated_numbers ( - sp_id bigint not null primary key, - allocated jsonb not null -); diff --git a/lib/harmony/harmonydb/sql/20231225-message-waits.sql b/lib/harmony/harmonydb/sql/20231225-message-waits.sql deleted file mode 100644 index 4143f3a56..000000000 --- a/lib/harmony/harmonydb/sql/20231225-message-waits.sql +++ /dev/null @@ -1,13 +0,0 @@ -create table message_waits ( - signed_message_cid text primary key, - waiter_machine_id int references harmony_machines (id) on delete set null, - - executed_tsk_cid text, - executed_tsk_epoch bigint, - executed_msg_cid text, - executed_msg_data jsonb, - - executed_rcpt_exitcode bigint, - executed_rcpt_return bytea, - executed_rcpt_gas_used bigint -) diff --git a/lib/harmony/harmonydb/sql/20240212-common-layers.sql b/lib/harmony/harmonydb/sql/20240212-common-layers.sql deleted file mode 100644 index cf72e1750..000000000 --- a/lib/harmony/harmonydb/sql/20240212-common-layers.sql +++ /dev/null @@ -1,42 +0,0 @@ -INSERT INTO harmony_config (title, config) VALUES - ('post', ' - [Subsystems] - EnableWindowPost = true - EnableWinningPost = true - '), - - ('gui', ' - [Subsystems] - EnableWebGui = true - '), - - ('seal', ' - [Subsystems] - EnableSealSDR = true - EnableSealSDRTrees = true - EnableSendPrecommitMsg = true - EnablePoRepProof = true - EnableSendCommitMsg = true - EnableMoveStorage = true - '), - - ('seal-gpu', ' - [Subsystems] - EnableSealSDRTrees = true - EnableSendPrecommitMsg = true - '), - ('seal-snark', ' - [Subsystems] - EnablePoRepProof = true - EnableSendCommitMsg = true - '), - ('sdr', ' - [Subsystems] - EnableSealSDR = true - '), - - ('storage', ' - [Subsystems] - EnableMoveStorage = true - ') - ON CONFLICT (title) DO NOTHING; -- SPs may have these names defined already. \ No newline at end of file diff --git a/lib/harmony/harmonydb/sql/20240228-piece-park.sql b/lib/harmony/harmonydb/sql/20240228-piece-park.sql deleted file mode 100644 index add0a4093..000000000 --- a/lib/harmony/harmonydb/sql/20240228-piece-park.sql +++ /dev/null @@ -1,37 +0,0 @@ -create table parked_pieces ( - id bigserial primary key, - created_at timestamp default current_timestamp, - - piece_cid text not null, - piece_padded_size bigint not null, - piece_raw_size bigint not null, - - complete boolean not null default false, - task_id bigint default null, - - cleanup_task_id bigint default null, - - -- NOTE: Following keys were dropped in 20240507-sdr-pipeline-fk-drop.sql - foreign key (task_id) references harmony_task (id) on delete set null, -- dropped - foreign key (cleanup_task_id) references harmony_task (id) on delete set null, -- dropped - - unique (piece_cid) -); - -/* - * This table is used to keep track of the references to the parked pieces - * so that we can delete them when they are no longer needed. - * - * All references into the parked_pieces table should be done through this table. - * - * data_url is optional for refs which also act as data sources. - */ -create table parked_piece_refs ( - ref_id bigserial primary key, - piece_id bigint not null, - - data_url text, - data_headers jsonb not null default '{}', - - foreign key (piece_id) references parked_pieces(id) on delete cascade -); diff --git a/lib/harmony/harmonydb/sql/20240317-web-summary-index.sql b/lib/harmony/harmonydb/sql/20240317-web-summary-index.sql deleted file mode 100644 index e1c7e7a79..000000000 --- a/lib/harmony/harmonydb/sql/20240317-web-summary-index.sql +++ /dev/null @@ -1,8 +0,0 @@ -/* Used for webui clusterMachineSummary */ --- NOTE: This index is changed in 20240420-web-task-indexes.sql -CREATE INDEX harmony_task_history_work_index - ON harmony_task_history (completed_by_host_and_port ASC, name ASC, result ASC, work_end DESC); - -/* Used for webui actorSummary sp wins */ -CREATE INDEX mining_tasks_won_sp_id_base_compute_time_index - ON mining_tasks (won ASC, sp_id ASC, base_compute_time DESC); diff --git a/lib/harmony/harmonydb/sql/20240401-storage-miner-filter.sql b/lib/harmony/harmonydb/sql/20240401-storage-miner-filter.sql deleted file mode 100644 index 27cecafe6..000000000 --- a/lib/harmony/harmonydb/sql/20240401-storage-miner-filter.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS allow_miners varchar DEFAULT ''; -ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS deny_miners varchar DEFAULT ''; -- comma separated list of miner addresses \ No newline at end of file diff --git a/lib/harmony/harmonydb/sql/20240402-sdr-pipeline-ddo-deal-info.sql b/lib/harmony/harmonydb/sql/20240402-sdr-pipeline-ddo-deal-info.sql deleted file mode 100644 index 2230ef574..000000000 --- a/lib/harmony/harmonydb/sql/20240402-sdr-pipeline-ddo-deal-info.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE sectors_sdr_initial_pieces - ADD COLUMN direct_start_epoch BIGINT; - -ALTER TABLE sectors_sdr_initial_pieces - ADD COLUMN direct_end_epoch BIGINT; - -ALTER TABLE sectors_sdr_initial_pieces - ADD COLUMN direct_piece_activation_manifest JSONB; diff --git a/lib/harmony/harmonydb/sql/20240404-machine_detail.sql b/lib/harmony/harmonydb/sql/20240404-machine_detail.sql deleted file mode 100644 index ae6de0951..000000000 --- a/lib/harmony/harmonydb/sql/20240404-machine_detail.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE harmony_machine_details ( - id SERIAL PRIMARY KEY, - tasks TEXT, - layers TEXT, - startup_time TIMESTAMP, - miners TEXT, - machine_id INTEGER, - FOREIGN KEY (machine_id) REFERENCES harmony_machines(id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX machine_details_machine_id ON harmony_machine_details(machine_id); - diff --git a/lib/harmony/harmonydb/sql/20240416-harmony_singleton_task.sql b/lib/harmony/harmonydb/sql/20240416-harmony_singleton_task.sql deleted file mode 100644 index d565cfa47..000000000 --- a/lib/harmony/harmonydb/sql/20240416-harmony_singleton_task.sql +++ /dev/null @@ -1,8 +0,0 @@ -create table harmony_task_singletons ( - task_name varchar(255) not null, - task_id bigint, - last_run_time timestamp, - - primary key (task_name), - foreign key (task_id) references harmony_task (id) on delete set null -); diff --git a/lib/harmony/harmonydb/sql/20240417-sector_index_gc.sql b/lib/harmony/harmonydb/sql/20240417-sector_index_gc.sql deleted file mode 100644 index e9771d9f3..000000000 --- a/lib/harmony/harmonydb/sql/20240417-sector_index_gc.sql +++ /dev/null @@ -1,13 +0,0 @@ -create table sector_path_url_liveness ( - storage_id text, - url text, - - last_checked timestamp not null, - last_live timestamp, - last_dead timestamp, - last_dead_reason text, - - primary key (storage_id, url), - - foreign key (storage_id) references storage_path (storage_id) on delete cascade -) diff --git a/lib/harmony/harmonydb/sql/20240420-web-task-indexes.sql b/lib/harmony/harmonydb/sql/20240420-web-task-indexes.sql deleted file mode 100644 index 33eb9e736..000000000 --- a/lib/harmony/harmonydb/sql/20240420-web-task-indexes.sql +++ /dev/null @@ -1,9 +0,0 @@ -DROP INDEX harmony_task_history_work_index; - -/* - This structure improves clusterMachineSummary query better than the old version, - while at the same time also being usable by clusterTaskHistorySummary (which wasn't - the case with the old index). - */ -create index harmony_task_history_work_index - on harmony_task_history (work_end desc, completed_by_host_and_port asc, name asc, result asc); diff --git a/lib/harmony/harmonydb/sql/20240425-sector_meta.sql b/lib/harmony/harmonydb/sql/20240425-sector_meta.sql deleted file mode 100644 index 4a83e0f95..000000000 --- a/lib/harmony/harmonydb/sql/20240425-sector_meta.sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE TABLE sectors_meta ( - sp_id BIGINT NOT NULL, - sector_num BIGINT NOT NULL, - - reg_seal_proof INT NOT NULL, - ticket_epoch BIGINT NOT NULL, - ticket_value BYTEA NOT NULL, - - orig_sealed_cid TEXT NOT NULL, - orig_unsealed_cid TEXT NOT NULL, - - cur_sealed_cid TEXT NOT NULL, - cur_unsealed_cid TEXT NOT NULL, - - msg_cid_precommit TEXT, - msg_cid_commit TEXT, - msg_cid_update TEXT, -- snapdeal update - - seed_epoch BIGINT NOT NULL, - seed_value BYTEA NOT NULL, - - PRIMARY KEY (sp_id, sector_num) -); - -CREATE TABLE sectors_meta_pieces ( - sp_id BIGINT NOT NULL, - sector_num BIGINT NOT NULL, - piece_num BIGINT NOT NULL, - - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, -- padded size - - requested_keep_data BOOLEAN NOT NULL, - raw_data_size BIGINT, -- null = piece_size.unpadded() - - start_epoch BIGINT, - orig_end_epoch BIGINT, - - f05_deal_id BIGINT, - ddo_pam jsonb, - - PRIMARY KEY (sp_id, sector_num, piece_num), - FOREIGN KEY (sp_id, sector_num) REFERENCES sectors_meta(sp_id, sector_num) ON DELETE CASCADE -); diff --git a/lib/harmony/harmonydb/sql/20240501-harmony-indexes.sql b/lib/harmony/harmonydb/sql/20240501-harmony-indexes.sql deleted file mode 100644 index 72a9b669a..000000000 --- a/lib/harmony/harmonydb/sql/20240501-harmony-indexes.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Harmony counts failed tasks by task_id, without this index we'd do a full scan on the history table. -CREATE INDEX harmony_task_history_task_id_result_index - ON harmony_task_history (task_id, result); - diff --git a/lib/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql b/lib/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql deleted file mode 100644 index daf7a4429..000000000 --- a/lib/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql +++ /dev/null @@ -1,12 +0,0 @@ -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_commit_msg_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_finalize_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_move_storage_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_porep_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_precommit_msg_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_sdr_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_c_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_d_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_r_fkey; - -ALTER TABLE parked_pieces DROP CONSTRAINT parked_pieces_cleanup_task_id_fkey; -ALTER TABLE parked_pieces DROP CONSTRAINT parked_pieces_task_id_fkey; diff --git a/lib/harmony/harmonydb/sql/20240508-open-deal-sectors.sql b/lib/harmony/harmonydb/sql/20240508-open-deal-sectors.sql deleted file mode 100644 index 2b1cd7880..000000000 --- a/lib/harmony/harmonydb/sql/20240508-open-deal-sectors.sql +++ /dev/null @@ -1,206 +0,0 @@ -ALTER TABLE sectors_sdr_initial_pieces - ADD COLUMN created_at TIMESTAMP NOT NULL DEFAULT current_timestamp; - -create table open_sector_pieces ( - sp_id bigint not null, - sector_number bigint not null, - - piece_index bigint not null, - piece_cid text not null, - piece_size bigint not null, -- padded size - - -- data source - data_url text not null, - data_headers jsonb not null default '{}', - data_raw_size bigint not null, - data_delete_on_finalize bool not null, - - -- deal info - f05_publish_cid text, - f05_deal_id bigint, - f05_deal_proposal jsonb, - f05_deal_start_epoch bigint, - f05_deal_end_epoch bigint, - - -- ddo deal info - -- added in 20240402-sdr-pipeline-ddo-deal-info.sql - direct_start_epoch bigint, - direct_end_epoch bigint, - direct_piece_activation_manifest jsonb, - - -- created_at added in 20240508-open-deal-sectors.sql - created_at timestamp NOT NULL DEFAULT current_timestamp, - - -- sectors_sdr_initial_pieces table is a copy of this - -- all alters should happen on both tables except constraints - - primary key (sp_id, sector_number, piece_index) -); - - -CREATE OR REPLACE FUNCTION insert_sector_market_piece( - v_sp_id bigint, - v_sector_number bigint, - v_piece_index bigint, - v_piece_cid text, - v_piece_size bigint, - v_data_url text, - v_data_headers jsonb, - v_data_raw_size bigint, - v_data_delete_on_finalize boolean, - v_f05_publish_cid text, - v_f05_deal_id bigint, - v_f05_deal_proposal jsonb, - v_f05_deal_start_epoch bigint, - v_f05_deal_end_epoch bigint -) RETURNS void AS $$ -BEGIN -INSERT INTO open_sector_pieces ( - sp_id, - sector_number, - piece_index, - created_at, - piece_cid, - piece_size, - data_url, - data_headers, - data_raw_size, - data_delete_on_finalize, - f05_publish_cid, - f05_deal_id, - f05_deal_proposal, - f05_deal_start_epoch, - f05_deal_end_epoch -) VALUES ( - v_sp_id, - v_sector_number, - v_piece_index, - NOW(), - v_piece_cid, - v_piece_size, - v_data_url, - v_data_headers, - v_data_raw_size, - v_data_delete_on_finalize, - v_f05_publish_cid, - v_f05_deal_id, - v_f05_deal_proposal, - v_f05_deal_start_epoch, - v_f05_deal_end_epoch - ) ON CONFLICT (sp_id, sector_number, piece_index) DO NOTHING; -IF NOT FOUND THEN - RAISE EXCEPTION 'Conflict detected for piece_index %', v_piece_index; -END IF; -END; -$$ LANGUAGE plpgsql; - - - -CREATE OR REPLACE FUNCTION insert_sector_ddo_piece( - v_sp_id bigint, - v_sector_number bigint, - v_piece_index bigint, - v_piece_cid text, - v_piece_size bigint, - v_data_url text, - v_data_headers jsonb, - v_data_raw_size bigint, - v_data_delete_on_finalize boolean, - v_direct_start_epoch bigint, - v_direct_end_epoch bigint, - v_direct_piece_activation_manifest jsonb -) RETURNS void AS $$ -BEGIN -INSERT INTO open_sector_pieces ( - sp_id, - sector_number, - piece_index, - created_at, - piece_cid, - piece_size, - data_url, - data_headers, - data_raw_size, - data_delete_on_finalize, - direct_start_epoch, - direct_end_epoch, - direct_piece_activation_manifest -) VALUES ( - v_sp_id, - v_sector_number, - v_piece_index, - NOW(), - v_piece_cid, - v_piece_size, - v_data_url, - v_data_headers, - v_data_raw_size, - v_data_delete_on_finalize, - v_direct_start_epoch, - v_direct_end_epoch, - v_direct_piece_activation_manifest - ) ON CONFLICT (sp_id, sector_number, piece_index) DO NOTHING; -IF NOT FOUND THEN - RAISE EXCEPTION 'Conflict detected for piece_index %', v_piece_index; -END IF; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION transfer_and_delete_open_piece(v_sp_id bigint, v_sector_number bigint) -RETURNS void AS $$ -BEGIN - -- Copy data from open_sector_pieces to sectors_sdr_initial_pieces -INSERT INTO sectors_sdr_initial_pieces ( - sp_id, - sector_number, - piece_index, - piece_cid, - piece_size, - data_url, - data_headers, - data_raw_size, - data_delete_on_finalize, - f05_publish_cid, - f05_deal_id, - f05_deal_proposal, - f05_deal_start_epoch, - f05_deal_end_epoch, - direct_start_epoch, - direct_end_epoch, - direct_piece_activation_manifest, - created_at -) -SELECT - sp_id, - sector_number, - piece_index, - piece_cid, - piece_size, - data_url, - data_headers, - data_raw_size, - data_delete_on_finalize, - f05_publish_cid, - f05_deal_id, - f05_deal_proposal, - f05_deal_start_epoch, - f05_deal_end_epoch, - direct_start_epoch, - direct_end_epoch, - direct_piece_activation_manifest, - created_at -FROM - open_sector_pieces -WHERE - sp_id = v_sp_id AND - sector_number = v_sector_number; - --- Check for successful insertion, then delete the corresponding row from open_sector_pieces -IF FOUND THEN -DELETE FROM open_sector_pieces -WHERE sp_id = v_sp_id AND sector_number = v_sector_number; -ELSE - RAISE EXCEPTION 'No data found to transfer for sp_id % and sector_number %', v_sp_id, v_sector_number; -END IF; -END; -$$ LANGUAGE plpgsql; diff --git a/lib/harmony/harmonydb/sql/20240522-ts-to-timestampz.sql b/lib/harmony/harmonydb/sql/20240522-ts-to-timestampz.sql deleted file mode 100644 index 85e2330eb..000000000 --- a/lib/harmony/harmonydb/sql/20240522-ts-to-timestampz.sql +++ /dev/null @@ -1,115 +0,0 @@ --- Convert timestamps in harmony_machines table -ALTER TABLE harmony_machines -ALTER COLUMN last_contact TYPE TIMESTAMPTZ - USING last_contact AT TIME ZONE 'UTC'; - --- Convert timestamps in harmony_task table -ALTER TABLE harmony_task -ALTER COLUMN update_time TYPE TIMESTAMPTZ - USING update_time AT TIME ZONE 'UTC'; - -ALTER TABLE harmony_task -ALTER COLUMN posted_time TYPE TIMESTAMPTZ - USING posted_time AT TIME ZONE 'UTC'; - --- Convert timestamps in harmony_task_history table -ALTER TABLE harmony_task_history -ALTER COLUMN posted TYPE TIMESTAMPTZ - USING posted AT TIME ZONE 'UTC'; - -ALTER TABLE harmony_task_history -ALTER COLUMN work_start TYPE TIMESTAMPTZ - USING work_start AT TIME ZONE 'UTC'; - -ALTER TABLE harmony_task_history -ALTER COLUMN work_end TYPE TIMESTAMPTZ - USING work_end AT TIME ZONE 'UTC'; - --- Convert timestamps in sector_location table -ALTER TABLE sector_location -ALTER COLUMN read_ts TYPE TIMESTAMPTZ - USING read_ts AT TIME ZONE 'UTC'; - -ALTER TABLE sector_location -ALTER COLUMN write_ts TYPE TIMESTAMPTZ - USING write_ts AT TIME ZONE 'UTC'; - --- Convert timestamps in storage_path table -ALTER TABLE storage_path -ALTER COLUMN last_heartbeat TYPE TIMESTAMPTZ - USING last_heartbeat AT TIME ZONE 'UTC'; - --- Convert timestamps in mining_tasks table -ALTER TABLE mining_tasks -ALTER COLUMN base_compute_time TYPE TIMESTAMPTZ - USING base_compute_time AT TIME ZONE 'UTC'; - -ALTER TABLE mining_tasks -ALTER COLUMN mined_at TYPE TIMESTAMPTZ - USING mined_at AT TIME ZONE 'UTC'; - -ALTER TABLE mining_tasks -ALTER COLUMN submitted_at TYPE TIMESTAMPTZ - USING submitted_at AT TIME ZONE 'UTC'; - --- Convert timestamps in itest_scratch table -ALTER TABLE itest_scratch -ALTER COLUMN update_time TYPE TIMESTAMPTZ - USING update_time AT TIME ZONE 'UTC'; - --- Convert timestamps in message_sends table -ALTER TABLE message_sends -ALTER COLUMN send_time TYPE TIMESTAMPTZ - USING send_time AT TIME ZONE 'UTC'; - --- Convert timestamps in message_send_locks table -ALTER TABLE message_send_locks -ALTER COLUMN claimed_at TYPE TIMESTAMPTZ - USING claimed_at AT TIME ZONE 'UTC'; - --- Convert timestamps in harmony_task_singletons table -ALTER TABLE harmony_task_singletons -ALTER COLUMN last_run_time TYPE TIMESTAMPTZ - USING last_run_time AT TIME ZONE 'UTC'; - --- Convert timestamps in sector_path_url_liveness table -ALTER TABLE sector_path_url_liveness -ALTER COLUMN last_checked TYPE TIMESTAMPTZ - USING last_checked AT TIME ZONE 'UTC'; - -ALTER TABLE sector_path_url_liveness -ALTER COLUMN last_live TYPE TIMESTAMPTZ - USING last_live AT TIME ZONE 'UTC'; - -ALTER TABLE sector_path_url_liveness -ALTER COLUMN last_dead TYPE TIMESTAMPTZ - USING last_dead AT TIME ZONE 'UTC'; - --- Convert timestamps in harmony_machine_details table -ALTER TABLE harmony_machine_details -ALTER COLUMN startup_time TYPE TIMESTAMPTZ - USING startup_time AT TIME ZONE 'UTC'; - --- Convert timestamps in open_sector_pieces table -ALTER TABLE open_sector_pieces -ALTER COLUMN created_at TYPE TIMESTAMPTZ - USING created_at AT TIME ZONE 'UTC'; - --- Convert timestamps in parked_pieces table -ALTER TABLE parked_pieces -ALTER COLUMN created_at TYPE TIMESTAMPTZ - USING created_at AT TIME ZONE 'UTC'; - --- Convert timestamps in sectors_sdr_pipeline table -ALTER TABLE sectors_sdr_pipeline -ALTER COLUMN create_time TYPE TIMESTAMPTZ - USING create_time AT TIME ZONE 'UTC'; - -ALTER TABLE sectors_sdr_pipeline -ALTER COLUMN failed_at TYPE TIMESTAMPTZ - USING failed_at AT TIME ZONE 'UTC'; - --- Convert timestamps in sectors_sdr_initial_pieces table -ALTER TABLE sectors_sdr_initial_pieces -ALTER COLUMN created_at TYPE TIMESTAMPTZ - USING created_at AT TIME ZONE 'UTC';