cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Trim harmonydb schema to what's needed here

Browse Source
This commit is contained in:
Łukasz Magiera 2024-05-27 18:52:37 +02:00 committed by Łukasz Magiera
parent 2a4b1ad62b
commit dce7f5d32d
23 changed files with 0 additions and 878 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
lib/harmony/harmonydb/sql/20230719-harmony.sql
View File

@ -1,51 +0,0 @@
/* For HarmonyTask base implementation. */
CREATE TABLE harmony_machines (
id SERIAL PRIMARY KEY NOT NULL,
last_contact TIMESTAMP NOT NULL DEFAULT current_timestamp,
host_and_port varchar(300) NOT NULL,
cpu INTEGER NOT NULL,
ram BIGINT NOT NULL,
gpu FLOAT NOT NULL
);
CREATE TABLE harmony_task (
id SERIAL PRIMARY KEY NOT NULL,
initiated_by INTEGER,
update_time TIMESTAMP NOT NULL DEFAULT current_timestamp,
posted_time TIMESTAMP NOT NULL,
owner_id INTEGER REFERENCES harmony_machines (id) ON DELETE SET NULL,
added_by INTEGER NOT NULL,
previous_task INTEGER,
name varchar(16) NOT NULL
);
COMMENT ON COLUMN harmony_task.initiated_by IS 'The task ID whose completion occasioned this task.';
COMMENT ON COLUMN harmony_task.owner_id IS 'The foreign key to harmony_machines.';
COMMENT ON COLUMN harmony_task.name IS 'The name of the task type.';
COMMENT ON COLUMN harmony_task.owner_id IS 'may be null if between owners or not yet taken';
COMMENT ON COLUMN harmony_task.update_time IS 'When it was last modified. not a heartbeat';
CREATE TABLE harmony_task_history (
id SERIAL PRIMARY KEY NOT NULL,
task_id INTEGER NOT NULL,
name VARCHAR(16) NOT NULL,
posted TIMESTAMP NOT NULL,
work_start TIMESTAMP NOT NULL,
work_end TIMESTAMP NOT NULL,
result BOOLEAN NOT NULL,
err varchar
);
COMMENT ON COLUMN harmony_task_history.result IS 'Use to detemine if this was a successful run.';
CREATE TABLE harmony_task_follow (
id SERIAL PRIMARY KEY NOT NULL,
owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
to_type VARCHAR(16) NOT NULL,
from_type VARCHAR(16) NOT NULL
);
CREATE TABLE harmony_task_impl (
id SERIAL PRIMARY KEY NOT NULL,
owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
name VARCHAR(16) NOT NULL
);

48
lib/harmony/harmonydb/sql/20230823-wdpost.sql
View File

@ -1,48 +0,0 @@
create table wdpost_partition_tasks
(
task_id bigint not null
constraint wdpost_partition_tasks_pk
primary key,
sp_id bigint not null,
proving_period_start bigint not null,
deadline_index bigint not null,
partition_index bigint not null,
constraint wdpost_partition_tasks_identity_key
unique (sp_id, proving_period_start, deadline_index, partition_index)
);
comment on column wdpost_partition_tasks.task_id is 'harmonytask task ID';
comment on column wdpost_partition_tasks.sp_id is 'storage provider ID';
comment on column wdpost_partition_tasks.proving_period_start is 'proving period start';
comment on column wdpost_partition_tasks.deadline_index is 'deadline index within the proving period';
comment on column wdpost_partition_tasks.partition_index is 'partition index within the deadline';
create table wdpost_proofs
(
sp_id bigint not null,
proving_period_start bigint not null,
deadline bigint not null,
partition bigint not null,
submit_at_epoch bigint not null,
submit_by_epoch bigint not null,
proof_params bytea,
submit_task_id bigint,
message_cid text,
constraint wdpost_proofs_identity_key
unique (sp_id, proving_period_start, deadline, partition)
);
create table wdpost_recovery_tasks
(
task_id bigint not null
constraint wdpost_recovery_tasks_pk
primary key,
sp_id bigint not null,
proving_period_start bigint not null,
deadline_index bigint not null,
partition_index bigint not null,
constraint wdpost_recovery_tasks_identity_key
unique (sp_id, proving_period_start, deadline_index, partition_index)
);

5
lib/harmony/harmonydb/sql/20230919-config.sql
View File

@ -1,5 +0,0 @@
CREATE TABLE harmony_config (
id SERIAL PRIMARY KEY NOT NULL,
title VARCHAR(300) UNIQUE NOT NULL,
config TEXT NOT NULL
);

55
lib/harmony/harmonydb/sql/20231103-chain_sends.sql
View File

@ -1,55 +0,0 @@
create table message_sends
(
from_key text not null,
to_addr text not null,
send_reason text not null,
send_task_id bigint not null,
unsigned_data bytea not null,
unsigned_cid text not null,
nonce bigint,
signed_data bytea,
signed_json jsonb,
signed_cid text,
send_time timestamp default null,
send_success boolean default null,
send_error text,
constraint message_sends_pk
primary key (send_task_id, from_key)
);
comment on column message_sends.from_key is 'text f[1/3/4]... address';
comment on column message_sends.to_addr is 'text f[0/1/2/3/4]... address';
comment on column message_sends.send_reason is 'optional description of send reason';
comment on column message_sends.send_task_id is 'harmony task id of the send task';
comment on column message_sends.unsigned_data is 'unsigned message data';
comment on column message_sends.unsigned_cid is 'unsigned message cid';
comment on column message_sends.nonce is 'assigned message nonce, set while the send task is executing';
comment on column message_sends.signed_data is 'signed message data, set while the send task is executing';
comment on column message_sends.signed_cid is 'signed message cid, set while the send task is executing';
comment on column message_sends.send_time is 'time when the send task was executed, set after pushing the message to the network';
comment on column message_sends.send_success is 'whether this message was broadcasted to the network already, null if not yet attempted, true if successful, false if failed';
comment on column message_sends.send_error is 'error message if send_success is false';
create unique index message_sends_success_index
on message_sends (from_key, nonce)
where send_success is not false;
comment on index message_sends_success_index is
'message_sends_success_index enforces sender/nonce uniqueness, it is a conditional index that only indexes rows where send_success is not false. This allows us to have multiple rows with the same sender/nonce, as long as only one of them was successfully broadcasted (true) to the network or is in the process of being broadcasted (null).';
create table message_send_locks
(
from_key text not null,
task_id bigint not null,
claimed_at timestamp not null,
constraint message_send_locks_pk
primary key (from_key)
);

39
lib/harmony/harmonydb/sql/20231110-mining_tasks.sql
View File

@ -1,39 +0,0 @@
create table mining_tasks
(
task_id bigint not null
constraint mining_tasks_pk
primary key,
sp_id bigint not null,
epoch bigint not null,
base_compute_time timestamp not null,
won bool not null default false,
mined_cid text,
mined_header jsonb,
mined_at timestamp,
submitted_at timestamp,
constraint mining_tasks_sp_epoch
unique (sp_id, epoch)
);
create table mining_base_block
(
id bigserial not null
constraint mining_base_block_pk
primary key,
task_id bigint not null
constraint mining_base_block_mining_tasks_task_id_fk
references mining_tasks
on delete cascade,
sp_id bigint,
block_cid text not null,
no_win bool not null default false,
constraint mining_base_block_pk2
unique (sp_id, task_id, block_cid)
);
CREATE UNIQUE INDEX mining_base_block_cid_k ON mining_base_block (sp_id, block_cid) WHERE no_win = false;

1
lib/harmony/harmonydb/sql/20231113-harmony_taskhistory_oops.sql
View File

@ -1 +0,0 @@
ALTER TABLE harmony_task_history ADD COLUMN completed_by_host_and_port varchar(300) NOT NULL;

8
lib/harmony/harmonydb/sql/20231120-testing1.sql
View File

@ -1,8 +0,0 @@
CREATE TABLE harmony_test (
task_id bigint
constraint harmony_test_pk
primary key,
options text,
result text
);
ALTER TABLE wdpost_proofs ADD COLUMN test_task_id bigint;

138
lib/harmony/harmonydb/sql/20231217-sdr-pipeline.sql
View File

@ -1,138 +0,0 @@
-- NOTE: task_ids can be the same between different task types and between different sectors
-- e.g. SN-supraseal doing 128 sdr/TreeC/TreeR with the same task_id
create table sectors_sdr_pipeline (
sp_id bigint not null,
sector_number bigint not null,
-- at request time
create_time timestamp not null default current_timestamp,
reg_seal_proof int not null,
-- sdr
ticket_epoch bigint,
ticket_value bytea,
task_id_sdr bigint,
after_sdr bool not null default false,
-- tree D
tree_d_cid text, -- commd from treeD compute, should match comm_d_cid
task_id_tree_d bigint,
after_tree_d bool not null default false,
-- tree C
task_id_tree_c bigint,
after_tree_c bool not null default false,
-- tree R
tree_r_cid text, -- commr from treeR compute
task_id_tree_r bigint,
after_tree_r bool not null default false,
-- precommit message sending
precommit_msg_cid text,
task_id_precommit_msg bigint,
after_precommit_msg bool not null default false,
-- precommit message wait
seed_epoch bigint,
precommit_msg_tsk bytea,
after_precommit_msg_success bool not null default false,
-- seed
seed_value bytea,
-- Commit (PoRep snark)
task_id_porep bigint,
porep_proof bytea,
after_porep bool not null default false,
-- Finalize (trim cache)
task_id_finalize bigint,
after_finalize bool not null default false,
-- MoveStorage (move data to storage)
task_id_move_storage bigint,
after_move_storage bool not null default false,
-- Commit message sending
commit_msg_cid text,
task_id_commit_msg bigint,
after_commit_msg bool not null default false,
-- Commit message wait
commit_msg_tsk bytea,
after_commit_msg_success bool not null default false,
-- Failure handling
failed bool not null default false,
failed_at timestamp,
failed_reason varchar(20) not null default '',
failed_reason_msg text not null default '',
-- foreign key
-- NOTE: Following keys were dropped in 20240507-sdr-pipeline-fk-drop.sql
foreign key (task_id_sdr) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_tree_d) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_tree_c) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_tree_r) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_precommit_msg) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_porep) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_finalize) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_move_storage) references harmony_task (id) on delete set null, -- dropped
foreign key (task_id_commit_msg) references harmony_task (id) on delete set null, -- dropped
-- constraints
primary key (sp_id, sector_number)
);
create table sectors_sdr_initial_pieces (
sp_id bigint not null,
sector_number bigint not null,
piece_index bigint not null,
piece_cid text not null,
piece_size bigint not null, -- padded size
-- data source
data_url text not null,
data_headers jsonb not null default '{}',
data_raw_size bigint not null,
data_delete_on_finalize bool not null,
-- deal info
f05_publish_cid text,
f05_deal_id bigint,
f05_deal_proposal jsonb,
f05_deal_start_epoch bigint,
f05_deal_end_epoch bigint,
-- ddo deal info
-- added in 20240402-sdr-pipeline-ddo-deal-info.sql
-- direct_start_epoch bigint,
-- direct_end_epoch bigint,
-- direct_piece_activation_manifest jsonb,
-- created_at added in 20240508-open-deal-sectors.sql
-- created_at timestamp,
-- open_sector_pieces table is a copy of this
-- all alters should happen on both tables except constraints
-- foreign key
foreign key (sp_id, sector_number) references sectors_sdr_pipeline (sp_id, sector_number) on delete cascade,
primary key (sp_id, sector_number, piece_index)
);
comment on column sectors_sdr_initial_pieces.piece_size is 'padded size of the piece';
create table sectors_allocated_numbers (
sp_id bigint not null primary key,
allocated jsonb not null
);

13
lib/harmony/harmonydb/sql/20231225-message-waits.sql
View File

@ -1,13 +0,0 @@
create table message_waits (
signed_message_cid text primary key,
waiter_machine_id int references harmony_machines (id) on delete set null,
executed_tsk_cid text,
executed_tsk_epoch bigint,
executed_msg_cid text,
executed_msg_data jsonb,
executed_rcpt_exitcode bigint,
executed_rcpt_return bytea,
executed_rcpt_gas_used bigint
)

42
lib/harmony/harmonydb/sql/20240212-common-layers.sql
View File

@ -1,42 +0,0 @@
INSERT INTO harmony_config (title, config) VALUES
('post', '
[Subsystems]
EnableWindowPost = true
EnableWinningPost = true
'),
('gui', '
[Subsystems]
EnableWebGui = true
'),
('seal', '
[Subsystems]
EnableSealSDR = true
EnableSealSDRTrees = true
EnableSendPrecommitMsg = true
EnablePoRepProof = true
EnableSendCommitMsg = true
EnableMoveStorage = true
'),
('seal-gpu', '
[Subsystems]
EnableSealSDRTrees = true
EnableSendPrecommitMsg = true
'),
('seal-snark', '
[Subsystems]
EnablePoRepProof = true
EnableSendCommitMsg = true
'),
('sdr', '
[Subsystems]
EnableSealSDR = true
'),
('storage', '
[Subsystems]
EnableMoveStorage = true
')
ON CONFLICT (title) DO NOTHING; -- SPs may have these names defined already.

37
lib/harmony/harmonydb/sql/20240228-piece-park.sql
View File

@ -1,37 +0,0 @@
create table parked_pieces (
id bigserial primary key,
created_at timestamp default current_timestamp,
piece_cid text not null,
piece_padded_size bigint not null,
piece_raw_size bigint not null,
complete boolean not null default false,
task_id bigint default null,
cleanup_task_id bigint default null,
-- NOTE: Following keys were dropped in 20240507-sdr-pipeline-fk-drop.sql
foreign key (task_id) references harmony_task (id) on delete set null, -- dropped
foreign key (cleanup_task_id) references harmony_task (id) on delete set null, -- dropped
unique (piece_cid)
);
/*
* This table is used to keep track of the references to the parked pieces
* so that we can delete them when they are no longer needed.
*
* All references into the parked_pieces table should be done through this table.
*
* data_url is optional for refs which also act as data sources.
*/
create table parked_piece_refs (
ref_id bigserial primary key,
piece_id bigint not null,
data_url text,
data_headers jsonb not null default '{}',
foreign key (piece_id) references parked_pieces(id) on delete cascade
);

8
lib/harmony/harmonydb/sql/20240317-web-summary-index.sql
View File

@ -1,8 +0,0 @@
/* Used for webui clusterMachineSummary */
-- NOTE: This index is changed in 20240420-web-task-indexes.sql
CREATE INDEX harmony_task_history_work_index
ON harmony_task_history (completed_by_host_and_port ASC, name ASC, result ASC, work_end DESC);
/* Used for webui actorSummary sp wins */
CREATE INDEX mining_tasks_won_sp_id_base_compute_time_index
ON mining_tasks (won ASC, sp_id ASC, base_compute_time DESC);

2
lib/harmony/harmonydb/sql/20240401-storage-miner-filter.sql
View File

@ -1,2 +0,0 @@
ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS allow_miners varchar DEFAULT '';
ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS deny_miners varchar DEFAULT ''; -- comma separated list of miner addresses

8
lib/harmony/harmonydb/sql/20240402-sdr-pipeline-ddo-deal-info.sql
View File

@ -1,8 +0,0 @@
ALTER TABLE sectors_sdr_initial_pieces
ADD COLUMN direct_start_epoch BIGINT;
ALTER TABLE sectors_sdr_initial_pieces
ADD COLUMN direct_end_epoch BIGINT;
ALTER TABLE sectors_sdr_initial_pieces
ADD COLUMN direct_piece_activation_manifest JSONB;

12
lib/harmony/harmonydb/sql/20240404-machine_detail.sql
View File

@ -1,12 +0,0 @@
CREATE TABLE harmony_machine_details (
id SERIAL PRIMARY KEY,
tasks TEXT,
layers TEXT,
startup_time TIMESTAMP,
miners TEXT,
machine_id INTEGER,
FOREIGN KEY (machine_id) REFERENCES harmony_machines(id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX machine_details_machine_id ON harmony_machine_details(machine_id);

8
lib/harmony/harmonydb/sql/20240416-harmony_singleton_task.sql
View File

@ -1,8 +0,0 @@
create table harmony_task_singletons (
task_name varchar(255) not null,
task_id bigint,
last_run_time timestamp,
primary key (task_name),
foreign key (task_id) references harmony_task (id) on delete set null
);

13
lib/harmony/harmonydb/sql/20240417-sector_index_gc.sql
View File

@ -1,13 +0,0 @@
create table sector_path_url_liveness (
storage_id text,
url text,
last_checked timestamp not null,
last_live timestamp,
last_dead timestamp,
last_dead_reason text,
primary key (storage_id, url),
foreign key (storage_id) references storage_path (storage_id) on delete cascade
)

9
lib/harmony/harmonydb/sql/20240420-web-task-indexes.sql
View File

@ -1,9 +0,0 @@
DROP INDEX harmony_task_history_work_index;
/*
This structure improves clusterMachineSummary query better than the old version,
while at the same time also being usable by clusterTaskHistorySummary (which wasn't
the case with the old index).
*/
create index harmony_task_history_work_index
on harmony_task_history (work_end desc, completed_by_host_and_port asc, name asc, result asc);

44
lib/harmony/harmonydb/sql/20240425-sector_meta.sql
View File

@ -1,44 +0,0 @@
CREATE TABLE sectors_meta (
sp_id BIGINT NOT NULL,
sector_num BIGINT NOT NULL,
reg_seal_proof INT NOT NULL,
ticket_epoch BIGINT NOT NULL,
ticket_value BYTEA NOT NULL,
orig_sealed_cid TEXT NOT NULL,
orig_unsealed_cid TEXT NOT NULL,
cur_sealed_cid TEXT NOT NULL,
cur_unsealed_cid TEXT NOT NULL,
msg_cid_precommit TEXT,
msg_cid_commit TEXT,
msg_cid_update TEXT, -- snapdeal update
seed_epoch BIGINT NOT NULL,
seed_value BYTEA NOT NULL,
PRIMARY KEY (sp_id, sector_num)
);
CREATE TABLE sectors_meta_pieces (
sp_id BIGINT NOT NULL,
sector_num BIGINT NOT NULL,
piece_num BIGINT NOT NULL,
piece_cid TEXT NOT NULL,
piece_size BIGINT NOT NULL, -- padded size
requested_keep_data BOOLEAN NOT NULL,
raw_data_size BIGINT, -- null = piece_size.unpadded()
start_epoch BIGINT,
orig_end_epoch BIGINT,
f05_deal_id BIGINT,
ddo_pam jsonb,
PRIMARY KEY (sp_id, sector_num, piece_num),
FOREIGN KEY (sp_id, sector_num) REFERENCES sectors_meta(sp_id, sector_num) ON DELETE CASCADE
);

4
lib/harmony/harmonydb/sql/20240501-harmony-indexes.sql
View File

@ -1,4 +0,0 @@
-- Harmony counts failed tasks by task_id, without this index we'd do a full scan on the history table.
CREATE INDEX harmony_task_history_task_id_result_index
ON harmony_task_history (task_id, result);

12
lib/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql
View File

@ -1,12 +0,0 @@
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_commit_msg_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_finalize_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_move_storage_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_porep_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_precommit_msg_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_sdr_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_c_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_d_fkey;
ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_r_fkey;
ALTER TABLE parked_pieces DROP CONSTRAINT parked_pieces_cleanup_task_id_fkey;
ALTER TABLE parked_pieces DROP CONSTRAINT parked_pieces_task_id_fkey;

206
lib/harmony/harmonydb/sql/20240508-open-deal-sectors.sql
View File

@ -1,206 +0,0 @@
ALTER TABLE sectors_sdr_initial_pieces
ADD COLUMN created_at TIMESTAMP NOT NULL DEFAULT current_timestamp;
create table open_sector_pieces (
sp_id bigint not null,
sector_number bigint not null,
piece_index bigint not null,
piece_cid text not null,
piece_size bigint not null, -- padded size
-- data source
data_url text not null,
data_headers jsonb not null default '{}',
data_raw_size bigint not null,
data_delete_on_finalize bool not null,
-- deal info
f05_publish_cid text,
f05_deal_id bigint,
f05_deal_proposal jsonb,
f05_deal_start_epoch bigint,
f05_deal_end_epoch bigint,
-- ddo deal info
-- added in 20240402-sdr-pipeline-ddo-deal-info.sql
direct_start_epoch bigint,
direct_end_epoch bigint,
direct_piece_activation_manifest jsonb,
-- created_at added in 20240508-open-deal-sectors.sql
created_at timestamp NOT NULL DEFAULT current_timestamp,
-- sectors_sdr_initial_pieces table is a copy of this
-- all alters should happen on both tables except constraints
primary key (sp_id, sector_number, piece_index)
);
CREATE OR REPLACE FUNCTION insert_sector_market_piece(
v_sp_id bigint,
v_sector_number bigint,
v_piece_index bigint,
v_piece_cid text,
v_piece_size bigint,
v_data_url text,
v_data_headers jsonb,
v_data_raw_size bigint,
v_data_delete_on_finalize boolean,
v_f05_publish_cid text,
v_f05_deal_id bigint,
v_f05_deal_proposal jsonb,
v_f05_deal_start_epoch bigint,
v_f05_deal_end_epoch bigint
) RETURNS void AS $$
BEGIN
INSERT INTO open_sector_pieces (
sp_id,
sector_number,
piece_index,
created_at,
piece_cid,
piece_size,
data_url,
data_headers,
data_raw_size,
data_delete_on_finalize,
f05_publish_cid,
f05_deal_id,
f05_deal_proposal,
f05_deal_start_epoch,
f05_deal_end_epoch
) VALUES (
v_sp_id,
v_sector_number,
v_piece_index,
NOW(),
v_piece_cid,
v_piece_size,
v_data_url,
v_data_headers,
v_data_raw_size,
v_data_delete_on_finalize,
v_f05_publish_cid,
v_f05_deal_id,
v_f05_deal_proposal,
v_f05_deal_start_epoch,
v_f05_deal_end_epoch
) ON CONFLICT (sp_id, sector_number, piece_index) DO NOTHING;
IF NOT FOUND THEN
RAISE EXCEPTION 'Conflict detected for piece_index %', v_piece_index;
END IF;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION insert_sector_ddo_piece(
v_sp_id bigint,
v_sector_number bigint,
v_piece_index bigint,
v_piece_cid text,
v_piece_size bigint,
v_data_url text,
v_data_headers jsonb,
v_data_raw_size bigint,
v_data_delete_on_finalize boolean,
v_direct_start_epoch bigint,
v_direct_end_epoch bigint,
v_direct_piece_activation_manifest jsonb
) RETURNS void AS $$
BEGIN
INSERT INTO open_sector_pieces (
sp_id,
sector_number,
piece_index,
created_at,
piece_cid,
piece_size,
data_url,
data_headers,
data_raw_size,
data_delete_on_finalize,
direct_start_epoch,
direct_end_epoch,
direct_piece_activation_manifest
) VALUES (
v_sp_id,
v_sector_number,
v_piece_index,
NOW(),
v_piece_cid,
v_piece_size,
v_data_url,
v_data_headers,
v_data_raw_size,
v_data_delete_on_finalize,
v_direct_start_epoch,
v_direct_end_epoch,
v_direct_piece_activation_manifest
) ON CONFLICT (sp_id, sector_number, piece_index) DO NOTHING;
IF NOT FOUND THEN
RAISE EXCEPTION 'Conflict detected for piece_index %', v_piece_index;
END IF;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION transfer_and_delete_open_piece(v_sp_id bigint, v_sector_number bigint)
RETURNS void AS $$
BEGIN
-- Copy data from open_sector_pieces to sectors_sdr_initial_pieces
INSERT INTO sectors_sdr_initial_pieces (
sp_id,
sector_number,
piece_index,
piece_cid,
piece_size,
data_url,
data_headers,
data_raw_size,
data_delete_on_finalize,
f05_publish_cid,
f05_deal_id,
f05_deal_proposal,
f05_deal_start_epoch,
f05_deal_end_epoch,
direct_start_epoch,
direct_end_epoch,
direct_piece_activation_manifest,
created_at
)
SELECT
sp_id,
sector_number,
piece_index,
piece_cid,
piece_size,
data_url,
data_headers,
data_raw_size,
data_delete_on_finalize,
f05_publish_cid,
f05_deal_id,
f05_deal_proposal,
f05_deal_start_epoch,
f05_deal_end_epoch,
direct_start_epoch,
direct_end_epoch,
direct_piece_activation_manifest,
created_at
FROM
open_sector_pieces
WHERE
sp_id = v_sp_id AND
sector_number = v_sector_number;
-- Check for successful insertion, then delete the corresponding row from open_sector_pieces
IF FOUND THEN
DELETE FROM open_sector_pieces
WHERE sp_id = v_sp_id AND sector_number = v_sector_number;
ELSE
RAISE EXCEPTION 'No data found to transfer for sp_id % and sector_number %', v_sp_id, v_sector_number;
END IF;
END;
$$ LANGUAGE plpgsql;

115
lib/harmony/harmonydb/sql/20240522-ts-to-timestampz.sql
View File

@ -1,115 +0,0 @@
-- Convert timestamps in harmony_machines table
ALTER TABLE harmony_machines
ALTER COLUMN last_contact TYPE TIMESTAMPTZ
USING last_contact AT TIME ZONE 'UTC';
-- Convert timestamps in harmony_task table
ALTER TABLE harmony_task
ALTER COLUMN update_time TYPE TIMESTAMPTZ
USING update_time AT TIME ZONE 'UTC';
ALTER TABLE harmony_task
ALTER COLUMN posted_time TYPE TIMESTAMPTZ
USING posted_time AT TIME ZONE 'UTC';
-- Convert timestamps in harmony_task_history table
ALTER TABLE harmony_task_history
ALTER COLUMN posted TYPE TIMESTAMPTZ
USING posted AT TIME ZONE 'UTC';
ALTER TABLE harmony_task_history
ALTER COLUMN work_start TYPE TIMESTAMPTZ
USING work_start AT TIME ZONE 'UTC';
ALTER TABLE harmony_task_history
ALTER COLUMN work_end TYPE TIMESTAMPTZ
USING work_end AT TIME ZONE 'UTC';
-- Convert timestamps in sector_location table
ALTER TABLE sector_location
ALTER COLUMN read_ts TYPE TIMESTAMPTZ
USING read_ts AT TIME ZONE 'UTC';
ALTER TABLE sector_location
ALTER COLUMN write_ts TYPE TIMESTAMPTZ
USING write_ts AT TIME ZONE 'UTC';
-- Convert timestamps in storage_path table
ALTER TABLE storage_path
ALTER COLUMN last_heartbeat TYPE TIMESTAMPTZ
USING last_heartbeat AT TIME ZONE 'UTC';
-- Convert timestamps in mining_tasks table
ALTER TABLE mining_tasks
ALTER COLUMN base_compute_time TYPE TIMESTAMPTZ
USING base_compute_time AT TIME ZONE 'UTC';
ALTER TABLE mining_tasks
ALTER COLUMN mined_at TYPE TIMESTAMPTZ
USING mined_at AT TIME ZONE 'UTC';
ALTER TABLE mining_tasks
ALTER COLUMN submitted_at TYPE TIMESTAMPTZ
USING submitted_at AT TIME ZONE 'UTC';
-- Convert timestamps in itest_scratch table
ALTER TABLE itest_scratch
ALTER COLUMN update_time TYPE TIMESTAMPTZ
USING update_time AT TIME ZONE 'UTC';
-- Convert timestamps in message_sends table
ALTER TABLE message_sends
ALTER COLUMN send_time TYPE TIMESTAMPTZ
USING send_time AT TIME ZONE 'UTC';
-- Convert timestamps in message_send_locks table
ALTER TABLE message_send_locks
ALTER COLUMN claimed_at TYPE TIMESTAMPTZ
USING claimed_at AT TIME ZONE 'UTC';
-- Convert timestamps in harmony_task_singletons table
ALTER TABLE harmony_task_singletons
ALTER COLUMN last_run_time TYPE TIMESTAMPTZ
USING last_run_time AT TIME ZONE 'UTC';
-- Convert timestamps in sector_path_url_liveness table
ALTER TABLE sector_path_url_liveness
ALTER COLUMN last_checked TYPE TIMESTAMPTZ
USING last_checked AT TIME ZONE 'UTC';
ALTER TABLE sector_path_url_liveness
ALTER COLUMN last_live TYPE TIMESTAMPTZ
USING last_live AT TIME ZONE 'UTC';
ALTER TABLE sector_path_url_liveness
ALTER COLUMN last_dead TYPE TIMESTAMPTZ
USING last_dead AT TIME ZONE 'UTC';
-- Convert timestamps in harmony_machine_details table
ALTER TABLE harmony_machine_details
ALTER COLUMN startup_time TYPE TIMESTAMPTZ
USING startup_time AT TIME ZONE 'UTC';
-- Convert timestamps in open_sector_pieces table
ALTER TABLE open_sector_pieces
ALTER COLUMN created_at TYPE TIMESTAMPTZ
USING created_at AT TIME ZONE 'UTC';
-- Convert timestamps in parked_pieces table
ALTER TABLE parked_pieces
ALTER COLUMN created_at TYPE TIMESTAMPTZ
USING created_at AT TIME ZONE 'UTC';
-- Convert timestamps in sectors_sdr_pipeline table
ALTER TABLE sectors_sdr_pipeline
ALTER COLUMN create_time TYPE TIMESTAMPTZ
USING create_time AT TIME ZONE 'UTC';
ALTER TABLE sectors_sdr_pipeline
ALTER COLUMN failed_at TYPE TIMESTAMPTZ
USING failed_at AT TIME ZONE 'UTC';
-- Convert timestamps in sectors_sdr_initial_pieces table
ALTER TABLE sectors_sdr_initial_pieces
ALTER COLUMN created_at TYPE TIMESTAMPTZ
USING created_at AT TIME ZONE 'UTC';