Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[batch] A procedure to rename job_groups_cancelled.id -> job_groups_cancelled.batch_id #14672

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+551 −27
Open

[batch] A procedure to rename job_groups_cancelled.id -> job_groups_cancelled.batch_id #14672

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
09f081c
A procedure for renaming a column (non-primary) in job_groups_cancell…
ichengchang Sep 6, 2024
45e0049
Added rename-job-groups-cancelled-column script to batch migration. R…
ichengchang Sep 6, 2024
fec05d4
revert files
ichengchang Sep 9, 2024
64dcea3
Recreate triggers and stored procedures after column name change.
ichengchang Sep 10, 2024
c6e3c66
Replaced job_groups_cancelled.id by job_groups_cancelled.batch_id, in…
ichengchang Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion batch/batch/driver/instance_collection/job_private.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ async def user_runnable_jobs(user, remaining) -> AsyncIterator[Dict[str, Any]]:
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down
4 changes: 2 additions & 2 deletions batch/batch/driver/instance_collection/pool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ async def regions_to_ready_cores_mcpu_from_estimated_job_queue(self) -> List[Tup
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE jobs.batch_id = job_group_self_and_ancestors.batch_id AND
jobs.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down Expand Up @@ -622,7 +622,7 @@ async def user_runnable_jobs(user):
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down
4 changes: 2 additions & 2 deletions batch/batch/driver/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1034,7 +1034,7 @@ async def check(tx):
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down Expand Up @@ -1312,7 +1312,7 @@ async def cancel_fast_failing_job_groups(app):
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down
6 changes: 3 additions & 3 deletions batch/batch/front_end/front_end.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -911,7 +911,7 @@ async def _create_job_group(
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_group_self_and_ancestors.batch_id = %s AND job_group_self_and_ancestors.job_group_id = %s;
""",
Expand Down Expand Up @@ -1984,7 +1984,7 @@ async def _get_job_group(app, batch_id: int, job_group_id: int) -> GetJobGroupRe
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down Expand Up @@ -2086,7 +2086,7 @@ async def close_batch(request, userdata):
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down
4 changes: 2 additions & 2 deletions batch/batch/front_end/query/query_v1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def parse_list_batches_query_v1(user: str, q: str, last_batch_id: Optional[int])
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down Expand Up @@ -171,7 +171,7 @@ def parse_list_job_groups_query_v1(
SELECT 1 AS cancelled
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE job_groups.batch_id = job_group_self_and_ancestors.batch_id AND
job_groups.job_group_id = job_group_self_and_ancestors.job_group_id
Expand Down
18 changes: 9 additions & 9 deletions batch/sql/estimated-current.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,7 +612,7 @@ BEGIN

SET job_group_cancelled = EXISTS (SELECT TRUE
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE batch_id = NEW.batch_id AND job_group_self_and_ancestors.job_group_id = NEW.job_group_id
LOCK IN SHARE MODE);
Expand Down Expand Up @@ -671,7 +671,7 @@ BEGIN

SET cur_job_group_cancelled = EXISTS (SELECT TRUE
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE batch_id = OLD.batch_id AND job_group_self_and_ancestors.job_group_id = OLD.job_group_id
LOCK IN SHARE MODE);
Expand Down Expand Up @@ -1107,7 +1107,7 @@ BEGIN

SET cur_cancelled = EXISTS (SELECT TRUE
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE batch_id = in_batch_id AND job_group_self_and_ancestors.job_group_id = in_job_group_id
FOR UPDATE);
Expand Down Expand Up @@ -1256,10 +1256,10 @@ BEGIN
WHERE batch_id = in_batch_id AND job_id = in_job_id
FOR UPDATE;

SELECT (jobs.cancelled OR job_groups_cancelled.id IS NOT NULL) AND NOT jobs.always_run
SELECT (jobs.cancelled OR job_groups_cancelled.batch_id IS NOT NULL) AND NOT jobs.always_run
INTO cur_job_cancel
FROM jobs
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.id = jobs.batch_id
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.batch_id = jobs.batch_id
WHERE batch_id = in_batch_id AND job_id = in_job_id
LOCK IN SHARE MODE;

Expand Down Expand Up @@ -1376,10 +1376,10 @@ BEGIN
WHERE batch_id = in_batch_id AND job_id = in_job_id
FOR UPDATE;

SELECT (jobs.cancelled OR job_groups_cancelled.id IS NOT NULL) AND NOT jobs.always_run
SELECT (jobs.cancelled OR job_groups_cancelled.batch_id IS NOT NULL) AND NOT jobs.always_run
INTO cur_job_cancel
FROM jobs
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.id = jobs.batch_id
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.batch_id = jobs.batch_id
WHERE batch_id = in_batch_id AND job_id = in_job_id
LOCK IN SHARE MODE;

Expand Down Expand Up @@ -1421,10 +1421,10 @@ BEGIN
WHERE batch_id = in_batch_id AND job_id = in_job_id
FOR UPDATE;

SELECT (jobs.cancelled OR job_groups_cancelled.id IS NOT NULL) AND NOT jobs.always_run
SELECT (jobs.cancelled OR job_groups_cancelled.batch_id IS NOT NULL) AND NOT jobs.always_run
INTO cur_job_cancel
FROM jobs
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.id = jobs.batch_id
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.batch_id = jobs.batch_id
WHERE batch_id = in_batch_id AND job_id = in_job_id
LOCK IN SHARE MODE;

Expand Down
6 changes: 3 additions & 3 deletions batch/sql/finalize-job-groups.sql
View file
Open in desktop
ichengchang marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ BEGIN

SET job_group_cancelled = EXISTS (SELECT TRUE
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE batch_id = NEW.batch_id AND job_group_self_and_ancestors.job_group_id = NEW.job_group_id
LOCK IN SHARE MODE);
Expand Down Expand Up @@ -142,7 +142,7 @@ BEGIN

SET cur_job_group_cancelled = EXISTS (SELECT TRUE
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE batch_id = OLD.batch_id AND job_group_self_and_ancestors.job_group_id = OLD.job_group_id
LOCK IN SHARE MODE);
Expand Down Expand Up @@ -318,7 +318,7 @@ BEGIN

SET cur_cancelled = EXISTS (SELECT TRUE
FROM job_group_self_and_ancestors
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.id AND
INNER JOIN job_groups_cancelled ON job_group_self_and_ancestors.batch_id = job_groups_cancelled.batch_id AND
job_group_self_and_ancestors.ancestor_id = job_groups_cancelled.job_group_id
WHERE batch_id = in_batch_id AND job_group_self_and_ancestors.job_group_id = in_job_group_id
FOR UPDATE);
Expand Down
27 changes: 27 additions & 0 deletions batch/sql/rename-job-groups-cancelled-column.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
mysql> SELECT * FROM INFORMATION_SCHEMA.INNODB_SYS_FOREIGN \G

(mysql 8.x or above)
mysql> SELECT * FROM INFORMATION_SCHEMA.INNODB_FOREIGN \G

*************************** XXX row ***************************
ID: batches/job_groups_cancelled_ibfk_1
FOR_NAME: batches/job_groups_cancelled
REF_NAME: batches/batches
N_COLS: 1
TYPE: 33
*************************** YYY row ***************************
ID: batches/job_groups_cancelled_ibfk_2
FOR_NAME: batches/job_groups_cancelled
REF_NAME: batches/job_groups
N_COLS: 2
TYPE: 33
*/

ALTER TABLE job_groups_cancelled DROP FOREIGN KEY job_groups_cancelled_ibfk_1;

Check failure on line 21 in batch/sql/rename-job-groups-cancelled-column.sql

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

batch/sql/rename-job-groups-cancelled-column.sql#L21 

syntax error at or near "FOREIGN"
ALTER TABLE job_groups_cancelled DROP FOREIGN KEY job_groups_cancelled_ibfk_2;

Check failure on line 22 in batch/sql/rename-job-groups-cancelled-column.sql

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

batch/sql/rename-job-groups-cancelled-column.sql#L22 

syntax error at or near "FOREIGN"
ALTER TABLE job_groups_cancelled DROP PRIMARY KEY;

Check failure on line 23 in batch/sql/rename-job-groups-cancelled-column.sql

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

batch/sql/rename-job-groups-cancelled-column.sql#L23 

syntax error at or near "PRIMARY"
ALTER TABLE job_groups_cancelled CHANGE COLUMN `id` `batch_id` BIGINT NOT NULL;

Check failure on line 24 in batch/sql/rename-job-groups-cancelled-column.sql

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

batch/sql/rename-job-groups-cancelled-column.sql#L24 

syntax error at or near "CHANGE"
ALTER TABLE job_groups_cancelled ADD PRIMARY KEY (`batch_id`, `job_group_id`),

Check failure on line 25 in batch/sql/rename-job-groups-cancelled-column.sql

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

batch/sql/rename-job-groups-cancelled-column.sql#L25 

syntax error at or near "`"
ADD FOREIGN KEY (`batch_id`) REFERENCES batches(id) ON DELETE CASCADE,
ADD FOREIGN KEY (`batch_id`, `job_group_id`) REFERENCES job_groups (`batch_id`, `job_group_id`) ON DELETE CASCADE;
12 changes: 6 additions & 6 deletions batch/sql/rename-job-groups-tables.sql
View file
Open in desktop
ichengchang marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -604,10 +604,10 @@ BEGIN
WHERE batch_id = in_batch_id AND job_id = in_job_id
FOR UPDATE;

SELECT (jobs.cancelled OR job_groups_cancelled.id IS NOT NULL) AND NOT jobs.always_run
SELECT (jobs.cancelled OR job_groups_cancelled.batch_id IS NOT NULL) AND NOT jobs.always_run
INTO cur_job_cancel
FROM jobs
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.id = jobs.batch_id
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.batch_id = jobs.batch_id
WHERE batch_id = in_batch_id AND job_id = in_job_id
LOCK IN SHARE MODE;

Expand Down Expand Up @@ -669,10 +669,10 @@ BEGIN
WHERE batch_id = in_batch_id AND job_id = in_job_id
FOR UPDATE;

SELECT (jobs.cancelled OR job_groups_cancelled.id IS NOT NULL) AND NOT jobs.always_run
SELECT (jobs.cancelled OR job_groups_cancelled.batch_id IS NOT NULL) AND NOT jobs.always_run
INTO cur_job_cancel
FROM jobs
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.id = jobs.batch_id
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.batch_id = jobs.batch_id
WHERE batch_id = in_batch_id AND job_id = in_job_id
LOCK IN SHARE MODE;

Expand Down Expand Up @@ -714,10 +714,10 @@ BEGIN
WHERE batch_id = in_batch_id AND job_id = in_job_id
FOR UPDATE;

SELECT (jobs.cancelled OR job_groups_cancelled.id IS NOT NULL) AND NOT jobs.always_run
SELECT (jobs.cancelled OR job_groups_cancelled.batch_id IS NOT NULL) AND NOT jobs.always_run
INTO cur_job_cancel
FROM jobs
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.id = jobs.batch_id
LEFT JOIN job_groups_cancelled ON job_groups_cancelled.batch_id = jobs.batch_id
WHERE batch_id = in_batch_id AND job_id = in_job_id
LOCK IN SHARE MODE;

Expand Down
3 changes: 3 additions & 0 deletions build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2294,6 +2294,9 @@ steps:
- name: fix-mark-job-complete-deadlocks
script: /io/sql/fix-mark-job-complete-deadlocks.sql
online: true
- name: rename-job-groups-cancelled-column
script: /io/sql/rename-job-groups-cancelled-column.sql
online: false
inputs:
- from: /repo/batch/sql
to: /io/sql
Expand Down