From 4e2f6fd71b99c37943e24957caa230c7c125b18d Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 14:25:49 -0500 Subject: [PATCH 01/10] vine: check the availability of staged inputs --- taskvine/src/manager/vine_manager.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 79c226baa4..526f950f63 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -3183,7 +3183,7 @@ static void vine_manager_create_recovery_tasks(struct vine_manager *q, struct vi LIST_ITERATE(t->output_mounts, m) { - if (m->file->type == VINE_TEMP) { + if (m->file->type == VINE_TEMP || m->file->type == VINE_FILE) { if (!recovery_task) { recovery_task = vine_task_copy(t); recovery_task->type = VINE_TASK_TYPE_RECOVERY; @@ -3253,8 +3253,14 @@ static int vine_manager_check_inputs_available(struct vine_manager *q, struct vi LIST_ITERATE(t->input_mounts, m) { struct vine_file *f = m->file; - if (f->type == VINE_FILE && f->state == VINE_FILE_STATE_PENDING) { - all_available = 0; + if (f->type == VINE_FILE) { + if (f->state == VINE_FILE_STATE_PENDING) { + all_available = 0; + } else if (vine_file_has_changed(f)) { + /* this only happens if the input file is missing in the staging directory */ + vine_manager_consider_recovery_task(q, f, f->recovery_task); + all_available = 0; + } } else if (f->type == VINE_TEMP && f->state == VINE_FILE_STATE_CREATED) { if (!vine_file_replica_table_exists_somewhere(q, f->cached_name)) { vine_manager_consider_recovery_task(q, f, f->recovery_task); From c43ba585d29c0a20ee849e7ef43827b2aa8389b4 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 14:47:58 -0500 Subject: [PATCH 02/10] check for created files --- taskvine/src/manager/vine_manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 526f950f63..232b9187ec 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -3256,7 +3256,7 @@ static int vine_manager_check_inputs_available(struct vine_manager *q, struct vi if (f->type == VINE_FILE) { if (f->state == VINE_FILE_STATE_PENDING) { all_available = 0; - } else if (vine_file_has_changed(f)) { + } else if (f->state == VINE_FILE_STATE_CREATED && vine_file_has_changed(f)) { /* this only happens if the input file is missing in the staging directory */ vine_manager_consider_recovery_task(q, f, f->recovery_task); all_available = 0; From db74e76978909b4ab92a9810ca88b4a66196d20e Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 14:56:56 -0500 Subject: [PATCH 03/10] revert changes --- taskvine/src/manager/vine_manager.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 232b9187ec..ac51b865c2 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -3183,7 +3183,7 @@ static void vine_manager_create_recovery_tasks(struct vine_manager *q, struct vi LIST_ITERATE(t->output_mounts, m) { - if (m->file->type == VINE_TEMP || m->file->type == VINE_FILE) { + if (m->file->type == VINE_TEMP) { if (!recovery_task) { recovery_task = vine_task_copy(t); recovery_task->type = VINE_TASK_TYPE_RECOVERY; @@ -3253,14 +3253,8 @@ static int vine_manager_check_inputs_available(struct vine_manager *q, struct vi LIST_ITERATE(t->input_mounts, m) { struct vine_file *f = m->file; - if (f->type == VINE_FILE) { - if (f->state == VINE_FILE_STATE_PENDING) { - all_available = 0; - } else if (f->state == VINE_FILE_STATE_CREATED && vine_file_has_changed(f)) { - /* this only happens if the input file is missing in the staging directory */ - vine_manager_consider_recovery_task(q, f, f->recovery_task); - all_available = 0; - } + if (f->type == VINE_FILE && f->state == VINE_FILE_STATE_PENDING) { + all_available = 0; } else if (f->type == VINE_TEMP && f->state == VINE_FILE_STATE_CREATED) { if (!vine_file_replica_table_exists_somewhere(q, f->cached_name)) { vine_manager_consider_recovery_task(q, f, f->recovery_task); @@ -6293,4 +6287,4 @@ char *vine_version_string() return cctools_version_string(); } -/* vim: set noexpandtab tabstop=8: */ +/* vim: set noexpandtab tabstop=8: */ \ No newline at end of file From 8bba9e3c082f2394c0fc9bac83c58e2793a19f23 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 14:57:13 -0500 Subject: [PATCH 04/10] revert changes --- taskvine/src/manager/vine_manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index ac51b865c2..79c226baa4 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -6287,4 +6287,4 @@ char *vine_version_string() return cctools_version_string(); } -/* vim: set noexpandtab tabstop=8: */ \ No newline at end of file +/* vim: set noexpandtab tabstop=8: */ From 3cb45969c9a27c44bb34726357d3080f92fb312a Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 17:09:07 -0500 Subject: [PATCH 05/10] vine: change &= to = --- taskvine/src/manager/vine_manager.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 79c226baa4..0578192b0e 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -1277,10 +1277,10 @@ static int fetch_outputs_from_worker(struct vine_manager *q, struct vine_worker_ default: /* Otherwise get all of the output files. */ if (!t->output_received) { - result &= retrieve_output(q, w, t); + result = retrieve_output(q, w, t); t->output_received = 1; } - result &= vine_manager_get_output_files(q, w, t); + result = vine_manager_get_output_files(q, w, t); break; } From 60cf40e25cef01e5c8dc5b355acb7a29543707d5 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 17:24:54 -0500 Subject: [PATCH 06/10] set t->output_received on success --- taskvine/src/manager/vine_manager.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 0578192b0e..a9ca71d52f 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -1278,7 +1278,9 @@ static int fetch_outputs_from_worker(struct vine_manager *q, struct vine_worker_ /* Otherwise get all of the output files. */ if (!t->output_received) { result = retrieve_output(q, w, t); - t->output_received = 1; + if (result == VINE_SUCCESS) { + t->output_received = 1; + } } result = vine_manager_get_output_files(q, w, t); break; From 4a78c60e1a78f2d4b51d51d4ad063e7fb8cfeb80 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 17:25:54 -0500 Subject: [PATCH 07/10] break when result is not success --- taskvine/src/manager/vine_manager.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index a9ca71d52f..dd4ebcebff 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -1281,6 +1281,9 @@ static int fetch_outputs_from_worker(struct vine_manager *q, struct vine_worker_ if (result == VINE_SUCCESS) { t->output_received = 1; } + else { + break; + } } result = vine_manager_get_output_files(q, w, t); break; From f654937b17450126a27921e6ac5cf32f476eb976 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 17:27:17 -0500 Subject: [PATCH 08/10] lint --- taskvine/src/manager/vine_manager.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index dd4ebcebff..ac3b494709 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -1280,8 +1280,7 @@ static int fetch_outputs_from_worker(struct vine_manager *q, struct vine_worker_ result = retrieve_output(q, w, t); if (result == VINE_SUCCESS) { t->output_received = 1; - } - else { + } else { break; } } From ad3773c19cdbc82222ed127ac0b9f0c35a313dc3 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 17:28:45 -0500 Subject: [PATCH 09/10] remove redund code --- taskvine/src/manager/vine_manager.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index ac3b494709..2ddf62269e 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -1280,11 +1280,9 @@ static int fetch_outputs_from_worker(struct vine_manager *q, struct vine_worker_ result = retrieve_output(q, w, t); if (result == VINE_SUCCESS) { t->output_received = 1; - } else { - break; + result = vine_manager_get_output_files(q, w, t); } } - result = vine_manager_get_output_files(q, w, t); break; } From dd3347830f2660bfd895e276ac595c2c4e476d27 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 26 Nov 2024 17:41:11 -0500 Subject: [PATCH 10/10] always do vine_manager_get_output_files --- taskvine/src/manager/vine_manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 2ddf62269e..a9ca71d52f 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -1280,9 +1280,9 @@ static int fetch_outputs_from_worker(struct vine_manager *q, struct vine_worker_ result = retrieve_output(q, w, t); if (result == VINE_SUCCESS) { t->output_received = 1; - result = vine_manager_get_output_files(q, w, t); } } + result = vine_manager_get_output_files(q, w, t); break; }