diff --git a/taskvine/src/manager/Makefile b/taskvine/src/manager/Makefile index 849cb50262..ec78e9d5fa 100644 --- a/taskvine/src/manager/Makefile +++ b/taskvine/src/manager/Makefile @@ -15,6 +15,7 @@ SOURCES = \ vine_task.c \ vine_file.c \ vine_mount.c \ + vine_symlink.c \ vine_txn_log.c \ vine_taskgraph_log.c \ vine_cached_name.c \ diff --git a/taskvine/src/manager/taskvine.h b/taskvine/src/manager/taskvine.h index 2f9c5c683e..553921b72a 100644 --- a/taskvine/src/manager/taskvine.h +++ b/taskvine/src/manager/taskvine.h @@ -446,6 +446,15 @@ feature. */ void vine_task_add_feature(struct vine_task *t, const char *name); +/** Add a symbolic link to the sandbox namespace of the task. +This can be used to access a file within a shared filesystem, +without causing the file to be transferred using the taskvine caching infrastructure. +@param t A task object; +@param name The name of the symlink; must be a relative path in the task sandbox. +@param target The target of the symlink; must be an absolute path to an outside filesystem. +*/ +void vine_task_add_symlink(struct vine_task *t, const char *name, const char *target); + /** Specify the priority of this task relative to others in the manager. Tasks with a higher priority value run first. If no priority is given, a task is placed at the end of the ready list, regardless of the priority. diff --git a/taskvine/src/manager/vine_manager_put.c b/taskvine/src/manager/vine_manager_put.c index 83dbaeb64f..2c3c97b812 100644 --- a/taskvine/src/manager/vine_manager_put.c +++ b/taskvine/src/manager/vine_manager_put.c @@ -11,6 +11,7 @@ See the file COPYING for details. #include "vine_file_replica_table.h" #include "vine_mount.h" #include "vine_protocol.h" +#include "vine_symlink.h" #include "vine_task.h" #include "vine_txn_log.h" #include "vine_worker_info.h" @@ -560,6 +561,18 @@ vine_result_code_t vine_manager_put_task( } } + if (t->symlink_list) { + struct vine_symlink *s; + LIST_ITERATE(t->symlink_list, s) + { + char name_encoded[PATH_MAX]; + char target_encoded[PATH_MAX]; + url_encode(s->name, name_encoded, PATH_MAX); + url_encode(s->target, target_encoded, PATH_MAX); + vine_manager_send(q, w, "symlink %s %s\n", name_encoded, target_encoded ); + } + } + // vine_manager_send returns the number of bytes sent, or a number less than // zero to indicate errors. We are lazy here, we only check the last // message we sent to the worker (other messages may have failed above). diff --git a/taskvine/src/manager/vine_task.c b/taskvine/src/manager/vine_task.c index c0ea564b27..06ee3c237b 100644 --- a/taskvine/src/manager/vine_task.c +++ b/taskvine/src/manager/vine_task.c @@ -9,6 +9,7 @@ See the file COPYING for details. #include "vine_file.h" #include "vine_manager.h" #include "vine_mount.h" +#include "vine_symlink.h" #include "vine_worker_info.h" #include "debug.h" @@ -51,6 +52,7 @@ struct vine_task *vine_task_create(const char *command_line) t->input_mounts = list_create(); t->output_mounts = list_create(); + t->symlink_list = list_create(); t->env_list = list_create(); t->feature_list = list_create(); @@ -177,6 +179,17 @@ static void vine_task_mount_list_copy(struct list *destination, struct list *lis } } +static void vine_task_symlink_list_copy(struct list *destination, struct list *list) +{ + struct vine_symlink *old_symlink, *new_symlink; + + LIST_ITERATE(list, old_symlink) + { + new_symlink = vine_symlink_copy(old_symlink); + list_push_tail(destination, new_symlink); + } +} + static void vine_task_string_list_copy(struct list *destination, struct list *string_list) { char *var; @@ -230,6 +243,7 @@ struct vine_task *vine_task_copy(const struct vine_task *task) vine_task_mount_list_copy(new->input_mounts, task->input_mounts); vine_task_mount_list_copy(new->output_mounts, task->output_mounts); + vine_task_symlink_list_copy(new->symlink_list, task->symlink_list); vine_task_string_list_copy(new->env_list, task->env_list); vine_task_string_list_copy(new->feature_list, task->feature_list); new->function_slots_requested = task->function_slots_requested; @@ -636,6 +650,11 @@ int vine_task_add_environment(struct vine_task *t, struct vine_file *f) return vine_task_add_execution_context(t, f); } +void vine_task_add_symlink(struct vine_task *t, const char *name, const char *target) +{ + list_push_tail(t->symlink_list,vine_symlink_create(name,target)); +} + int vine_task_add_execution_context(struct vine_task *t, struct vine_file *context_file) { if (!context_file) { @@ -754,6 +773,9 @@ void vine_task_delete(struct vine_task *t) list_clear(t->output_mounts, (void *)vine_mount_delete); list_delete(t->output_mounts); + list_clear(t->symlink_list, (void*)vine_symlink_delete); + list_delete(t->symlink_list); + list_clear(t->env_list, (void *)free); list_delete(t->env_list); diff --git a/taskvine/src/manager/vine_task.h b/taskvine/src/manager/vine_task.h index 4c53ece509..af586f839c 100644 --- a/taskvine/src/manager/vine_task.h +++ b/taskvine/src/manager/vine_task.h @@ -59,8 +59,9 @@ struct vine_task { int function_slots_requested; /**< If this is a LibraryTask, the number of function slots requested by the user. -1 causes the number of slots to match the number of cores. */ vine_task_func_exec_mode_t func_exec_mode; /**< If this a LibraryTask, the execution mode of its functions. */ - struct list *input_mounts; /**< The mounted files expected as inputs. */ - struct list *output_mounts; /**< The mounted files expected as outputs. */ + struct list *input_mounts; /**< The mounted files expected as inputs. */ + struct list *output_mounts; /**< The mounted files expected as outputs. */ + struct list *symlink_list; /**< Additional symlinks to add to the sandbox namespace. */ struct list *env_list; /**< Environment variables applied to the task. */ struct list *feature_list; /**< User-defined features this task requires. (See vine_worker's --feature option.) */ diff --git a/taskvine/src/worker/vine_sandbox.c b/taskvine/src/worker/vine_sandbox.c index 060869d591..6d0dad7b68 100644 --- a/taskvine/src/worker/vine_sandbox.c +++ b/taskvine/src/worker/vine_sandbox.c @@ -9,6 +9,7 @@ See the file COPYING for details. #include "vine_cache_file.h" #include "vine_file.h" #include "vine_mount.h" +#include "vine_symlink.h" #include "vine_task.h" #include "vine_worker.h" @@ -134,6 +135,7 @@ int vine_sandbox_stagein(struct vine_process *p, struct vine_cache *cache) int result = 1; struct vine_mount *m; + struct vine_symlink *s; /* For each input mount, stage it into the sandbox. */ @@ -144,6 +146,17 @@ int vine_sandbox_stagein(struct vine_process *p, struct vine_cache *cache) break; } + /* For each requested symlink, create it in the sandbox */ + + LIST_ITERATE(t->symlink_list, s) + { + result = symlink(s->target,s->name); + if(result!=0) { + debug(D_VINE,"unable to symlink %s -> %s: %s",s->name,s->target,strerror(errno)); + break; + } + } + /* If any of the output mounts have the MKDIR flag, then create those empty dirs. */ LIST_ITERATE(t->output_mounts, m) diff --git a/taskvine/src/worker/vine_worker.c b/taskvine/src/worker/vine_worker.c index dd2eddc555..643afbb263 100644 --- a/taskvine/src/worker/vine_worker.c +++ b/taskvine/src/worker/vine_worker.c @@ -834,6 +834,8 @@ static struct vine_task *do_task_body(struct link *manager, int task_id, time_t char localname[VINE_LINE_MAX]; char taskname[VINE_LINE_MAX]; char taskname_encoded[VINE_LINE_MAX]; + char target[VINE_LINE_MAX]; + char target_encoded[VINE_LINE_MAX]; char library_name[VINE_LINE_MAX]; char category[VINE_LINE_MAX]; int flags, length; @@ -873,6 +875,10 @@ static struct vine_task *do_task_body(struct link *manager, int task_id, time_t url_decode(taskname_encoded, taskname, VINE_LINE_MAX); vine_hack_do_not_compute_cached_name = 1; vine_task_add_output_file(task, localname, taskname, flags); + } else if (sscanf(line, "symlink %s %s", taskname_encoded, target_encoded)==2) { + url_decode(taskname_encoded, taskname, VINE_LINE_MAX); + url_decode(target_encoded, target, VINE_LINE_MAX); + vine_task_add_symlink(task,taskname,target); } else if (sscanf(line, "cores %" PRId64, &n)) { vine_task_set_cores(task, n); } else if (sscanf(line, "memory %" PRId64, &n)) {