From f2ef679e1e724d64c99fa3db633e41d9a525004a Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 11 Apr 2024 14:46:20 -0400 Subject: [PATCH 01/21] Use standard buttons for history import/switch in the view interface --- client/src/components/History/HistoryView.vue | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/src/components/History/HistoryView.vue b/client/src/components/History/HistoryView.vue index 9fa3061b4e05..b8bad9242b5e 100644 --- a/client/src/components/History/HistoryView.vue +++ b/client/src/components/History/HistoryView.vue @@ -8,7 +8,6 @@ Import this history From 1ed2af1a6900a3b1c1950cb5f10a39c73d1530f5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 12 Apr 2024 13:25:01 +0200 Subject: [PATCH 02/21] Fix get_content_as_text for compressed text datatypes Fixes: ``` UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte File "starlette/applications.py", line 123, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/middleware/errors.py", line 186, in __call__ raise exc File "starlette/middleware/errors.py", line 164, in __call__ await self.app(scope, receive, _send) File "starlette_context/middleware/raw_middleware.py", line 92, in __call__ await self.app(scope, receive, send_wrapper) File "starlette/middleware/base.py", line 189, in __call__ with collapse_excgroups(): File "contextlib.py", line 155, in __exit__ self.gen.throw(typ, value, traceback) File "starlette/_utils.py", line 93, in collapse_excgroups raise exc File "starlette/middleware/base.py", line 191, in __call__ response = await self.dispatch_func(request, call_next) File "galaxy/webapps/galaxy/fast_app.py", line 108, in add_x_frame_options response = await call_next(request) File "starlette/middleware/base.py", line 165, in call_next raise app_exc File "starlette/middleware/base.py", line 151, in coro await self.app(scope, receive_or_disconnect, send_no_error) File "starlette/middleware/exceptions.py", line 62, in __call__ await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 758, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/routing.py", line 778, in app await route.handle(scope, receive, send) File "starlette/routing.py", line 299, in handle await self.app(scope, receive, send) File "starlette/routing.py", line 79, in app await wrap_app_handling_exceptions(app, request)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 74, in app response = await func(request) File "fastapi/routing.py", line 278, in app raw_response = await run_endpoint_function( File "fastapi/routing.py", line 193, in run_endpoint_function return await run_in_threadpool(dependant.call, **values) File "starlette/concurrency.py", line 42, in run_in_threadpool return await anyio.to_thread.run_sync(func, *args) File "anyio/to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( File "anyio/_backends/_asyncio.py", line 2144, in run_sync_in_worker_thread return await future File "anyio/_backends/_asyncio.py", line 851, in run result = context.run(func, *args) File "galaxy/webapps/galaxy/api/datasets.py", line 192, in get_content_as_text return self.service.get_content_as_text(trans, dataset_id) File "galaxy/webapps/galaxy/services/datasets.py", line 643, in get_content_as_text truncated, dataset_data = self.hda_manager.text_data(hda, preview=True) File "galaxy/managers/hdas.py", line 310, in text_data hda_data = open(hda.get_file_name()).read(MAX_PEEK_SIZE) File "", line 322, in decode ``` from https://sentry.galaxyproject.org/share/issue/9eb8e5b692b94700ac9b304b6d1c2418/ --- lib/galaxy/managers/hdas.py | 9 ++++++--- lib/galaxy_test/api/test_datasets.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index e600ab311cd2..3be812fcf0e8 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -68,6 +68,7 @@ MinimalManagerApp, StructuredApp, ) +from galaxy.util.compression_utils import get_fileobj log = logging.getLogger(__name__) @@ -303,11 +304,13 @@ def text_data(self, hda, preview=True): # For now, cannot get data from non-text datasets. if not isinstance(hda.datatype, datatypes.data.Text): return truncated, hda_data - if not os.path.exists(hda.get_file_name()): + file_path = hda.get_file_name() + if not os.path.exists(file_path): return truncated, hda_data - truncated = preview and os.stat(hda.get_file_name()).st_size > MAX_PEEK_SIZE - hda_data = open(hda.get_file_name()).read(MAX_PEEK_SIZE) + truncated = preview and os.stat(file_path).st_size > MAX_PEEK_SIZE + with get_fileobj(file_path) as fh: + hda_data = fh.read(MAX_PEEK_SIZE) return truncated, hda_data # .... annotatable diff --git a/lib/galaxy_test/api/test_datasets.py b/lib/galaxy_test/api/test_datasets.py index fdd139d78640..3c8c9daf3420 100644 --- a/lib/galaxy_test/api/test_datasets.py +++ b/lib/galaxy_test/api/test_datasets.py @@ -12,6 +12,7 @@ one_hda_model_store_dict, TEST_SOURCE_URI, ) +from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base.api_asserts import assert_has_keys from galaxy_test.base.decorators import ( @@ -356,6 +357,15 @@ def test_get_content_as_text(self, history_id): self._assert_has_key(get_content_as_text_response.json(), "item_data") assert get_content_as_text_response.json().get("item_data") == contents + def test_get_content_as_text_with_compressed_text_data(self, history_id): + test_data_resolver = TestDataResolver() + with open(test_data_resolver.get_filename("1.fasta.gz"), mode="rb") as fh: + hda1 = self.dataset_populator.new_dataset(history_id, content=fh, ftype="fasta.gz", wait=True) + get_content_as_text_response = self._get(f"datasets/{hda1['id']}/get_content_as_text") + self._assert_status_code_is(get_content_as_text_response, 200) + self._assert_has_key(get_content_as_text_response.json(), "item_data") + assert ">hg17" in get_content_as_text_response.json().get("item_data") + def test_anon_get_content_as_text(self, history_id): contents = "accessible data" hda1 = self.dataset_populator.new_dataset(history_id, content=contents, wait=True) From 9cd781e12fd41cc4a740330df6fe0f732171d604 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 12 Apr 2024 14:52:58 +0200 Subject: [PATCH 03/21] Filter workflow outputs by type in markdown directives --- .../src/components/Markdown/MarkdownToolBox.vue | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/client/src/components/Markdown/MarkdownToolBox.vue b/client/src/components/Markdown/MarkdownToolBox.vue index c8f7341f6fed..d382dad028b7 100644 --- a/client/src/components/Markdown/MarkdownToolBox.vue +++ b/client/src/components/Markdown/MarkdownToolBox.vue @@ -259,18 +259,25 @@ export default { }); return steps; }, - getOutputs() { + getOutputs(filterByType = undefined) { const outputLabels = []; this.steps && Object.values(this.steps).forEach((step) => { step.workflow_outputs.forEach((workflowOutput) => { if (workflowOutput.label) { - outputLabels.push(workflowOutput.label); + if (!filterByType || this.stepOutputMatchesType(step, workflowOutput, filterByType)) { + outputLabels.push(workflowOutput.label); + } } }); }); return outputLabels; }, + stepOutputMatchesType(step, workflowOutput, type) { + return Boolean( + step.outputs.find((output) => output.name === workflowOutput.output_name && output.type === type) + ); + }, getArgumentTitle(argumentName) { return ( argumentName[0].toUpperCase() + @@ -331,13 +338,13 @@ export default { onHistoryDatasetId(argumentName) { this.selectedArgumentName = argumentName; this.selectedType = "history_dataset_id"; - this.selectedLabels = this.getOutputs(); + this.selectedLabels = this.getOutputs("data"); this.selectedShow = true; }, onHistoryCollectionId(argumentName) { this.selectedArgumentName = argumentName; this.selectedType = "history_dataset_collection_id"; - this.selectedLabels = this.getOutputs(); + this.selectedLabels = this.getOutputs("collection"); this.selectedShow = true; }, onWorkflowId(argumentName) { From 136ae7544c1295d5a4af39664880c14b2537ba07 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 12 Apr 2024 14:53:43 -0400 Subject: [PATCH 04/21] In history view, add info message for when history import is complete --- client/src/components/History/HistoryView.vue | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/src/components/History/HistoryView.vue b/client/src/components/History/HistoryView.vue index b8bad9242b5e..103909b0a9f7 100644 --- a/client/src/components/History/HistoryView.vue +++ b/client/src/components/History/HistoryView.vue @@ -25,6 +25,8 @@ + History imported and set to your active history. + - + @@ -67,6 +69,7 @@ export default { data() { return { selectedCollections: [], + copySuccess: false, }; }, computed: { @@ -127,6 +130,9 @@ export default { onViewCollection(collection) { this.selectedCollections = [...this.selectedCollections, collection]; }, + copyOkay() { + this.copySuccess = true; + }, }, }; From bf60ea9fc78c08e1c0d3291bd31dcfd91aa09775 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Feb 2024 16:07:03 -0500 Subject: [PATCH 05/21] Fix bug: call unique() on result, not select stmt --- lib/galaxy/tools/actions/upload_common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index a345abde954a..a5460c08efa0 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -441,7 +441,6 @@ def active_folders(trans, folder): select(LibraryFolder) .filter_by(parent=folder, deleted=False) .options(joinedload(LibraryFolder.actions)) - .unique() .order_by(LibraryFolder.name) ) - return trans.sa_session.scalars(stmt).all() + return trans.sa_session.scalars(stmt).unique().all() From 92dfa466f54535eea184e793b58cc30eefcde010 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 13 Apr 2024 13:52:22 +0200 Subject: [PATCH 06/21] Fix None passed to LengthValidator It is valid to pass None as a default value to the validator. It should just return ValueError, which it will do with this change via the inner validator. Fixes https://github.com/galaxyproject/galaxy/issues/17961 Ideally we'd add type annotations, but with the current structure and instantiation order it seems impossible to do this in a meaningful way. If we refactored the validators to be functions, and validators functions were registered per parameter type we could have narrow types on the validator signature. --- lib/galaxy/tools/parameters/validation.py | 4 +++- test/unit/app/tools/test_parameter_validation.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 143b1c187029..cf02d86c56b5 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -186,7 +186,9 @@ def __init__(self, message, length_min, length_max, negate): super().__init__(message, range_min=length_min, range_max=length_max, negate=negate) def validate(self, value, trans=None): - super().validate(len(value), trans) + if value is None: + raise ValueError("No value provided") + super().validate(len(value) if value else 0, trans) class DatasetOkValidator(Validator): diff --git a/test/unit/app/tools/test_parameter_validation.py b/test/unit/app/tools/test_parameter_validation.py index 2e0bf0ffe857..bddb0e325309 100644 --- a/test/unit/app/tools/test_parameter_validation.py +++ b/test/unit/app/tools/test_parameter_validation.py @@ -186,6 +186,14 @@ def test_LengthValidator(self): p.validate("bar") p.validate("f") p.validate("foobarbaz") + p = self._parameter_for( + xml=""" + + +""" + ) + with self.assertRaisesRegex(ValueError, "No value provided"): + p.validate(None) def test_InRangeValidator(self): p = self._parameter_for( From e680f627449cf34f6beba6f94f83952681088c43 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 15 Apr 2024 16:06:38 +0200 Subject: [PATCH 07/21] Raise exception if collection element has unknown extension These might be typo's or the result of bugs as in https://github.com/galaxyproject/galaxy/issues/17938. Fixes: ``` Exception caught while attempting to execute tool with id 'toolshed.g2.bx.psu.edu/repos/iuc/mothur_make_contigs/mothur_make_contigs/1.39.5.1': AttributeError: 'NoneType' object has no attribute 'matches_any' File "galaxy/tools/__init__.py", line 1968, in handle_single_execution rval = self.execute( File "galaxy/tools/__init__.py", line 2065, in execute return self.tool_action.execute( File "galaxy/tools/actions/__init__.py", line 418, in execute ) = self._collect_inputs(tool, trans, incoming, history, current_user_roles, collection_info) File "galaxy/tools/actions/__init__.py", line 352, in _collect_inputs inp_data, all_permissions = self._collect_input_datasets( File "galaxy/tools/actions/__init__.py", line 292, in _collect_input_datasets tool.visit_inputs(param_values, visitor) File "galaxy/tools/__init__.py", line 1792, in visit_inputs visit_input_values(self.inputs, values, callback) File "galaxy/tools/parameters/__init__.py", line 211, in visit_input_values visit_input_values( File "galaxy/tools/parameters/__init__.py", line 227, in visit_input_values callback_helper( File "galaxy/tools/parameters/__init__.py", line 162, in callback_helper new_value = callback(**args) File "galaxy/tools/actions/__init__.py", line 259, in visitor if not datatype.matches_any(input.formats): ``` into something more useful for users. --- lib/galaxy/tools/actions/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 8403e0137bbe..8afad126269d 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -17,8 +17,12 @@ from packaging.version import Version from galaxy import model -from galaxy.exceptions import ItemAccessibilityException +from galaxy.exceptions import ( + ItemAccessibilityException, + RequestParameterInvalidException, +) from galaxy.job_execution.actions.post import ActionBox +from galaxy.managers.context import ProvidesHistoryContext from galaxy.model import ( HistoryDatasetAssociation, Job, @@ -98,7 +102,7 @@ def _collect_input_datasets( self, tool, param_values, - trans, + trans: ProvidesHistoryContext, history, current_user_roles=None, dataset_collection_elements=None, @@ -256,6 +260,10 @@ def process_dataset(data, formats=None): for ext in extensions: if ext: datatype = trans.app.datatypes_registry.get_datatype_by_extension(ext) + if not datatype: + raise RequestParameterInvalidException( + f"Extension '{ext}' unknown, cannot use dataset collection as input" + ) if not datatype.matches_any(input.formats): conversion_required = True break From 7804b18f8eb690ef631be32578eab2bee5e07d0d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 15 Apr 2024 18:06:32 +0200 Subject: [PATCH 08/21] Don't attempt to commit in dry_run mode Fixes https://github.com/galaxyproject/galaxy/issues/17959: ``` NotNullViolation: null value in column "workflow_step_id" of relation "workflow_output" violates not-null constraint DETAIL: Failing row contains (3843428, null, multiqc_fastqc_html, multiqc_fastqc_html, 1d8a0c348b6f446eb4d84770365f570f). File "sqlalchemy/engine/base.py", line 1890, in _execute_context self.dialect.do_executemany( File "sqlalchemy/dialects/postgresql/psycopg2.py", line 982, in do_executemany context._psycopg2_fetched_rows = xtras.execute_values( File "psycopg2/extras.py", line 1299, in execute_values cur.execute(b''.join(parts)) IntegrityError: (psycopg2.errors.NotNullViolation) null value in column "workflow_step_id" of relation "workflow_output" violates not-null constraint DETAIL: Failing row contains (3843428, null, multiqc_fastqc_html, multiqc_fastqc_html, 1d8a0c348b6f446eb4d84770365f570f). [SQL: INSERT INTO workflow_output (workflow_step_id, output_name, label, uuid) VALUES (%(workflow_step_id)s, %(output_name)s, %(label)s, %(uuid)s) RETURNING workflow_output.id] [parameters: ({'workflow_step_id': None, 'output_name': 'multiqc_fastqc_html', 'label': 'multiqc_fastqc_html', 'uuid': '1d8a0c348b6f446eb4d84770365f570f'}, {'workflow_step_id': None, 'output_name': 'multiqc_cutadapt_html', 'label': 'multiqc_cutadapt_html', 'uuid': '11fef12f5cf24904b31dac7a277f5fda'}, {'workflow_step_id': None, 'output_name': 'multiqc_star_html', 'label': 'multiqc_star_html', 'uuid': '383da02d5a4b4af7ac2933b9cfad316f'}, {'workflow_step_id': None, 'output_name': 'STAR_BAM', 'label': 'STAR_BAM', 'uuid': '6aed76c8a4a5491bacd2f995f16df86b'}, {'workflow_step_id': None,... File "starlette/applications.py", line 123, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/middleware/errors.py", line 186, in __call__ raise exc File "starlette/middleware/errors.py", line 164, in __call__ await self.app(scope, receive, _send) File "starlette_context/middleware/raw_middleware.py", line 92, in __call__ await self.app(scope, receive, send_wrapper) File "starlette/middleware/base.py", line 189, in __call__ with collapse_excgroups(): File "contextlib.py", line 155, in __exit__ self.gen.throw(typ, value, traceback) File "starlette/_utils.py", line 93, in collapse_excgroups raise exc File "starlette/middleware/base.py", line 191, in __call__ response = await self.dispatch_func(request, call_next) File "galaxy/webapps/galaxy/fast_app.py", line 108, in add_x_frame_options response = await call_next(request) File "starlette/middleware/base.py", line 165, in call_next raise app_exc File "starlette/middleware/base.py", line 151, in coro await self.app(scope, receive_or_disconnect, send_no_error) File "starlette/middleware/exceptions.py", line 62, in __call__ await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 758, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/routing.py", line 778, in app await route.handle(scope, receive, send) File "starlette/routing.py", line 299, in handle await self.app(scope, receive, send) File "starlette/routing.py", line 79, in app await wrap_app_handling_exceptions(app, request)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 74, in app response = await func(request) File "fastapi/routing.py", line 278, in app raw_response = await run_endpoint_function( File "fastapi/routing.py", line 193, in run_endpoint_function return await run_in_threadpool(dependant.call, **values) File "starlette/concurrency.py", line 42, in run_in_threadpool return await anyio.to_thread.run_sync(func, *args) File "anyio/to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( File "anyio/_backends/_asyncio.py", line 2144, in run_sync_in_worker_thread return await future File "anyio/_backends/_asyncio.py", line 851, in run result = context.run(func, *args) File "galaxy/webapps/galaxy/api/workflows.py", line 987, in refactor return self.service.refactor(trans, workflow_id, payload, instance or False) File "galaxy/webapps/galaxy/services/workflows.py", line 218, in refactor return self._workflow_contents_manager.refactor(trans, stored_workflow, payload) File "galaxy/managers/workflows.py", line 1963, in refactor refactored_workflow, action_executions = self.do_refactor(trans, stored_workflow, refactor_request) File "galaxy/managers/workflows.py", line 1948, in do_refactor refactored_workflow, errors = self.update_workflow_from_raw_description( File "galaxy/managers/workflows.py", line 728, in update_workflow_from_raw_description trans.tag_handler.set_tags_from_list( File "galaxy/model/tags.py", line 94, in set_tags_from_list self.apply_item_tags(user, item, unicodify(new_tags_str, "utf-8"), flush=flush) File "galaxy/model/tags.py", line 265, in apply_item_tags self.apply_item_tag(user, item, name, value, flush=flush) File "galaxy/model/tags.py", line 249, in apply_item_tag self.sa_session.commit() File "", line 2, in commit File "sqlalchemy/orm/session.py", line 1454, in commit self._transaction.commit(_to_root=self.future) File "sqlalchemy/orm/session.py", line 832, in commit self._prepare_impl() File "sqlalchemy/orm/session.py", line 811, in _prepare_impl self.session.flush() File "sqlalchemy/orm/session.py", line 3449, in flush self._flush(objects) File "sqlalchemy/orm/session.py", line 3588, in _flush with util.safe_reraise(): File "sqlalchemy/util/langhelpers.py", line 70, in __exit__ compat.raise_( File "sqlalchemy/util/compat.py", line 211, in raise_ raise exception File "sqlalchemy/orm/session.py", line 3549, in _flush flush_context.execute() File "sqlalchemy/orm/unitofwork.py", line 456, in execute rec.execute(self) File "sqlalchemy/orm/unitofwork.py", line 630, in execute util.preloaded.orm_persistence.save_obj( File "sqlalchemy/orm/persistence.py", line 245, in save_obj _emit_insert_statements( File "sqlalchemy/orm/persistence.py", line 1156, in _emit_insert_statements c = connection._execute_20( File "sqlalchemy/engine/base.py", line 1710, in _execute_20 return meth(self, args_10style, kwargs_10style, execution_options) File "sqlalchemy/sql/elements.py", line 334, in _execute_on_connection return connection._execute_clauseelement( File "sqlalchemy/engine/base.py", line 1577, in _execute_clauseelement ret = self._execute_context( File "sqlalchemy/engine/base.py", line 1953, in _execute_context self._handle_dbapi_exception( File "sqlalchemy/engine/base.py", line 2134, in _handle_dbapi_exception util.raise_( File "sqlalchemy/util/compat.py", line 211, in raise_ raise exception File "sqlalchemy/engine/base.py", line 1890, in _execute_context self.dialect.do_executemany( File "sqlalchemy/dialects/postgresql/psycopg2.py", line 982, in do_executemany context._psycopg2_fetched_rows = xtras.execute_values( File "psycopg2/extras.py", line 1299, in execute_values cur.execute(b''.join(parts)) ``` broken in https://github.com/galaxyproject/galaxy/commit/b00a94c1f7972dcd1060069fe83fece7b8943165 --- lib/galaxy/managers/workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index d4c46b8b7752..23f02f65f937 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -707,7 +707,8 @@ def update_workflow_from_raw_description( trans.tag_handler.set_tags_from_list( trans.user, stored_workflow, - data.get("tags", []), + data["tags"], + flush=False, ) if workflow_update_options.update_stored_workflow_attributes: From c7c0bafcf795b18260c79998ff0cc4abb4eab250 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 15 Apr 2024 19:30:31 +0200 Subject: [PATCH 09/21] Use or copy StoredWorkflow when copying step Fixes https://sentry.galaxyproject.org/share/issue/509cadd762e749e1a782bb4806161ed3/: ``` AttributeError: 'NoneType' object has no attribute 'latest_workflow' File "starlette/applications.py", line 123, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/middleware/errors.py", line 186, in __call__ raise exc File "starlette/middleware/errors.py", line 164, in __call__ await self.app(scope, receive, _send) File "starlette_context/middleware/raw_middleware.py", line 92, in __call__ await self.app(scope, receive, send_wrapper) File "starlette/middleware/base.py", line 189, in __call__ with collapse_excgroups(): File "contextlib.py", line 155, in __exit__ self.gen.throw(typ, value, traceback) File "starlette/_utils.py", line 93, in collapse_excgroups raise exc File "starlette/middleware/base.py", line 191, in __call__ response = await self.dispatch_func(request, call_next) File "galaxy/webapps/galaxy/fast_app.py", line 108, in add_x_frame_options response = await call_next(request) File "starlette/middleware/base.py", line 165, in call_next raise app_exc File "starlette/middleware/base.py", line 151, in coro await self.app(scope, receive_or_disconnect, send_no_error) File "starlette/middleware/exceptions.py", line 62, in __call__ await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 758, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/routing.py", line 778, in app await route.handle(scope, receive, send) File "starlette/routing.py", line 299, in handle await self.app(scope, receive, send) File "starlette/routing.py", line 79, in app await wrap_app_handling_exceptions(app, request)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 74, in app response = await func(request) File "fastapi/routing.py", line 278, in app raw_response = await run_endpoint_function( File "fastapi/routing.py", line 193, in run_endpoint_function return await run_in_threadpool(dependant.call, **values) File "starlette/concurrency.py", line 42, in run_in_threadpool return await anyio.to_thread.run_sync(func, *args) File "anyio/to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( File "anyio/_backends/_asyncio.py", line 2144, in run_sync_in_worker_thread return await future File "anyio/_backends/_asyncio.py", line 851, in run result = context.run(func, *args) File "galaxy/webapps/galaxy/api/workflows.py", line 987, in refactor return self.service.refactor(trans, workflow_id, payload, instance or False) File "galaxy/webapps/galaxy/services/workflows.py", line 218, in refactor return self._workflow_contents_manager.refactor(trans, stored_workflow, payload) File "galaxy/managers/workflows.py", line 1963, in refactor refactored_workflow, action_executions = self.do_refactor(trans, stored_workflow, refactor_request) File "galaxy/managers/workflows.py", line 1947, in do_refactor action_executions = refactor_executor.refactor(refactor_request) File "galaxy/workflow/refactor/execute.py", line 77, in refactor refactor_method(action, execution) File "galaxy/workflow/refactor/execute.py", line 374, in _apply_upgrade_subworkflow content_id = trans.security.encode_id(stored_workflow.latest_workflow.id) ``` You can reproduce this by creating a subworkflow, renaming the parent workflow and then attempting to update the subworkflow step. --- lib/galaxy/model/__init__.py | 18 ++++++++++++++---- test/unit/data/test_galaxy_mapping.py | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7e582782e911..7c239fd720cd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8002,10 +8002,20 @@ def copy_to(self, copied_step, step_mapping, user=None): copied_step.annotations = annotations if subworkflow := self.subworkflow: - copied_subworkflow = subworkflow.copy() - copied_step.subworkflow = copied_subworkflow - for subworkflow_step, copied_subworkflow_step in zip(subworkflow.steps, copied_subworkflow.steps): - subworkflow_step_mapping[subworkflow_step.id] = copied_subworkflow_step + stored_subworkflow = subworkflow.stored_workflow + if stored_subworkflow and stored_subworkflow.user == user: + # This should be fine and reduces the number of stored subworkflows + copied_step.subworkflow = subworkflow + else: + # Can this even happen, building a workflow with a subworkflow you don't own ? + copied_subworkflow = subworkflow.copy() + stored_workflow = StoredWorkflow( + user, name=copied_subworkflow.name, workflow=copied_subworkflow, hidden=True + ) + copied_subworkflow.stored_workflow = stored_workflow + copied_step.subworkflow = copied_subworkflow + for subworkflow_step, copied_subworkflow_step in zip(subworkflow.steps, copied_subworkflow.steps): + subworkflow_step_mapping[subworkflow_step.id] = copied_subworkflow_step for old_conn, new_conn in zip(self.input_connections, copied_step.input_connections): new_conn.input_step_input = copied_step.get_or_add_input(old_conn.input_name) diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 7e6c35d757f1..aa4ff3a86b3a 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -881,6 +881,9 @@ def test_workflows(self): assert loaded_invocation assert loaded_invocation.history.id == history_id + # recover user after expunge + user = loaded_invocation.history.user + step_1, step_2 = loaded_invocation.workflow.steps assert not step_1.subworkflow From ae64de668d85291e28bbcfbe36613c7d01b1612a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:54:10 +0200 Subject: [PATCH 10/21] Fix token retrieval for invenio Do not force to use the Vault and rely on the config template value. --- lib/galaxy/files/sources/_rdm.py | 34 +++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/files/sources/_rdm.py b/lib/galaxy/files/sources/_rdm.py index fb33cf444579..4f841871974c 100644 --- a/lib/galaxy/files/sources/_rdm.py +++ b/lib/galaxy/files/sources/_rdm.py @@ -1,6 +1,5 @@ import logging from typing import ( - cast, List, NamedTuple, Optional, @@ -138,11 +137,10 @@ class RDMFilesSource(BaseFilesSource): def __init__(self, **kwd: Unpack[FilesSourceProperties]): props = self._parse_common_config_opts(kwd) - base_url = props.get("url", None) + base_url = props.get("url") if not base_url: raise Exception("URL for RDM repository must be provided in configuration") self._repository_url = base_url - self._token = props.get("token", None) self._props = props self._repository_interactor = self.get_repository_interactor(base_url) @@ -150,10 +148,6 @@ def __init__(self, **kwd: Unpack[FilesSourceProperties]): def repository(self) -> RDMRepositoryInteractor: return self._repository_interactor - @property - def token(self) -> Optional[str]: - return self._token if self._token and not self._token.startswith("$") else None - def get_repository_interactor(self, repository_url: str) -> RDMRepositoryInteractor: """Returns an interactor compatible with the given repository URL. @@ -190,25 +184,19 @@ def get_error_msg(details: str) -> str: def get_record_id_from_path(self, source_path: str) -> str: return self.parse_path(source_path, record_id_only=True).record_id - def _serialization_props(self, user_context: OptionalUserContext = None) -> RDMFilesSourceProperties: + def _serialization_props(self, user_context: OptionalUserContext = None): effective_props = {} for key, val in self._props.items(): effective_props[key] = self._evaluate_prop(val, user_context=user_context) - effective_props["url"] = self._repository_url - effective_props["token"] = self.safe_get_authorization_token(user_context) - return cast(RDMFilesSourceProperties, effective_props) + return effective_props def get_authorization_token(self, user_context: OptionalUserContext) -> str: - token = self.token - if not token and user_context: - vault = user_context.user_vault if user_context else None - token = vault.read_secret(f"preferences/{self.id}/token") if vault else None - if token is None: - raise AuthenticationRequired(f"No authorization token provided in user's settings for '{self.label}'") + token = None + if user_context: + effective_props = self._serialization_props(user_context) + token = effective_props.get("token") + if not token: + raise AuthenticationRequired( + f"Please provide a personal access token in your user's preferences for '{self.label}'" + ) return token - - def safe_get_authorization_token(self, user_context: OptionalUserContext) -> Optional[str]: - try: - return self.get_authorization_token(user_context) - except AuthenticationRequired: - return None From c02750ec7e4a8c314aa28d545b8844c228e7e41a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:10:53 +0200 Subject: [PATCH 11/21] Fix public_name retrieval for invenio --- lib/galaxy/files/sources/_rdm.py | 9 ++++++++- lib/galaxy/files/sources/invenio.py | 30 +++++++++++++++++------------ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/files/sources/_rdm.py b/lib/galaxy/files/sources/_rdm.py index 4f841871974c..14f7e9e1daa0 100644 --- a/lib/galaxy/files/sources/_rdm.py +++ b/lib/galaxy/files/sources/_rdm.py @@ -25,6 +25,7 @@ class RDMFilesSourceProperties(FilesSourceProperties): url: str token: str + public_name: str class RecordFilename(NamedTuple): @@ -79,7 +80,9 @@ def get_files_in_record( """ raise NotImplementedError() - def create_draft_record(self, title: str, user_context: OptionalUserContext = None): + def create_draft_record( + self, title: str, public_name: Optional[str] = None, user_context: OptionalUserContext = None + ): """Creates a draft record (directory) in the repository with basic metadata. The metadata is usually just the title of the record and the user that created it. @@ -200,3 +203,7 @@ def get_authorization_token(self, user_context: OptionalUserContext) -> str: f"Please provide a personal access token in your user's preferences for '{self.label}'" ) return token + + def get_public_name(self, user_context: OptionalUserContext) -> Optional[str]: + effective_props = self._serialization_props(user_context) + return effective_props.get("public_name") diff --git a/lib/galaxy/files/sources/invenio.py b/lib/galaxy/files/sources/invenio.py index 3ddf4fee107b..6e22f4107e7c 100644 --- a/lib/galaxy/files/sources/invenio.py +++ b/lib/galaxy/files/sources/invenio.py @@ -135,7 +135,8 @@ def _create_entry( user_context: OptionalUserContext = None, opts: Optional[FilesSourceOptions] = None, ) -> Entry: - record = self.repository.create_draft_record(entry_data["name"], user_context=user_context) + public_name = self.get_public_name(user_context) + record = self.repository.create_draft_record(entry_data["name"], public_name, user_context=user_context) return { "uri": self.repository.to_plugin_uri(record["id"]), "name": record["metadata"]["title"], @@ -198,9 +199,11 @@ def get_files_in_record( response_data = self._get_response(user_context, request_url) return self._get_record_files_from_response(record_id, response_data) - def create_draft_record(self, title: str, user_context: OptionalUserContext = None) -> RemoteDirectory: + def create_draft_record( + self, title: str, public_name: Optional[str] = None, user_context: OptionalUserContext = None + ) -> RemoteDirectory: today = datetime.date.today().isoformat() - creator = self._get_creator_from_user_context(user_context) + creator = self._get_creator_from_public_name(public_name) create_record_request = { "files": {"enabled": True}, "metadata": { @@ -360,10 +363,9 @@ def _get_record_files_from_response(self, record_id: str, response: dict) -> Lis ) return rval - def _get_creator_from_user_context(self, user_context: OptionalUserContext): - public_name = self.get_user_preference_by_key("public_name", user_context) - family_name = "Galaxy User" + def _get_creator_from_public_name(self, public_name: Optional[str] = None) -> Creator: given_name = "Anonymous" + family_name = "Galaxy User" if public_name: tokens = public_name.split(", ") if len(tokens) == 2: @@ -371,12 +373,16 @@ def _get_creator_from_user_context(self, user_context: OptionalUserContext): given_name = tokens[1] else: given_name = public_name - return {"person_or_org": {"family_name": family_name, "given_name": given_name, "type": "personal"}} - - def get_user_preference_by_key(self, key: str, user_context: OptionalUserContext): - preferences = user_context.preferences if user_context else None - value = preferences.get(f"{self.plugin.id}|{key}", None) if preferences else None - return value + return { + "person_or_org": { + "name": f"{given_name} {family_name}", + "family_name": family_name, + "given_name": given_name, + "type": "personal", + "identifiers": [], + }, + "affiliations": [], + } def _get_response( self, user_context: OptionalUserContext, request_url: str, params: Optional[Dict[str, Any]] = None From 133730c221ddd308c3e13108342c91896033c1e6 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:24:52 +0200 Subject: [PATCH 12/21] Fix sample file source for Invenio --- lib/galaxy/config/sample/file_sources_conf.yml.sample | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index 76b1a8616903..b21bb1339b60 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -199,10 +199,14 @@ doc: Make sure to define this generic drs file source if you have defined any other drs file sources, or stock drs download capability will be disabled. - type: inveniordm - id: invenio - doc: Invenio RDM turn-key research data management repository - label: Invenio RDM Demo Repository + id: invenio_sandbox + doc: This is the Sandbox instance of Invenio. It is used for testing purposes only, content is NOT preserved. DOIs created in this instance are not real and will not resolve. + label: Invenio RDM Sandbox Repository (TESTING ONLY) url: https://inveniordm.web.cern.ch/ + token: ${user.user_vault.read_secret('preferences/invenio_sandbox/token')} + # token: ${user.preferences['invenio_sandbox|token']} # Alternatively use this for retrieving the token from user preferences instead of the Vault + public_name: ${user.preferences['invenio_sandbox|public_name']} + writable: true - type: onedata id: onedata1 From 0f8c379c486a93d8e9447b2cc8a3b42a38a65e67 Mon Sep 17 00:00:00 2001 From: Ahmed Awan Date: Mon, 15 Apr 2024 19:03:40 -0500 Subject: [PATCH 13/21] Use the new column-select component in `FormData` for multiple select This allows for easier multiselect with added ease for range select. Fixes https://github.com/galaxyproject/galaxy/issues/17947 --- .../components/Form/Elements/FormData/FormData.vue | 9 ++++++++- .../Form/Elements/FormSelectMany/FormSelectMany.vue | 12 ++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/client/src/components/Form/Elements/FormData/FormData.vue b/client/src/components/Form/Elements/FormData/FormData.vue index 87ecd98f4928..2322b9b17ab8 100644 --- a/client/src/components/Form/Elements/FormData/FormData.vue +++ b/client/src/components/Form/Elements/FormData/FormData.vue @@ -14,6 +14,7 @@ import { orList } from "@/utils/strings"; import type { DataOption } from "./types"; import { BATCH, SOURCE, VARIANTS } from "./variants"; +import FormSelection from "../FormSelection.vue"; import FormSelect from "@/components/Form/Elements/FormSelect.vue"; library.add(faCopy, faFile, faFolder, faCaretDown, faCaretUp, faExclamation, faLink, faUnlink); @@ -502,7 +503,7 @@ const noOptionsWarningMessage = computed(() => { { +