From 4a92363190646ed64b4ab01141fd07ea6c6cba4a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 13 Sep 2024 17:18:18 +0200 Subject: [PATCH 01/13] Prevent optional output to required input connection in editor Fixes https://github.com/galaxyproject/galaxy/issues/18791 --- .../components/Workflow/Editor/NodeOutput.vue | 2 +- .../Workflow/Editor/modules/terminals.test.ts | 10 +++ .../Workflow/Editor/modules/terminals.ts | 3 + .../Editor/test-data/parameter_steps.json | 64 +++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/client/src/components/Workflow/Editor/NodeOutput.vue b/client/src/components/Workflow/Editor/NodeOutput.vue index d76c17e44345..1ce9500f6c34 100644 --- a/client/src/components/Workflow/Editor/NodeOutput.vue +++ b/client/src/components/Workflow/Editor/NodeOutput.vue @@ -300,7 +300,7 @@ const outputDetails = computed(() => { const outputType = collectionType && collectionType.isCollection && collectionType.collectionType ? `output is ${collectionTypeToDescription(collectionType)}` - : `output is ${terminal.value.type || "dataset"}`; + : `output is ${terminal.value.optional ? "optional " : ""}${terminal.value.type || "dataset"}`; if (isMultiple.value) { if (!collectionType) { collectionType = NULL_COLLECTION_TYPE_DESCRIPTION; diff --git a/client/src/components/Workflow/Editor/modules/terminals.test.ts b/client/src/components/Workflow/Editor/modules/terminals.test.ts index 844c9114bb14..7cc4fb0e26c2 100644 --- a/client/src/components/Workflow/Editor/modules/terminals.test.ts +++ b/client/src/components/Workflow/Editor/modules/terminals.test.ts @@ -298,6 +298,16 @@ describe("canAccept", () => { "Cannot attach a text parameter to a integer input" ); }); + it("rejects optional integer to required parameter connection", () => { + const integerInputParam = terminals["multi data"]!["advanced|advanced_threshold"] as InputParameterTerminal; + const optionalIntegerOutputParam = terminals["optional integer parameter input"]![ + "output" + ] as OutputParameterTerminal; + expect(integerInputParam.canAccept(optionalIntegerOutputParam).canAccept).toBe(false); + expect(integerInputParam.canAccept(optionalIntegerOutputParam).reason).toBe( + "Cannot attach an optional output to a required parameter" + ); + }); it("rejects data to parameter connection", () => { const dataOut = terminals["data input"]!["output"] as OutputTerminal; const integerInputParam = terminals["multi data"]!["advanced|advanced_threshold"] as InputParameterTerminal; diff --git a/client/src/components/Workflow/Editor/modules/terminals.ts b/client/src/components/Workflow/Editor/modules/terminals.ts index 7ed030b537e2..cb86c796a9c5 100644 --- a/client/src/components/Workflow/Editor/modules/terminals.ts +++ b/client/src/components/Workflow/Editor/modules/terminals.ts @@ -540,6 +540,9 @@ export class InputParameterTerminal extends BaseInputTerminal { const effectiveThisType = this.effectiveType(this.type); const otherType = ("type" in other && other.type) || "data"; const effectiveOtherType = this.effectiveType(otherType); + if (!this.optional && other.optional) { + return new ConnectionAcceptable(false, `Cannot attach an optional output to a required parameter`); + } const canAccept = effectiveThisType === effectiveOtherType; return new ConnectionAcceptable( canAccept, diff --git a/client/src/components/Workflow/Editor/test-data/parameter_steps.json b/client/src/components/Workflow/Editor/test-data/parameter_steps.json index d3cff2cf8d98..bb0ad17cd901 100644 --- a/client/src/components/Workflow/Editor/test-data/parameter_steps.json +++ b/client/src/components/Workflow/Editor/test-data/parameter_steps.json @@ -906,5 +906,69 @@ "left": 12.8729248046875, "top": 501.7839660644531 } + }, + "19": { + "id": 19, + "type": "parameter_input", + "label": "optional text parameter input", + "content_id": null, + "name": "Input parameter", + "tool_state": { + "parameter_definition": "{\"__current_case__\": 0, \"optional\": {\"__current_case__\": 1, \"optional\": \"true\"}, \"parameter_type\": \"text\", \"restrictions\": {\"__current_case__\": 0, \"how\": \"none\"}}", + "__page__": null, + "__rerun_remap_job_id__": null + }, + "errors": null, + "inputs": [], + "outputs": [ + { + "name": "output", + "label": "optional text parameter input", + "type": "text", + "optional": true, + "parameter": true + } + ], + "annotation": "", + "post_job_actions": {}, + "uuid": "71c030ff-a29f-4e32-a735-37471e7a3f12", + "workflow_outputs": [], + "input_connections": {}, + "position": { + "left": 95.0506591796875, + "top": 867.0427551269531 + } + }, + "19": { + "id": 19, + "type": "parameter_input", + "label": "optional integer parameter input", + "content_id": null, + "name": "Input parameter", + "tool_state": { + "parameter_definition": "{\"__current_case__\": 1, \"optional\": {\"__current_case__\": 1, \"optional\": \"true\"}, \"parameter_type\": \"integer\"}", + "__page__": null, + "__rerun_remap_job_id__": null + }, + "errors": null, + "inputs": [], + "outputs": [ + { + "name": "output", + "label": "integer parameter input", + "type": "integer", + "optional": true, + "parameter": true + } + ], + "annotation": "", + "post_job_actions": {}, + "uuid": "634f06f1-b279-4160-9390-a004018e92ce", + "workflow_outputs": [], + "input_connections": {}, + "position": { + "left": 148.72970581054688, + "top": 620.5069274902344 + } } } From 103879e3094ac885a738e52e03a943d394babc91 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 16 Sep 2024 17:03:38 +0200 Subject: [PATCH 02/13] fix for https://github.com/galaxyproject/galaxy/issues/18816 --- lib/galaxy/authnz/psa_authnz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index 96ca33c6cd53..75c5edd6d35e 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -135,9 +135,10 @@ def _setup_idp(self, oidc_backend_config): self.config["KEY"] = oidc_backend_config.get("client_id") self.config["SECRET"] = oidc_backend_config.get("client_secret") self.config["TENANT_ID"] = oidc_backend_config.get("tenant_id") - self.config["OIDC_ENDPOINT"] = oidc_backend_config.get("oidc_endpoint") self.config["redirect_uri"] = oidc_backend_config.get("redirect_uri") self.config["EXTRA_SCOPES"] = oidc_backend_config.get("extra_scopes") + if oidc_backend_config.get("oidc_endpoint"): + self.config["OIDC_ENDPOINT"] = oidc_backend_config["oidc_endpoint"] if oidc_backend_config.get("prompt") is not None: self.config[setting_name("AUTH_EXTRA_ARGUMENTS")]["prompt"] = oidc_backend_config.get("prompt") if oidc_backend_config.get("api_url") is not None: From ec48f50877461143b011de4d85a167eea47cc30d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 16 Sep 2024 18:01:14 +0200 Subject: [PATCH 03/13] Fix wrong celery_app config on job and workflow handlers Fixes https://github.com/galaxyproject/galaxy/issues/18727 / https://sentry.galaxyproject.org/share/issue/8091a452f32645b89b37976362779532/: ``` Message (59965185/49195886) Job wrapper finish method failed Stack Trace Newest OperationalError: unable to open database file File "sqlalchemy/engine/base.py", line 146, in __init__ self._dbapi_connection = engine.raw_connection() File "sqlalchemy/engine/base.py", line 3300, in raw_connection return self.pool.connect() File "sqlalchemy/pool/base.py", line 449, in connect return _ConnectionFairy._checkout(self) File "sqlalchemy/pool/base.py", line 1263, in _checkout fairy = _ConnectionRecord.checkout(pool) File "sqlalchemy/pool/base.py", line 712, in checkout rec = pool._do_get() File "sqlalchemy/pool/impl.py", line 179, in _do_get with util.safe_reraise(): File "sqlalchemy/util/langhelpers.py", line 146, in __exit__ raise exc_value.with_traceback(exc_tb) File "sqlalchemy/pool/impl.py", line 177, in _do_get return self._create_connection() File "sqlalchemy/pool/base.py", line 390, in _create_connection return _ConnectionRecord(self) File "sqlalchemy/pool/base.py", line 674, in __init__ self.__connect() File "sqlalchemy/pool/base.py", line 900, in __connect with util.safe_reraise(): File "sqlalchemy/util/langhelpers.py", line 146, in __exit__ raise exc_value.with_traceback(exc_tb) File "sqlalchemy/pool/base.py", line 896, in __connect self.dbapi_connection = connection = pool._invoke_creator(self) File "sqlalchemy/engine/create.py", line 643, in connect return dialect.connect(*cargs, **cparams) File "sqlalchemy/engine/default.py", line 620, in connect return self.loaded_dbapi.connect(*cargs, **cparams) OperationalError: (sqlite3.OperationalError) unable to open database file (Background on this error at: https://sqlalche.me/e/20/e3q8) File "kombu/connection.py", line 472, in _reraise_as_library_errors yield File "kombu/connection.py", line 556, in _ensured return fun(*args, **kwargs) File "kombu/messaging.py", line 202, in _publish [maybe_declare(entity) for entity in declare] File "kombu/messaging.py", line 202, in [maybe_declare(entity) for entity in declare] File "kombu/messaging.py", line 107, in maybe_declare return maybe_declare(entity, self.channel, retry, **retry_policy) File "kombu/common.py", line 113, in maybe_declare return _maybe_declare(entity, channel) File "kombu/common.py", line 153, in _maybe_declare entity.declare(channel=channel) File "kombu/entity.py", line 617, in declare self._create_queue(nowait=nowait, channel=channel) File "kombu/entity.py", line 626, in _create_queue self.queue_declare(nowait=nowait, passive=False, channel=channel) File "kombu/entity.py", line 655, in queue_declare ret = channel.queue_declare( File "kombu/transport/virtual/base.py", line 537, in queue_declare self._new_queue(queue, **kwargs) File "kombu/transport/sqlalchemy/__init__.py", line 159, in _new_queue self._get_or_create(queue) File "kombu/transport/sqlalchemy/__init__.py", line 138, in _get_or_create obj = self.session.query(self.queue_cls) \ File "kombu/transport/sqlalchemy/__init__.py", line 133, in session _, Session = self._open() File "kombu/transport/sqlalchemy/__init__.py", line 125, in _open metadata.create_all(engine) File "sqlalchemy/sql/schema.py", line 5857, in create_all bind._run_ddl_visitor( File "sqlalchemy/engine/base.py", line 3250, in _run_ddl_visitor with self.begin() as conn: File "contextlib.py", line 137, in __enter__ return next(self.gen) File "sqlalchemy/engine/base.py", line 3240, in begin with self.connect() as conn: File "sqlalchemy/engine/base.py", line 3276, in connect return self._connection_cls(self) File "sqlalchemy/engine/base.py", line 148, in __init__ Connection._handle_dbapi_exception_noconnection( File "sqlalchemy/engine/base.py", line 2440, in _handle_dbapi_exception_noconnection raise sqlalchemy_exception.with_traceback(exc_info[2]) from e File "sqlalchemy/engine/base.py", line 146, in __init__ self._dbapi_connection = engine.raw_connection() File "sqlalchemy/engine/base.py", line 3300, in raw_connection return self.pool.connect() File "sqlalchemy/pool/base.py", line 449, in connect return _ConnectionFairy._checkout(self) File "sqlalchemy/pool/base.py", line 1263, in _checkout fairy = _ConnectionRecord.checkout(pool) File "sqlalchemy/pool/base.py", line 712, in checkout rec = pool._do_get() File "sqlalchemy/pool/impl.py", line 179, in _do_get with util.safe_reraise(): File "sqlalchemy/util/langhelpers.py", line 146, in __exit__ raise exc_value.with_traceback(exc_tb) File "sqlalchemy/pool/impl.py", line 177, in _do_get return self._create_connection() File "sqlalchemy/pool/base.py", line 390, in _create_connection return _ConnectionRecord(self) File "sqlalchemy/pool/base.py", line 674, in __init__ self.__connect() File "sqlalchemy/pool/base.py", line 900, in __connect with util.safe_reraise(): File "sqlalchemy/util/langhelpers.py", line 146, in __exit__ raise exc_value.with_traceback(exc_tb) File "sqlalchemy/pool/base.py", line 896, in __connect self.dbapi_connection = connection = pool._invoke_creator(self) File "sqlalchemy/engine/create.py", line 643, in connect return dialect.connect(*cargs, **cparams) File "sqlalchemy/engine/default.py", line 620, in connect return self.loaded_dbapi.connect(*cargs, **cparams) OperationalError: (sqlite3.OperationalError) unable to open database file (Background on this error at: https://sqlalche.me/e/20/e3q8) File "galaxy/jobs/runners/__init__.py", line 677, in _finish_or_resubmit_job job_wrapper.finish( File "galaxy/jobs/__init__.py", line 2070, in finish task_wrapper.delay() File "celery/canvas.py", line 353, in delay return self.apply_async(partial_args, partial_kwargs) File "celery/canvas.py", line 400, in apply_async return _apply(args, kwargs, **options) File "celery/app/task.py", line 594, in apply_async return app.send_task( File "celery/app/base.py", line 801, in send_task amqp.send_task_message(P, name, message, **options) File "celery/app/amqp.py", line 518, in send_task_message ret = producer.publish( File "kombu/messaging.py", line 186, in publish return _publish( File "kombu/connection.py", line 553, in _ensured with self._reraise_as_library_errors(): File "contextlib.py", line 155, in __exit__ self.gen.throw(typ, value, traceback) File "kombu/connection.py", line 476, in _reraise_as_library_errors raise ConnectionError(str(exc)) from exc ``` The issue was that we aren't setting `GALAXY_CONFIG_FILE` if the job and workflow handlers are started with `-c path/to/galaxy.yml`, and the celery config heuristic isn't looking at sys.argv. --- lib/galaxy/celery/__init__.py | 2 +- scripts/galaxy_main.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 11432481529c..120df3c28906 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -220,7 +220,7 @@ def config_celery_app(config, celery_app): if config.celery_conf: celery_app.conf.update(config.celery_conf) # Handle special cases - if not celery_app.conf.broker_url: + if not config.celery_conf.get("broker_url"): celery_app.conf.broker_url = config.amqp_internal_connection diff --git a/scripts/galaxy_main.py b/scripts/galaxy_main.py index 02ef73928604..85d6b954dbb7 100755 --- a/scripts/galaxy_main.py +++ b/scripts/galaxy_main.py @@ -87,8 +87,15 @@ def load_galaxy_app(config_builder, config_env=False, log=None, attach_to_pools= kwds = config_builder.app_kwds() kwds = load_app_properties(**kwds) from galaxy.app import UniverseApplication + from galaxy.celery import ( + celery_app, + config_celery_app, + ) app = UniverseApplication(global_conf=config_builder.global_conf(), attach_to_pools=attach_to_pools, **kwds) + # Update celery app, which might not have taken into account the config file if set via the `-c` argument. + config_celery_app(app.config, celery_app) + app.database_heartbeat.start() app.application_stack.log_startup() return app From fd8675ed8013ec5a1268a600f74150dc661dd062 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 17 Sep 2024 13:01:03 +0200 Subject: [PATCH 04/13] Start job handler only after building InteractiveToolManager This should fix https://github.com/galaxyproject/galaxy/issues/18822 --- lib/galaxy/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 7668fb54be6a..357e544db27b 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -814,21 +814,21 @@ def __init__(self, **kwargs) -> None: ) self.application_stack.register_postfork_function(self.prune_history_audit_task.start) self.haltables.append(("HistoryAuditTablePruneTask", self.prune_history_audit_task.shutdown)) - # Start the job manager - self.application_stack.register_postfork_function(self.job_manager.start) self.proxy_manager = ProxyManager(self.config) # Must be initialized after job_config. self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self) self.trs_proxy = self._register_singleton(TrsProxy, TrsProxy(self.config)) + # We need InteractiveToolManager before the job handler starts + self.interactivetool_manager = InteractiveToolManager(self) + # Start the job manager + self.application_stack.register_postfork_function(self.job_manager.start) # Must be initialized after any component that might make use of stack messaging is configured. Alternatively if # it becomes more commonly needed we could create a prefork function registration method like we do with # postfork functions. self.application_stack.init_late_prefork() - self.interactivetool_manager = InteractiveToolManager(self) - # Configure handling of signals handlers = {} if self.heartbeat: From 7042d73994c32b446c655736e0664e05cb137c87 Mon Sep 17 00:00:00 2001 From: Ales Krenek Date: Tue, 17 Sep 2024 11:35:39 +0000 Subject: [PATCH 05/13] enable extra_scopes config parameter also for Keycloak backend --- lib/galaxy/authnz/custos_authnz.py | 3 +++ lib/galaxy/authnz/managers.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index 806f6e1cafba..5236d84924b9 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -62,6 +62,7 @@ class CustosAuthnzConfiguration: pkce_support: bool accepted_audiences: List[str] extra_params: Optional[dict] + extra_scopes: List[str] authorization_endpoint: Optional[str] token_endpoint: Optional[str] end_session_endpoint: Optional[str] @@ -98,6 +99,7 @@ def __init__(self, provider, oidc_config, oidc_backend_config, idphint=None): ) ), extra_params={}, + extra_scopes=oidc_backend_config.get("extra_scopes",[]), authorization_endpoint=None, token_endpoint=None, end_session_endpoint=None, @@ -156,6 +158,7 @@ def _get_provider_specific_scopes(self): def authenticate(self, trans, idphint=None): base_authorize_url = self.config.authorization_endpoint scopes = ["openid", "email", "profile"] + scopes.extend(self.config.extra_scopes) scopes.extend(self._get_provider_specific_scopes()) oauth2_session = self._create_oauth2_session(scope=scopes) nonce = generate_nonce() diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 7af14e776895..0e6d27d6589c 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -213,6 +213,8 @@ def _parse_custos_config(self, config_xml): rtv["ca_bundle"] = config_xml.find("ca_bundle").text if config_xml.find("icon") is not None: rtv["icon"] = config_xml.find("icon").text + if config_xml.find("extra_scopes") is not None: + rtv["extra_scopes"] = listify(config_xml.find("extra_scopes").text) if config_xml.find("pkce_support") is not None: rtv["pkce_support"] = asbool(config_xml.find("pkce_support").text) if config_xml.find("accepted_audiences") is not None: From 0344210cb939b66d3c119992a152afd1136ea502 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 17 Sep 2024 13:10:11 +0200 Subject: [PATCH 06/13] Fix ``named cursor is not valid anymore`` Fixes https://github.com/galaxyproject/galaxy/issues/18782 / https://sentry.galaxyproject.org/share/issue/dca23c8d10e743509648cf6092a1dc15/: ``` Message Error closing cursor Stack Trace Newest ProgrammingError: named cursor isn't valid anymore File "sqlalchemy/engine/cursor.py", line 1255, in fetchmany new = dbapi_cursor.fetchmany(size - lb) ProgrammingError: named cursor isn't valid anymore File "sqlalchemy/engine/base.py", line 2213, in _safe_close_cursor cursor.close() ``` --- lib/galaxy/managers/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 98cdf2408cb6..85bf8d255511 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -1119,7 +1119,7 @@ def get_jobs_to_check_at_startup(session: galaxy_scoped_session, track_jobs_in_d # Filter out the jobs of inactive users. stmt = stmt.outerjoin(User).filter(or_((Job.user_id == null()), (User.active == true()))) - return session.scalars(stmt) + return session.scalars(stmt).all() def get_job(session, *where_clauses): From 244ff927ca1af900b6d805d96f2fceba9526c2da Mon Sep 17 00:00:00 2001 From: Ales Krenek Date: Tue, 17 Sep 2024 15:29:09 +0200 Subject: [PATCH 07/13] make flake happy --- lib/galaxy/authnz/custos_authnz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index 5236d84924b9..4974ff57a122 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -99,7 +99,7 @@ def __init__(self, provider, oidc_config, oidc_backend_config, idphint=None): ) ), extra_params={}, - extra_scopes=oidc_backend_config.get("extra_scopes",[]), + extra_scopes=oidc_backend_config.get("extra_scopes", []), authorization_endpoint=None, token_endpoint=None, end_session_endpoint=None, From 2b4b7ed37f3b4b349afebdf2c6489979bb85d5d9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 17 Sep 2024 17:16:59 +0200 Subject: [PATCH 08/13] Maybe fix flaky navigation in v2 ts tests The stack trace is this: ``` __ TestSimplePriorInstallation.test_0005_create_convert_repository[chromium] ___ self = def test_0005_create_convert_repository(self): """Create and populate convert_chars_0150.""" global running_standalone category = self.create_category(name=category_name, description=category_description) > self.login(email=common.test_user_1_email, username=common.test_user_1_name) lib/tool_shed/test/functional/test_1170_prior_installation_required.py:50: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ lib/tool_shed/test/base/twilltestcase.py:776: in login previously_created, username_taken, invalid_username = self.create( lib/tool_shed/test/base/twilltestcase.py:714: in create self.visit_url("/user/create", params) lib/tool_shed/test/base/twilltestcase.py:850: in visit_url return self._browser.visit_url(url, allowed_codes=allowed_codes) lib/tool_shed/test/base/playwrightbrowser.py:31: in visit_url response = self._page.goto(url) .venv/lib/python3.8/site-packages/playwright/sync_api/_generated.py:8851: in goto self._sync( .venv/lib/python3.8/site-packages/playwright/_impl/_page.py:524: in goto return await self._main_frame.goto(**locals_to_params(locals())) .venv/lib/python3.8/site-packages/playwright/_impl/_frame.py:145: in goto await self._channel.send("goto", locals_to_params(locals())) .venv/lib/python3.8/site-packages/playwright/_impl/_connection.py:59: in send return await self._connection.wrap_api_call( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = cb = . at 0x7f87af067550> is_internal = False async def wrap_api_call( self, cb: Callable[[], Any], is_internal: bool = False ) -> Any: if self._api_zone.get(): return await cb() task = asyncio.current_task(self._loop) st: List[inspect.FrameInfo] = getattr(task, "__pw_stack__", inspect.stack()) parsed_st = _extract_stack_trace_information_from_stack(st, is_internal) self._api_zone.set(parsed_st) try: return await cb() except Exception as error: > raise rewrite_error(error, f"{parsed_st['apiName']}: {error}") from None E playwright._impl._errors.Error: Page.goto: Navigation to "http://127.0.0.1:8723/user/create?cntrller=user&use_panels=False" is interrupted by another navigation to "http://127.0.0.1:8723/logout_success" E Call log: E navigating to "http://127.0.0.1:8723/user/create?cntrller=user&use_panels=False", waiting until "load" ``` --- lib/tool_shed/test/base/playwrightbrowser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tool_shed/test/base/playwrightbrowser.py b/lib/tool_shed/test/base/playwrightbrowser.py index 01949ff1eb96..cc32cb49f2a5 100644 --- a/lib/tool_shed/test/base/playwrightbrowser.py +++ b/lib/tool_shed/test/base/playwrightbrowser.py @@ -173,6 +173,7 @@ def logout_if_logged_in(self, assert_logged_out=True): logout_locator = self._page.locator(Locators.toolbar_logout) if logout_locator.is_visible(): logout_locator.click() + self._page.wait_for_load_state("networkidle") if assert_logged_out: self.expect_not_logged_in() From 52f34453782d7e75c25898fdebf1d2a18c0a3948 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:53:50 +0200 Subject: [PATCH 09/13] Make all fields optional for HelpForumPost Unfortunately, the OpenAPI schema of Discourse (https://docs.discourse.org/openapi.json) does not specify the proper type for posts in the "/search.json" response. There is at least one case where the post name is null (https://help.galaxyproject.org//search.json?q=tags:circos+tool-help%20status:solved) so let's play safe. --- client/src/api/schema/schema.ts | 16 ++++++++-------- lib/galaxy/schema/help.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 1577170c9663..af2fc0a5b8bb 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -7031,17 +7031,17 @@ export interface components { * Avatar Template * @description The avatar template of the user. */ - avatar_template: string; + avatar_template: string | null; /** * Blurb * @description The blurb of the post. */ - blurb: string; + blurb: string | null; /** * Created At * @description The creation date of the post. */ - created_at: string; + created_at: string | null; /** * Id * @description The ID of the post. @@ -7051,27 +7051,27 @@ export interface components { * Like Count * @description The number of likes of the post. */ - like_count: number; + like_count: number | null; /** * Name * @description The name of the post. */ - name: string; + name: string | null; /** * Post Number * @description The post number of the post. */ - post_number: number; + post_number: number | null; /** * Topic Id * @description The ID of the topic of the post. */ - topic_id: number; + topic_id: number | null; /** * Username * @description The username of the post author. */ - username: string; + username: string | null; [key: string]: unknown | undefined; }; /** diff --git a/lib/galaxy/schema/help.py b/lib/galaxy/schema/help.py index ebe2da9370db..6ecb83513b49 100644 --- a/lib/galaxy/schema/help.py +++ b/lib/galaxy/schema/help.py @@ -27,14 +27,14 @@ class HelpForumPost(HelpTempBaseModel): """Model for a post in the help forum.""" id: Annotated[int, Field(description="The ID of the post.")] - name: Annotated[str, Field(description="The name of the post.")] - username: Annotated[str, Field(description="The username of the post author.")] - avatar_template: Annotated[str, Field(description="The avatar template of the user.")] - created_at: Annotated[str, Field(description="The creation date of the post.")] - like_count: Annotated[int, Field(description="The number of likes of the post.")] - blurb: Annotated[str, Field(description="The blurb of the post.")] - post_number: Annotated[int, Field(description="The post number of the post.")] - topic_id: Annotated[int, Field(description="The ID of the topic of the post.")] + name: Annotated[Optional[str], Field(description="The name of the post.")] + username: Annotated[Optional[str], Field(description="The username of the post author.")] + avatar_template: Annotated[Optional[str], Field(description="The avatar template of the user.")] + created_at: Annotated[Optional[str], Field(description="The creation date of the post.")] + like_count: Annotated[Optional[int], Field(description="The number of likes of the post.")] + blurb: Annotated[Optional[str], Field(description="The blurb of the post.")] + post_number: Annotated[Optional[int], Field(description="The post number of the post.")] + topic_id: Annotated[Optional[int], Field(description="The ID of the topic of the post.")] class HelpForumTopic(Model): From efa5ada463ea60cb450487da79fb7e7663526e94 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 17 Sep 2024 18:34:57 +0300 Subject: [PATCH 10/13] Return null instead of empty string in unique label validator --- .../Workflow/Editor/composables/useUniqueLabelError.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/src/components/Workflow/Editor/composables/useUniqueLabelError.ts b/client/src/components/Workflow/Editor/composables/useUniqueLabelError.ts index 331c1a70b7dc..84276f2998ae 100644 --- a/client/src/components/Workflow/Editor/composables/useUniqueLabelError.ts +++ b/client/src/components/Workflow/Editor/composables/useUniqueLabelError.ts @@ -6,11 +6,9 @@ export function useUniqueLabelError( workflowStateStore: ReturnType, label: string | null | undefined ) { - const error = ref(""); + const error = ref(null); if (label && workflowStateStore.workflowOutputs[label]) { error.value = `Duplicate label ${label}. Please fix this before saving the workflow.`; - } else { - error.value = ""; } return error; } From 64e74b8b48c8bef160fdc579c6f83ae337e27c4c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 18 Sep 2024 09:46:59 +0200 Subject: [PATCH 11/13] Tighten TRS url check Fixes https://sentry.galaxyproject.org/share/issue/5b8d495edd6545278d4d93d0ed69ac3a/: ``` Message Uncaught exception in exposed API method: Stack Trace Newest gaierror: [Errno -2] Name or service not known File "urllib3/connection.py", line 174, in _new_conn conn = connection.create_connection( File "urllib3/util/connection.py", line 72, in create_connection for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): File "socket.py", line 962, in getaddrinfo for res in _socket.getaddrinfo(host, port, family, type, proto, flags): NewConnectionError: : Failed to establish a new connection: [Errno -2] Name or service not known File "urllib3/connectionpool.py", line 715, in urlopen httplib_response = self._make_request( File "urllib3/connectionpool.py", line 416, in _make_request conn.request(method, url, **httplib_request_kw) File "urllib3/connection.py", line 244, in request super(HTTPConnection, self).request(method, url, body=body, headers=headers) File "http/client.py", line 1286, in request self._send_request(method, url, body, headers, encode_chunked) File "http/client.py", line 1332, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "http/client.py", line 1281, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "http/client.py", line 1041, in _send_output self.send(msg) File "http/client.py", line 979, in send self.connect() File "urllib3/connection.py", line 205, in connect conn = self._new_conn() File "urllib3/connection.py", line 186, in _new_conn raise NewConnectionError( MaxRetryError: HTTPConnectionPool(host='https', port=80): Max retries exceeded with url: //training.galaxyproject.org/training-material//api/ga4gh/trs/v2/tools/variant-analysis-microbial-variants/versions/microbial_variant_calling/GALAXY/descriptor (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known')) File "requests/adapters.py", line 486, in send resp = conn.urlopen( File "urllib3/connectionpool.py", line 799, in urlopen retries = retries.increment( File "urllib3/util/retry.py", line 592, in increment raise MaxRetryError(_pool, url, error or ResponseError(cause)) ConnectionError: HTTPConnectionPool(host='https', port=80): Max retries exceeded with url: //training.galaxyproject.org/training-material//api/ga4gh/trs/v2/tools/variant-analysis-microbial-variants/versions/microbial_variant_calling/GALAXY/descriptor (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known')) File "galaxy/web/framework/decorators.py", line 346, in decorator rval = func(self, trans, *args, **kwargs) File "galaxy/webapps/galaxy/api/workflows.py", line 261, in create archive_data = server.get_version_descriptor(trs_tool_id, trs_version_id) File "galaxy/workflow/trs_proxy.py", line 115, in get_version_descriptor return self._get(trs_api_url)["content"] File "galaxy/workflow/trs_proxy.py", line 126, in _get response = requests.get(url, params=params, timeout=DEFAULT_SOCKET_TIMEOUT) File "galaxy/util/requests.py", line 29, in wrapper rval = f(*args, **kwargs) File "requests/api.py", line 73, in get return request("get", url, params=params, **kwargs) File "requests/api.py", line 59, in request return session.request(method=method, url=url, **kwargs) File "requests/sessions.py", line 589, in request resp = self.send(prep, **send_kwargs) File "requests/sessions.py", line 703, in send r = adapter.send(request, **kwargs) File "requests/adapters.py", line 519, in send raise ConnectionError(e, request=request) ``` which happened for https://usegalaxy.org/workflows/trs_import?run_form=true&trs_url=http%3A%2F%2Flocalhost%3A4000%2F%2Ftraining-material%2F%2Fapi%2Fga4gh%2Ftrs%2Fv2%2Ftools%2Fvariant-analysis-microbial-variants%2Fversions%2Fmicrobial_variant_calling" --- lib/galaxy/webapps/galaxy/api/workflows.py | 6 ++-- lib/galaxy/workflow/trs_proxy.py | 18 ++++++++++-- test/unit/workflows/test_trs_proxy.py | 32 ++++++++++++++-------- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 64d1d2a4c0ba..8ebfcc98440c 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -243,7 +243,9 @@ def create(self, trans: GalaxyWebTransaction, payload=None, **kwd): trs_version_id = None import_source = None if "trs_url" in payload: - parts = self.app.trs_proxy.match_url(payload["trs_url"]) + parts = self.app.trs_proxy.match_url( + payload["trs_url"], trans.app.config.fetch_url_allowlist_ips + ) if parts: server = self.app.trs_proxy.server_from_url(parts["trs_base_url"]) trs_tool_id = parts["tool_id"] @@ -251,7 +253,7 @@ def create(self, trans: GalaxyWebTransaction, payload=None, **kwd): payload["trs_tool_id"] = trs_tool_id payload["trs_version_id"] = trs_version_id else: - raise exceptions.MessageException("Invalid TRS URL.") + raise exceptions.RequestParameterInvalidException(f"Invalid TRS URL {payload['trs_url']}.") else: trs_server = payload.get("trs_server") server = self.app.trs_proxy.get_server(trs_server) diff --git a/lib/galaxy/workflow/trs_proxy.py b/lib/galaxy/workflow/trs_proxy.py index e465127a99ad..2e6d1acaa310 100644 --- a/lib/galaxy/workflow/trs_proxy.py +++ b/lib/galaxy/workflow/trs_proxy.py @@ -2,15 +2,21 @@ import os import re import urllib.parse +from typing import List import yaml -from galaxy.exceptions import MessageException +from galaxy.exceptions import ( + MessageException, + RequestParameterInvalidException, +) +from galaxy.files.uris import validate_non_local from galaxy.util import ( asbool, DEFAULT_SOCKET_TIMEOUT, requests, ) +from galaxy.util.config_parsers import IpAllowedListEntryT from galaxy.util.search import parse_filters log = logging.getLogger(__name__) @@ -73,7 +79,15 @@ def get_server(self, trs_server): def server_from_url(self, trs_url): return TrsServer(trs_url) - def match_url(self, url): + def match_url(self, url, ip_allowlist: List[IpAllowedListEntryT]): + if url.lstrip().startswith("file://"): + # requests doesn't know what to do with file:// anyway, but just in case we swap + # out the implementation + raise RequestParameterInvalidException("Invalid TRS URL %s", url) + validate_non_local(url, ip_allowlist=ip_allowlist or []) + return self._match_url(url) + + def _match_url(self, url): if matches := re.match(TRS_URL_REGEX, url): match_dict = matches.groupdict() match_dict["tool_id"] = urllib.parse.unquote(match_dict["tool_id"]) diff --git a/test/unit/workflows/test_trs_proxy.py b/test/unit/workflows/test_trs_proxy.py index 6296dea99688..179d95625d80 100644 --- a/test/unit/workflows/test_trs_proxy.py +++ b/test/unit/workflows/test_trs_proxy.py @@ -4,6 +4,7 @@ import pytest import yaml +from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.workflow.trs_proxy import ( GA4GH_GALAXY_DESCRIPTOR, parse_search_kwds, @@ -49,9 +50,9 @@ def test_proxy(): def test_match_url(): proxy = TrsProxy() - valid_dockstore = proxy.match_url( + valid_dockstore = proxy._match_url( "https://dockstore.org/api/ga4gh/trs/v2/tools/" - "quay.io%2Fcollaboratory%2Fdockstore-tool-bedtools-genomecov/versions/0.3" + "quay.io%2Fcollaboratory%2Fdockstore-tool-bedtools-genomecov/versions/0.3", ) assert valid_dockstore assert valid_dockstore["trs_base_url"] == "https://dockstore.org/api" @@ -59,9 +60,9 @@ def test_match_url(): assert valid_dockstore["tool_id"] == "quay.io/collaboratory/dockstore-tool-bedtools-genomecov" assert valid_dockstore["version_id"] == "0.3" - valid_dockstore_unescaped = proxy.match_url( + valid_dockstore_unescaped = proxy._match_url( "https://dockstore.org/api/ga4gh/trs/v2/tools/" - "#workflow/github.com/jmchilton/galaxy-workflow-dockstore-example-1/mycoolworkflow/versions/master" + "#workflow/github.com/jmchilton/galaxy-workflow-dockstore-example-1/mycoolworkflow/versions/master", ) assert valid_dockstore_unescaped assert valid_dockstore_unescaped["trs_base_url"] == "https://dockstore.org/api" @@ -71,13 +72,13 @@ def test_match_url(): ) assert valid_dockstore_unescaped["version_id"] == "master" - valid_workflow_hub = proxy.match_url("https://workflowhub.eu/ga4gh/trs/v2/tools/344/versions/1") + valid_workflow_hub = proxy._match_url("https://workflowhub.eu/ga4gh/trs/v2/tools/344/versions/1") assert valid_workflow_hub assert valid_workflow_hub["trs_base_url"] == "https://workflowhub.eu" assert valid_workflow_hub["tool_id"] == "344" assert valid_workflow_hub["version_id"] == "1" - valid_arbitrary_trs = proxy.match_url( + valid_arbitrary_trs = proxy._match_url( "https://my-trs-server.golf/stuff/ga4gh/trs/v2/tools/hello-world/versions/version-1" ) assert valid_arbitrary_trs @@ -85,26 +86,33 @@ def test_match_url(): assert valid_arbitrary_trs["tool_id"] == "hello-world" assert valid_arbitrary_trs["version_id"] == "version-1" - ignore_extra = proxy.match_url( - "https://workflowhub.eu/ga4gh/trs/v2/tools/344/versions/1/CWL/descriptor/ro-crate-metadata.json" + ignore_extra = proxy._match_url( + "https://workflowhub.eu/ga4gh/trs/v2/tools/344/versions/1/CWL/descriptor/ro-crate-metadata.json", ) assert ignore_extra assert ignore_extra["trs_base_url"] == "https://workflowhub.eu" assert ignore_extra["tool_id"] == "344" assert ignore_extra["version_id"] == "1" - invalid = proxy.match_url("https://workflowhub.eu/workflows/1") + invalid = proxy._match_url("https://workflowhub.eu/workflows/1") assert invalid is None - missing_version = proxy.match_url("https://workflowhub.eu/ga4gh/trs/v2/tools/344") + missing_version = proxy._match_url("https://workflowhub.eu/ga4gh/trs/v2/tools/344") assert missing_version is None - blank = proxy.match_url("") + blank = proxy.match_url("", []) assert blank is None - not_url = proxy.match_url("1234") + not_url = proxy.match_url("1234", []) assert not_url is None + expected_exception = None + try: + proxy.match_url("https://localhost", []) + except ConfigDoesNotAllowException as e: + expected_exception = e + assert expected_exception, "matching url against localhost should fail" + def test_server_from_url(): proxy = TrsProxy() From 31189ad20429420d780684879bb3911b4c59fd2f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 18 Sep 2024 17:23:56 +0200 Subject: [PATCH 12/13] Skip metric collection if job working directory doesn't exist That happens if the user stops a job before it's queued (or if we have a job rule preventing queuing). Fixes https://sentry.galaxyproject.org/share/issue/8b462ee0d50a4de8bb9ca2c50f049b42/: ``` Message Could not recover job metrics Stack Trace Newest JobMappingException: This tool is currently being beta-tested and your account has not been given access. File "galaxy/jobs/mapper.py", line 214, in __handle_rule job_destination = self.__invoke_expand_function( File "galaxy/jobs/mapper.py", line 125, in __invoke_expand_function return expand_function(**actual_args) File "tpv/rules/gateway.py", line 51, in map_tool_to_destination return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job, job_wrapper, resource_params, File "tpv/core/mapper.py", line 168, in map_to_destination evaluated_entity = self.match_combine_evaluate_entities(context, tool, user) File "tpv/core/mapper.py", line 138, in match_combine_evaluate_entities evaluated_entity = combined_entity.evaluate_rules(context) File "tpv/core/entities.py", line 489, in evaluate_rules rule = rule.evaluate(context) File "tpv/core/entities.py", line 750, in evaluate raise JobMappingException( JobMappingException: This tool is currently being beta-tested and your account has not been given access. File "galaxy/jobs/handler.py", line 719, in __verify_job_ready job_destination = job_wrapper.job_destination File "galaxy/jobs/__init__.py", line 2576, in job_destination return self.job_runner_mapper.get_job_destination(self.params) File "galaxy/jobs/mapper.py", line 276, in get_job_destination return self.__cache_job_destination(params) File "galaxy/jobs/mapper.py", line 257, in __cache_job_destination self.cached_job_destination = self.__determine_job_destination( File "galaxy/jobs/mapper.py", line 241, in __determine_job_destination job_destination = self.__handle_dynamic_job_destination(raw_job_destination) File "galaxy/jobs/mapper.py", line 204, in __handle_dynamic_job_destination return self.__handle_rule(expand_function, destination) File "galaxy/jobs/mapper.py", line 219, in __handle_rule raise e File "galaxy/jobs/mapper.py", line 208, in __handle_rule job_destination = self.__invoke_expand_function(rule_function, destination) File "galaxy/jobs/mapper.py", line 125, in __invoke_expand_function return expand_function(**actual_args) File "tpv/rules/gateway.py", line 51, in map_tool_to_destination return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job, job_wrapper, resource_params, File "tpv/core/mapper.py", line 168, in map_to_destination evaluated_entity = self.match_combine_evaluate_entities(context, tool, user) File "tpv/core/mapper.py", line 138, in match_combine_evaluate_entities evaluated_entity = combined_entity.evaluate_rules(context) File "tpv/core/entities.py", line 489, in evaluate_rules rule = rule.evaluate(context) File "tpv/core/entities.py", line 750, in evaluate raise JobMappingException( ObjectNotFound: objectstore, _call_method failed: _get_filename on , kwargs: {'base_dir': 'job_work', 'dir_only': True, 'obj_dir': True} File "galaxy/jobs/__init__.py", line 2163, in _collect_metrics job_metrics_directory = self.working_directory File "galaxy/jobs/__init__.py", line 1317, in working_directory self.__working_directory = self.app.object_store.get_filename( File "galaxy/objectstore/__init__.py", line 475, in get_filename return self._invoke("get_filename", obj, **kwargs) File "galaxy/objectstore/__init__.py", line 451, in _invoke return self.__getattribute__(f"_{delegate}")(obj=obj, **kwargs) File "galaxy/objectstore/__init__.py", line 1009, in _get_filename return self._call_method("_get_filename", obj, ObjectNotFound, True, **kwargs) File "galaxy/objectstore/__init__.py", line 1281, in _call_method raise default( ``` Minor follow up to https://github.com/galaxyproject/galaxy/pull/18809 --- lib/galaxy/jobs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 34d3550dc1ae..4ad932893b76 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1414,7 +1414,7 @@ def fail( message = str(message) working_directory_exists = self.working_directory_exists() - if not job.tasks: + if not job.tasks and working_directory_exists: # If job was composed of tasks, don't attempt to recollect statistics self._collect_metrics(job, job_metrics_directory) From 8609cf51bb7fea4d937cb39c5b1d7be3cfeefd39 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 18 Sep 2024 19:08:13 +0200 Subject: [PATCH 13/13] Extend on disk checks to running, queued and error states Fixes https://sentry.galaxyproject.org/share/issue/c1c4c9b0ab324b4ea4f98c3b8c402d21/: ``` Message Uncaught Exception Stack Trace Newest FileNotFoundError: [Errno 2] No such file or directory: '' File "galaxy/web/framework/middleware/error.py", line 167, in __call__ app_iter = self.application(environ, sr_checker) File "galaxy/web/framework/middleware/statsd.py", line 29, in __call__ req = self.application(environ, start_response) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.11/site-packages/paste/httpexceptions.py", line 635, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/base.py", line 176, in __call__ return self.handle_request(request_id, path_info, environ, start_response) File "galaxy/web/framework/base.py", line 271, in handle_request body = method(trans, **kwargs) File "galaxy/webapps/galaxy/controllers/dataset.py", line 155, in display display_data, headers = data.datatype.display_data( File "galaxy/datatypes/tabular.py", line 226, in display_data chunk=self.get_chunk(trans, dataset, 0), File "galaxy/datatypes/tabular.py", line 149, in get_chunk ck_data, last_read = self._read_chunk(trans, dataset, offset, ck_size) File "galaxy/datatypes/tabular.py", line 159, in _read_chunk with compression_utils.get_fileobj(dataset.get_file_name()) as f: File "galaxy/util/compression_utils.py", line 79, in get_fileobj return get_fileobj_raw(filename, mode, compressed_formats)[1] File "galaxy/util/compression_utils.py", line 139, in get_fileobj_raw return compressed_format, open(filename, mode, encoding="utf-8") ``` --- lib/galaxy/managers/datasets.py | 12 ++++++++-- .../webapps/galaxy/controllers/dataset.py | 24 +------------------ 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 0bbb4f4cbaf6..472e6689639e 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -497,8 +497,8 @@ def ensure_dataset_on_disk(self, trans, dataset): raise exceptions.ItemDeletionException("The dataset you are attempting to view has been deleted.") elif dataset.state == Dataset.states.UPLOAD: raise exceptions.Conflict("Please wait until this dataset finishes uploading before attempting to view it.") - elif dataset.state == Dataset.states.NEW: - raise exceptions.Conflict("The dataset you are attempting to view is new and has no data.") + elif dataset.state in (Dataset.states.NEW, Dataset.states.QUEUED): + raise exceptions.Conflict(f"The dataset you are attempting to view is {dataset.state} and has no data.") elif dataset.state == Dataset.states.DISCARDED: raise exceptions.ItemDeletionException("The dataset you are attempting to view has been discarded.") elif dataset.state == Dataset.states.DEFERRED: @@ -509,6 +509,14 @@ def ensure_dataset_on_disk(self, trans, dataset): raise exceptions.Conflict( "The dataset you are attempting to view is in paused state. One of the inputs for the job that creates this dataset has failed." ) + elif dataset.state == Dataset.states.RUNNING: + if not self.app.object_store.exists(dataset.dataset): + raise exceptions.Conflict( + "The dataset you are attempting to view is still being created and has no data yet." + ) + elif dataset.state == Dataset.states.ERROR: + if not self.app.object_store.exists(dataset.dataset): + raise exceptions.RequestParameterInvalidException("The dataset is in error and has no data.") def ensure_can_change_datatype(self, dataset: model.DatasetInstance, raiseException: bool = True) -> bool: if not dataset.datatype.is_datatype_change_allowed(): diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index efcfe5ae2c2d..f9ff2a413893 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -25,7 +25,6 @@ HDAManager, ) from galaxy.managers.histories import HistoryManager -from galaxy.model import Dataset from galaxy.model.base import transaction from galaxy.model.item_attrs import ( UsesAnnotations, @@ -108,24 +107,7 @@ def _check_dataset(self, trans, hda_id): raise web.httpexceptions.HTTPNotFound(f"Invalid reference dataset id: {str(hda_id)}.") if not self._can_access_dataset(trans, data): return trans.show_error_message("You are not allowed to access this dataset") - if data.purged or data.dataset.purged: - return trans.show_error_message("The dataset you are attempting to view has been purged.") - elif data.deleted and not (trans.user_is_admin or (data.history and trans.get_user() == data.user)): - return trans.show_error_message("The dataset you are attempting to view has been deleted.") - elif data.state == Dataset.states.UPLOAD: - return trans.show_error_message( - "Please wait until this dataset finishes uploading before attempting to view it." - ) - elif data.state == Dataset.states.DISCARDED: - return trans.show_error_message("The dataset you are attempting to view has been discarded.") - elif data.state == Dataset.states.DEFERRED: - return trans.show_error_message( - "The dataset you are attempting to view has deferred data. You can only use this dataset as input for jobs." - ) - elif data.state == Dataset.states.PAUSED: - return trans.show_error_message( - "The dataset you are attempting to view is in paused state. One of the inputs for the job that creates this dataset has failed." - ) + self.app.hda_manager.ensure_dataset_on_disk(trans, data) return data @web.expose @@ -133,8 +115,6 @@ def display( self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd ): data = self._check_dataset(trans, dataset_id) - if not isinstance(data, trans.app.model.DatasetInstance): - return data if "hdca" in kwd: raise RequestParameterInvalidException("Invalid request parameter 'hdca' encountered.") if hdca_id := kwd.get("hdca_id", None): @@ -478,8 +458,6 @@ def imp(self, trans, dataset_id=None, **kwd): def display_by_username_and_slug(self, trans, username, slug, filename=None, preview=True, **kwargs): """Display dataset by username and slug; because datasets do not yet have slugs, the slug is the dataset's id.""" dataset = self._check_dataset(trans, slug) - if not isinstance(dataset, trans.app.model.DatasetInstance): - return dataset # Filename used for composite types. if filename: return self.display(trans, dataset_id=slug, filename=filename)