Merge branch 'release_24.1' into dev

client/src/api/schema/schema.ts

@@ -10535,17 +10535,17 @@ export interface components {
              * Avatar Template
              * @description The avatar template of the user.
              */
-            avatar_template: string;
+            avatar_template: string | null;
             /**
              * Blurb
              * @description The blurb of the post.
              */
-            blurb: string;
+            blurb: string | null;
             /**
              * Created At
              * @description The creation date of the post.
              */
-            created_at: string;
+            created_at: string | null;
             /**
              * Id
              * @description The ID of the post.
@@ -10555,27 +10555,27 @@ export interface components {
              * Like Count
              * @description The number of likes of the post.
              */
-            like_count: number;
+            like_count: number | null;
             /**
              * Name
              * @description The name of the post.
              */
-            name: string;
+            name: string | null;
             /**
              * Post Number
              * @description The post number of the post.
              */
-            post_number: number;
+            post_number: number | null;
             /**
              * Topic Id
              * @description The ID of the topic of the post.
              */
-            topic_id: number;
+            topic_id: number | null;
             /**
              * Username
              * @description The username of the post author.
              */
-            username: string;
+            username: string | null;
         } & {
             [key: string]: unknown;
         };

client/src/components/Workflow/Editor/NodeOutput.vue

@@ -310,7 +310,7 @@ const outputDetails = computed(() => {
     const outputType =
         collectionType && collectionType.isCollection && collectionType.collectionType
             ? `output is ${collectionTypeToDescription(collectionType)}`
-            : `output is ${terminal.value.type || "dataset"}`;
+            : `output is  ${terminal.value.optional ? "optional " : ""}${terminal.value.type || "dataset"}`;
     if (isMultiple.value) {
         if (!collectionType) {
             collectionType = NULL_COLLECTION_TYPE_DESCRIPTION;

client/src/components/Workflow/Editor/composables/useUniqueLabelError.ts

@@ -6,11 +6,9 @@ export function useUniqueLabelError(
     workflowStateStore: ReturnType<typeof useWorkflowStepStore>,
     label: string | null | undefined
 ) {
-    const error = ref("");
+    const error = ref<string | null>(null);
     if (label && workflowStateStore.workflowOutputs[label]) {
         error.value = `Duplicate label ${label}. Please fix this before saving the workflow.`;
-    } else {
-        error.value = "";
     }
     return error;
 }

client/src/components/Workflow/Editor/modules/terminals.test.ts

@@ -304,6 +304,16 @@ describe("canAccept", () => {
             "Cannot attach a text parameter to a integer input"
         );
     });
+    it("rejects optional integer to required parameter connection", () => {
+        const integerInputParam = terminals["multi data"]!["advanced|advanced_threshold"] as InputParameterTerminal;
+        const optionalIntegerOutputParam = terminals["optional integer parameter input"]![
+            "output"
+        ] as OutputParameterTerminal;
+        expect(integerInputParam.canAccept(optionalIntegerOutputParam).canAccept).toBe(false);
+        expect(integerInputParam.canAccept(optionalIntegerOutputParam).reason).toBe(
+            "Cannot attach an optional output to a required parameter"
+        );
+    });
     it("rejects data to parameter connection", () => {
         const dataOut = terminals["data input"]!["output"] as OutputTerminal;
         const integerInputParam = terminals["multi data"]!["advanced|advanced_threshold"] as InputParameterTerminal;

client/src/components/Workflow/Editor/modules/terminals.ts

@@ -551,6 +551,9 @@ export class InputParameterTerminal extends BaseInputTerminal {
         const effectiveThisType = this.effectiveType(this.type);
         const otherType = ("type" in other && other.type) || "data";
         const effectiveOtherType = this.effectiveType(otherType);
+        if (!this.optional && other.optional) {
+            return new ConnectionAcceptable(false, `Cannot attach an optional output to a required parameter`);
+        }
         const canAccept = effectiveThisType === effectiveOtherType;
         if (!this.multiple && other.multiple) {
             return new ConnectionAcceptable(

client/src/components/Workflow/Editor/test-data/parameter_steps.json

@@ -910,5 +910,69 @@
       "left": 12.8729248046875,
       "top": 501.7839660644531
     }
+  },
+  "19": {
+    "id": 19,
+    "type": "parameter_input",
+    "label": "optional text parameter input",
+    "content_id": null,
+    "name": "Input parameter",
+    "tool_state": {
+      "parameter_definition": "{\"__current_case__\": 0, \"optional\": {\"__current_case__\": 1, \"optional\": \"true\"}, \"parameter_type\": \"text\", \"restrictions\": {\"__current_case__\": 0, \"how\": \"none\"}}",
+      "__page__": null,
+      "__rerun_remap_job_id__": null
+    },
+    "errors": null,
+    "inputs": [],
+    "outputs": [
+      {
+        "name": "output",
+        "label": "optional text parameter input",
+        "type": "text",
+        "optional": true,
+        "parameter": true
+      }
+    ],
+    "annotation": "",
+    "post_job_actions": {},
+    "uuid": "71c030ff-a29f-4e32-a735-37471e7a3f12",
+    "workflow_outputs": [],
+    "input_connections": {},
+    "position": {
+      "left": 95.0506591796875,
+      "top": 867.0427551269531
+    }
+  },
+  "19": {
+    "id": 19,
+    "type": "parameter_input",
+    "label": "optional integer parameter input",
+    "content_id": null,
+    "name": "Input parameter",
+    "tool_state": {
+      "parameter_definition": "{\"__current_case__\": 1, \"optional\": {\"__current_case__\": 1, \"optional\": \"true\"}, \"parameter_type\": \"integer\"}",
+      "__page__": null,
+      "__rerun_remap_job_id__": null
+    },
+    "errors": null,
+    "inputs": [],
+    "outputs": [
+      {
+        "name": "output",
+        "label": "integer parameter input",
+        "type": "integer",
+        "optional": true,
+        "parameter": true
+      }
+    ],
+    "annotation": "",
+    "post_job_actions": {},
+    "uuid": "634f06f1-b279-4160-9390-a004018e92ce",
+    "workflow_outputs": [],
+    "input_connections": {},
+    "position": {
+      "left": 148.72970581054688,
+      "top": 620.5069274902344
+    }
   }
 }

lib/galaxy/app.py

@@ -837,21 +837,21 @@ def __init__(self, **kwargs) -> None:
             )
             self.application_stack.register_postfork_function(self.prune_history_audit_task.start)
             self.haltables.append(("HistoryAuditTablePruneTask", self.prune_history_audit_task.shutdown))
-        # Start the job manager
-        self.application_stack.register_postfork_function(self.job_manager.start)
         self.proxy_manager = ProxyManager(self.config)
 
         # Must be initialized after job_config.
         self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self)
 
         self.trs_proxy = self._register_singleton(TrsProxy, TrsProxy(self.config))
+        # We need InteractiveToolManager before the job handler starts
+        self.interactivetool_manager = InteractiveToolManager(self)
+        # Start the job manager
+        self.application_stack.register_postfork_function(self.job_manager.start)
         # Must be initialized after any component that might make use of stack messaging is configured. Alternatively if
         # it becomes more commonly needed we could create a prefork function registration method like we do with
         # postfork functions.
         self.application_stack.init_late_prefork()
 
-        self.interactivetool_manager = InteractiveToolManager(self)
-
         # Configure handling of signals
         handlers = {}
         if self.heartbeat:

lib/galaxy/authnz/custos_authnz.py

@@ -62,6 +62,7 @@ class CustosAuthnzConfiguration:
     pkce_support: bool
     accepted_audiences: List[str]
     extra_params: Optional[dict]
+    extra_scopes: List[str]
     authorization_endpoint: Optional[str]
     token_endpoint: Optional[str]
     end_session_endpoint: Optional[str]
@@ -98,6 +99,7 @@ def __init__(self, provider, oidc_config, oidc_backend_config, idphint=None):
                 )
             ),
             extra_params={},
+            extra_scopes=oidc_backend_config.get("extra_scopes", []),
             authorization_endpoint=None,
             token_endpoint=None,
             end_session_endpoint=None,
@@ -156,6 +158,7 @@ def _get_provider_specific_scopes(self):
     def authenticate(self, trans, idphint=None):
         base_authorize_url = self.config.authorization_endpoint
         scopes = ["openid", "email", "profile"]
+        scopes.extend(self.config.extra_scopes)
         scopes.extend(self._get_provider_specific_scopes())
         oauth2_session = self._create_oauth2_session(scope=scopes)
         nonce = generate_nonce()

lib/galaxy/authnz/managers.py

@@ -201,6 +201,8 @@ def _parse_custos_config(self, config_xml):
             rtv["ca_bundle"] = config_xml.find("ca_bundle").text
         if config_xml.find("icon") is not None:
             rtv["icon"] = config_xml.find("icon").text
+        if config_xml.find("extra_scopes") is not None:
+            rtv["extra_scopes"] = listify(config_xml.find("extra_scopes").text)
         if config_xml.find("pkce_support") is not None:
             rtv["pkce_support"] = asbool(config_xml.find("pkce_support").text)
         if config_xml.find("accepted_audiences") is not None:

lib/galaxy/authnz/psa_authnz.py

@@ -135,9 +135,10 @@ def _setup_idp(self, oidc_backend_config):
         self.config["KEY"] = oidc_backend_config.get("client_id")
         self.config["SECRET"] = oidc_backend_config.get("client_secret")
         self.config["TENANT_ID"] = oidc_backend_config.get("tenant_id")
-        self.config["OIDC_ENDPOINT"] = oidc_backend_config.get("oidc_endpoint")
         self.config["redirect_uri"] = oidc_backend_config.get("redirect_uri")
         self.config["EXTRA_SCOPES"] = oidc_backend_config.get("extra_scopes")
+        if oidc_backend_config.get("oidc_endpoint"):
+            self.config["OIDC_ENDPOINT"] = oidc_backend_config["oidc_endpoint"]
         if oidc_backend_config.get("prompt") is not None:
             self.config[setting_name("AUTH_EXTRA_ARGUMENTS")]["prompt"] = oidc_backend_config.get("prompt")
         if oidc_backend_config.get("api_url") is not None:

lib/galaxy/celery/__init__.py

@@ -220,7 +220,7 @@ def config_celery_app(config, celery_app):
     if config.celery_conf:
         celery_app.conf.update(config.celery_conf)
     # Handle special cases
-    if not celery_app.conf.broker_url:
+    if not config.celery_conf.get("broker_url"):
         celery_app.conf.broker_url = config.amqp_internal_connection
 
 

lib/galaxy/jobs/__init__.py

@@ -1422,7 +1422,7 @@ def fail(
         message = str(message)
         working_directory_exists = self.working_directory_exists()
 
-        if not job.tasks:
+        if not job.tasks and working_directory_exists:
             # If job was composed of tasks, don't attempt to recollect statistics
             self._collect_metrics(job, job_metrics_directory)
 

lib/galaxy/managers/datasets.py

@@ -479,8 +479,8 @@ def ensure_dataset_on_disk(self, trans, dataset):
             raise exceptions.ItemDeletionException("The dataset you are attempting to view has been deleted.")
         elif dataset.state == Dataset.states.UPLOAD:
             raise exceptions.Conflict("Please wait until this dataset finishes uploading before attempting to view it.")
-        elif dataset.state == Dataset.states.NEW:
-            raise exceptions.Conflict("The dataset you are attempting to view is new and has no data.")
+        elif dataset.state in (Dataset.states.NEW, Dataset.states.QUEUED):
+            raise exceptions.Conflict(f"The dataset you are attempting to view is {dataset.state} and has no data.")
         elif dataset.state == Dataset.states.DISCARDED:
             raise exceptions.ItemDeletionException("The dataset you are attempting to view has been discarded.")
         elif dataset.state == Dataset.states.DEFERRED:
@@ -491,6 +491,14 @@ def ensure_dataset_on_disk(self, trans, dataset):
             raise exceptions.Conflict(
                 "The dataset you are attempting to view is in paused state. One of the inputs for the job that creates this dataset has failed."
             )
+        elif dataset.state == Dataset.states.RUNNING:
+            if not self.app.object_store.exists(dataset.dataset):
+                raise exceptions.Conflict(
+                    "The dataset you are attempting to view is still being created and has no data yet."
+                )
+        elif dataset.state == Dataset.states.ERROR:
+            if not self.app.object_store.exists(dataset.dataset):
+                raise exceptions.RequestParameterInvalidException("The dataset is in error and has no data.")
 
     def ensure_can_change_datatype(self, dataset: DatasetInstance, raiseException: bool = True) -> bool:
         if not dataset.datatype.is_datatype_change_allowed():

lib/galaxy/managers/jobs.py

@@ -1146,7 +1146,7 @@ def get_jobs_to_check_at_startup(session: galaxy_scoped_session, track_jobs_in_d
         # Filter out the jobs of inactive users.
         stmt = stmt.outerjoin(User).filter(or_((Job.user_id == null()), (User.active == true())))
 
-    return session.scalars(stmt)
+    return session.scalars(stmt).all()
 
 
 def get_job(session, *where_clauses):

lib/galaxy/schema/help.py

@@ -27,14 +27,14 @@ class HelpForumPost(HelpTempBaseModel):
     """Model for a post in the help forum."""
 
     id: Annotated[int, Field(description="The ID of the post.")]
-    name: Annotated[str, Field(description="The name of the post.")]
-    username: Annotated[str, Field(description="The username of the post author.")]
-    avatar_template: Annotated[str, Field(description="The avatar template of the user.")]
-    created_at: Annotated[str, Field(description="The creation date of the post.")]
-    like_count: Annotated[int, Field(description="The number of likes of the post.")]
-    blurb: Annotated[str, Field(description="The blurb of the post.")]
-    post_number: Annotated[int, Field(description="The post number of the post.")]
-    topic_id: Annotated[int, Field(description="The ID of the topic of the post.")]
+    name: Annotated[Optional[str], Field(description="The name of the post.")]
+    username: Annotated[Optional[str], Field(description="The username of the post author.")]
+    avatar_template: Annotated[Optional[str], Field(description="The avatar template of the user.")]
+    created_at: Annotated[Optional[str], Field(description="The creation date of the post.")]
+    like_count: Annotated[Optional[int], Field(description="The number of likes of the post.")]
+    blurb: Annotated[Optional[str], Field(description="The blurb of the post.")]
+    post_number: Annotated[Optional[int], Field(description="The post number of the post.")]
+    topic_id: Annotated[Optional[int], Field(description="The ID of the topic of the post.")]
 
 
 class HelpForumTopic(Model):

lib/galaxy/webapps/galaxy/api/workflows.py

@@ -244,15 +244,17 @@ def create(self, trans: GalaxyWebTransaction, payload=None, **kwd):
                     trs_version_id = None
                     import_source = None
                     if "trs_url" in payload:
-                        parts = self.app.trs_proxy.match_url(payload["trs_url"])
+                        parts = self.app.trs_proxy.match_url(
+                            payload["trs_url"], trans.app.config.fetch_url_allowlist_ips
+                        )
                         if parts:
                             server = self.app.trs_proxy.server_from_url(parts["trs_base_url"])
                             trs_tool_id = parts["tool_id"]
                             trs_version_id = parts["version_id"]
                             payload["trs_tool_id"] = trs_tool_id
                             payload["trs_version_id"] = trs_version_id
                         else:
-                            raise exceptions.MessageException("Invalid TRS URL.")
+                            raise exceptions.RequestParameterInvalidException(f"Invalid TRS URL {payload['trs_url']}.")
                     else:
                         trs_server = payload.get("trs_server")
                         server = self.app.trs_proxy.get_server(trs_server)

lib/galaxy/webapps/galaxy/controllers/dataset.py

@@ -28,7 +28,6 @@
     HDAManager,
 )
 from galaxy.managers.histories import HistoryManager
-from galaxy.model import Dataset
 from galaxy.model.base import transaction
 from galaxy.model.item_attrs import (
     UsesAnnotations,
@@ -108,33 +107,14 @@ def _check_dataset(self, trans, hda_id):
             raise web.httpexceptions.HTTPNotFound(f"Invalid reference dataset id: {str(hda_id)}.")
         if not self._can_access_dataset(trans, data):
             return trans.show_error_message("You are not allowed to access this dataset")
-        if data.purged or data.dataset.purged:
-            return trans.show_error_message("The dataset you are attempting to view has been purged.")
-        elif data.deleted and not (trans.user_is_admin or (data.history and trans.get_user() == data.user)):
-            return trans.show_error_message("The dataset you are attempting to view has been deleted.")
-        elif data.state == Dataset.states.UPLOAD:
-            return trans.show_error_message(
-                "Please wait until this dataset finishes uploading before attempting to view it."
-            )
-        elif data.state == Dataset.states.DISCARDED:
-            return trans.show_error_message("The dataset you are attempting to view has been discarded.")
-        elif data.state == Dataset.states.DEFERRED:
-            return trans.show_error_message(
-                "The dataset you are attempting to view has deferred data. You can only use this dataset as input for jobs."
-            )
-        elif data.state == Dataset.states.PAUSED:
-            return trans.show_error_message(
-                "The dataset you are attempting to view is in paused state. One of the inputs for the job that creates this dataset has failed."
-            )
+        self.app.hda_manager.ensure_dataset_on_disk(trans, data)
         return data
 
     @web.expose
     def display(
         self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd
     ):
         data = self._check_dataset(trans, dataset_id)
-        if not isinstance(data, trans.app.model.DatasetInstance):
-            return data
         if "hdca" in kwd:
             raise RequestParameterInvalidException("Invalid request parameter 'hdca' encountered.")
         if hdca_id := kwd.get("hdca_id", None):
@@ -455,8 +435,6 @@ def _get_dataset_for_edit(self, trans, dataset_id):
     def display_by_username_and_slug(self, trans, username, slug, filename=None, preview=True, **kwargs):
         """Display dataset by username and slug; because datasets do not yet have slugs, the slug is the dataset's id."""
         dataset = self._check_dataset(trans, slug)
-        if not isinstance(dataset, trans.app.model.DatasetInstance):
-            return dataset
         # Filename used for composite types.
         if filename:
             return self.display(trans, dataset_id=slug, filename=filename)