EH: CS-612 allow to overwrite the PE allocation rule on the submission command line

doc/markdown/man/man1/submit.include.md

@@ -1259,6 +1259,24 @@ If this option is specified and the priority is not 0 then this value will be pa
 instances as parameter with the name *p*. (see `-jsv` option above or find more information concerning JSV in 
 xxqs_name_sxx_jsv(1))
 
+## -par *allocation_rule*
+
+Available for `qsub`, `qsh`, `qrsh`, `qlogin` and `qalter` only.
+
+Allows overwriting the allocation rule defined in the parallel environment, see xxqs_name_sxx_pe(1), attribute `allocation_rule`.
+
+The allocation rule can be overwritten globally, as well as for master and/or slave scope, when combined with the `-scope` option.
+
+Examples:
+
+`qsub -pe <pe_name> 4 -par $round_robin` will overwrite the allocation rule defined in the PE globally.
+
+`qsub -pe <pe_name> 4 -scope master -par 1` will overwrite the allocation rule defined in the PE for the master scope.
+
+`qsub -pe <pe_name> 5 -scope master -par 1 -q master.q -scope slave -par 4 -q slave.q` will overwrite the allocation rule defined in the PE
+with `1` for the master scope, and `4` for the slave scope. This is a typical use case for jobs which requires
+the master task to run on a separate host / in a different queue instance than the slave tasks.
+
 ## -pe *parallel_environment* n\[-\[m\]\]\|\[-\]m,...  
 
 Available for `qsub`, `qsh`, `qrsh`, `qlogin` and `qalter` only.

source/clients/common/ocs_client_job.cc

@@ -337,6 +337,16 @@ void cull_show_job(const lListElem *job, int flags, bool show_binding) {
             printf("%-33s", str_attrib);
             uni_print_list(stdout, nullptr, 0, lp, fields, delis, FLG_NO_DELIS_STRINGS);
          }
+
+         const char *allocation_rule = lGetString(jrs, JRS_allocation_rule);
+         if (allocation_rule != nullptr) {
+            if (str_scope == nullptr) {
+               str_attrib = sge_dstring_sprintf(&dstr_attrib, "allocation_rule:");
+            } else {
+               str_attrib = sge_dstring_sprintf(&dstr_attrib, "%s_allocation_rule:", str_scope);
+            }
+            printf("%-33s%s\n", str_attrib, allocation_rule);
+         }
       }
    }
 

source/clients/qalter/ocs_qalter.cc

@@ -27,7 +27,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -701,6 +701,17 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
          nm_set(job_field, JB_project);
       }
 
+      while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-par"))) {
+         const char *allocation_rule = lGetString(ep, SPA_switch_arg);
+         u_long32 scope = lGetChar(ep, SPA_argval_lCharT);
+         lListElem *jrs = job_get_or_create_request_setRW(job, scope);
+         if (jrs != nullptr) {
+            lSetString(jrs, JRS_allocation_rule, allocation_rule);
+         }
+         lRemoveElem(cmdline, &ep);
+         nm_set(job_field, JB_request_set_list);
+      }
+
       while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-pe"))) {
          lSetString(job, JB_pe, lGetString(ep, SPA_argval_lStringT));
          /* put sublist from parsing into job */
@@ -808,7 +819,6 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
             nm_set(job_field, JB_job_args);
          }
          lSetList(job, JB_job_args, lp);
-
       }
 
       /* context switches are sensitive to order */
@@ -829,8 +839,9 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
             lRemoveElem(cmdline, &ep);
             ep = temp;
             nm_set(job_field, JB_context);
-         } else
+         } else {
             ep = lNextRW(ep);
+         }
    }
 
    /* complain about unused options */
@@ -862,8 +873,10 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
       DRETURN(answer);
    }
 
-/* printf("=============== lWriteElemTo(job, stdout); ==================\n"); */
-/* lWriteElemTo(job, stdout); */
+#if 0
+   printf("=============== lWriteElemTo(job, stdout); ==================\n");
+   lWriteElemTo(job, stdout);
+#endif
 
 
    /* 

source/clients/qsh/ocs_qsh.cc

@@ -29,7 +29,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2024-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -2373,8 +2373,8 @@ static void remove_unknown_opts(lList *lp, u_long32 jb_now, int tightly_integrat
             strcmp(cp, "-hold_jid") && strcmp(cp, "-hold_jid_ad") && strcmp(cp, "-h") &&
             strcmp(cp, "-l") && strcmp(cp, "-m") && strcmp(cp, "-masterq") &&
             strcmp(cp, "-N") && strcmp(cp, "-noshell") && strcmp(cp, "-now") &&
-            strcmp(cp, "-notify") && strcmp(cp, "-P") &&
-            strcmp(cp, "-p") && strcmp(cp, "-pe") && strcmp(cp, "-q") && strcmp(cp, "-v") &&
+            strcmp(cp, "-notify") && strcmp(cp, "-P") && strcmp(cp, "-p") && strcmp(cp, "-par") &&
+            strcmp(cp, "-pe") && strcmp(cp, "-q") && strcmp(cp, "-v") &&
             strcmp(cp, "-V") && strcmp(cp, "-display") && strcmp(cp, "-verify") &&
             strcmp(cp, "-soft") && strcmp(cp, "-M") && strcmp(cp, "-verbose") &&
             strcmp(cp, "-ac") && strcmp(cp, "-dc") && strcmp(cp, "-sc") && strcmp(cp, "-scope") &&

source/clients/qsh/ocs_qsh_parse.cc

@@ -27,7 +27,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -493,6 +493,16 @@ lList *cull_parse_qsh_parameter(u_long32 prog_number, u_long32 uid, const char *
       lRemoveElem(cmdline, &ep);
    }
 
+   while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-par"))) {
+      const char *allocation_rule = lGetString(ep, SPA_switch_arg);
+      u_long32 scope = lGetChar(ep, SPA_argval_lCharT);
+      lListElem *jrs = job_get_or_create_request_setRW(*pjob, scope);
+      if (jrs != nullptr) {
+         lSetString(jrs, JRS_allocation_rule, allocation_rule);
+      }
+      lRemoveElem(cmdline, &ep);
+   }
+
    while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-js"))) {
       lSetUlong(*pjob, JB_jobshare, lGetUlong(ep, SPA_argval_lUlongT));
       lRemoveElem(cmdline, &ep);

source/common/msg_common.h

@@ -30,7 +30,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -50,9 +50,10 @@
 #define MSG_TABLE_EV_POOL                          "POOL"
 #define MSG_TABLE_SIZE                             "SIZE"
 
-#define MSG_GDI_ARGUMENTSYNTAX_QA_CATEGORY_ID          "cat_id                  category ID"
+#define MSG_GDI_ARGUMENTSYNTAX_OA_ALLOCATION_RULE      "allocation_rule         <int> | $pe_slots | $fill_up | $round_robin"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_ACCOUNT_STRING       "account_string          account_name"
-#define MSG_GDI_ARGUMENTSYNTAX_QA_BINDING_FILTER_STR   "topology_string         topology string where lower case letters show masked units"
+#define MSG_GDI_ARGUMENTSYNTAX_OA_BINDING_FILTER_STR   "topology_string         topology string where lower case letters show masked units"
+#define MSG_GDI_ARGUMENTSYNTAX_OA_CATEGORY_ID          "cat_id                  category ID"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_COMPLEX_LIST         "complex_list            complex[,complex,...]"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_CONTEXT_LIST         "context_list            variable[=value][,variable[=value],...]"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_CKPT_SEL             "ckpt_selector           `n' `s' `m' `x' <interval> "
@@ -965,7 +966,7 @@
 #define MSG_SEC_DELCREDNOBIN_US           _MESSAGE(60413, _("could not delete credentials for job " sge_u32 " - " SFN " binary does not exist"))
 #define MSG_SEC_PUTCREDSTDERR_S           _MESSAGE(60414, _("put_cred stderr: " SFN))
 #define MSG_SEC_NOSTARTCMD4GETCRED_SU     _MESSAGE(60415, _("can't start command " SFQ " for job " sge_u32 " to get credentials"))
-#define MSG_PE_ALLOCRULE_S                _MESSAGE(60416, _("incorrect allocation_rule " SFQ ))
+#define MSG_PE_ALLOCRULE_S                _MESSAGE(60416, _("invalid allocation_rule " SFQ ))
 #define MSG_GDI_OUTOFMEMORY               _MESSAGE(60418, _("out of memory"))
 #define MSG_COM_UNPACKINT_I               _MESSAGE(60419, _("unpacking integer %d failed"))
 #define MSG_COM_UNPACKOBJ_S               _MESSAGE(60420, _("unpacking a " SFN " object failed"))
@@ -1110,7 +1111,10 @@
 
 #define MSG_PARSE_BSORT_CONTRA_S          _MESSAGE(60744, _("contradicting ascending and descending sort order in binding sort " SFQ))
 
-#define MSG_GDI_USAGE_when_OPT              "[-when now|on_reschedule]"
-#define MSG_GDI_UTEXT_when_OPT              _MESSAGE(60745, _("apply modification immediately or on job rescheduling"))
+#define MSG_GDI_USAGE_when_OPT            "[-when now|on_reschedule]"
+#define MSG_GDI_UTEXT_when_OPT            _MESSAGE(60745, _("apply modification immediately or on job rescheduling"))
+
+#define MSG_GDI_USAGE_PAR_OPT             "[-par allocation_rule]"
+#define MSG_GDI_UTEXT_PAR_OPT             _MESSAGE(60746, _("set the parallel job allocation rule"))
 
 // clang-format on

source/common/parse_job_cull.cc

@@ -27,7 +27,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -621,6 +621,16 @@ lList *cull_parse_job_parameter(u_long32 uid, const char *username, const char *
       lRemoveElem(cmdline, &ep);
    }
 
+   while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-par"))) {
+      const char *allocation_rule = lGetString(ep, SPA_switch_arg);
+      u_long32 scope = lGetChar(ep, SPA_argval_lCharT);
+      lListElem *jrs = job_get_or_create_request_setRW(*pjob, scope);
+      if (jrs != nullptr) {
+         lSetString(jrs, JRS_allocation_rule, allocation_rule);
+      }
+      lRemoveElem(cmdline, &ep);
+   }
+
    while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-pe"))) {
       lSetString(*pjob, JB_pe, lGetString(ep, SPA_argval_lStringT));
       lSwapList(*pjob, JB_pe_range, ep, SPA_argval_lListT);

source/common/parse_qsub.cc

@@ -37,6 +37,7 @@
 #include <cstdlib>
 #include <unistd.h>
 
+#include "uti/config_file.h"
 #include "uti/sge_dstring.h"
 #include "uti/sge_log.h"
 #include "uti/sge_parse_num_par.h"
@@ -1540,7 +1541,35 @@ lList *cull_parse_cmdline(
          continue;
       }
 
-/*----------------------------------------------------------------------------*/
+      /*-----------------------------------------------------------------------------*/
+      /* "-par" */
+
+      if (strcmp("-par", *sp) == 0) {
+
+         DPRINTF("\"%s\"\n", *sp);
+
+         sp++;
+         if (*sp == nullptr) {
+            answer_list_add_sprintf(&answer, STATUS_ESEMANTIC, ANSWER_QUALITY_ERROR,
+                                    MSG_PARSE_XOPTIONMUSTHAVEARGUMENT_S, "-par");
+            DRETURN(answer);
+         }
+
+         // check if the given allocation rule (e.g., $pe_slots) is correct
+         if (replace_params(*sp, nullptr, 0, pe_alloc_rule_variables) != 0) {
+            answer_list_add_sprintf(&answer, STATUS_ESEMANTIC, ANSWER_QUALITY_ERROR,
+                                    MSG_PARSE_INVALIDOPTIONARGUMENT_SS, "-par", *sp);
+            DRETURN(answer);
+         }
+
+         ep_opt = sge_add_arg(pcmdline, par_OPT, lStringT, *(sp - 1), *sp);
+         lSetChar(ep_opt, SPA_argval_lCharT, scope_flag);
+
+         sp++;
+         continue;
+      }
+
+      /*----------------------------------------------------------------------------*/
       if (!strcmp("-pe", *sp)) {
          lList *pe_range = nullptr;
          dstring d_arg = DSTRING_INIT;

source/common/sge_options.cc

@@ -29,7 +29,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -533,6 +533,8 @@ unsigned short sge_options[][ALL_OPT + 1] =
  {0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1},
 /* when_OPT */
  {0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
+/* par_OPT */
+ {0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1},
 /*
   n  q  q  q  q  q  q  q  q  q  q  q  q  q  q  e  q  q  q  q  n  A
   o  a  c  d  h  m  m  r  r  s  s  r  l  s  s  x  e  r  r  r  o  L

source/common/sge_options.h

@@ -30,7 +30,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -309,6 +309,7 @@ enum {
    binstance_OPT, //< binding instance
 
    when_OPT, //< for qalter set if changes shall be applied on a running job or only after rescheduling
+   par_OPT,  //< submit / qalter parameter to set the PE allocation rule per scope
 };
 
 /* macros used in parsing */

source/common/usage.cc

@@ -29,7 +29,7 @@
  * 
  *  Portions of this code are Copyright 2011 Univa Inc.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -55,9 +55,7 @@ static void usage_silent(FILE *fp);
 
 static int marker[OA__END];
 
-void mark_argument_syntax(
-int argument_number 
-) {
+void mark_argument_syntax(int argument_number) {
    marker[argument_number] = 1;
 }
 
@@ -68,6 +66,8 @@ static const char* get_argument_syntax(u_long32 prog_number, int nr)
    {
      case OA_ACCOUNT_STRING:
        return MSG_GDI_ARGUMENTSYNTAX_OA_ACCOUNT_STRING;
+     case OA_ALLOCATION_RULE:
+       return MSG_GDI_ARGUMENTSYNTAX_OA_ALLOCATION_RULE;
      case OA_COMPLEX_LIST:
        return MSG_GDI_ARGUMENTSYNTAX_OA_COMPLEX_LIST;
      case OA_CONTEXT_LIST:
@@ -150,7 +150,7 @@ static const char* get_argument_syntax(u_long32 prog_number, int nr)
      case OA_JOB_TASK_LIST:
          return MSG_GDI_ARGUMENTSYNTAX_OA_JOB_TASK_LIST; 
      case OA_BINDING_FILTER:
-         return MSG_GDI_ARGUMENTSYNTAX_QA_BINDING_FILTER_STR;
+         return MSG_GDI_ARGUMENTSYNTAX_OA_BINDING_FILTER_STR;
      case OA_JOB_TASKS:
          if (prog_number == QRESUB) {
             return MSG_GDI_ARGUMENTSYNTAX_OA_JOB_TASKS_RESUB;
@@ -203,7 +203,7 @@ static const char* get_argument_syntax(u_long32 prog_number, int nr)
      case OA_TASK_CONCURRENCY:
          return MSG_GDI_ARGUMENTSYNTAX_OA_TASK_CONCURRENCY;
      case OA_CATEGORY_ID:
-         return MSG_GDI_ARGUMENTSYNTAX_QA_CATEGORY_ID;
+         return MSG_GDI_ARGUMENTSYNTAX_OA_CATEGORY_ID;
      default:
          break; 
    }
@@ -991,8 +991,13 @@ void sge_usage(u_long32 prog_number, FILE *fp) {
       MARK(OA_PRIORITY);
    }
 
+   if (VALID_OPT(par_OPT, prog_number)) {
+      PRINTITD(MSG_GDI_USAGE_PAR_OPT, MSG_GDI_UTEXT_PAR_OPT);
+      MARK(OA_ALLOCATION_RULE);
+   }
+
    if (VALID_OPT(pe_OPT, prog_number)) {
-      PRINTITD(MSG_GDI_USAGE_pe_OPT_PE_NAME_SLOT_RANGE, 
+      PRINTITD(MSG_GDI_USAGE_pe_OPT_PE_NAME_SLOT_RANGE,
          MSG_GDI_UTEXT_pe_OPT_PE_NAME_SLOT_RANGE );
       MARK(OA_SLOT_RANGE);
    }

source/common/usage.h

@@ -28,7 +28,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -48,6 +48,7 @@
 */
 enum {
    OA_ACCOUNT_STRING,
+   OA_ALLOCATION_RULE,
    OA_CATEGORY_ID,
    OA_COMPLEX_LIST,
    OA_CONTEXT_LIST,