From 82c787c8614daf9d4fb935a54f18736d61d2df00 Mon Sep 17 00:00:00 2001 From: stxue1 <122345910+stxue1@users.noreply.github.com> Date: Tue, 10 Sep 2024 09:57:57 -0700 Subject: [PATCH] Add `--memoryIsProduct` for clusters that multiply requested memory and cores (#5080) * Add command line argument * typo * Update documentation --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/running/cliOptions.rst | 4 ++++ src/toil/batchSystems/abstractGridEngineBatchSystem.py | 3 ++- src/toil/batchSystems/options.py | 4 ++++ src/toil/common.py | 4 ++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/running/cliOptions.rst b/docs/running/cliOptions.rst index e4a8a96ecc..c5f678c1b3 100644 --- a/docs/running/cliOptions.rst +++ b/docs/running/cliOptions.rst @@ -199,6 +199,10 @@ levels in toil are based on priority from the logging module: 'h_vmem=MEMORY' to the qsub call, and instead rely on TOIL_GRIDGENGINE_ARGS to supply alternative arguments. Requires that TOIL_GRIDGENGINE_ARGS be set. + --memoryIsProduct + If the batch system understands requested memory as a product of the requested + memory and the number of cores, set this flag to properly allocate memory. This + can be fairly common with grid engine clusters (Ex: SGE, PBS, Torque). --runCwlInternalJobsOnWorkers Whether to run CWL internal jobs (e.g. CWLScatter) on the worker nodes instead of the primary node. If false diff --git a/src/toil/batchSystems/abstractGridEngineBatchSystem.py b/src/toil/batchSystems/abstractGridEngineBatchSystem.py index 7a1a9d9bca..5f7d570ccd 100644 --- a/src/toil/batchSystems/abstractGridEngineBatchSystem.py +++ b/src/toil/batchSystems/abstractGridEngineBatchSystem.py @@ -128,7 +128,8 @@ def createJobs(self, newJob: JobTuple) -> bool: len(self.runningJobs) < int(self.boss.config.max_jobs): activity = True jobID, cpu, memory, command, jobName, environment, gpus = self.waitingJobs.pop(0) - + if self.boss.config.memory_is_product and cpu > 1: + memory = memory // cpu # prepare job submission command subLine = self.prepareSubmission(cpu, memory, jobID, command, jobName, environment, gpus) logger.debug("Running %r", subLine) diff --git a/src/toil/batchSystems/options.py b/src/toil/batchSystems/options.py index b33f9971fb..bfab8e5718 100644 --- a/src/toil/batchSystems/options.py +++ b/src/toil/batchSystems/options.py @@ -185,6 +185,10 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) - "systems such as gridengine, htcondor, torque, slurm, and lsf." ) + parser.add_argument('--memoryIsProduct', dest='memory_is_product', default=False, action="store_true", + help="If the batch system understands requested memory as a product of the requested memory and the number" + "of cores, set this flag to properly allocate memory.") + for name in get_batch_systems(): # All the batch systems are responsible for adding their own options # with the add_options class method. diff --git a/src/toil/common.py b/src/toil/common.py index 5c4083eefe..62a052be9f 100644 --- a/src/toil/common.py +++ b/src/toil/common.py @@ -243,6 +243,8 @@ class Config: # CWL cwl: bool + memory_is_product: bool + def __init__(self) -> None: # only default options that are not CLI options defined here (thus CLI options are centralized) self.cwl = False # will probably remove later @@ -417,6 +419,8 @@ def set_option(option_name: str, set_option("logLevel") set_option("colored_logs") + set_option("memory_is_product") + # Apply overrides as highest priority # Override workDir with value of TOIL_WORKDIR_OVERRIDE if it exists if os.getenv('TOIL_WORKDIR_OVERRIDE') is not None: