From 243e76ec95cef7280ca6973dfbaf5c55479f0613 Mon Sep 17 00:00:00 2001 From: Per Unneberg Date: Mon, 16 May 2022 12:47:43 +0200 Subject: [PATCH 1/4] Deprecate advanced argument option Fixes #91 --- .github/workflows/slurm.yaml | 1 - ChangeLog.md | 1 + README.md | 101 +++--------- cookiecutter.json | 1 - tests/conftest.py | 3 - tests/test_cookie.py | 5 +- tests/test_slurm_advanced.py | 64 -------- tests/test_utils.py | 144 +++++------------- {{cookiecutter.profile_name}}/CookieCutter.py | 8 - {{cookiecutter.profile_name}}/settings.json | 3 +- {{cookiecutter.profile_name}}/slurm-submit.py | 7 +- {{cookiecutter.profile_name}}/slurm_utils.py | 105 ------------- 12 files changed, 68 insertions(+), 375 deletions(-) delete mode 100644 tests/test_slurm_advanced.py diff --git a/.github/workflows/slurm.yaml b/.github/workflows/slurm.yaml index c41dc97..ba24bc7 100644 --- a/.github/workflows/slurm.yaml +++ b/.github/workflows/slurm.yaml @@ -68,5 +68,4 @@ jobs: pytest -v -s tests/test_cookie.py pytest -v -s tests/test_utils.py pytest -v -s tests/test_slurm.py --slow - pytest -v -s tests/test_slurm_advanced.py --slow pytest -v -s tests/test_sidecar.py diff --git a/ChangeLog.md b/ChangeLog.md index 343c6c9..245e0de 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -4,6 +4,7 @@ ### Changes +- deprecate advanced argument conversion (#91, PR #93) - add support for sidecar (PR #85) ## 2021-03-10 diff --git a/README.md b/README.md index 59ed090..852553f 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,10 @@ account: $ cookiecutter https://github.com/Snakemake-Profiles/slurm.git profile_name [slurm]: slurm.my_account sbatch_defaults []: account=my_account no-requeue exclusive - Select advanced_argument_conversion: - 1 - no - 2 - yes + cluster_sidecar_help: [Use cluster sidecar. NB! Requires snakemake >= 7.0! Enter to continue...] + Select cluster_sidecar: + 1 - yes + 2 - no Choose from 1, 2 [1]: cluster_name []: cluster_config_help: [The use of cluster-config is discouraged. Rather, set snakemake CLI options in the profile configuration file (see snakemake documentation on best practices). Enter to continue...] @@ -122,9 +123,10 @@ create a profile that uses a specified cluster: $ cookiecutter slurm profile_name [slurm]: slurm.dusk sbatch_defaults []: account=my_account - Select advanced_argument_conversion: - 1 - no - 2 - yes + cluster_sidecar_help: [Use cluster sidecar. NB! Requires snakemake >= 7.0! Enter to continue...] + Select cluster_sidecar: + 1 - yes + 2 - no Choose from 1, 2 [1]: cluster_name []: dusk cluster_config_help: [The use of cluster-config is discouraged. Rather, set snakemake CLI options in the profile configuration file (see snakemake documentation on best practices). Enter to continue...] @@ -139,27 +141,6 @@ with `sbatch --parsable --account=my_account --cluster=dusk`. In addition, the `slurm-status.py` script will check for jobs in the `dusk` cluster job queue. -### Example 3: project setup using advanced argument conversion (WARNING: experimental feature!) - -As a final example, assume we want to use advanced argument -conversion: - - $ cookiecutter slurm - profile_name [slurm]: slurm.convert_args - sbatch_defaults []: account=my_account - Select advanced_argument_conversion: - 1 - no - 2 - yes - Choose from 1, 2 [1]: 2 - cluster_name []: - cluster_config_help: [The use of cluster-config is discouraged. Rather, set snakemake CLI options in the profile configuration file (see snakemake documentation on best practices). Enter to continue...] - cluster_config []: - -The command `snakemake --profile slurm.convert_args ...` will now -submit jobs with `sbatch --parsable --account=my_account`. The -advanced argument conversion feature will attempt to adjust memory -settings and number of cpus to comply with the cluster configuration. -See the section below ## Profile details @@ -169,11 +150,6 @@ See the section below Snakemake option. * `sbatch_defaults` : List of default arguments to sbatch, e.g.: `qos=short time=60`. -* `advanced_argument_conversion` : If True, try to adjust/constrain - mem, time, nodes and ntasks (i.e. cpus) to parsed or default - partition after converting resources. This may fail due to - heterogeneous slurm setups, i.e. code adjustments will likely be - necessary. * `cluster_name` : some HPCs define multiple SLURM clusters. Set the cluster name, leave empty to use the default. This will add the `--cluster` string to the sbatch defaults, and adjust @@ -208,7 +184,7 @@ names](https://slurm.schedmd.com/sbatch.html): 4) Profile `cluster_config` file entries 5) `--cluster-config` parsed to Snakemake (deprecated since Snakemake 5.10) 6) Snakemake CLI resource configuration in profile configuration file -7) Any other argument conversion (experimental, currently time, ntasks and mem) if `advanced_argument_conversion` is True. + ### Rule specific resource configuration In addition to Snakemake CLI resource configuration, resources can be @@ -227,39 +203,6 @@ configuration follows: partition = "debug" -### Advanced argument conversion (EXPERIMENTAL) -By default, Snakefile resources are provided as-is to the sbatch -submission step. Although `sbatch` does adjust options to match -cluster configuration, it will throw an error if resources exceed -available cluster resources. For instance, if the memory is set larger -than the maximum memory of any node, `sbatch` will exit with the -message - - sbatch: error: CPU count per node can not be satisfied - sbatch: error: Batch job submission failed: Requested node configuration is not available - -By choosing the advanced argument conversion upon creating a profile, -an attempt will be made to adjust memory, cpu and time settings if -these do not comply with the cluster configuration. As an example, -consider a rule with the following resources and threads: - - rule bwa_mem: - resources: - mem_mb = lambda wildcards, attempt: attempt * 8000, - runtime = lambda wildcards, attempt: attempt * 1200 - threads: 1 - -Assume further that the available cores provide 6400MB memory per -core. If the job reaches a peak memory (8000MB), it will likely be -terminated. The advanced argument conversion will compare the -requested memory requirements to those available (defined as memory -per cpu times number of requested cpus) and adjust the number of cpus -accordingly. - -Other checks are also performed to make sure memory, runtime, and -number of cpus don't exceed the maximum values specificed by the -cluster configuration. - ### Cluster configuration file @@ -347,22 +290,24 @@ copies any user-provided data file to the test directory. Finally, the `smk_runner` fixture provides an instance of the `tests/wrapper.SnakemakeRunner` class for running Snakemake tests. -As an example, `tests/test_slurm_advanced.py` defines a fixture -`profile` that uses the `cookie_factory` fixture factory to create an -slurm profile that uses advanced argument conversion: +As an example, `tests/test_slurm.py` defines a fixture +`sidecar_profile` that uses the `cookie_factory` fixture factory to +create an slurm profile that parametrizes use of sidecar argument: - @pytest.fixture - def profile(cookie_factory, data): - cookie_factory(advanced="yes") + @pytest.fixture(params=["yes", "no"]) + def sidecar_profile(cookie_factory, data, request): + cookie_factory(cluster_sidecar=request.param) -The test `tests/test_slurm_advanced.py::test_adjust_runtime` depends +The test `tests/test_slurm.py::test_profile_status_running` depends on this fixture and `smk_runner`: - def test_adjust_runtime(smk_runner, profile): - smk_runner.make_target( - "timeout.txt", options=f"--cluster-config {smk_runner.cluster_config}" - ) - + def test_profile_status_running(smk_runner, sidecar_profile): + """Test that slurm-status.py catches RUNNING status""" + ... + smk_runner.make_target( + "timeout.txt", options=opts, profile=None, asynchronous=True + ) # noqa: E501 + ... The `make_target` method makes a snakemake target with additional options passed to Snakemake. diff --git a/cookiecutter.json b/cookiecutter.json index 4c5c20e..083e3e6 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -1,7 +1,6 @@ { "profile_name": "slurm", "sbatch_defaults": "", - "advanced_argument_conversion": ["no", "yes"], "cluster_sidecar_help": "Use cluster sidecar. NB! Requires snakemake >= 7.0! Enter to continue...", "cluster_sidecar": ["yes", "no"], "cluster_name": "", diff --git a/tests/conftest.py b/tests/conftest.py index 61b5060..a9051a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,7 +99,6 @@ def cookie_factory(tmpdir_factory, _cookiecutter_config_file, datadir): Args: sbatch_defaults (str): sbatch defaults for cookie - advanced (str): use advanced argument conversion ("no" or "yes") cluster_sidecar (str): use sidecar to monitor job status cluster_name (str): set cluster name cluster_config (str): cluster configuration file @@ -118,7 +117,6 @@ def cookie_factory(tmpdir_factory, _cookiecutter_config_file, datadir): def _cookie_factory( sbatch_defaults=_sbatch_defaults, - advanced="no", cluster_sidecar="yes", cluster_name=None, cluster_config=None, @@ -130,7 +128,6 @@ def _cookie_factory( c._new_output_dir = lambda: str(datadir) extra_context = { "sbatch_defaults": sbatch_defaults, - "advanced_argument_conversion": advanced, "cluster_sidecar": cluster_sidecar, } if cluster_name is not None: diff --git a/tests/test_cookie.py b/tests/test_cookie.py index 595c2e9..d433480 100644 --- a/tests/test_cookie.py +++ b/tests/test_cookie.py @@ -26,7 +26,6 @@ def test_cookiecutter(cookies, monkeypatch): assert CookieCutter.CLUSTER_NAME == "" assert CookieCutter.CLUSTER_CONFIG == "" assert CookieCutter.get_cluster_option() == "" - assert CookieCutter.get_advanced_argument_conversion() is False sys.modules.pop("CookieCutter") @@ -34,13 +33,11 @@ def test_cookiecutter_extra_context(cookies, monkeypatch): result = cookies.bake(template=str(pytest.cookie_template), extra_context={"sbatch_defaults": "account=foo", "cluster_name": "dusk", - "cluster_config": "slurm.yaml", - "advanced_argument_conversion": "yes"}) + "cluster_config": "slurm.yaml"}) monkeypatch.syspath_prepend(str(result.project_path)) from CookieCutter import CookieCutter assert CookieCutter.SBATCH_DEFAULTS == "account=foo" assert CookieCutter.CLUSTER_NAME == "dusk" assert CookieCutter.CLUSTER_CONFIG == "slurm.yaml" assert CookieCutter.get_cluster_option() == "--cluster=dusk" - assert CookieCutter.get_advanced_argument_conversion() is True sys.modules.pop("CookieCutter") diff --git a/tests/test_slurm_advanced.py b/tests/test_slurm_advanced.py deleted file mode 100644 index 0820d2f..0000000 --- a/tests/test_slurm_advanced.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -import pytest - - -@pytest.fixture -def profile(cookie_factory, data): - cookie_factory(advanced="yes") - - -@pytest.mark.skipci -@pytest.mark.slow -@pytest.mark.docker -def test_adjust_runtime(smk_runner, profile): - smk_runner.make_target( - "timeout.txt", options=f"--cluster-config {smk_runner.cluster_config}" - ) - m = smk_runner.check_jobstatus(r"(?P\d+)", "-o TimeLimitRaw -n", which=1) - assert int(m.group("timelimit")) == 2 - - -@pytest.mark.skipci -@pytest.mark.timeout(60) -@pytest.mark.docker -def test_adjust_memory(smk_runner, profile): - smk_runner.make_target( - "memory.txt", options=f"--cluster-config {smk_runner.cluster_config}" - ) - m = smk_runner.check_jobstatus(r"(?P\d+)", "-o ReqMem -n") - assert int(m.group("mem")) == 500 - - -@pytest.mark.skipci -@pytest.mark.timeout(60) -@pytest.mark.docker -def test_memory_with_constraint(smk_runner, profile): - smk_runner.make_target( - "memory_with_constraint.txt", - options=f"--cluster-config {smk_runner.cluster_config}", - ) - m = smk_runner.check_jobstatus(r"(?P\d+)", "-o ReqMem -n") - assert int(m.group("mem")) == 800 - - -@pytest.mark.skipci -@pytest.mark.timeout(60) -@pytest.mark.docker -def test_cluster_short_queue(smk_runner, profile): - smk_runner.make_target( - "short_queue.txt", - options=f"--cluster-config {smk_runner.cluster_config}", - ) - assert smk_runner.check_jobstatus("debug", "-n -o Partition") - - -@pytest.mark.skipci -def test_si_units(smk_runner, profile): - """Test that setting memory with si units works""" - _, output = smk_runner.make_target( - "siunit.txt", - options=f"--cluster-config {smk_runner.cluster_config}", - stream=False - ) - assert "requested memory (1000)" in (smk_runner.output) diff --git a/tests/test_utils.py b/tests/test_utils.py index b880b0a..fef5e88 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -13,106 +13,44 @@ import slurm_utils # noqa: E402 -# Profile has not been installed so need to mock calls -@pytest.fixture -def mock_cookiecutter_cluster_option(monkeypatch): - def cluster_option(): - return "" - monkeypatch.setattr(CookieCutter, "get_cluster_option", cluster_option) - - -@pytest.fixture -def mock_cookiecutter_named_cluster_option(monkeypatch): - def cluster_option(): - return "--cluster=linux" - monkeypatch.setattr(CookieCutter, "get_cluster_option", cluster_option) - - -@pytest.fixture -def mock_get_cluster_config(smk_runner, - monkeypatch): - if isinstance(smk_runner._container, Container): - class MockPopen: - def __init__(self, cmd, **kwargs): - self._res = smk_runner.exec_run(cmd) - - def communicate(self): - return (self._res.output, None) - - monkeypatch.setattr(subprocess, "Popen", MockPopen) - - -@pytest.fixture -def mock_get_default_partition(smk_runner, - monkeypatch): - if isinstance(smk_runner._container, Container): - def mock_res(cmd, **kwargs): - res = smk_runner.exec_run(cmd) - return res.output - monkeypatch.setattr(subprocess, "check_output", mock_res) - - -def test_time_to_minutes(): - minutes = slurm_utils.time_to_minutes("foo") - assert minutes is None - minutes = slurm_utils.time_to_minutes("10-00:00:10") - assert minutes == 14401 - minutes = slurm_utils.time_to_minutes("10:00:00") - assert minutes == 600 - minutes = slurm_utils.time_to_minutes("100:00") - assert minutes == 100 - minutes = slurm_utils.time_to_minutes("20") - assert minutes == 20 - - -def test_cluster_configuration(request, - mock_get_cluster_config, - mock_cookiecutter_cluster_option): - partition = request.config.getoption("--partition") - df = slurm_utils._get_cluster_configuration(partition) - assert re.match(r"^\d+$", str(df["TIMELIMIT_MINUTES"][0])) - - -def test_named_cluster_configuration(request, - mock_get_cluster_config, - mock_cookiecutter_named_cluster_option): - partition = request.config.getoption("--partition") - df = slurm_utils._get_cluster_configuration(partition) - assert re.match(r"^\d+$", str(df["TIMELIMIT_MINUTES"][0])) - - -def test_argument_conversion(request, - mock_get_cluster_config, - mock_cookiecutter_cluster_option, - mock_get_default_partition): - partition = request.config.getoption("--partition") - config = slurm_utils._get_cluster_configuration(partition) - args = {"mem": min(config["MEMORY"]) / min(config["CPUS"]) * 2} - d = slurm_utils.advanced_argument_conversion(args) - assert d["cpus-per-task"] == 2 - max_cpus = max(config["CPUS"]) - args = {"cpus-per-task": max_cpus * 2} - d = slurm_utils.advanced_argument_conversion(args) - assert d["cpus-per-task"] == max_cpus - - -@pytest.mark.docker -def test_default_partition(mock_get_default_partition, - mock_cookiecutter_cluster_option): - p = slurm_utils._get_default_partition() - assert p == "normal" - - -def test_si_units(): - m = slurm_utils._convert_units_to_mb(1000) - assert m == 1000 - m = slurm_utils._convert_units_to_mb("1000K") - assert m == 1 - m = slurm_utils._convert_units_to_mb("1000M") - assert m == 1000 - m = slurm_utils._convert_units_to_mb("1000G") - assert m == 1e6 - m = slurm_utils._convert_units_to_mb("1000T") - assert m == 1e9 - with pytest.raises(SystemExit): - m = slurm_utils._convert_units_to_mb("1000E") +# # Profile has not been installed so need to mock calls +# @pytest.fixture +# def mock_cookiecutter_cluster_option(monkeypatch): +# def cluster_option(): +# return "" +# monkeypatch.setattr(CookieCutter, "get_cluster_option", cluster_option) + + +# @pytest.fixture +# def mock_cookiecutter_named_cluster_option(monkeypatch): +# def cluster_option(): +# return "--cluster=linux" +# monkeypatch.setattr(CookieCutter, "get_cluster_option", cluster_option) + + +# def test_time_to_minutes(): +# minutes = slurm_utils.time_to_minutes("foo") +# assert minutes is None +# minutes = slurm_utils.time_to_minutes("10-00:00:10") +# assert minutes == 14401 +# minutes = slurm_utils.time_to_minutes("10:00:00") +# assert minutes == 600 +# minutes = slurm_utils.time_to_minutes("100:00") +# assert minutes == 100 +# minutes = slurm_utils.time_to_minutes("20") +# assert minutes == 20 + + +# def test_si_units(): +# m = slurm_utils._convert_units_to_mb(1000) +# assert m == 1000 +# m = slurm_utils._convert_units_to_mb("1000K") +# assert m == 1 +# m = slurm_utils._convert_units_to_mb("1000M") +# assert m == 1000 +# m = slurm_utils._convert_units_to_mb("1000G") +# assert m == 1e6 +# m = slurm_utils._convert_units_to_mb("1000T") +# assert m == 1e9 +# with pytest.raises(SystemExit): +# m = slurm_utils._convert_units_to_mb("1000E") diff --git a/{{cookiecutter.profile_name}}/CookieCutter.py b/{{cookiecutter.profile_name}}/CookieCutter.py index 19d61df..2000dd7 100644 --- a/{{cookiecutter.profile_name}}/CookieCutter.py +++ b/{{cookiecutter.profile_name}}/CookieCutter.py @@ -14,7 +14,6 @@ class CookieCutter: SBATCH_DEFAULTS = settings['SBATCH_DEFAULTS'] CLUSTER_NAME = settings['CLUSTER_NAME'] CLUSTER_CONFIG = settings['CLUSTER_CONFIG'] - ADVANCED_ARGUMENT_CONVERSION = settings['ADVANCED_ARGUMENT_CONVERSION'] @staticmethod def get_cluster_option() -> str: @@ -22,10 +21,3 @@ def get_cluster_option() -> str: if cluster != "": return f"--cluster={cluster}" return "" - - @staticmethod - def get_advanced_argument_conversion() -> bool: - val = {"yes": True, "no": False}[ - CookieCutter.ADVANCED_ARGUMENT_CONVERSION - ] - return val diff --git a/{{cookiecutter.profile_name}}/settings.json b/{{cookiecutter.profile_name}}/settings.json index 2055eda..e5c4199 100644 --- a/{{cookiecutter.profile_name}}/settings.json +++ b/{{cookiecutter.profile_name}}/settings.json @@ -1,6 +1,5 @@ { "SBATCH_DEFAULTS": "{{cookiecutter.sbatch_defaults}}", "CLUSTER_NAME": "{{cookiecutter.cluster_name}}", - "CLUSTER_CONFIG": "{{cookiecutter.cluster_config}}", - "ADVANCED_ARGUMENT_CONVERSION": "{{cookiecutter.advanced_argument_conversion}}" + "CLUSTER_CONFIG": "{{cookiecutter.cluster_config}}" } diff --git a/{{cookiecutter.profile_name}}/slurm-submit.py b/{{cookiecutter.profile_name}}/slurm-submit.py index 18bc470..d56a70d 100755 --- a/{{cookiecutter.profile_name}}/slurm-submit.py +++ b/{{cookiecutter.profile_name}}/slurm-submit.py @@ -36,7 +36,6 @@ def register_with_sidecar(jobid): SBATCH_DEFAULTS = CookieCutter.SBATCH_DEFAULTS CLUSTER = CookieCutter.get_cluster_option() CLUSTER_CONFIG = CookieCutter.CLUSTER_CONFIG -ADVANCED_ARGUMENT_CONVERSION = CookieCutter.get_advanced_argument_conversion() RESOURCE_MAPPING = { "time": ("time", "runtime", "walltime"), @@ -69,11 +68,7 @@ def register_with_sidecar(jobid): # 5) cluster_config options sbatch_options.update(job_properties.get("cluster", {})) -# 6) Advanced conversion of parameters -if ADVANCED_ARGUMENT_CONVERSION: - sbatch_options = slurm_utils.advanced_argument_conversion(sbatch_options) - -# 7) Format pattern in snakemake style +# 6) Format pattern in snakemake style sbatch_options = slurm_utils.format_values(sbatch_options, job_properties) # ensure sbatch output dirs exist diff --git a/{{cookiecutter.profile_name}}/slurm_utils.py b/{{cookiecutter.profile_name}}/slurm_utils.py index 261ab1c..68181f5 100644 --- a/{{cookiecutter.profile_name}}/slurm_utils.py +++ b/{{cookiecutter.profile_name}}/slurm_utils.py @@ -189,59 +189,6 @@ def submit_job(jobscript, **sbatch_options): return jobid -def advanced_argument_conversion(arg_dict): - """Experimental adjustment of sbatch arguments to the given or default partition.""" - # Currently not adjusting for multiple node jobs - nodes = int(arg_dict.get("nodes", 1)) - if nodes > 1: - return arg_dict - partition = arg_dict.get("partition", None) or _get_default_partition() - constraint = arg_dict.get("constraint", None) - ncpus = int(arg_dict.get("cpus-per-task", 1)) - runtime = arg_dict.get("time", None) - memory = _convert_units_to_mb(arg_dict.get("mem", 0)) - config = _get_cluster_configuration(partition, constraint, memory) - mem = arg_dict.get("mem", ncpus * min(config["MEMORY_PER_CPU"])) - mem = _convert_units_to_mb(mem) - if mem > max(config["MEMORY"]): - logger.info( - f"requested memory ({mem}) > max memory ({max(config['MEMORY'])}); " - "adjusting memory settings" - ) - mem = max(config["MEMORY"]) - - # Calculate available memory as defined by the number of requested - # cpus times memory per cpu - AVAILABLE_MEM = ncpus * min(config["MEMORY_PER_CPU"]) - # Add additional cpus if memory is larger than AVAILABLE_MEM - if mem > AVAILABLE_MEM: - logger.info( - f"requested memory ({mem}) > " - f"ncpus x MEMORY_PER_CPU ({AVAILABLE_MEM}); " - "trying to adjust number of cpus up" - ) - ncpus = int(math.ceil(mem / min(config["MEMORY_PER_CPU"]))) - if ncpus > max(config["CPUS"]): - logger.info( - f"ncpus ({ncpus}) > available cpus ({max(config['CPUS'])}); " - "adjusting number of cpus down" - ) - ncpus = min(int(max(config["CPUS"])), ncpus) - adjusted_args = {"mem": int(mem), "cpus-per-task": ncpus} - - # Update time. If requested time is larger than maximum allowed time, reset - if runtime: - runtime = time_to_minutes(runtime) - time_limit = max(config["TIMELIMIT_MINUTES"]) - if runtime > time_limit: - logger.info(f"time (runtime) > time limit {time_limit}; " "adjusting time down") - adjusted_args["time"] = time_limit - - # update and return - arg_dict.update(adjusted_args) - return arg_dict - - timeformats = [ re.compile(r"^(?P\d+)-(?P\d+):(?P\d+):(?P\d+)$"), re.compile(r"^(?P\d+)-(?P\d+):(?P\d+)$"), @@ -279,55 +226,3 @@ def time_to_minutes(time): ) assert minutes > 0, "minutes has to be greater than 0" return minutes - - -def _get_default_partition(): - """Retrieve default partition for cluster""" - cluster = CookieCutter.get_cluster_option() - cmd = f"sinfo -O partition {cluster}" - res = sp.check_output(cmd.split()) - m = re.search(r"(?P\S+)\*", res.decode(), re.M) - partition = m.group("partition") - return partition - - -def _get_cluster_configuration(partition, constraints=None, memory=0): - """Retrieve cluster configuration. - - Retrieve cluster configuration for a partition filtered by - constraints, memory and cpus - - """ - try: - import pandas as pd - except ImportError: - print( - "Error: currently advanced argument conversion " "depends on 'pandas'.", file=sys.stderr - ) - sys.exit(1) - - if constraints: - constraint_set = set(constraints.split(",")) - cluster = CookieCutter.get_cluster_option() - cmd = f"sinfo -e -o %all -p {partition} {cluster}".split() - try: - output = sp.Popen(" ".join(cmd), shell=True, stdout=sp.PIPE).communicate() - except Exception as e: - print(e) - raise - data = re.sub("^CLUSTER:.+\n", "", re.sub(" \\|", "|", output[0].decode())) - df = pd.read_csv(StringIO(data), sep="|") - try: - df["TIMELIMIT_MINUTES"] = df["TIMELIMIT"].apply(time_to_minutes) - df["MEMORY_PER_CPU"] = df["MEMORY"] / df["CPUS"] - df["FEATURE_SET"] = df["AVAIL_FEATURES"].str.split(",").apply(set) - except Exception as e: - print(e) - raise - if constraints: - constraint_set = set(constraints.split(",")) - i = df["FEATURE_SET"].apply(lambda x: len(x.intersection(constraint_set)) > 0) - df = df.loc[i] - memory = min(_convert_units_to_mb(memory), max(df["MEMORY"])) - df = df.loc[df["MEMORY"] >= memory] - return df From bf2cef4d810aa9e213e5c9a3d0f19cc02270e46c Mon Sep 17 00:00:00 2001 From: Per Unneberg Date: Mon, 16 May 2022 15:11:08 +0200 Subject: [PATCH 2/4] Pin cryptograph to avoid deprecated algorithm error Algorithm blowfish-cdc has been removed from cryptography >=37.0.0, which causes CI to fail. Until paramiko is updated this should avoid this issue. --- test-environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test-environment.yml b/test-environment.yml index b87d2ae..5dba0de 100644 --- a/test-environment.yml +++ b/test-environment.yml @@ -15,3 +15,4 @@ dependencies: - pyflakes - snakemake - urllib3 + - cryptography <=37.0.0 From e9ebf764303c0693211df944838f05d861e665a6 Mon Sep 17 00:00:00 2001 From: Per Unneberg Date: Mon, 16 May 2022 16:35:39 +0200 Subject: [PATCH 3/4] Replace deprecated project with project_path --- tests/test_cookie.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_cookie.py b/tests/test_cookie.py index d433480..c2f43ce 100644 --- a/tests/test_cookie.py +++ b/tests/test_cookie.py @@ -7,15 +7,15 @@ def test_bake_project(cookies, sidecar): result = cookies.bake(template=str(pytest.cookie_template), extra_context={"cluster_sidecar": sidecar}) - cfg = result.project / "config.yaml" + cfg = result.project_path / "config.yaml" if sidecar == "yes": - assert "cluster-sidecar: \"slurm-sidecar.py\"\n" in cfg.readlines() + assert "cluster-sidecar: \"slurm-sidecar.py\"\n" in cfg.read_text() else: - assert "cluster-sidecar: \"slurm-sidecar.py\"" not in cfg.readlines() + assert "cluster-sidecar: \"slurm-sidecar.py\"" not in cfg.read_text() assert result.exit_code == 0 assert result.exception is None - assert result.project.basename == "slurm" - assert result.project.isdir() + assert result.project_path.name == "slurm" + assert result.project_path.is_dir() def test_cookiecutter(cookies, monkeypatch): From 86d6c6fbbb731a9b7766043ae24a75fd5fc7633e Mon Sep 17 00:00:00 2001 From: Per Unneberg Date: Mon, 16 May 2022 18:49:42 +0200 Subject: [PATCH 4/4] Uncomment utils tests --- tests/test_utils.py | 67 ++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index fef5e88..bc02f06 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -13,44 +13,29 @@ import slurm_utils # noqa: E402 -# # Profile has not been installed so need to mock calls -# @pytest.fixture -# def mock_cookiecutter_cluster_option(monkeypatch): -# def cluster_option(): -# return "" -# monkeypatch.setattr(CookieCutter, "get_cluster_option", cluster_option) - - -# @pytest.fixture -# def mock_cookiecutter_named_cluster_option(monkeypatch): -# def cluster_option(): -# return "--cluster=linux" -# monkeypatch.setattr(CookieCutter, "get_cluster_option", cluster_option) - - -# def test_time_to_minutes(): -# minutes = slurm_utils.time_to_minutes("foo") -# assert minutes is None -# minutes = slurm_utils.time_to_minutes("10-00:00:10") -# assert minutes == 14401 -# minutes = slurm_utils.time_to_minutes("10:00:00") -# assert minutes == 600 -# minutes = slurm_utils.time_to_minutes("100:00") -# assert minutes == 100 -# minutes = slurm_utils.time_to_minutes("20") -# assert minutes == 20 - - -# def test_si_units(): -# m = slurm_utils._convert_units_to_mb(1000) -# assert m == 1000 -# m = slurm_utils._convert_units_to_mb("1000K") -# assert m == 1 -# m = slurm_utils._convert_units_to_mb("1000M") -# assert m == 1000 -# m = slurm_utils._convert_units_to_mb("1000G") -# assert m == 1e6 -# m = slurm_utils._convert_units_to_mb("1000T") -# assert m == 1e9 -# with pytest.raises(SystemExit): -# m = slurm_utils._convert_units_to_mb("1000E") +def test_time_to_minutes(): + minutes = slurm_utils.time_to_minutes("foo") + assert minutes is None + minutes = slurm_utils.time_to_minutes("10-00:00:10") + assert minutes == 14401 + minutes = slurm_utils.time_to_minutes("10:00:00") + assert minutes == 600 + minutes = slurm_utils.time_to_minutes("100:00") + assert minutes == 100 + minutes = slurm_utils.time_to_minutes("20") + assert minutes == 20 + + +def test_si_units(): + m = slurm_utils._convert_units_to_mb(1000) + assert m == 1000 + m = slurm_utils._convert_units_to_mb("1000K") + assert m == 1 + m = slurm_utils._convert_units_to_mb("1000M") + assert m == 1000 + m = slurm_utils._convert_units_to_mb("1000G") + assert m == 1e6 + m = slurm_utils._convert_units_to_mb("1000T") + assert m == 1e9 + with pytest.raises(SystemExit): + m = slurm_utils._convert_units_to_mb("1000E")