Merge pull request #92 from Snakemake-Profiles/slurm-sidecar-ci-update

Slurm sidecar ci update

.github/workflows/slurm.yaml

@@ -1,6 +1,6 @@
 name: Test SnakemakeProfiles/slurm
 env:
-  SNAKEMAKE_IMAGE: quay.io/biocontainers/snakemake:6.15.5--hdfd78af_0
+  SNAKEMAKE_IMAGE: quay.io/biocontainers/snakemake:7.3.2--hdfd78af_0
   SLURM_IMAGE: giovtorres/docker-centos7-slurm:20.11.8
   DOCKER_COMPOSE: tests/docker-compose.yaml
 
@@ -69,3 +69,4 @@ jobs:
           pytest -v -s tests/test_utils.py
           pytest -v -s tests/test_slurm.py --slow
           pytest -v -s tests/test_slurm_advanced.py --slow
+          pytest -v -s tests/test_sidecar.py

.gitignore

@@ -1,2 +1,9 @@
 __pycache__
 .pytest
+
+# pycharm
+.idea/
+
+# text editors
+*.sw?
+*~

ChangeLog.md

@@ -1,5 +1,11 @@
 # ChangeLog
 
+## 2022-05-12
+
+### Changes
+
+- add support for sidecar (PR #85)
+
 ## 2021-03-10
 
 ### Issues

README.md

@@ -4,12 +4,14 @@
 
 This cookiecutter provides a template Snakemake profile for
 configuring Snakemake to run on the [SLURM Workload
-Manager](https://slurm.schedmd.com/). The profile defines three
-scripts
+Manager](https://slurm.schedmd.com/). The profile defines the
+following scripts
 
 1. `slurm-submit.py` - submits a jobscript to slurm
 2. `slurm-jobscript.sh` - a template jobscript
 3. `slurm-status.py` - checks the status of jobs in slurm
+4. `slurm-sidecar.py` - run a Snakemake cluster sidecar for caching
+   queries to Slurm's controller/database daemons
 
 and a configuration file `config.yaml` that defines default values for
 snakemake command line arguments. The default `config.yaml` file is
@@ -18,6 +20,7 @@ snakemake command line arguments. The default `config.yaml` file is
     jobscript: "slurm-jobscript.sh"
     cluster: "slurm-submit.py"
     cluster-status: "slurm-status.py"
+    cluster-sidecar: "slurm-sidecar.py"
     max-jobs-per-second: 1
     max-status-checks-per-second: 10
     local-cores: 1
@@ -27,7 +30,8 @@ Given an installed profile `profile_name`, when snakemake is run with
 `--profile profile_name`, the configuration above would imply the
 following snakemake call:
 
-    snakemake --jobscript slurm-jobscript.sh --cluster slurm-submit.py --cluster-status slurm-status.py --restart-times 3 --max-jobs-per-second 1 --max-status-checks-per-second 10 --local-cores 1 --latency-wait 60
+    snakemake --jobscript slurm-jobscript.sh --cluster slurm-submit.py --cluster-status slurm-status.py \
+        --restart-times 3 --max-jobs-per-second 1 --max-status-checks-per-second 10 --local-cores 1 --latency-wait 60
 
 plus any additional options to snakemake that the user has applied.
 

cookiecutter.json

@@ -2,6 +2,8 @@
     "profile_name": "slurm",
     "sbatch_defaults": "",
     "advanced_argument_conversion": ["no", "yes"],
+    "cluster_sidecar_help": "Use cluster sidecar. NB! Requires snakemake >= 7.0! Enter to continue...",
+    "cluster_sidecar": ["yes", "no"],
     "cluster_name": "",
     "cluster_config_help": "The use of cluster-config is discouraged. Rather, set snakemake CLI options in the profile configuration file (see snakemake documentation on best practices). Enter to continue...",
     "cluster_config": ""

tests/conftest.py

@@ -100,6 +100,7 @@ def cookie_factory(tmpdir_factory, _cookiecutter_config_file, datadir):
     Args:
       sbatch_defaults (str): sbatch defaults for cookie
       advanced (str): use advanced argument conversion ("no" or "yes")
+      cluster_sidecar (str): use sidecar to monitor job status
       cluster_name (str): set cluster name
       cluster_config (str): cluster configuration file
       yamlconfig (dict): dictionary of snakemake options with values
@@ -118,6 +119,7 @@ def cookie_factory(tmpdir_factory, _cookiecutter_config_file, datadir):
     def _cookie_factory(
         sbatch_defaults=_sbatch_defaults,
         advanced="no",
+        cluster_sidecar="yes",
         cluster_name=None,
         cluster_config=None,
         yamlconfig=_yamlconfig_default,
@@ -129,6 +131,7 @@ def _cookie_factory(
         extra_context = {
             "sbatch_defaults": sbatch_defaults,
             "advanced_argument_conversion": advanced,
+            "cluster_sidecar": cluster_sidecar,
         }
         if cluster_name is not None:
             extra_context["cluster_name"] = cluster_name

tests/deploystack.sh

@@ -6,7 +6,7 @@
 DOCKER_COMPOSE=${DOCKER_COMPOSE:=docker-compose.yaml}
 
 # Images
-SNAKEMAKE_IMAGE=${SNAKEMAKE_IMAGE:=quay.io/biocontainers/snakemake:6.15.5--hdfd78af_0}
+SNAKEMAKE_IMAGE=${SNAKEMAKE_IMAGE:=quay.io/biocontainers/snakemake:7.3.2--hdfd78af_0}
 SLURM_IMAGE=${SLURM_IMAGE:=giovtorres/docker-centos7-slurm:20.11.8}
 
 docker pull $SNAKEMAKE_IMAGE

tests/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   snakemake:
-    image: quay.io/biocontainers/snakemake:6.15.5--hdfd78af_0
+    image: quay.io/biocontainers/snakemake:7.3.2--hdfd78af_0
     hostname: slurmctl
     command: /bin/bash
     deploy:

tests/mock-slurm/bin/sacct

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+cat <"EOF"
+1044785|RUNNING|0:0
+1044785.extern|RUNNING|0:0
+1044785.0|RUNNING|0:0
+EOF

tests/mock-slurm/bin/squeue

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+cat <<"EOF"
+JOBID,STATE
+1044785,RUNNING
+1044875,RUNNING
+EOF

tests/test_cookie.py

@@ -3,8 +3,15 @@
 import pytest
 
 
-def test_bake_project(cookies):
-    result = cookies.bake(template=str(pytest.cookie_template))
+@pytest.mark.parametrize("sidecar", ["yes", "no"])
+def test_bake_project(cookies, sidecar):
+    result = cookies.bake(template=str(pytest.cookie_template),
+                          extra_context={"cluster_sidecar": sidecar})
+    cfg = result.project / "config.yaml"
+    if sidecar == "yes":
+        assert "cluster-sidecar: \"slurm-sidecar.py\"\n" in cfg.readlines()
+    else:
+        assert "cluster-sidecar: \"slurm-sidecar.py\"" not in cfg.readlines()
     assert result.exit_code == 0
     assert result.exception is None
     assert result.project.basename == "slurm"

tests/test_sidecar.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import json
+import os
+import signal
+import subprocess
+import tempfile
+import time
+
+import pytest
+
+
+@pytest.mark.slow
+@pytest.mark.timeout(60)
+def test_cluster_sidecar_smoke():
+    env = dict(os.environ)
+    env["PATH"] = (
+        os.path.realpath(os.path.dirname(__file__) + "/mock-slurm/bin") + ":" + env.get("PATH")
+    )
+    path_sidecar_py = os.path.realpath(
+        os.path.dirname(__file__) + "/../{{cookiecutter.profile_name}}/slurm-sidecar.py"
+    )
+    with tempfile.TemporaryFile("w+t") as tmpf:
+        with subprocess.Popen(["python", path_sidecar_py], env=env, text=True, stdout=tmpf) as proc:
+            time.sleep(2)
+            os.kill(proc.pid, signal.SIGTERM)
+        tmpf.seek(0)
+        stdout = tmpf.read()
+    the_vars = json.loads(stdout.splitlines()[0])
+    assert "server_port" in the_vars
+    assert "server_secret" in the_vars
+    assert proc.returncode == 0

tests/test_slurm.py

@@ -4,13 +4,18 @@
 
 
 @pytest.fixture
-def profile(cookie_factory, data):
+def profile(cookie_factory, data, request):
     cookie_factory()
 
 
+@pytest.fixture(params=["yes", "no"])
+def sidecar_profile(cookie_factory, data, request):
+    cookie_factory(cluster_sidecar=request.param)
+
+
 @pytest.mark.slow
 @pytest.mark.skipci
-def test_no_timeout(smk_runner, profile):
+def test_no_timeout(smk_runner, sidecar_profile):
     """Test that rule that updates runtime doesn't timeout"""
     smk_runner.make_target("timeout.txt")
     assert "Trying to restart" in smk_runner.output
@@ -19,7 +24,7 @@ def test_no_timeout(smk_runner, profile):
 
 
 @pytest.mark.slow
-def test_timeout(smk_runner, profile):
+def test_timeout(smk_runner, sidecar_profile):
     """Test that rule excessive runtime resources times out"""
     opts = (
         f'--cluster "sbatch --parsable -p {smk_runner.partition} {pytest.account} '
@@ -32,7 +37,7 @@ def test_timeout(smk_runner, profile):
     assert smk_runner.check_jobstatus("TIMEOUT|NODE_FAIL")
 
 
-def test_profile_status_running(smk_runner, profile):
+def test_profile_status_running(smk_runner, sidecar_profile):
     """Test that slurm-status.py catches RUNNING status"""
     opts = (
         f'--cluster "sbatch --parsable -p {smk_runner.partition}'

tests/wrapper.py

@@ -115,7 +115,7 @@ def make_target(self, target, stream=True, asynchronous=False, **kwargs):
         cmd = (
             f"{self.exe} -c '{self.pp} && "
             + f"{self.snakemake} -s {self.snakefile} "
-            + f"{options} --nolock "
+            + f"{options} --nolock --default-resources mem_mb=100 "
             + f"-j {self._num_cores} {self.workdir} {force} {target} {prof} {jn}'"
         )
 

{{cookiecutter.profile_name}}/config.yaml

@@ -2,6 +2,10 @@ restart-times: 3
 jobscript: "slurm-jobscript.sh"
 cluster: "slurm-submit.py"
 cluster-status: "slurm-status.py"
+{%- if cookiecutter.cluster_sidecar == "yes" %}
+cluster-sidecar: "slurm-sidecar.py"
+{%- endif %}
+cluster-cancel: "scancel"
 max-jobs-per-second: 1
 max-status-checks-per-second: 10
 local-cores: 1