Merge pull request #153 from ENCODE-DCC/dev

v2.1.1
ENCODE-DCC · Nov 20, 2021 · cddba79 · cddba79
2 parents 0a8b36d + 0e65408
commit cddba79
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 10 deletions.
diff --git a/caper/__init__.py b/caper/__init__.py
@@ -2,4 +2,4 @@
 from .caper_runner import CaperRunner
 
 __all__ = ['CaperClient', 'CaperClientSubmit', 'CaperRunner']
-__version__ = '2.1.0'
+__version__ = '2.1.1'
diff --git a/caper/caper_init.py b/caper/caper_init.py
@@ -24,7 +24,13 @@
 # Metadata DB for call-caching (reusing previous outputs):
 # Cromwell supports restarting workflows based on a metadata DB
 # DB is in-memory by default
-db=in-memory
+#db=in-memory
+
+# If you use 'caper server' then you can use one unified '--file-db'
+# for all submitted workflows. In such case, uncomment the following two lines
+# and defined file-db as an absolute path to store metadata of all workflows
+#db=file
+#file-db=
 
 # If you use 'caper run' and want to use call-caching:
 # Make sure to define different 'caper run ... --db file --file-db DB_PATH'
@@ -33,11 +39,6 @@
 # then Caper will collect/re-use previous outputs without running the same task again
 # Previous outputs will be simply hard/soft-linked.
 
-# If you use 'caper server' then you can use one unified '--file-db'
-# for all submitted workflows. In such case, uncomment the following two lines
-# and defined file-db as an absolute path to store metadata of all workflows
-#db=file
-#file-db=
 """
 
 CONF_CONTENTS_LOCAL_HASH_STRAT = """

diff --git a/caper/cromwell_backend.py b/caper/cromwell_backend.py
@@ -772,7 +772,8 @@ class CromwellBackendSlurm(CromwellBackendHpc):
     )
     SLURM_CHECK_ALIVE = dedent(
         """
-        for ITER in 1 2 3; do
+        for ITER in 1 2 3
+        do
             CHK_ALIVE=$(squeue --noheader -j ${job_id} --format=%i | grep ${job_id})
             if [ -z "$CHK_ALIVE" ]
             then
@@ -801,14 +802,16 @@ class CromwellBackendSlurm(CromwellBackendHpc):
         {submit}
         EOF
 
-        for ITER in 1 2 3; do
+        for ITER in 1 2 3
+        do
             sbatch --export=ALL -J ${{job_name}} -D ${{cwd}} -o ${{out}} -e ${{err}} \\
                 ${{'-p ' + slurm_partition}} ${{'--account ' + slurm_account}} \\
                 {slurm_resource_param} \\
                 ${{slurm_extra_param}} \\
-                ${{script}}.caper && break
+                ${{script}}.caper && exit 0
             sleep 30
         done
+        exit 1
     """
     )
     DEFAULT_SLURM_RESOURCE_PARAM = (

diff --git a/caper/cromwell_metadata.py b/caper/cromwell_metadata.py
@@ -208,6 +208,7 @@ def troubleshoot_call(call_name, call, parent_call_names):
                 job_id = call.get('jobId')
                 stdout = call.get('stdout')
                 stderr = call.get('stderr')
+                strerr_background = stderr + '.background'
                 run_start = None
                 run_end = None
                 for event in call.get('executionEvents', []):
@@ -245,6 +246,11 @@ def troubleshoot_call(call_name, call, parent_call_names):
                             help_msg += 'STDOUT_CONTENTS=\n{s}\n'.format(
                                 s=AutoURI(stdout).read()
                             )
+                    if strerr_background:
+                        if AutoURI(strerr_background).exists:
+                            help_msg += 'STDERR_BACKGROUND_CONTENTS=\n{s}\n'.format(
+                                s=AutoURI(strerr_background).read()
+                            )
 
                 return help_msg