Run python module under Condor (#325)

* Use python -m under Condor

* Drop which() and resolve CI errors

gwsumm/batch.py

@@ -37,7 +37,7 @@
 from gwdetchar import cli
 
 from . import __version__
-from .utils import (mkdir, which)
+from .utils import mkdir
 
 __author__ = 'Duncan Macleod <duncan.macleod@ligo.org>'
 __credits__ = 'Alex Urban <alexander.urban@ligo.org>'
@@ -53,9 +53,9 @@ class GWSummaryJob(pipeline.CondorDAGJob):
     """
     logtag = '$(cluster)-$(process)'
 
-    def __init__(self, universe, executable, tag='gw_summary',
+    def __init__(self, universe, tag='gw_summary',
                  subdir=None, logdir=None, **cmds):
-        pipeline.CondorDAGJob.__init__(self, universe, executable)
+        pipeline.CondorDAGJob.__init__(self, universe, sys.executable)
         if subdir:
             subdir = os.path.abspath(subdir)
             self.set_sub_file(os.path.join(subdir, '%s.sub' % (tag)))
@@ -73,37 +73,42 @@ def __init__(self, universe, executable, tag='gw_summary',
                 getattr(self, 'set_%s' % key.lower())(val)
             else:
                 self.add_condor_cmd(key, val)
-        # add sub-command option
-        self._command = None
+        # add python module sub-command
+        self._command = ' '.join(['-m', __package__])
 
     def add_opt(self, opt, value=''):
         pipeline.CondorDAGJob.add_opt(self, opt, str(value))
     add_opt.__doc__ = pipeline.CondorDAGJob.add_opt.__doc__
 
     def set_command(self, command):
-        self._command = command
+        self._command = ' '.join([
+            self._command,
+            command,
+        ])
 
     def get_command(self):
         return self._command
 
     def write_sub_file(self):
         pipeline.CondorDAGJob.write_sub_file(self)
-        if self.get_command():
-            with open(self.get_sub_file(), 'r') as f:
-                sub = f.read()
-            sub = sub.replace('arguments = "', 'arguments = " %s'
-                              % self.get_command())
-            with open(self.get_sub_file(), 'w') as f:
-                f.write(sub)
+        # insert positional arguments in the right place
+        with open(self.get_sub_file(), 'r') as f:
+            sub = f.read()
+        sub = sub.replace(
+            'arguments = "',
+            'arguments = " {0}'.format(self.get_command()),
+        )
+        with open(self.get_sub_file(), 'w') as f:
+            f.write(sub)
 
 
 class GWSummaryDAGNode(pipeline.CondorDAGNode):
     def get_cmd_line(self):
-        cmd = pipeline.CondorDAGNode.get_cmd_line(self)
-        if self.job().get_command():
-            return '%s %s' % (self.job().get_command(), cmd)
-        else:
-            return cmd
+        # merge positional arguments with options
+        return ' '.join([
+            self.job().get_command(),
+            pipeline.CondorDAGNode.get_cmd_line(self),
+        ])
 
 
 # -- parse command-line -------------------------------------------------------
@@ -196,14 +201,6 @@ def create_parser():
     )
 
     # HTCondor options
-    htcopts.add_argument(
-        '-x',
-        '--executable',
-        action='store',
-        type=str,
-        default=which('gw_summary'),
-        help="Path to gw_summary executable, default: %(default)s",
-    )
     htcopts.add_argument(
         '-u',
         '--universe',
@@ -497,7 +494,6 @@ def main(args=None):
     dag.set_dag_file(os.path.join(outdir, args.file_tag))
 
     universe = args.universe
-    executable = args.executable
 
     # -- parse condor commands ----------------------
 
@@ -530,12 +526,12 @@ def main(args=None):
     jobs = []
     if not args.skip_html_wrapper:
         htmljob = GWSummaryJob(
-            'local', executable, subdir=outdir, logdir=logdir,
+            'local', subdir=outdir, logdir=logdir,
             tag='%s_local' % args.file_tag, **condorcmds)
         jobs.append(htmljob)
     if not args.html_wrapper_only:
         datajob = GWSummaryJob(
-            universe, executable, subdir=outdir, logdir=logdir,
+            universe, subdir=outdir, logdir=logdir,
             tag=args.file_tag, **condorcmds)
         jobs.append(datajob)
 

gwsumm/tests/test_utils.py

@@ -29,11 +29,6 @@
 
 import pytest
 
-try:
-    from unittest import mock
-except ImportError:
-    import mock
-
 from .. import (utils, globalv)
 
 __author__ = 'Duncan Macleod <duncan.macleod@ligo.org>'
@@ -80,20 +75,6 @@ def test_nat_sorted():
         '1', '2', '10', 'B', 'a']
 
 
-def test_which():
-    environ = os.environ.copy()
-    os.environ['PATH'] = '/usr/bin:/bin'
-    try:
-        with mock.patch('os.path.isfile', return_value=True), \
-                mock.patch('os.access', return_value=True):
-            assert utils.which('/bin/bash') == '/bin/bash'
-            assert utils.which('bash') == '/usr/bin/bash'
-        with mock.patch('os.path.isfile', return_value=False):
-            assert utils.which('blah') is None
-    finally:
-        os.environ = environ
-
-
 @pytest.mark.parametrize('chan, mask', [
     ('L1:TEST-ODC_CHANNEL_OUT_DQ', 'L1:TEST-ODC_CHANNEL_BITMASK'),
     ('L1:TEST-ODC_CHANNEL_OUTMON', 'L1:TEST-ODC_CHANNEL_BITMASK'),

gwsumm/utils.py

@@ -116,22 +116,6 @@ def alphanum_key(key):
     return sorted(iterable, key=alphanum_key)
 
 
-def which(program):
-    """Find full path of executable program
-    """
-    def is_exe(fpath):
-        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-    fpath, fname = os.path.split(program)
-    if fpath:
-        if is_exe(program):
-            return program
-    else:
-        for path in os.environ["PATH"].split(os.pathsep):
-            exe_file = os.path.join(path, program)
-            if is_exe(exe_file):
-                return exe_file
-
-
 _re_odc = re.compile('(OUTMON|OUT_DQ|LATCH)')