From 8a6790ec916a0e520a16028cd26454fe6efc2599 Mon Sep 17 00:00:00 2001 From: "Alex L. Urban" Date: Sat, 16 Jan 2021 12:37:41 -0500 Subject: [PATCH] Run python module under Condor (#325) * Use python -m under Condor * Drop which() and resolve CI errors --- gwsumm/batch.py | 54 ++++++++++++++++++-------------------- gwsumm/tests/test_utils.py | 19 -------------- gwsumm/utils.py | 16 ----------- 3 files changed, 25 insertions(+), 64 deletions(-) diff --git a/gwsumm/batch.py b/gwsumm/batch.py index 98267e4e..499390bc 100644 --- a/gwsumm/batch.py +++ b/gwsumm/batch.py @@ -37,7 +37,7 @@ from gwdetchar import cli from . import __version__ -from .utils import (mkdir, which) +from .utils import mkdir __author__ = 'Duncan Macleod ' __credits__ = 'Alex Urban ' @@ -53,9 +53,9 @@ class GWSummaryJob(pipeline.CondorDAGJob): """ logtag = '$(cluster)-$(process)' - def __init__(self, universe, executable, tag='gw_summary', + def __init__(self, universe, tag='gw_summary', subdir=None, logdir=None, **cmds): - pipeline.CondorDAGJob.__init__(self, universe, executable) + pipeline.CondorDAGJob.__init__(self, universe, sys.executable) if subdir: subdir = os.path.abspath(subdir) self.set_sub_file(os.path.join(subdir, '%s.sub' % (tag))) @@ -73,37 +73,42 @@ def __init__(self, universe, executable, tag='gw_summary', getattr(self, 'set_%s' % key.lower())(val) else: self.add_condor_cmd(key, val) - # add sub-command option - self._command = None + # add python module sub-command + self._command = ' '.join(['-m', __package__]) def add_opt(self, opt, value=''): pipeline.CondorDAGJob.add_opt(self, opt, str(value)) add_opt.__doc__ = pipeline.CondorDAGJob.add_opt.__doc__ def set_command(self, command): - self._command = command + self._command = ' '.join([ + self._command, + command, + ]) def get_command(self): return self._command def write_sub_file(self): pipeline.CondorDAGJob.write_sub_file(self) - if self.get_command(): - with open(self.get_sub_file(), 'r') as f: - sub = f.read() - sub = sub.replace('arguments = "', 'arguments = " %s' - % self.get_command()) - with open(self.get_sub_file(), 'w') as f: - f.write(sub) + # insert positional arguments in the right place + with open(self.get_sub_file(), 'r') as f: + sub = f.read() + sub = sub.replace( + 'arguments = "', + 'arguments = " {0}'.format(self.get_command()), + ) + with open(self.get_sub_file(), 'w') as f: + f.write(sub) class GWSummaryDAGNode(pipeline.CondorDAGNode): def get_cmd_line(self): - cmd = pipeline.CondorDAGNode.get_cmd_line(self) - if self.job().get_command(): - return '%s %s' % (self.job().get_command(), cmd) - else: - return cmd + # merge positional arguments with options + return ' '.join([ + self.job().get_command(), + pipeline.CondorDAGNode.get_cmd_line(self), + ]) # -- parse command-line ------------------------------------------------------- @@ -196,14 +201,6 @@ def create_parser(): ) # HTCondor options - htcopts.add_argument( - '-x', - '--executable', - action='store', - type=str, - default=which('gw_summary'), - help="Path to gw_summary executable, default: %(default)s", - ) htcopts.add_argument( '-u', '--universe', @@ -497,7 +494,6 @@ def main(args=None): dag.set_dag_file(os.path.join(outdir, args.file_tag)) universe = args.universe - executable = args.executable # -- parse condor commands ---------------------- @@ -530,12 +526,12 @@ def main(args=None): jobs = [] if not args.skip_html_wrapper: htmljob = GWSummaryJob( - 'local', executable, subdir=outdir, logdir=logdir, + 'local', subdir=outdir, logdir=logdir, tag='%s_local' % args.file_tag, **condorcmds) jobs.append(htmljob) if not args.html_wrapper_only: datajob = GWSummaryJob( - universe, executable, subdir=outdir, logdir=logdir, + universe, subdir=outdir, logdir=logdir, tag=args.file_tag, **condorcmds) jobs.append(datajob) diff --git a/gwsumm/tests/test_utils.py b/gwsumm/tests/test_utils.py index 295e0079..7235911c 100644 --- a/gwsumm/tests/test_utils.py +++ b/gwsumm/tests/test_utils.py @@ -29,11 +29,6 @@ import pytest -try: - from unittest import mock -except ImportError: - import mock - from .. import (utils, globalv) __author__ = 'Duncan Macleod ' @@ -80,20 +75,6 @@ def test_nat_sorted(): '1', '2', '10', 'B', 'a'] -def test_which(): - environ = os.environ.copy() - os.environ['PATH'] = '/usr/bin:/bin' - try: - with mock.patch('os.path.isfile', return_value=True), \ - mock.patch('os.access', return_value=True): - assert utils.which('/bin/bash') == '/bin/bash' - assert utils.which('bash') == '/usr/bin/bash' - with mock.patch('os.path.isfile', return_value=False): - assert utils.which('blah') is None - finally: - os.environ = environ - - @pytest.mark.parametrize('chan, mask', [ ('L1:TEST-ODC_CHANNEL_OUT_DQ', 'L1:TEST-ODC_CHANNEL_BITMASK'), ('L1:TEST-ODC_CHANNEL_OUTMON', 'L1:TEST-ODC_CHANNEL_BITMASK'), diff --git a/gwsumm/utils.py b/gwsumm/utils.py index 3ce56bf7..4676866e 100644 --- a/gwsumm/utils.py +++ b/gwsumm/utils.py @@ -116,22 +116,6 @@ def alphanum_key(key): return sorted(iterable, key=alphanum_key) -def which(program): - """Find full path of executable program - """ - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - _re_odc = re.compile('(OUTMON|OUT_DQ|LATCH)')