From 48485cda8c4265ccbd0994e16eab5431f0c65ac1 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Wed, 12 Jul 2017 16:03:14 -0500 Subject: [PATCH 1/3] Ensure that we get the absolute *status.py path --- src/scripts/pbs_status.py | 2 +- src/scripts/slurm_status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripts/pbs_status.py b/src/scripts/pbs_status.py index 57a2d8d3..8fd30cf6 100755 --- a/src/scripts/pbs_status.py +++ b/src/scripts/pbs_status.py @@ -43,7 +43,7 @@ import pickle import csv -sys.path.insert(0, os.path.dirname(__file__)) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import blah cache_timeout = 60 diff --git a/src/scripts/slurm_status.py b/src/scripts/slurm_status.py index c3c1a580..16b1f3bb 100644 --- a/src/scripts/slurm_status.py +++ b/src/scripts/slurm_status.py @@ -43,7 +43,7 @@ import pickle import csv -sys.path.insert(0, os.path.dirname(__file__)) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import blah cache_timeout = 60 From 87286a79173e2c4613425b5e8cd22bcfc03e74d3 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Wed, 12 Jul 2017 18:49:21 -0500 Subject: [PATCH 2/3] Properly load blah.config (SOFTWARE-2628) Previous method didn't work because the options in blah.config weren't ever exported so they never made it ot the environment --- src/scripts/blah.py | 34 +++++++++++++++++++++------------- src/scripts/pbs_status.py | 7 ++++--- src/scripts/slurm_status.py | 2 +- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/scripts/blah.py b/src/scripts/blah.py index 5c5a3e41..d0f9bea4 100644 --- a/src/scripts/blah.py +++ b/src/scripts/blah.py @@ -1,18 +1,26 @@ """Common functions for BLAH python scripts""" import os -import subprocess +from ConfigParser import RawConfigParser +from io import StringIO -def load_env(config_dir): - """Load blah.config into the environment""" - load_config_path = os.path.join(config_dir, 'blah_load_config.sh') - command = ['bash', '-c', 'source %s && env' % load_config_path] - try: - config_proc = subprocess.Popen(command, stdout=subprocess.PIPE) - config_out, _ = config_proc.communicate() +class BlahConfigParser(RawConfigParser, object): + + def __init__(self, path='/etc/blah.config'): + # RawConfigParser requires ini-style [section headers] but since + # blah.config is also used as a shell script we need to fake one + self.header = 'blahp' + with open(path) as f: + config = f.read() + vfile = StringIO(u'[%s]\n%s' % (self.header, config)) + + super(BlahConfigParser, self).__init__() + # TODO: readfp() is replaced by read_file() in Python 3.2+ + self.readfp(vfile) + + def items(self): + return super(BlahConfigParser, self).items(self.header) + + def get(self, option): + return super(BlahConfigParser, self).get(self.header, option) - for line in config_out.splitlines(): - (key, _, val) = line.partition('=') - os.environ[key] = val - except IOError: - pass diff --git a/src/scripts/pbs_status.py b/src/scripts/pbs_status.py index 8fd30cf6..044904bc 100755 --- a/src/scripts/pbs_status.py +++ b/src/scripts/pbs_status.py @@ -233,7 +233,7 @@ def qstat(jobid=""): starttime = time.time() log("Starting qstat.") command = (qstat_bin, '-f') - if os.environ.get('pbs_pro').lower() != 'yes': + if config.get('pbs_pro').lower() != 'yes': command += ('-1',) # -1 conflicts with -f in PBS Pro if jobid: command += (jobid,) @@ -358,7 +358,7 @@ def get_qstat_location(): if _qstat_location_cache != None: return _qstat_location_cache try: - cmd = os.path.join(os.environ['pbs_binpath'], 'qstat') + cmd = os.path.join(config.get('pbs_binpath'), 'qstat') except KeyError: cmd = 'which qstat' child_stdout = os.popen(cmd) @@ -527,7 +527,8 @@ def main(): jobid = jobid_arg.split("/")[-1].split(".")[0] config_dir = os.path.dirname(os.path.abspath(__file__)) - blah.load_env(config_dir) + global config + config = blah.BlahConfigParser() log("Checking cache for jobid %s" % jobid) cache_contents = None diff --git a/src/scripts/slurm_status.py b/src/scripts/slurm_status.py index 16b1f3bb..8fb9e553 100644 --- a/src/scripts/slurm_status.py +++ b/src/scripts/slurm_status.py @@ -336,7 +336,7 @@ def get_slurm_location(program): if _slurm_location_cache != None: return os.path.join(_slurm_location_cache, program) try: - cmd = os.path.join(os.environ['slurm_binpath'], program) + cmd = os.path.join(config.get('slurm_binpath'), program) except KeyError: cmd = 'which %s' % program child_stdout = os.popen(cmd) From 273f7bc3c6bb7f5b90e6a0908d8840797f1f764e Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Thu, 13 Jul 2017 14:00:42 -0500 Subject: [PATCH 3/3] Fix bugs in constructing the path to pbs/slurm query commands --- src/scripts/blah.py | 5 ++++- src/scripts/pbs_status.py | 14 ++++++++------ src/scripts/slurm_status.py | 13 +++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/scripts/blah.py b/src/scripts/blah.py index d0f9bea4..679add99 100644 --- a/src/scripts/blah.py +++ b/src/scripts/blah.py @@ -22,5 +22,8 @@ def items(self): return super(BlahConfigParser, self).items(self.header) def get(self, option): - return super(BlahConfigParser, self).get(self.header, option) + # ConfigParser happily includes quotes in value strings, which we + # happily allow in /etc/blah.config. This causes failures when joining + # paths, for example. + return super(BlahConfigParser, self).get(self.header, option).strip('"\'') diff --git a/src/scripts/pbs_status.py b/src/scripts/pbs_status.py index 044904bc..01664f70 100755 --- a/src/scripts/pbs_status.py +++ b/src/scripts/pbs_status.py @@ -357,15 +357,17 @@ def get_qstat_location(): global _qstat_location_cache if _qstat_location_cache != None: return _qstat_location_cache + try: - cmd = os.path.join(config.get('pbs_binpath'), 'qstat') + location = os.path.join(config.get('pbs_binpath'), 'qstat') except KeyError: cmd = 'which qstat' - child_stdout = os.popen(cmd) - output = child_stdout.read() - location = output.split("\n")[0].strip() - if child_stdout.close(): - raise Exception("Unable to determine qstat location: %s" % output) + child_stdout = os.popen(cmd) + output = child_stdout.read() + location = output.split("\n")[0].strip() + if child_stdout.close(): + raise Exception("Unable to determine qstat location: %s" % output) + _qstat_location_cache = location return location diff --git a/src/scripts/slurm_status.py b/src/scripts/slurm_status.py index 8fb9e553..b462a298 100644 --- a/src/scripts/slurm_status.py +++ b/src/scripts/slurm_status.py @@ -336,14 +336,15 @@ def get_slurm_location(program): if _slurm_location_cache != None: return os.path.join(_slurm_location_cache, program) try: - cmd = os.path.join(config.get('slurm_binpath'), program) + location = os.path.join(config.get('slurm_binpath'), program) except KeyError: cmd = 'which %s' % program - child_stdout = os.popen(cmd) - output = child_stdout.read() - location = output.split("\n")[0].strip() - if child_stdout.close(): - raise Exception("Unable to determine scontrol location: %s" % output) + child_stdout = os.popen(cmd) + output = child_stdout.read() + location = output.split("\n")[0].strip() + if child_stdout.close(): + raise Exception("Unable to determine scontrol location: %s" % output) + _slurm_location_cache = os.path.dirname(location) return location