|
| 1 | +#!/usr/bin/env python |
| 2 | +# -*- coding: utf-8 -*- |
| 3 | +# vim: nu:ai:ts=4:sw=4 |
| 4 | + |
| 5 | +# |
| 6 | +# Copyright (C) 2024 Joseph Areeda <joseph.areeda@ligo.org> |
| 7 | +# |
| 8 | +# This program is free software: you can redistribute it and/or modify |
| 9 | +# it under the terms of the GNU General Public License as published by |
| 10 | +# the Free Software Foundation, either version 3 of the License, or |
| 11 | +# (at your option) any later version. |
| 12 | +# |
| 13 | +# This program is distributed in the hope that it will be useful, |
| 14 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | +# GNU General Public License for more details. |
| 17 | +# |
| 18 | +# You should have received a copy of the GNU General Public License |
| 19 | +# along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 20 | +# |
| 21 | + |
| 22 | +""" |
| 23 | +The situation is that we run DAGs with many omicron jobs, some of which fail for data dependent reasons that |
| 24 | +are valid and permanent but others are transient like network issues that could be resolved with a retry. |
| 25 | +
|
| 26 | +This program isun as a post script to allow us to retry the job but return a success code even if it fails |
| 27 | +repeatedly so that the DAG completes. |
| 28 | +""" |
| 29 | +import textwrap |
| 30 | +import time |
| 31 | + |
| 32 | +start_time = time.time() |
| 33 | + |
| 34 | +import argparse |
| 35 | +import logging |
| 36 | +from pathlib import Path |
| 37 | +import sys |
| 38 | +import traceback |
| 39 | + |
| 40 | +try: |
| 41 | + from ._version import __version__ |
| 42 | +except ImportError: |
| 43 | + __version__ = '0.0.0' |
| 44 | + |
| 45 | +__author__ = 'joseph areeda' |
| 46 | +__email__ = 'joseph.areeda@ligo.org' |
| 47 | +__process_name__ = Path(__file__).name |
| 48 | + |
| 49 | +logger = None |
| 50 | + |
| 51 | + |
| 52 | +def parser_add_args(parser): |
| 53 | + """ |
| 54 | + Set up command parser |
| 55 | + :param argparse.ArgumentParser parser: |
| 56 | + :return: None but parser object is updated |
| 57 | + """ |
| 58 | + parser.add_argument('-v', '--verbose', action='count', default=1, |
| 59 | + help='increase verbose output') |
| 60 | + parser.add_argument('-V', '--version', action='version', |
| 61 | + version=__version__) |
| 62 | + parser.add_argument('-q', '--quiet', default=False, action='store_true', |
| 63 | + help='show only fatal errors') |
| 64 | + parser.add_argument('--return-code', help='Program return code') |
| 65 | + parser.add_argument('--max-retry', help='condor max retry value') |
| 66 | + parser.add_argument('--retry', help='current try starting at 0') |
| 67 | + parser.add_argument('--log', help='Path for a copy of our logger output') |
| 68 | + |
| 69 | + |
| 70 | +def main(): |
| 71 | + global logger |
| 72 | + |
| 73 | + log_file_format = "%(asctime)s - %(levelname)s, %(pathname)s:%(lineno)d: %(message)s" |
| 74 | + log_file_date_format = '%m-%d %H:%M:%S' |
| 75 | + logging.basicConfig(format=log_file_format, datefmt=log_file_date_format) |
| 76 | + logger = logging.getLogger(__process_name__) |
| 77 | + logger.setLevel(logging.DEBUG) |
| 78 | + |
| 79 | + epilog = textwrap.dedent(""" |
| 80 | + This progam is designed to be run as a post script in a Condor DAG. For available arguments see: |
| 81 | + https://htcondor.readthedocs.io/en/latest/automated-workflows/dagman-scripts.html#special-script-argument-macros |
| 82 | + A typical lne in the DAG might look like: |
| 83 | + python omicron_post_script.py -vvv --return $(RETURN) --retry $(RETRY) --max-retry $(MAX_RETRIES) --log |
| 84 | + <path_to_log> |
| 85 | + """) |
| 86 | + |
| 87 | + parser = argparse.ArgumentParser(description=__doc__, prog=__process_name__, epilog=epilog, |
| 88 | + formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| 89 | + parser_add_args(parser) |
| 90 | + args = parser.parse_args() |
| 91 | + verbosity = 0 if args.quiet else args.verbose |
| 92 | + |
| 93 | + if verbosity < 1: |
| 94 | + logger.setLevel(logging.CRITICAL) |
| 95 | + elif verbosity < 2: |
| 96 | + logger.setLevel(logging.INFO) |
| 97 | + else: |
| 98 | + logger.setLevel(logging.DEBUG) |
| 99 | + |
| 100 | + if args.log: |
| 101 | + log = Path(args.log) |
| 102 | + log.parent.mkdir(0o775, exist_ok=True, parents=True) |
| 103 | + file_handler = logging.FileHandler(log, mode='a') |
| 104 | + log_formatter = logging.Formatter(log_file_format, datefmt=log_file_date_format) |
| 105 | + file_handler.setFormatter(log_formatter) |
| 106 | + logger.addHandler(file_handler) |
| 107 | + |
| 108 | + me = Path(__file__).name |
| 109 | + logger.info(f'--------- Running {str(me)}') |
| 110 | + # debugging? |
| 111 | + logger.debug(f'{__process_name__} version: {__version__} called with arguments:') |
| 112 | + for k, v in args.__dict__.items(): |
| 113 | + logger.debug(' {} = {}'.format(k, v)) |
| 114 | + |
| 115 | + ret = int(args.return_code) |
| 116 | + retry = int(args.retry) |
| 117 | + max_retry = int(args.max_retry) |
| 118 | + ret = ret if retry < max_retry or ret == 0 else 0 |
| 119 | + logger.info(f'returning {ret}') |
| 120 | + return ret |
| 121 | + |
| 122 | + |
| 123 | +if __name__ == "__main__": |
| 124 | + try: |
| 125 | + ret = main() |
| 126 | + except (ValueError, TypeError, OSError, NameError, ArithmeticError, RuntimeError) as ex: |
| 127 | + print(ex, file=sys.stderr) |
| 128 | + traceback.print_exc(file=sys.stderr) |
| 129 | + ret = 21 |
| 130 | + |
| 131 | + if logger is None: |
| 132 | + logging.basicConfig() |
| 133 | + logger = logging.getLogger(__process_name__) |
| 134 | + logger.setLevel(logging.DEBUG) |
| 135 | + # report our run time |
| 136 | + logger.info(f'Elapsed time: {time.time() - start_time:.1f}s') |
| 137 | + sys.exit(ret) |
0 commit comments