From 51a128a8994672489e7bb35246be58e6befe0f0a Mon Sep 17 00:00:00 2001 From: Ian Harry Date: Wed, 12 Jul 2023 09:50:15 +0100 Subject: [PATCH 1/4] Remove most X509/globus links and hooks from PyCBC (#4375) * Remove most X509/globus links and hooks from PyCBC * Need to update this one as well * Don't need these options now * Update docs/workflow/pycbc_make_coinc_search_workflow.rst Co-authored-by: Gareth S Cabourn Davies --------- Co-authored-by: Gareth S Cabourn Davies --- bin/pycbc_submit_dax | 89 +------------------ docs/hwinj.rst | 2 +- docs/workflow/datafind.rst | 2 +- docs/workflow/initialization.rst | 2 +- .../pycbc_make_offline_search_workflow.rst | 23 +---- examples/search/submit.sh | 2 +- pycbc/frame/frame.py | 2 +- pycbc/workflow/configuration.py | 3 +- pycbc/workflow/core.py | 2 +- pycbc/workflow/datafind.py | 9 -- .../pegasus_files/pegasus-properties.conf | 4 +- pycbc/workflow/pegasus_sites.py | 6 +- pycbc/workflow/pegasus_workflow.py | 34 ------- 13 files changed, 15 insertions(+), 165 deletions(-) diff --git a/bin/pycbc_submit_dax b/bin/pycbc_submit_dax index 4f9bd3ef20e..8f33203a946 100644 --- a/bin/pycbc_submit_dax +++ b/bin/pycbc_submit_dax @@ -17,9 +17,7 @@ exec 2>&1 LOCAL_PEGASUS_DIR="" PEGASUS_PROPERTIES="" -NO_CREATE_PROXY=0 NO_QUERY_DB=0 -NO_GRID="" SUBMIT_DAX="--submit" HTML_ENTITIES="{\"\'\": ''', '(': '(', ')': ')', '+': '+', '\"': '"'}" @@ -54,7 +52,6 @@ while true ; do "") shift 2 ;; *) echo $2 >> extra-properties.conf ; shift 2 ;; esac ;; - -K|--no-create-proxy) NO_CREATE_PROXY=1 ; shift ;; -Q|--no-query-db) NO_QUERY_DB=1 ; shift ;; -n|--no-submit) SUBMIT_DAX="" ; shift ;; -l|--local-dir) @@ -62,7 +59,6 @@ while true ; do "") shift 2 ;; *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;; esac ;; - -G|--no-grid) NO_GRID="--forward nogrid" ; shift ;; -h|--help) echo "usage: pycbc_submit_dax [-h] [optional arguments]" echo @@ -72,13 +68,9 @@ while true ; do echo " the pegasus properties file" echo " -P, --append-pegasus-property STRING add the extra property" echo " specified by the argument" - echo " -K, --no-create-proxy Do not run ligo-proxy-init and assume" - echo " that the user has a valid grid proxy" echo " -n, --no-submit Plan the DAX but do not submit it" echo " -l, --local-dir Directory to put condor files under" echo " -Q, --no-query-db Don't query the pegasus DB." - echo " -G, --no-grid Disable checks for grid proxy and" - echo " GLOBUS_LOCATION in pegasus-plan" echo echo "If the environment variable TMPDIR is set then this is prepended to the " echo "path to the temporary workflow execute directory passed to pegasus-plan." @@ -95,49 +87,6 @@ while true ; do esac done -if [ $NO_CREATE_PROXY == 0 ]; then - # Force the user to create a new grid proxy - LIGO_USER_NAME="" - while true; do - read -p "Enter your LIGO.ORG username in (e.g. albert.einstein): " LIGO_USER_NAME - echo - if [ ! -z $LIGO_USER_NAME ] ; then - break - fi - done - unset X509_USER_PROXY - ligo-proxy-init -p $LIGO_USER_NAME || exit 1 -else - if [ ! -z ${X509_USER_PROXY} ] ; then - if [ -f ${X509_USER_PROXY} ] ; then - cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u` - fi - unset X509_USER_PROXY - fi -fi - -if [ -z "${NO_GRID}" ] ; then - #Check that the proxy is valid - set +e - ecp-cert-info -exists - RESULT=$? - set -e - if [ ${RESULT} -eq 0 ] ; then - PROXY_TYPE=`ecp-cert-info -type | tr -d ' '` - if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then - ecp-cert-info - else - cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig - grid-proxy-init -hours 276 -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig - rm -f /tmp/x509up_u`id -u`.orig - ecp-cert-info - fi - else - echo "Error: Could not find a valid grid proxy to submit workflow." - exit 1 - fi -fi - #Make a directory for the submit files SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX` @@ -170,7 +119,7 @@ cat extra-properties.conf >> pegasus-properties.conf # it deserves! STORED_PLANNER_ARGS=`cat additional_planner_args.dat` -pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX $NO_GRID ${STORED_PLANNER_ARGS} +pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX ${STORED_PLANNER_ARGS} echo @@ -186,42 +135,6 @@ chmod 755 debug echo "pegasus-remove $SUBMIT_DIR/work \$@" > stop chmod 755 stop -if [ -z "${NO_GRID}" ] ; then - cat << EOF > start -#!/bin/bash - -if [ -f /tmp/x509up_u\`id -u\` ] ; then - unset X509_USER_PROXY -else - if [ ! -z \${X509_USER_PROXY} ] ; then - if [ -f \${X509_USER_PROXY} ] ; then - cp -a \${X509_USER_PROXY} /tmp/x509up_u\`id -u\` - fi - fi - unset X509_USER_PROXY -fi - -# Check that the proxy is valid -ecp-cert-info -exists -RESULT=\${?} -if [ \${RESULT} -eq 0 ] ; then - PROXY_TYPE=\`ecp-cert-info -type | tr -d ' '\` - if [ x\${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then - ecp-cert-info - else - cp /tmp/x509up_u\`id -u\` /tmp/x509up_u\`id -u\`.orig - grid-proxy-init -hours 276 -cert /tmp/x509up_u\`id -u\`.orig -key /tmp/x509up_u\`id -u\`.orig - rm -f /tmp/x509up_u\`id -u\`.orig - ecp-cert-info - fi -else - echo "Error: Could not find a valid grid proxy to submit workflow." - exit 1 -fi - -EOF -fi - echo "pegasus-run $SUBMIT_DIR/work \$@" > start chmod 755 start diff --git a/docs/hwinj.rst b/docs/hwinj.rst index f09c89a448c..11da93a62c7 100644 --- a/docs/hwinj.rst +++ b/docs/hwinj.rst @@ -181,7 +181,7 @@ You can plot the ASCII waveform files with an X11 connection. It's strongly reco where ``${OUTPUT_PATH}`` is the path to the output plot. -If you are using ``ssh`` or ``gsissh`` to log into a cluster, you can provide the ``-Y`` option to open an X11 connection. For example :: +If you are using ``ssh`` to log into a cluster, you can provide the ``-Y`` option to open an X11 connection. For example :: gsissh -Y ldas-pcdev1.ligo.caltech.edu diff --git a/docs/workflow/datafind.rst b/docs/workflow/datafind.rst index 94d2de364a8..b3e4d288ad2 100644 --- a/docs/workflow/datafind.rst +++ b/docs/workflow/datafind.rst @@ -83,7 +83,7 @@ When using any of the AT_RUNTIME sub-modules the following other configuration o * datafind-X1-frame-type = NAME - REQUIRED. Where X1 is replaced by the ifo name for each ifo. The NAME should be the full frame type, which is used when querying the database. * datafind-ligo-datafind-server = URL - OPTIONAL. If provided use this server when querying for frames. If not provided, which is recommended for most applications, then the LIGO_DATAFIND_SERVER environment variable will be used to determine this. -* datafind-backup-datafind-server = URL - OPTIONAL. This option is only available when using AT_RUNTIME_SINGLE_FRAMES or AT_RUNTIME_MULTIPLE_FRAMES. If given it will query a second datafind server (ie. a remote server) using gsiftp urltypes. This will then allow frames to be associated with both a file:// and gsiftp:// url, in the case that your local site is missing a frame file, or the file is not accessible, pegasus will copy the file from gsiftp://. **NOTE** This will not catch the case that the frame file is available at the **start** of a workflow but goes missing later. Pegasus can copy **all** frame files around at the start of the workflow, but you may not want this (remove symlink option from the basic_pegasus.conf if you want this). +* datafind-backup-datafind-server = URL - OPTIONAL. This option is only available when using AT_RUNTIME_SINGLE_FRAMES or AT_RUNTIME_MULTIPLE_FRAMES. If given it will query a second datafind server (ie. a remote server) using gsiftp urltypes. This will then allow frames to be associated with both a file:// and gsiftp:// url, in the case that your local site is missing a frame file, or the file is not accessible, pegasus will copy the file from gsiftp://. **NOTE** This will not catch the case that the frame file is available at the **start** of a workflow but goes missing later. Pegasus can copy **all** frame files around at the start of the workflow, but you may not want this (remove symlink option from the basic_pegasus.conf if you want this). **WARNING** gsiftp copying is largely deprecated. This option can still work, if you have a valid X509 certificate, but if this is used in a production setting we should investigate the use case. Please contact Ian if using this! When using the PREGENERATED sub-module the following configuartion options apply in the [workflow-datafind] section: diff --git a/docs/workflow/initialization.rst b/docs/workflow/initialization.rst index c4c516666d2..ece0ff1c401 100644 --- a/docs/workflow/initialization.rst +++ b/docs/workflow/initialization.rst @@ -193,7 +193,7 @@ This section should contain the names of each of the executables that will be us tmpltbank = /full/path/to/lalapps_tmpltbank inspiral = /full/path/to/lalapps_inspiral -Note that one can give gsiftp or http/https paths here and the workflow generator will download the code to the workflow directory when it is run. +Note that one can give remote URLs here and the workflow generator will download the code to the workflow directory when it is run. One can also give a URL indicating singularity as the scheme. This will indicate that the executable will be run within a singularity container, and therefore the executables would not be directly accessible from the head node:: diff --git a/docs/workflow/pycbc_make_offline_search_workflow.rst b/docs/workflow/pycbc_make_offline_search_workflow.rst index 3a195b9f640..e7694321d8a 100644 --- a/docs/workflow/pycbc_make_offline_search_workflow.rst +++ b/docs/workflow/pycbc_make_offline_search_workflow.rst @@ -643,9 +643,9 @@ full URL where the file can be found, and ``SITE`` is the site on which that URL resides. The URI in the ``PHYSICAL_FILE_URL`` can be any of the URIs that Pegasus -recognizes. The URIs ``file://``, ``gsiftp://``, and ``http://`` are likely +recognizes. The URIs ``file://``, ``https://`` are likely the most useful. Pegasus will take care of adding transfer jobs for -``gsiftp://`` and ``http://`` URIs, if the data is not available locally. +``https://`` URIs, if the data is not available locally. The string ``SITE`` is a hint that tells Pegasus on which site the ``PHYSICAL_FILE_URL`` can be found. The ``SITE`` string should be one of the @@ -672,7 +672,7 @@ might not be obvious from the hostname in the ``PHYSICAL_FILE_URL``. The following rule should be helpful when chosing the ``SITE`` string: * If you are re-using a file that is available locally with a ``file://`` URI in its ``PHYSICAL_FILE_URL`` (or has an implicit ``file://`` URI since the ``PHYSICAL_FILE_URL`` starts with a ``/``) then the string ``SITE`` should be set to ``local``. -* If you are re-using a file from another cluster, e.g. you are on the Syracuse cluster and want to re-use data from AEI Atlas cluster, then the string ``SITE`` should be set to ``remote`` for that file. In this case, the URI in ``PHYSICAL_FILE_URL`` will be either ``gsiftp://`` or ``http://`` depending on how the file can be accessed. +* If you are re-using a file from another cluster, e.g. you are on the Syracuse cluster and want to re-use data from AEI Atlas cluster, then the string ``SITE`` should be set to ``remote`` for that file. In this case, the URI in ``PHYSICAL_FILE_URL`` will begin with the scheme (e.g. ``https://``) depending on how the file can be accessed. To illustrate this, an example of a simple cache file containing four files for re-use from the ``local`` site is:: @@ -685,23 +685,6 @@ Note that the ``LOGICAL_FILE_NAME`` for the veto files is just the name of the file, but for the two inspiral files it contains the subdirectory that the workflow uses to organize the files by GPS time. In the case of this file Pegasus will delete from the workflow the jobs that create the files ``H1-VETOTIME_CAT3-1169107218-1066800.xml``, ``L1-VETOTIME_CAT3-1169107218-1066800.xml``, ``116912/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf``, and ``116912/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf`` when it plans the workflow. Insted, the data will be re-used from the URLs specified in the cache. Since ``site="local"`` for these files, Pegasus expects that the files all exist on the host where the workflow is run from. -To re-use data from a remote cluster, the URLs must contain a file transfer -mechanism and the ``SITE`` should be set to ``remote``. For example, if the -files listed in the example above are available on -``sugwg-condor.phy.syr.edu`` and you want to re-use them in a workflow on the -AEI Atlas cluster, then the cache file would contain:: - - H1-VETOTIME_CAT3-1169107218-1066800.xml gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/results/1._analysis_time/1.01_segment_data/H1-VETOTIME_CAT3-1169107218-1066800.xml pool="remote" - L1-VETOTIME_CAT3-1169107218-1066800.xml gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/results/1._analysis_time/1.01_segment_data/L1-VETOTIME_CAT3-1169107218-1066800.xml pool="remote" - 116912/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/full_data/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf pool="remote" - 116912/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/full_data/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf pool="remote" - -Note that the URL now contains ``gsiftp://sugwg-condor.phy.syr.edu`` rather -than ``file://localhost`` and the files are listes as ``pool="remote"`` rather -than ``pool="local"``. Pegasus will re-use these data files adding in -file transfer jobs to the workflow to move them into the appropriate -locations. - Once a cache file has been constructed, to enable data re-use, you follow the standard instructions for planning and submitting the workflow in the section :ref:`coincworkflowplan`, but add the ``--cache-file`` argument that points to diff --git a/examples/search/submit.sh b/examples/search/submit.sh index 9b9411b1449..0bd1f677bb6 100644 --- a/examples/search/submit.sh +++ b/examples/search/submit.sh @@ -1,3 +1,3 @@ -pycbc_submit_dax --no-grid --no-create-proxy \ +pycbc_submit_dax \ --local-dir ./ \ --no-query-db \ diff --git a/pycbc/frame/frame.py b/pycbc/frame/frame.py index 5b4cc9201a8..08658e380cd 100644 --- a/pycbc/frame/frame.py +++ b/pycbc/frame/frame.py @@ -272,7 +272,7 @@ def frame_paths(frame_type, start_time, end_time, server=None, url_type='file'): attempt is made to use a local datafind server. url_type : string Returns only frame URLs with a particular scheme or head such - as "file" or "gsiftp". Default is "file", which queries locally + as "file" or "https". Default is "file", which queries locally stored frames. Option can be disabled if set to None. Returns ------- diff --git a/pycbc/workflow/configuration.py b/pycbc/workflow/configuration.py index 5a58f5fef6e..35c4038ce4a 100644 --- a/pycbc/workflow/configuration.py +++ b/pycbc/workflow/configuration.py @@ -90,8 +90,7 @@ def resolve_url(url, directory=None, permissions=None, copy_to_cwd=True): output_fp.close() else: - # TODO: We could support other schemes such as gsiftp by - # calling out to globus-url-copy + # TODO: We could support other schemes as needed errmsg = "Unknown URL scheme: %s\n" % (u.scheme) errmsg += "Currently supported are: file, http, and https." raise ValueError(errmsg) diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index e8019ec0d19..46536b476e8 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -197,7 +197,7 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None, # need to do anything here, as I cannot easily check it exists. exe_path = exe_url.path else: - # Could be http, gsiftp, etc. so it needs fetching if run now + # Could be http, https, etc. so it needs fetching if run now self.needs_fetching = True if self.needs_fetching and not self.installed: err_msg = "Non-file path URLs cannot be used unless the " diff --git a/pycbc/workflow/datafind.py b/pycbc/workflow/datafind.py index 7eef3b3dc8c..948cd0f350d 100644 --- a/pycbc/workflow/datafind.py +++ b/pycbc/workflow/datafind.py @@ -751,15 +751,6 @@ def convert_cachelist_to_filelist(datafindcache_list): # Datafind returned a URL valid on the osg as well # so add the additional PFNs to allow OSG access. currFile.add_pfn(frame.url, site='osg') - currFile.add_pfn(frame.url.replace( - 'file:///cvmfs/oasis.opensciencegrid.org/', - 'root://xrootd-local.unl.edu/user/'), site='osg') - currFile.add_pfn(frame.url.replace( - 'file:///cvmfs/oasis.opensciencegrid.org/', - 'gsiftp://red-gridftp.unl.edu/user/'), site='osg') - currFile.add_pfn(frame.url.replace( - 'file:///cvmfs/oasis.opensciencegrid.org/', - 'gsiftp://ldas-grid.ligo.caltech.edu/hdfs/'), site='osg') elif frame.url.startswith( 'file:///cvmfs/gwosc.osgstorage.org/'): # Datafind returned a URL valid on the osg as well diff --git a/pycbc/workflow/pegasus_files/pegasus-properties.conf b/pycbc/workflow/pegasus_files/pegasus-properties.conf index fe78a9c3bf4..06fd37bc9fe 100644 --- a/pycbc/workflow/pegasus_files/pegasus-properties.conf +++ b/pycbc/workflow/pegasus_files/pegasus-properties.conf @@ -17,14 +17,12 @@ pegasus.dir.storage.mapper.replica=File pegasus.dir.storage.mapper.replica.file=output.map # Add Replica selection options so that it will try URLs first, then -# XrootD for OSG, then gridftp, then anything else +# XrootD for OSG, then anything else # FIXME: This feels like a *site* property, not a global pegasus.selector.replica=Regex pegasus.selector.replica.regex.rank.1=file://(?!.*(cvmfs)).* pegasus.selector.replica.regex.rank.2=file:///cvmfs/.* pegasus.selector.replica.regex.rank.3=root://.* -pegasus.selector.replica.regex.rank.4=gsiftp://red-gridftp.unl.edu.* -pegasus.selector.replica.regex.rank.5=gridftp://.* pegasus.selector.replica.regex.rank.6=.\* dagman.maxpre=1 diff --git a/pycbc/workflow/pegasus_sites.py b/pycbc/workflow/pegasus_sites.py index a9a5a195f78..59209b55678 100644 --- a/pycbc/workflow/pegasus_sites.py +++ b/pycbc/workflow/pegasus_sites.py @@ -202,7 +202,9 @@ def add_osg_site(sitecat, cp): site.add_profiles(Namespace.CONDOR, key="should_transfer_files", value="Yes") site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output", - value="ON_EXIT_OR_EVICT") + value="ON_SUCCESS") + site.add_profiles(Namespace.CONDOR, key="success_exit_code", + value="0") site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid", value="True") site.add_profiles(Namespace.CONDOR, key="getenv", @@ -211,8 +213,6 @@ def add_osg_site(sitecat, cp): value="False") site.add_profiles(Namespace.CONDOR, key="+SingularityCleanEnv", value="True") - site.add_profiles(Namespace.CONDOR, key="use_x509userproxy", - value="True") site.add_profiles(Namespace.CONDOR, key="Requirements", value="(HAS_SINGULARITY =?= TRUE) && " "(HAS_LIGO_FRAMES =?= True) && " diff --git a/pycbc/workflow/pegasus_workflow.py b/pycbc/workflow/pegasus_workflow.py index 9edcb941435..b2fc880ced4 100644 --- a/pycbc/workflow/pegasus_workflow.py +++ b/pycbc/workflow/pegasus_workflow.py @@ -38,37 +38,6 @@ PEGASUS_FILE_DIRECTORY = os.path.join(os.path.dirname(__file__), 'pegasus_files') -GRID_START_TEMPLATE = '''#!/bin/bash - -if [ -f /tmp/x509up_u`id -u` ] ; then - unset X509_USER_PROXY -else - if [ ! -z ${X509_USER_PROXY} ] ; then - if [ -f ${X509_USER_PROXY} ] ; then - cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u` - fi - fi - unset X509_USER_PROXY -fi - -# Check that the proxy is valid -ecp-cert-info -exists -RESULT=${?} -if [ ${RESULT} -eq 0 ] ; then - PROXY_TYPE=`ecp-cert-info -type | tr -d ' '` - if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then - ecp-cert-info - else - cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig - grid-proxy-init -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig - rm -f /tmp/x509up_u`id -u`.orig - ecp-cert-info - fi -else - echo "Error: Could not find a valid grid proxy to submit workflow." - exit 1 -fi -''' class ProfileShortcuts(object): """ Container of common methods for setting pegasus profile information @@ -731,9 +700,6 @@ def plan_and_submit(self, submit_now=True): fp.write('pegasus-remove {}/work $@'.format(submitdir)) with open('start', 'w') as fp: - if self.cp.has_option('pegasus_profile', 'pycbc|check_grid'): - fp.write(GRID_START_TEMPLATE) - fp.write('\n') fp.write('pegasus-run {}/work $@'.format(submitdir)) os.chmod('status', 0o755) From 433dfd81287dfa47683487aadd8095e6ea7e9b16 Mon Sep 17 00:00:00 2001 From: Neeresh Kumar Date: Wed, 12 Jul 2023 15:02:18 -0400 Subject: [PATCH 2/4] fftw_fix (#4418) * Allow pycbc inspiral to run with only numpy fft * Remove Unneeded function from npfft.py --------- Co-authored-by: Neeresh Kumar Perla --- bin/pycbc_inspiral | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/bin/pycbc_inspiral b/bin/pycbc_inspiral index a372de80040..81d6b373256 100644 --- a/bin/pycbc_inspiral +++ b/bin/pycbc_inspiral @@ -301,19 +301,21 @@ def template_triggers(t_num): return out_vals_all, tparam with ctx: - # The following FFTW specific options needed to wait until - # we were inside the scheme context. + if opt.fft_backends == 'fftw': - # Import system wisdom. - if opt.fftw_import_system_wisdom: - fft.fftw.import_sys_wisdom() + # The following FFTW specific options needed to wait until + # we were inside the scheme context. - # Read specified user-provided wisdom files - if opt.fftw_input_float_wisdom_file is not None: - fft.fftw.import_single_wisdom_from_filename(opt.fftw_input_float_wisdom_file) + # Import system wisdom. + if opt.fftw_import_system_wisdom: + fft.fftw.import_sys_wisdom() - if opt.fftw_input_double_wisdom_file is not None: - fft.fftw.import_double_wisdom_from_filename(opt.fftw_input_double_wisdom_file) + # Read specified user-provided wisdom files + if opt.fftw_input_float_wisdom_file is not None: + fft.fftw.import_single_wisdom_from_filename(opt.fftw_input_float_wisdom_file) + + if opt.fftw_input_double_wisdom_file is not None: + fft.fftw.import_double_wisdom_from_filename(opt.fftw_input_double_wisdom_file) flow = opt.low_frequency_cutoff flen = strain_segments.freq_len @@ -499,10 +501,11 @@ event_mgr.save_performance(ncores, len(segments), len(bank), run_time, tsetup) logging.info("Writing out triggers") event_mgr.write_events(opt.output) -if opt.fftw_output_float_wisdom_file: - fft.fftw.export_single_wisdom_to_filename(opt.fftw_output_float_wisdom_file) +if opt.fft_backends == 'fftw': + if opt.fftw_output_float_wisdom_file: + fft.fftw.export_single_wisdom_to_filename(opt.fftw_output_float_wisdom_file) -if opt.fftw_output_double_wisdom_file: - fft.fftw.export_double_wisdom_to_filename(opt.fftw_output_double_wisdom_file) + if opt.fftw_output_double_wisdom_file: + fft.fftw.export_double_wisdom_to_filename(opt.fftw_output_double_wisdom_file) logging.info("Finished") From 3cf9b5273bd33c833d6b35bf88205f511719e5cf Mon Sep 17 00:00:00 2001 From: Gareth S Cabourn Davies Date: Mon, 17 Jul 2023 08:20:38 +0100 Subject: [PATCH 3/4] Add argument verification for singles (#4365) * Add argument verification for singles * Apply suggestions from code review * remove --enable-single-detector-background * Some more option-checking, and making fixed ifar option actually possible! * fix options in example * CC, typo * CC --- bin/pycbc_live | 10 ++-- examples/live/check_results.py | 7 ++- examples/live/run.sh | 3 +- pycbc/events/single.py | 92 +++++++++++++++++++++++++++------- 4 files changed, 86 insertions(+), 26 deletions(-) diff --git a/bin/pycbc_live b/bin/pycbc_live index 76929fcce8f..95b279cfa5d 100755 --- a/bin/pycbc_live +++ b/bin/pycbc_live @@ -870,8 +870,6 @@ parser.add_argument('--ifar-double-followup-threshold', type=float, required=Tru parser.add_argument('--pvalue-combination-livetime', type=float, required=True, help="Livetime used for p-value combination with followup " "detectors, in years") -parser.add_argument('--enable-single-detector-background', action='store_true', default=False) - parser.add_argument('--enable-gracedb-upload', action='store_true', default=False, help='Upload triggers to GraceDB') parser.add_argument('--enable-production-gracedb-upload', action='store_true', default=False, @@ -932,8 +930,11 @@ mchirp_area.insert_args(parser) livepau.insert_live_pastro_option_group(parser) args = parser.parse_args() + scheme.verify_processing_options(args, parser) fft.verify_fft_options(args, parser) +ifos = set(args.channel_name.keys()) +analyze_singles = LiveSingle.verify_args(args, parser, ifos) if args.output_background is not None and len(args.output_background) != 2: parser.error('--output-background takes two parameters: period and path') @@ -960,7 +961,6 @@ if bank.min_f_lower < args.low_frequency_cutoff: 'minimum f_lower across all templates ' '({} Hz)'.format(args.low_frequency_cutoff, bank.min_f_lower)) -ifos = set(args.channel_name.keys()) logging.info('Analyzing data from detectors %s', ppdets(ifos)) evnt = LiveEventManager(args, bank) @@ -1036,7 +1036,7 @@ with ctx: evnt.data_readers = data_reader # create single-detector background "estimators" - if args.enable_single_detector_background and evnt.rank == 0: + if analyze_singles and evnt.rank == 0: sngl_estimator = {ifo: LiveSingle.from_cli(args, ifo) for ifo in ifos} @@ -1175,7 +1175,7 @@ with ctx: evnt.check_coincs(list(results.keys()), best_coinc, psds) # Check for singles - if args.enable_single_detector_background: + if analyze_singles: evnt.check_singles(results, psds) gates = {ifo: data_reader[ifo].gate_params for ifo in data_reader} diff --git a/examples/live/check_results.py b/examples/live/check_results.py index 413e643ac62..31089efbe28 100755 --- a/examples/live/check_results.py +++ b/examples/live/check_results.py @@ -11,6 +11,7 @@ from pycbc.io.ligolw import LIGOLWContentHandler from ligo.lw.utils import load_filename as load_xml_doc from ligo.lw import lsctables +from pycbc import conversions as conv def close(a, b, c): @@ -130,7 +131,8 @@ def check_found_events(args): # create field array to store properties of triggers dtype = [('mass1', float), ('mass2', float), ('spin1z', float), ('spin2z', float), - ('tc', float), ('net_snr', float)] + ('tc', float), ('net_snr', float), + ('ifar', float)] trig_props = FieldArray(n_found, dtype=dtype) # store properties of found triggers @@ -139,18 +141,21 @@ def check_found_events(args): xmldoc = load_xml_doc( ctrigfp, False, contenthandler=LIGOLWContentHandler) si_table = lsctables.SnglInspiralTable.get_table(xmldoc) + ci_table = lsctables.CoincInspiralTable.get_table(xmldoc) trig_props['tc'][x] = si_table[0].end trig_props['mass1'][x] = si_table[0].mass1 trig_props['mass2'][x] = si_table[0].mass2 trig_props['spin1z'][x] = si_table[0].spin1z trig_props['spin2z'][x] = si_table[0].spin2z + trig_props['ifar'][x] = conv.sec_to_year(1 / ci_table[0].combined_far) snr_list = si_table.getColumnByName('snr').asarray() trig_props['net_snr'][x] = sum(snr_list ** 2) ** 0.5 log.info('Single-detector SNRs: %s', snr_list) log.info('Network SNR: %f', trig_props['net_snr'][x]) + log.info('IFAR: %f', trig_props['ifar'][x]) log.info('Merger time: %f', trig_props['tc'][x]) log.info('Mass 1: %f', trig_props['mass1'][x]) log.info('Mass 2: %f', trig_props['mass2'][x]) diff --git a/examples/live/run.sh b/examples/live/run.sh index 249382f8296..3cda2773664 100755 --- a/examples/live/run.sh +++ b/examples/live/run.sh @@ -195,12 +195,11 @@ python -m mpi4py `which pycbc_live` \ --src-class-eff-to-lum-distance 0.74899 \ --src-class-lum-distance-to-delta -0.51557 -0.32195 \ --run-snr-optimization \ ---enable-single-detector-background \ +--sngl-ifar-est-dist conservative \ --single-newsnr-threshold 9 \ --single-duration-threshold 7 \ --single-reduced-chisq-threshold 2 \ --single-fit-file single_trigger_fits.hdf \ ---sngl-ifar-est-dist conservative \ --verbose # note that, at this point, some SNR optimization processes may still be diff --git a/pycbc/events/single.py b/pycbc/events/single.py index 266e853de48..5312466fb57 100644 --- a/pycbc/events/single.py +++ b/pycbc/events/single.py @@ -57,29 +57,84 @@ def insert_args(parser): 'coefficients and counts for specific ' 'single trigger IFAR fitting.') parser.add_argument('--sngl-ifar-est-dist', nargs='+', - default='conservative', - choices=['conservative', 'mean', 'fixed'], action=MultiDetOptionAction, help='Which trigger distribution to use when ' 'calculating IFAR of single triggers. ' - 'Default conservative. Can be given as ' - 'a single value or as detector-value pairs, ' - 'e.g. H1:mean L1:mean V1:conservative') + 'Can be given as a single value or as ' + 'detector-value pairs, e.g. H1:mean ' + 'L1:mean V1:conservative') + + @staticmethod + def verify_args(args, parser, ifos): + sngl_opts = [args.single_reduced_chisq_threshold, + args.single_duration_threshold, + args.single_newsnr_threshold, + args.sngl_ifar_est_dist] + sngl_opts_str = ("--single-reduced-chisq-threshold, " + "--single-duration-threshold, " + "--single-newsnr-threshold, " + "--sngl-ifar-est-dist") + + if any(sngl_opts) and not all(sngl_opts): + parser.error(f"Single detector trigger options ({sngl_opts_str}) " + "must either all be given or none.") + + if args.enable_single_detector_upload \ + and not args.enable_gracedb_upload: + parser.error("--enable-single-detector-upload requires " + "--enable-gracedb-upload to be set.") + + sngl_optional_opts = [args.single_fixed_ifar, + args.single_fit_file] + sngl_optional_opts_str = ("--single-fixed-ifar, " + "--single-fit-file") + if any(sngl_optional_opts) and not all(sngl_opts): + parser.error("Optional singles options " + f"({sngl_optional_opts_str}) given but no " + f"required options ({sngl_opts_str}) are.") + + for ifo in ifos: + # Check which option(s) are needed for each IFO and if they exist: + + # Notes for the logic here: + # args.sngl_ifar_est_dist.default_set is True if single value has + # been set to be the same for all values + # bool(args.sngl_ifar_est_dist) is True if option is given + if args.sngl_ifar_est_dist and \ + not args.sngl_ifar_est_dist.default_set \ + and not args.sngl_ifar_est_dist[ifo]: + # Option has been given, different for each IFO, + # and this one is not present + parser.error("All IFOs required in --single-ifar-est-dist " + "if IFO-specific options are given.") + + if not args.sngl_ifar_est_dist[ifo] == 'fixed': + if not args.single_fit_file: + # Fixed IFAR option doesnt need the fits file + parser.error(f"Single detector trigger fits file must be " + "given if --single-ifar-est-dist is not " + f"fixed for all ifos (at least {ifo} has " + f"option {args.sngl_ifar_est_dist[ifo]}).") + if ifo in args.single_fixed_ifar: + parser.error(f"Value {args.single_fixed_ifar[ifo]} given " + f"for {ifo} in --single-fixed-ifar, but " + f"--single-ifar-est-dist for {ifo} " + f"is {args.sngl_ifar_est_dist[ifo]}, not " + "fixed.") + else: + # Check that the fixed IFAR value has actually been + # given if using this instead of a distribution + if not args.single_fixed_ifar[ifo]: + parser.error(f"--single-fixed-ifar must be " + "given if --single-ifar-est-dist is fixed. " + f"This is true for at least {ifo}.") + + # Return value is a boolean whether we are analysing singles or not + # The checks already performed mean that all(sngl_opts) is okay + return all(sngl_opts) @classmethod def from_cli(cls, args, ifo): - sngl_opts_required = all([args.single_fit_file, - args.single_reduced_chisq_threshold, - args.single_duration_threshold, - args.single_newsnr_threshold]) - if args.enable_single_detector_background and not sngl_opts_required: - raise RuntimeError("Single detector trigger options " - "(--single-fit-file, " - "--single-reduced-chisq-threshold, " - "--single-duration-threshold, " - "--single-newsnr-threshold) " - "must ALL be given if single detector " - "background is enabled") return cls( ifo, newsnr_threshold=args.single_newsnr_threshold[ifo], reduced_chisq_threshold=args.single_reduced_chisq_threshold[ifo], @@ -139,7 +194,8 @@ def check(self, trigs, data_reader): return candidate def calculate_ifar(self, sngl_ranking, duration): - if self.fixed_ifar: + logging.info("Calculating IFAR") + if self.fixed_ifar and self.ifo in self.fixed_ifar: return self.fixed_ifar[self.ifo] try: From b2c67fd7240243a8b08338feea2f0d56f75e89ad Mon Sep 17 00:00:00 2001 From: Gareth S Cabourn Davies Date: Mon, 17 Jul 2023 10:49:35 +0100 Subject: [PATCH 4/4] Move versioning page generation into its own executable (#4431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Move versioning page generation into its own executable * Allow duplicated executables * 🤦 * CC changes * CC * save_fig_with_metadata cannot be imported from versioning * Don't like spaces in arguments * Ian's comments * Dont pass executables as files * Same type output from get_code_version_numbers --- bin/plotting/pycbc_page_foreground | 2 +- bin/plotting/pycbc_page_versioning | 67 ++++++++++++++++++ bin/pygrb/pycbc_pygrb_pp_workflow | 10 ++- .../pycbc_make_bank_verifier_workflow | 9 ++- .../pycbc_make_inference_inj_workflow | 8 ++- .../pycbc_make_inference_plots_workflow | 10 ++- bin/workflows/pycbc_make_inference_workflow | 9 ++- .../pycbc_make_offline_search_workflow | 11 +-- examples/search/executables.ini | 1 + examples/search/plotting.ini | 1 + .../inference/small_test/workflow.ini | 5 ++ pycbc/results/versioning.py | 51 ++------------ pycbc/workflow/__init__.py | 1 + pycbc/workflow/versioning.py | 70 +++++++++++++++++++ 14 files changed, 190 insertions(+), 65 deletions(-) create mode 100755 bin/plotting/pycbc_page_versioning create mode 100644 pycbc/workflow/versioning.py diff --git a/bin/plotting/pycbc_page_foreground b/bin/plotting/pycbc_page_foreground index 18e7c4f7a8c..c6ebb116f16 100755 --- a/bin/plotting/pycbc_page_foreground +++ b/bin/plotting/pycbc_page_foreground @@ -9,7 +9,7 @@ from pycbc.io import hdf import h5py, logging from pycbc.pnutils import mass1_mass2_to_mchirp_eta import pycbc.results, pycbc.results.followup -from pycbc.results.versioning import save_fig_with_metadata +from pycbc.results import save_fig_with_metadata import pycbc.version import sys diff --git a/bin/plotting/pycbc_page_versioning b/bin/plotting/pycbc_page_versioning new file mode 100755 index 00000000000..387d58af9b9 --- /dev/null +++ b/bin/plotting/pycbc_page_versioning @@ -0,0 +1,67 @@ +#!/bin/env python + +""" +Create the html files needed to describe the versioning +information for a set of libraries and executables in +pycbc results pages +""" + +import argparse, logging +import pycbc.version +from pycbc import init_logging +from pycbc.results import (save_fig_with_metadata, html_escape, + get_library_version_info, get_code_version_numbers) + +parser = argparse.ArgumentParser() +parser.add_argument('--version', action="version", + version=pycbc.version.git_verbose_msg) +parser.add_argument('--verbose', action='store_true') +parser.add_argument('--executables', nargs='+', required=True, + help="List of executables to provide version " + "information for") +parser.add_argument('--executables-names', nargs='+', required=True, + help="Names of the executables, must be in the " + "same order as --executables-files") +parser.add_argument("--output-file", required=True, + help="The directory for output html snippets") +args = parser.parse_args() + +init_logging(args.verbose) + +if not len(args.executables) == len(args.executables_names): + raise parser.error("--executables-files and executables-names must be " + "the same number of arguments") + + +logging.info("Getting version information for libraries") +library_list = get_library_version_info() +html_text = '' +for curr_lib in library_list: + lib_name = curr_lib['Name'] + logging.info(f"Getting {lib_name} information") + html_text += f'

{lib_name} Version Information

:
\n' + for key, value in curr_lib.items(): + html_text += '
  • %s : %s
    • \n' % (key, value) + + +code_version_dict = get_code_version_numbers( + args.executables_names, + args.executables +) + +html_text += f'

    Version Information from Executables

    :
    \n' +for key, value in code_version_dict.items(): + html_text += '
  • %s:
    %s


    • \n' \ + % (key, str(value).replace('@', '@')) + +kwds = { + 'render-function' : 'render_text', + 'title' : 'Version Information', +} + +save_fig_with_metadata( + html_escape(html_text), + args.output_file, + **kwds +) +logging.info("Done") diff --git a/bin/pygrb/pycbc_pygrb_pp_workflow b/bin/pygrb/pycbc_pygrb_pp_workflow index 4bc5c297865..7e51112c303 100644 --- a/bin/pygrb/pycbc_pygrb_pp_workflow +++ b/bin/pygrb/pycbc_pygrb_pp_workflow @@ -32,9 +32,9 @@ import os import pycbc.version from pycbc import init_logging import pycbc.workflow as _workflow -from pycbc.results import create_versioning_page, layout +from pycbc.results import layout from pycbc.results.pygrb_postprocessing_utils import extract_ifos -from pycbc.results.versioning import save_fig_with_metadata +from pycbc.results import save_fig_with_metadata __author__ = "Francesco Pannarale " __version__ = pycbc.version.git_verbose_msg @@ -387,7 +387,11 @@ ini_file = _workflow.FileList([_workflow.File(wflow.ifos, '', layout.single_layout(base, ini_file) # Create versioning information -create_versioning_page(rdir['workflow/version'], wflow.cp) +wf.make_versioning_page( + _workflow, + wflow.cp, + rdir['workflow/version'], +) # Create the final log file log_file_html = _workflow.File(wflow.ifos, 'WORKFLOW-LOG', wflow.analysis_time, diff --git a/bin/workflows/pycbc_make_bank_verifier_workflow b/bin/workflows/pycbc_make_bank_verifier_workflow index 02db0858481..ebc5e64a60f 100644 --- a/bin/workflows/pycbc_make_bank_verifier_workflow +++ b/bin/workflows/pycbc_make_bank_verifier_workflow @@ -31,7 +31,7 @@ from ligo import segments import pycbc.version import pycbc.workflow as wf -from pycbc.results import (create_versioning_page, static_table, layout) +from pycbc.results import (static_table, layout) from pycbc.workflow.jobsetup import (select_generic_executable, int_gps_time_to_str, PycbcCreateInjectionsExecutable, @@ -382,7 +382,12 @@ for tag in sorted(output_pointinjs): layout.single_layout(conf_dir, conf_file) # Create versioning information -create_versioning_page(rdir['workflow/version'], workflow.cp) +wf.make_versioning_page( + workflow, + workflow.cp, + rdir['workflow/version'], +) + wf.make_results_web_page(workflow, os.path.join(os.getcwd(), rdir.base), explicit_dependencies=plotting_nodes) diff --git a/bin/workflows/pycbc_make_inference_inj_workflow b/bin/workflows/pycbc_make_inference_inj_workflow index adc9a77e3a6..5de7054679a 100644 --- a/bin/workflows/pycbc_make_inference_inj_workflow +++ b/bin/workflows/pycbc_make_inference_inj_workflow @@ -355,8 +355,12 @@ if do_pp_test: # add to the main workflow workflow += pp_workflow -# create versioning HTML pages -results.create_versioning_page(rdir["workflow/version"], workflow.cp) +# Create versioning information +wf.make_versioning_page( + workflow, + workflow.cp, + rdir['workflow/version'], +) # create node for making HTML pages plotting.make_results_web_page(finalize_workflow, diff --git a/bin/workflows/pycbc_make_inference_plots_workflow b/bin/workflows/pycbc_make_inference_plots_workflow index af626b04f45..505d2191e0b 100644 --- a/bin/workflows/pycbc_make_inference_plots_workflow +++ b/bin/workflows/pycbc_make_inference_plots_workflow @@ -246,9 +246,13 @@ for num_event, event in enumerate(events): layout.single_layout(rdir['detector_sensitivity'], [psd_plot], unique=str(num_event).zfill(zpad), title=label, collapse=True) - -# create versioning HTML pages -results.create_versioning_page(rdir["workflow/version"], container.cp) + +# Create versioning information +wf.make_versioning_page( + workflow, + container.cp, + rdir['workflow/version'], +) # create node for making HTML pages plotting.make_results_web_page(finalize_workflow, diff --git a/bin/workflows/pycbc_make_inference_workflow b/bin/workflows/pycbc_make_inference_workflow index b6c7c6ae107..9978bce38fa 100644 --- a/bin/workflows/pycbc_make_inference_workflow +++ b/bin/workflows/pycbc_make_inference_workflow @@ -37,6 +37,7 @@ from pycbc.workflow import configuration from pycbc.workflow import core from pycbc.workflow import datafind from pycbc.workflow import plotting +from pycbc.workflow import versioning from pycbc import __version__ import pycbc.workflow.inference_followups as inffu from pycbc.workflow.jobsetup import PycbcInferenceExecutable @@ -307,8 +308,12 @@ for num_event, event in enumerate(events): workflow += sub_workflow -# create versioning HTML pages -results.create_versioning_page(rdir["workflow/version"], container.cp) +# Create versioning information +versioning.make_versioning_page( + workflow, + container.cp, + rdir['workflow/version'], +) # create node for making HTML pages plotting.make_results_web_page(finalize_workflow, diff --git a/bin/workflows/pycbc_make_offline_search_workflow b/bin/workflows/pycbc_make_offline_search_workflow index 32d9815532b..e5e34719f1e 100755 --- a/bin/workflows/pycbc_make_offline_search_workflow +++ b/bin/workflows/pycbc_make_offline_search_workflow @@ -34,8 +34,7 @@ import os, argparse, logging import configparser as ConfigParser from ligo import segments import numpy, lal, datetime, itertools -from pycbc.results import create_versioning_page, static_table, layout -from pycbc.results.versioning import save_fig_with_metadata +from pycbc.results import static_table, layout, save_fig_with_metadata from pycbc.results.metadata import html_escape @@ -804,10 +803,12 @@ ini_file = wf.FileList([wf.File(workflow.ifos, '', workflow.analysis_time, file_url='file://' + ini_file_path)]) layout.single_layout(base, ini_file) - # Create versioning information -create_versioning_page(rdir['workflow/version'], container.cp) - +wf.make_versioning_page( + workflow, + container.cp, + rdir['workflow/version'], +) ############################ Finalization #################################### diff --git a/examples/search/executables.ini b/examples/search/executables.ini index 4fd71bcfdff..681e61ae7f2 100644 --- a/examples/search/executables.ini +++ b/examples/search/executables.ini @@ -23,6 +23,7 @@ page_ifar_catalog = ${which:pycbc_ifar_catalog} page_injections = ${which:pycbc_page_injtable} page_segplot = ${which:pycbc_page_segplot} page_segtable = ${which:pycbc_page_segtable} +page_versioning = ${which:pycbc_page_versioning} page_vetotable = ${which:pycbc_page_vetotable} plot_bank = ${which:pycbc_plot_bank_bins} plot_binnedhist = ${which:pycbc_fit_sngls_binned} diff --git a/examples/search/plotting.ini b/examples/search/plotting.ini index 5b9e1191aef..3a543f9f364 100644 --- a/examples/search/plotting.ini +++ b/examples/search/plotting.ini @@ -242,3 +242,4 @@ num-to-write = 2 [plot_gating] [plot_snrratehist] [plot_throughput] +[page_versioning] diff --git a/examples/workflow/inference/small_test/workflow.ini b/examples/workflow/inference/small_test/workflow.ini index 506c0807ae6..2560c07128d 100644 --- a/examples/workflow/inference/small_test/workflow.ini +++ b/examples/workflow/inference/small_test/workflow.ini @@ -61,6 +61,7 @@ table_summary = ${which:pycbc_inference_table_summary} create_fits_file = ${which:pycbc_inference_create_fits} plot_spectrum = ${which:pycbc_plot_psd_file} results_page = ${which:pycbc_make_html_page} +page_versioning = ${which:pycbc_page_versioning} ; diagnostic plots: at the moment, there are none for Dynesty [pegasus_profile] @@ -106,6 +107,10 @@ z-arg = snr ; page. The dyn-range-factor needs to be set to 1. dyn-range-factor = 1 +[page_versioning] +; This finds version information for all libraries and scripts used by the +; workflow, and puts them into a file for use in results pages + [results_page] ; This sets settings for creating the results page. You may want to change ; the analysis title, to make it more descriptive. diff --git a/pycbc/results/versioning.py b/pycbc/results/versioning.py index 678df9c6fe0..64536377d22 100644 --- a/pycbc/results/versioning.py +++ b/pycbc/results/versioning.py @@ -17,10 +17,8 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import logging -import os import subprocess import urllib.parse -from pycbc.results import save_fig_with_metadata, html_escape import lal, lalframe import pycbc.version, glue.git_version @@ -118,21 +116,7 @@ def add_info_new_version(info_dct, curr_module, extra_str): return library_list -def write_library_information(path): - library_list = get_library_version_info() - for curr_lib in library_list: - lib_name = curr_lib['Name'] - text = '' - for key, value in curr_lib.items(): - text+='
  • %s : %s
    • \n' %(key,value) - kwds = {'render-function' : 'render_text', - 'title' : '%s Version Information'%lib_name, - } - - save_fig_with_metadata(html_escape(text), - os.path.join(path,'%s_version_information.html' %(lib_name)), **kwds) - -def get_code_version_numbers(cp): +def get_code_version_numbers(executable_names, executable_files): """Will extract the version information from the executables listed in the executable section of the supplied ConfigParser object. @@ -143,9 +127,9 @@ def get_code_version_numbers(cp): version string for each executable. """ code_version_dict = {} - for _, value in cp.items('executables'): + for exe_name, value in zip(executable_names, executable_files): value = urllib.parse.urlparse(value) - _, exe_name = os.path.split(value.path) + logging.info("Getting version info for %s", exe_name) version_string = None if value.scheme in ['gsiftp', 'http', 'https']: code_version_dict[exe_name] = "Using bundle downloaded from %s" % value @@ -160,7 +144,7 @@ def get_code_version_numbers(cp): version_string = subprocess.check_output( [value.path, '--version'], stderr=subprocess.STDOUT - ) + ).decode() except subprocess.CalledProcessError: version_string = "Executable fails on {} --version" version_string = version_string.format(value.path) @@ -168,30 +152,3 @@ def get_code_version_numbers(cp): version_string = "Executable doesn't seem to exist(!)" code_version_dict[exe_name] = version_string return code_version_dict - -def write_code_versions(path, cp): - code_version_dict = get_code_version_numbers(cp) - html_text = '' - for key,value in code_version_dict.items(): - # value might be a str or a bytes object in python3. python2 is happy - # to combine these objects (or uniocde and str, their equivalents) - # but python3 is not. - try: - value = value.decode() - except AttributeError: - pass - html_text+= '
  • %s:
    %s


    • \n' \ - % (key, str(value).replace('@', '@')) - kwds = {'render-function' : 'render_text', - 'title' : 'Version Information from Executables', - } - save_fig_with_metadata(html_escape(html_text), - os.path.join(path,'version_information_from_executables.html'), **kwds) - -def create_versioning_page(path, cp): - logging.info("Entering versioning module") - if not os.path.exists(path): - os.mkdir(path) - write_library_information(path) - write_code_versions(path, cp) - logging.info("Leaving versioning module") diff --git a/pycbc/workflow/__init__.py b/pycbc/workflow/__init__.py index e2196d98172..be7ebdf0b53 100644 --- a/pycbc/workflow/__init__.py +++ b/pycbc/workflow/__init__.py @@ -44,6 +44,7 @@ from pycbc.workflow.plotting import * from pycbc.workflow.minifollowups import * from pycbc.workflow.dq import * +from pycbc.workflow.versioning import * # Set the pycbc workflow specific pegasus configuration and planning files from pycbc.workflow.pegasus_workflow import PEGASUS_FILE_DIRECTORY diff --git a/pycbc/workflow/versioning.py b/pycbc/workflow/versioning.py new file mode 100644 index 00000000000..ec70cd1ee0a --- /dev/null +++ b/pycbc/workflow/versioning.py @@ -0,0 +1,70 @@ +# Copyright (C) 2023 Gareth Cabourn Davies +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# ============================================================================= +# +# Preamble +# +# ============================================================================= +# +""" +Module to generate/manage the executable used for version information +in workflows +""" +import os +from pycbc.workflow.core import Executable + + +class VersioningExecutable(Executable): + """ + Executable for getting version information + """ + current_retention_level = Executable.FINAL_RESULT + + +def make_versioning_page(workflow, config_parser, out_dir, tags=None): + """ + Make executable for versioning information + """ + vers_exe = VersioningExecutable( + workflow.cp, + 'page_versioning', + out_dir=out_dir, + ifos=workflow.ifos, + tags=tags, + ) + node = vers_exe.create_node() + config_names = [] + exes = [] + for name, path in config_parser.items('executables'): + exe_to_test = os.path.basename(path) + if exe_to_test in exes: + # executable is already part of the list, + # find which index and add the name to the + # one already stored + path_idx = exes.index(exe_to_test) + name_orig = config_names[path_idx] + config_names[path_idx] = f"{name_orig},{name}" + else: + config_names.append(name) + exes.append(exe_to_test) + node.add_list_opt('--executables', exes) + node.add_list_opt('--executables-names', config_names) + node.new_output_file_opt(workflow.analysis_time, '.html', '--output-file') + workflow.add_node(node) + + return node