Merge branch 'master' into live_options

bin/plotting/pycbc_page_foreground

@@ -9,7 +9,7 @@ from pycbc.io import hdf
 import h5py, logging
 from pycbc.pnutils import mass1_mass2_to_mchirp_eta
 import pycbc.results, pycbc.results.followup
-from pycbc.results.versioning import save_fig_with_metadata
+from pycbc.results import save_fig_with_metadata
 import pycbc.version
 import sys
 

bin/plotting/pycbc_page_versioning

@@ -0,0 +1,67 @@
+#!/bin/env python
+
+"""
+Create the html files needed to describe the versioning
+information for a set of libraries and executables in
+pycbc results pages
+"""
+
+import argparse, logging
+import pycbc.version
+from pycbc import init_logging
+from pycbc.results import (save_fig_with_metadata, html_escape,
+    get_library_version_info, get_code_version_numbers)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--version', action="version",
+                    version=pycbc.version.git_verbose_msg)
+parser.add_argument('--verbose', action='store_true')
+parser.add_argument('--executables', nargs='+', required=True,
+                    help="List of executables to provide version "
+                         "information for")
+parser.add_argument('--executables-names', nargs='+', required=True,
+                    help="Names of the executables, must be in the "
+                         "same order as --executables-files")
+parser.add_argument("--output-file", required=True,
+                    help="The directory for output html snippets")
+args = parser.parse_args()
+
+init_logging(args.verbose)
+
+if not len(args.executables) == len(args.executables_names):
+    raise parser.error("--executables-files and executables-names must be "
+                       "the same number of arguments")
+
+
+logging.info("Getting version information for libraries")
+library_list = get_library_version_info()
+html_text = ''
+for curr_lib in library_list:
+    lib_name = curr_lib['Name']
+    logging.info(f"Getting {lib_name} information")
+    html_text += f'<h2>{lib_name} Version Information</h2>:<br>\n'
+    for key, value in curr_lib.items():
+        html_text += '<li> %s : %s </li>\n' % (key, value)
+
+
+code_version_dict = get_code_version_numbers(
+    args.executables_names,
+    args.executables
+)
+
+html_text += f'<h2>Version Information from Executables</h2>:<br>\n'
+for key, value in code_version_dict.items():
+    html_text += '<li><b>%s</b>:<br><pre>%s</pre></li><hr><br><br>\n' \
+        % (key, str(value).replace('@', '&#64;'))
+
+kwds = {
+    'render-function' : 'render_text',
+    'title' : 'Version Information',
+}
+
+save_fig_with_metadata(
+    html_escape(html_text),
+    args.output_file,
+    **kwds
+)
+logging.info("Done")

bin/pycbc_inspiral

@@ -301,19 +301,21 @@ def template_triggers(t_num):
     return out_vals_all, tparam
 
 with ctx:
-    # The following FFTW specific options needed to wait until
-    # we were inside the scheme context.
+    if opt.fft_backends == 'fftw':
 
-    # Import system wisdom.
-    if opt.fftw_import_system_wisdom:
-        fft.fftw.import_sys_wisdom()
+        # The following FFTW specific options needed to wait until
+        # we were inside the scheme context.
 
-    # Read specified user-provided wisdom files
-    if opt.fftw_input_float_wisdom_file is not None:
-        fft.fftw.import_single_wisdom_from_filename(opt.fftw_input_float_wisdom_file)
+        # Import system wisdom.
+        if opt.fftw_import_system_wisdom:
+            fft.fftw.import_sys_wisdom()
 
-    if opt.fftw_input_double_wisdom_file is not None:
-        fft.fftw.import_double_wisdom_from_filename(opt.fftw_input_double_wisdom_file)
+        # Read specified user-provided wisdom files
+        if opt.fftw_input_float_wisdom_file is not None:
+            fft.fftw.import_single_wisdom_from_filename(opt.fftw_input_float_wisdom_file)
+
+        if opt.fftw_input_double_wisdom_file is not None:
+            fft.fftw.import_double_wisdom_from_filename(opt.fftw_input_double_wisdom_file)
 
     flow = opt.low_frequency_cutoff
     flen = strain_segments.freq_len
@@ -499,10 +501,11 @@ event_mgr.save_performance(ncores, len(segments), len(bank), run_time, tsetup)
 logging.info("Writing out triggers")
 event_mgr.write_events(opt.output)
 
-if opt.fftw_output_float_wisdom_file:
-    fft.fftw.export_single_wisdom_to_filename(opt.fftw_output_float_wisdom_file)
+if opt.fft_backends == 'fftw':
+    if opt.fftw_output_float_wisdom_file:
+        fft.fftw.export_single_wisdom_to_filename(opt.fftw_output_float_wisdom_file)
 
-if opt.fftw_output_double_wisdom_file:
-    fft.fftw.export_double_wisdom_to_filename(opt.fftw_output_double_wisdom_file)
+    if opt.fftw_output_double_wisdom_file:
+        fft.fftw.export_double_wisdom_to_filename(opt.fftw_output_double_wisdom_file)
 
 logging.info("Finished")

bin/pycbc_live

@@ -879,8 +879,6 @@ parser.add_argument('--ifar-double-followup-threshold', type=float, required=Tru
 parser.add_argument('--pvalue-combination-livetime', type=float, required=True,
                     help="Livetime used for p-value combination with followup "
                          "detectors, in years")
-parser.add_argument('--enable-single-detector-background', action='store_true', default=False)
-
 parser.add_argument('--enable-gracedb-upload', action='store_true', default=False,
                     help='Upload triggers to GraceDB')
 parser.add_argument('--enable-production-gracedb-upload', action='store_true', default=False,
@@ -940,8 +938,11 @@ livepau.insert_live_pastro_option_group(parser)
 snr_optimizer.insert_snr_optimizer_options(parser)
 
 args = parser.parse_args()
+
 scheme.verify_processing_options(args, parser)
 fft.verify_fft_options(args, parser)
+ifos = set(args.channel_name.keys())
+analyze_singles = LiveSingle.verify_args(args, parser, ifos)
 
 if args.output_background is not None and len(args.output_background) != 2:
     parser.error('--output-background takes two parameters: period and path')
@@ -968,7 +969,6 @@ if bank.min_f_lower < args.low_frequency_cutoff:
                  'minimum f_lower across all templates '
                  '({} Hz)'.format(args.low_frequency_cutoff, bank.min_f_lower))
 
-ifos = set(args.channel_name.keys())
 logging.info('Analyzing data from detectors %s', ppdets(ifos))
 
 evnt = LiveEventManager(args, bank)
@@ -1044,7 +1044,7 @@ with ctx:
     evnt.data_readers = data_reader
 
     # create single-detector background "estimators"
-    if args.enable_single_detector_background and evnt.rank == 0:
+    if analyze_singles and evnt.rank == 0:
         sngl_estimator = {ifo: LiveSingle.from_cli(args, ifo)
                           for ifo in ifos}
 
@@ -1183,7 +1183,7 @@ with ctx:
                 evnt.check_coincs(list(results.keys()), best_coinc, psds)
 
             # Check for singles
-            if args.enable_single_detector_background:
+            if analyze_singles:
                 evnt.check_singles(results, psds)
 
             gates = {ifo: data_reader[ifo].gate_params for ifo in data_reader}

bin/pycbc_submit_dax

@@ -17,9 +17,7 @@ exec 2>&1
 
 LOCAL_PEGASUS_DIR=""
 PEGASUS_PROPERTIES=""
-NO_CREATE_PROXY=0
 NO_QUERY_DB=0
-NO_GRID=""
 SUBMIT_DAX="--submit"
 HTML_ENTITIES="{\"\'\": ''', '(': '(', ')': ')', '+': '+', '\"': '"'}"
 
@@ -54,15 +52,13 @@ while true ; do
         "") shift 2 ;;
         *) echo $2 >> extra-properties.conf ; shift 2 ;;
       esac ;;
-    -K|--no-create-proxy) NO_CREATE_PROXY=1 ; shift ;;
     -Q|--no-query-db) NO_QUERY_DB=1 ; shift ;;
     -n|--no-submit) SUBMIT_DAX="" ; shift ;;
     -l|--local-dir)
       case "$2" in
         "") shift 2 ;;
         *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;;
       esac ;;
-    -G|--no-grid) NO_GRID="--forward nogrid" ; shift ;;
     -h|--help)
       echo "usage: pycbc_submit_dax [-h] [optional arguments]"
       echo
@@ -72,13 +68,9 @@ while true ; do
       echo "                               the pegasus properties file"
       echo "  -P, --append-pegasus-property STRING add the extra property"
       echo "                                          specified by the argument"
-      echo "  -K, --no-create-proxy   Do not run ligo-proxy-init and assume"
-      echo "                             that the user has a valid grid proxy"
       echo "  -n, --no-submit         Plan the DAX but do not submit it"
       echo "  -l, --local-dir         Directory to put condor files under"
       echo "  -Q, --no-query-db       Don't query the pegasus DB."
-      echo "  -G, --no-grid           Disable checks for grid proxy and"
-      echo "                             GLOBUS_LOCATION in pegasus-plan"
       echo
       echo "If the environment variable TMPDIR is set then this is prepended to the "
       echo "path to the temporary workflow execute directory passed to pegasus-plan."
@@ -95,49 +87,6 @@ while true ; do
   esac
 done
 
-if [ $NO_CREATE_PROXY == 0 ]; then
-  # Force the user to create a new grid proxy
-  LIGO_USER_NAME=""
-  while true; do
-    read -p "Enter your LIGO.ORG username in (e.g. albert.einstein): " LIGO_USER_NAME
-    echo
-    if [ ! -z $LIGO_USER_NAME ] ; then
-      break
-    fi
-  done
-  unset X509_USER_PROXY
-  ligo-proxy-init -p $LIGO_USER_NAME || exit 1
-else
-  if [ ! -z ${X509_USER_PROXY} ] ; then
-    if [ -f ${X509_USER_PROXY} ] ; then
-      cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u`
-    fi
-  unset X509_USER_PROXY
-  fi
-fi
-
-if [ -z "${NO_GRID}" ] ; then
-  #Check that the proxy is valid
-  set +e
-  ecp-cert-info -exists
-  RESULT=$?
-  set -e
-  if [ ${RESULT} -eq 0 ] ; then
-    PROXY_TYPE=`ecp-cert-info -type | tr -d ' '`
-    if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then
-      ecp-cert-info
-    else
-      cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig
-      grid-proxy-init -hours 276 -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig
-      rm -f /tmp/x509up_u`id -u`.orig
-      ecp-cert-info
-    fi
-  else
-    echo "Error: Could not find a valid grid proxy to submit workflow."
-    exit 1
-  fi
-fi
-
 #Make a directory for the submit files
 SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX`
 
@@ -170,7 +119,7 @@ cat extra-properties.conf >> pegasus-properties.conf
 # it deserves!
 STORED_PLANNER_ARGS=`cat additional_planner_args.dat`
 
-pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX $NO_GRID ${STORED_PLANNER_ARGS}
+pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX ${STORED_PLANNER_ARGS}
 
 echo
 
@@ -186,42 +135,6 @@ chmod 755 debug
 echo "pegasus-remove $SUBMIT_DIR/work \$@" > stop
 chmod 755 stop
 
-if [ -z "${NO_GRID}" ] ; then
-  cat << EOF > start
-#!/bin/bash
-
-if [ -f /tmp/x509up_u\`id -u\` ] ; then
-  unset X509_USER_PROXY
-else
-  if [ ! -z \${X509_USER_PROXY} ] ; then
-    if [ -f \${X509_USER_PROXY} ] ; then
-      cp -a \${X509_USER_PROXY} /tmp/x509up_u\`id -u\`
-    fi
-  fi
-  unset X509_USER_PROXY
-fi
-
-# Check that the proxy is valid
-ecp-cert-info -exists
-RESULT=\${?}
-if [ \${RESULT} -eq 0 ] ; then
-  PROXY_TYPE=\`ecp-cert-info -type | tr -d ' '\`
-  if [ x\${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then
-    ecp-cert-info
-  else
-    cp /tmp/x509up_u\`id -u\` /tmp/x509up_u\`id -u\`.orig
-    grid-proxy-init -hours 276 -cert /tmp/x509up_u\`id -u\`.orig -key /tmp/x509up_u\`id -u\`.orig
-    rm -f /tmp/x509up_u\`id -u\`.orig
-    ecp-cert-info
-  fi
-else
-  echo "Error: Could not find a valid grid proxy to submit workflow."
-  exit 1
-fi
-
-EOF
-fi
-
 echo "pegasus-run $SUBMIT_DIR/work \$@" > start
 
 chmod 755 start

bin/pygrb/pycbc_pygrb_pp_workflow

@@ -32,9 +32,9 @@ import os
 import pycbc.version
 from pycbc import init_logging
 import pycbc.workflow as _workflow
-from pycbc.results import create_versioning_page, layout
+from pycbc.results import layout
 from pycbc.results.pygrb_postprocessing_utils import extract_ifos
-from pycbc.results.versioning import save_fig_with_metadata
+from pycbc.results import save_fig_with_metadata
 
 __author__ = "Francesco Pannarale  <francesco.pannarale@ligo.org>"
 __version__ = pycbc.version.git_verbose_msg
@@ -387,7 +387,11 @@ ini_file = _workflow.FileList([_workflow.File(wflow.ifos, '',
 layout.single_layout(base, ini_file)
 
 # Create versioning information
-create_versioning_page(rdir['workflow/version'], wflow.cp)
+wf.make_versioning_page(
+    _workflow,
+    wflow.cp,
+    rdir['workflow/version'],
+)
 
 # Create the final log file
 log_file_html = _workflow.File(wflow.ifos, 'WORKFLOW-LOG', wflow.analysis_time,

bin/workflows/pycbc_make_bank_verifier_workflow

@@ -31,7 +31,7 @@ from ligo import segments
 
 import pycbc.version
 import pycbc.workflow as wf
-from pycbc.results import (create_versioning_page, static_table, layout)
+from pycbc.results import (static_table, layout)
 from pycbc.workflow.jobsetup import (select_generic_executable,
                                      int_gps_time_to_str,
                                      PycbcCreateInjectionsExecutable,
@@ -382,7 +382,12 @@ for tag in sorted(output_pointinjs):
 layout.single_layout(conf_dir, conf_file)
 
 # Create versioning information
-create_versioning_page(rdir['workflow/version'], workflow.cp)
+wf.make_versioning_page(
+    workflow,
+    workflow.cp,
+    rdir['workflow/version'],
+)
+
 
 wf.make_results_web_page(workflow, os.path.join(os.getcwd(), rdir.base),
                          explicit_dependencies=plotting_nodes)

bin/workflows/pycbc_make_inference_inj_workflow

@@ -355,8 +355,12 @@ if do_pp_test:
     # add to the main workflow
     workflow += pp_workflow
 
-# create versioning HTML pages
-results.create_versioning_page(rdir["workflow/version"], workflow.cp)
+# Create versioning information
+wf.make_versioning_page(
+    workflow,
+    workflow.cp,
+    rdir['workflow/version'],
+)
 
 # create node for making HTML pages
 plotting.make_results_web_page(finalize_workflow,

bin/workflows/pycbc_make_inference_plots_workflow

@@ -246,9 +246,13 @@ for num_event, event in enumerate(events):
     layout.single_layout(rdir['detector_sensitivity'], [psd_plot],
                          unique=str(num_event).zfill(zpad),
                          title=label, collapse=True)
-
-# create versioning HTML pages
-results.create_versioning_page(rdir["workflow/version"], container.cp)
+
+# Create versioning information
+wf.make_versioning_page(
+    workflow,
+    container.cp,
+    rdir['workflow/version'],
+)
 
 # create node for making HTML pages
 plotting.make_results_web_page(finalize_workflow,