Added tables to dq results pages (gwastro#4633)

* Added tables to dq results pages

* Address Tom's comments and fix bin_trigger_rates_dq

* Fix decimal formatting

* Final change to formatting

bin/all_sky_search/pycbc_bin_trigger_rates_dq

@@ -46,6 +46,18 @@ logging.info('Start')
 
 ifo, flag_name = args.flag_name.split(':')
 
+if args.gating_windows:
+    gate_times = []
+    with h5.File(args.trig_file, 'r') as trig_file:
+        logging.info('Getting gated times')
+        try:
+            gating_types = trig_file[f'{ifo}/gating'].keys()
+            for gt in gating_types:
+                gate_times += list(trig_file[f'{ifo}/gating/{gt}/time'][:])
+            gate_times = np.unique(gate_times)
+        except KeyError:
+            logging.warning('No gating found in trigger file')
+
 trigs = SingleDetTriggers(
     args.trig_file,
     ifo,
@@ -134,6 +146,7 @@ with h5.File(args.output_file, 'w') as f:
             frac_dt = abs(segs) / livetime
             dq_rates[state] = frac_eff / frac_dt
         bin_grp['dq_rates'] = dq_rates
+        bin_grp['num_triggers'] = len(trig_times_bin)
 
     # save dq state segments
     for dq_state, segs in dq_state_segs_dict.items():
@@ -142,7 +155,10 @@ with h5.File(args.output_file, 'w') as f:
         starts, ends = segments_to_start_end(segs)
         dq_grp['segment_starts'] = starts
         dq_grp['segment_ends'] = ends
+        dq_grp['livetime'] = abs(segs)
 
     f.attrs['stat'] = f'{ifo}-dq_stat_info'
+    f.attrs['sngl_ranking'] = args.sngl_ranking
+    f.attrs['sngl_ranking_threshold'] = args.stat_threshold
 
 logging.info('Done!')

bin/plotting/pycbc_page_dq_table

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+""" Make a table of dq state information
+"""
+import sys
+import argparse
+import h5py as h5
+import numpy as np
+
+import pycbc
+import pycbc.results
+from pycbc.version import git_verbose_msg as version
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--version", action="version", version=version)
+parser.add_argument('--ifo', required=True)
+parser.add_argument('--dq-file', required=True)
+parser.add_argument('--verbose', action='count')
+parser.add_argument('--output-file')
+args = parser.parse_args()
+
+dq_states = {
+    'dq_state_0': 'Clean',
+    'dq_state_1': 'DQ Flag',
+    'dq_state_2': 'Autogating',
+}
+
+f = h5.File(args.dq_file, 'r')
+grp = f[args.ifo]['dq_segments']
+
+livetimes = []
+total_livetime = 0
+for dq_state in dq_states:
+    livetime = grp[dq_state]['livetime'][()]
+    livetimes.append(livetime)
+    total_livetime += livetime
+livetimes.append(total_livetime)
+
+frac_livetimes = [lt / total_livetime for lt in livetimes]
+state_names = list(dq_states.values()) + ['Total']
+columns = [state_names, livetimes, frac_livetimes]
+columns = [np.array(c) for c in columns]
+col_names = ['DQ State', 'Livetime', '% of Livetime']
+
+format_strings = [None, '0.0', '0.00%']
+
+html_table = pycbc.results.html_table(columns, col_names,
+                                      page_size=len(state_names),
+                                      format_strings=format_strings)
+title = f'{args.ifo} DQ State Livetimes'
+caption = 'Table of DQ state livetimes'
+
+pycbc.results.save_fig_with_metadata(
+    str(html_table), args.output_file, title=title,
+    caption=caption, cmd=' '.join(sys.argv))

bin/plotting/pycbc_page_template_bin_table

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+""" Make a table of template bin information
+"""
+import sys
+import argparse
+import h5py as h5
+import numpy as np
+
+import pycbc
+import pycbc.results
+from pycbc.version import git_verbose_msg as version
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--version", action="version", version=version)
+parser.add_argument('--ifo', required=True)
+parser.add_argument('--dq-file', required=True)
+parser.add_argument('--verbose', action='count')
+parser.add_argument('--output-file')
+args = parser.parse_args()
+
+f = h5.File(args.dq_file, 'r')
+grp = f[args.ifo]['bins']
+bin_names = list(grp.keys())
+
+sngl_ranking = f.attrs['sngl_ranking']
+sngl_thresh = f.attrs['sngl_ranking_threshold']
+
+livetime = 0
+seg_grp = f[args.ifo]['dq_segments']
+for k in seg_grp.keys():
+    livetime += seg_grp[k]['livetime'][()]
+
+num_templates = []
+num_triggers = []
+total_templates = 0
+total_triggers = 0
+for bin_name in bin_names:
+    bin_grp = grp[bin_name]
+
+    n_tmp = len(bin_grp['tids'][:])
+    num_templates.append(n_tmp)
+    total_templates += n_tmp
+
+    n_trig = bin_grp['num_triggers'][()]
+    num_triggers.append(n_trig)
+    total_triggers += n_trig
+
+bin_names.append('Total')
+num_triggers.append(total_triggers)
+num_templates.append(total_templates)
+frac_triggers = [n / total_triggers for n in num_triggers]
+frac_templates = [n / total_templates for n in num_templates]
+trigger_rate = [n / livetime for n in num_triggers]
+
+col_names = ['Template Bin', 'Number of Templates', '% of Templates',
+             'Number of Loud Triggers', '% of Loud Triggers',
+             'Loud Trigger Rate (Hz)']
+columns = [bin_names, num_templates, frac_templates,
+           num_triggers, frac_triggers, trigger_rate]
+columns = [np.array(c) for c in columns]
+
+format_strings = [None, '#', '0.000%', '#', '0.0%', '0.00E0']
+
+html_table = pycbc.results.html_table(columns, col_names,
+                                      page_size=len(bin_names),
+                                      format_strings=format_strings)
+title = f'{args.ifo} DQ Template Bin Information'
+caption = 'Table of information about template bins ' \
+           + 'used for DQ trigger rate calculations. ' \
+           + 'Loud triggers are defined as those with ' \
+           + f'{sngl_ranking} > {sngl_thresh}.'
+
+pycbc.results.save_fig_with_metadata(
+    str(html_table), args.output_file, title=title,
+    caption=caption, cmd=' '.join(sys.argv))

bin/plotting/pycbc_plot_dq_flag_likelihood

@@ -77,7 +77,10 @@ ax2.set_ylabel('DQ Log Likelihood Penalty')
 ax2.set_ylim(numpy.log(ymin), numpy.log(ymax))
 
 # add meta data and save figure
-plot_title = f'{ifo} DQ Trigger Rates'
+plot_title = f'{ifo}:{args.dq_label} DQ Trigger Rates'
+
+ax.set_title(plot_title)
+
 plot_caption = 'The log likelihood correction \
       during during each dq state for each template bin.'
 pycbc.results.save_fig_with_metadata(

bin/workflows/pycbc_make_offline_search_workflow

@@ -561,19 +561,25 @@ for key in final_bg_files:
 
 # DQ log likelihood plots
 for dqf, dql in zip(dqfiles, dqfile_labels):
+    ifo = dqf.ifo
+    outdir = rdir[f'data_quality/{dqf.ifo}_DQ_results']
+
     # plot rates when flag was on
-    outdir = rdir[f'data_quality/{dqf.ifo}_{dql}_trigger_rates']
     wf.make_dq_flag_trigger_rate_plot(workflow, dqf, dql, outdir, tags=[dql])
 
-# DQ background binning plot
-ifo_bins = workflow.cp.get_subsections('bin_templates')
-for ifo in ifo_bins:
+    # make table of dq segment info
+    wf.make_dq_segment_table(workflow, dqf, outdir, tags=[dql])
+
+    # plot background bins
     background_bins = workflow.cp.get_opt_tags(
         'bin_templates', 'background-bins', tags=[ifo])
-    wf.make_template_plot(workflow, hdfbank, rdir['data_quality'],
+    wf.make_template_plot(workflow, hdfbank, outdir,
                           bins=background_bins,
                           tags=[ifo, 'dq_bins'] + bank_tags)
 
+    # make table of template bin info
+    wf.make_template_bin_table(workflow, dqf, outdir, tags=[ifo])
+
 ############################## Setup the injection runs #######################
 
 splitbank_files_inj = wf.setup_splittable_workflow(workflow, [hdfbank],

pycbc/workflow/plotting.py

@@ -534,7 +534,6 @@ def make_singles_plot(workflow, trig_files, bank_file, veto_file, veto_name,
 def make_dq_flag_trigger_rate_plot(workflow, dq_file, dq_label, out_dir, tags=None):
     tags = [] if tags is None else tags
     makedir(out_dir)
-    files = FileList([])
     node = PlotExecutable(workflow.cp, 'plot_dq_flag_likelihood',
                           ifos=dq_file.ifo, out_dir=out_dir,
                           tags=tags).create_node()
@@ -543,5 +542,29 @@ def make_dq_flag_trigger_rate_plot(workflow, dq_file, dq_label, out_dir, tags=No
     node.add_opt('--ifo', dq_file.ifo)
     node.new_output_file_opt(dq_file.segment, '.png', '--output-file')
     workflow += node
-    files += node.output_files
-    return files
+    return node.output_files[0]
+
+
+def make_dq_segment_table(workflow, dq_file, out_dir, tags=None):
+    tags = [] if tags is None else tags
+    makedir(out_dir)
+    node = PlotExecutable(workflow.cp, 'page_dq_table', ifos=dq_file.ifo,
+                          out_dir=out_dir, tags=tags).create_node()
+    node.add_input_opt('--dq-file', dq_file)
+    node.add_opt('--ifo', dq_file.ifo)
+    node.new_output_file_opt(dq_file.segment, '.html', '--output-file')
+    workflow += node
+    return node.output_files[0]
+
+
+def make_template_bin_table(workflow, dq_file, out_dir, tags=None):
+    tags = [] if tags is None else tags
+    makedir(out_dir)
+    node = PlotExecutable(workflow.cp, 'page_template_bin_table',
+                          ifos=dq_file.ifo, out_dir=out_dir,
+                          tags=tags).create_node()
+    node.add_input_opt('--dq-file', dq_file)
+    node.add_opt('--ifo', dq_file.ifo)
+    node.new_output_file_opt(dq_file.segment, '.html', '--output-file')
+    workflow += node
+    return node.output_files[0]