diff --git a/bin/all_sky_search/pycbc_bin_trigger_rates_dq b/bin/all_sky_search/pycbc_bin_trigger_rates_dq index 25234a687f0..ef069d9a6b1 100644 --- a/bin/all_sky_search/pycbc_bin_trigger_rates_dq +++ b/bin/all_sky_search/pycbc_bin_trigger_rates_dq @@ -46,6 +46,18 @@ logging.info('Start') ifo, flag_name = args.flag_name.split(':') +if args.gating_windows: + gate_times = [] + with h5.File(args.trig_file, 'r') as trig_file: + logging.info('Getting gated times') + try: + gating_types = trig_file[f'{ifo}/gating'].keys() + for gt in gating_types: + gate_times += list(trig_file[f'{ifo}/gating/{gt}/time'][:]) + gate_times = np.unique(gate_times) + except KeyError: + logging.warning('No gating found in trigger file') + trigs = SingleDetTriggers( args.trig_file, ifo, @@ -134,6 +146,7 @@ with h5.File(args.output_file, 'w') as f: frac_dt = abs(segs) / livetime dq_rates[state] = frac_eff / frac_dt bin_grp['dq_rates'] = dq_rates + bin_grp['num_triggers'] = len(trig_times_bin) # save dq state segments for dq_state, segs in dq_state_segs_dict.items(): @@ -142,7 +155,10 @@ with h5.File(args.output_file, 'w') as f: starts, ends = segments_to_start_end(segs) dq_grp['segment_starts'] = starts dq_grp['segment_ends'] = ends + dq_grp['livetime'] = abs(segs) f.attrs['stat'] = f'{ifo}-dq_stat_info' + f.attrs['sngl_ranking'] = args.sngl_ranking + f.attrs['sngl_ranking_threshold'] = args.stat_threshold logging.info('Done!') diff --git a/bin/plotting/pycbc_page_dq_table b/bin/plotting/pycbc_page_dq_table new file mode 100644 index 00000000000..83b39b01569 --- /dev/null +++ b/bin/plotting/pycbc_page_dq_table @@ -0,0 +1,54 @@ +#!/usr/bin/env python +""" Make a table of dq state information +""" +import sys +import argparse +import h5py as h5 +import numpy as np + +import pycbc +import pycbc.results +from pycbc.version import git_verbose_msg as version + +parser = argparse.ArgumentParser() +parser.add_argument("--version", action="version", version=version) +parser.add_argument('--ifo', required=True) +parser.add_argument('--dq-file', required=True) +parser.add_argument('--verbose', action='count') +parser.add_argument('--output-file') +args = parser.parse_args() + +dq_states = { + 'dq_state_0': 'Clean', + 'dq_state_1': 'DQ Flag', + 'dq_state_2': 'Autogating', +} + +f = h5.File(args.dq_file, 'r') +grp = f[args.ifo]['dq_segments'] + +livetimes = [] +total_livetime = 0 +for dq_state in dq_states: + livetime = grp[dq_state]['livetime'][()] + livetimes.append(livetime) + total_livetime += livetime +livetimes.append(total_livetime) + +frac_livetimes = [lt / total_livetime for lt in livetimes] +state_names = list(dq_states.values()) + ['Total'] +columns = [state_names, livetimes, frac_livetimes] +columns = [np.array(c) for c in columns] +col_names = ['DQ State', 'Livetime', '% of Livetime'] + +format_strings = [None, '0.0', '0.00%'] + +html_table = pycbc.results.html_table(columns, col_names, + page_size=len(state_names), + format_strings=format_strings) +title = f'{args.ifo} DQ State Livetimes' +caption = 'Table of DQ state livetimes' + +pycbc.results.save_fig_with_metadata( + str(html_table), args.output_file, title=title, + caption=caption, cmd=' '.join(sys.argv)) diff --git a/bin/plotting/pycbc_page_template_bin_table b/bin/plotting/pycbc_page_template_bin_table new file mode 100644 index 00000000000..ef2f7ffb2aa --- /dev/null +++ b/bin/plotting/pycbc_page_template_bin_table @@ -0,0 +1,75 @@ +#!/usr/bin/env python +""" Make a table of template bin information +""" +import sys +import argparse +import h5py as h5 +import numpy as np + +import pycbc +import pycbc.results +from pycbc.version import git_verbose_msg as version + +parser = argparse.ArgumentParser() +parser.add_argument("--version", action="version", version=version) +parser.add_argument('--ifo', required=True) +parser.add_argument('--dq-file', required=True) +parser.add_argument('--verbose', action='count') +parser.add_argument('--output-file') +args = parser.parse_args() + +f = h5.File(args.dq_file, 'r') +grp = f[args.ifo]['bins'] +bin_names = list(grp.keys()) + +sngl_ranking = f.attrs['sngl_ranking'] +sngl_thresh = f.attrs['sngl_ranking_threshold'] + +livetime = 0 +seg_grp = f[args.ifo]['dq_segments'] +for k in seg_grp.keys(): + livetime += seg_grp[k]['livetime'][()] + +num_templates = [] +num_triggers = [] +total_templates = 0 +total_triggers = 0 +for bin_name in bin_names: + bin_grp = grp[bin_name] + + n_tmp = len(bin_grp['tids'][:]) + num_templates.append(n_tmp) + total_templates += n_tmp + + n_trig = bin_grp['num_triggers'][()] + num_triggers.append(n_trig) + total_triggers += n_trig + +bin_names.append('Total') +num_triggers.append(total_triggers) +num_templates.append(total_templates) +frac_triggers = [n / total_triggers for n in num_triggers] +frac_templates = [n / total_templates for n in num_templates] +trigger_rate = [n / livetime for n in num_triggers] + +col_names = ['Template Bin', 'Number of Templates', '% of Templates', + 'Number of Loud Triggers', '% of Loud Triggers', + 'Loud Trigger Rate (Hz)'] +columns = [bin_names, num_templates, frac_templates, + num_triggers, frac_triggers, trigger_rate] +columns = [np.array(c) for c in columns] + +format_strings = [None, '#', '0.000%', '#', '0.0%', '0.00E0'] + +html_table = pycbc.results.html_table(columns, col_names, + page_size=len(bin_names), + format_strings=format_strings) +title = f'{args.ifo} DQ Template Bin Information' +caption = 'Table of information about template bins ' \ + + 'used for DQ trigger rate calculations. ' \ + + 'Loud triggers are defined as those with ' \ + + f'{sngl_ranking} > {sngl_thresh}.' + +pycbc.results.save_fig_with_metadata( + str(html_table), args.output_file, title=title, + caption=caption, cmd=' '.join(sys.argv)) diff --git a/bin/plotting/pycbc_plot_dq_flag_likelihood b/bin/plotting/pycbc_plot_dq_flag_likelihood index a1d7beed5c1..ed4a6a1231f 100644 --- a/bin/plotting/pycbc_plot_dq_flag_likelihood +++ b/bin/plotting/pycbc_plot_dq_flag_likelihood @@ -77,7 +77,10 @@ ax2.set_ylabel('DQ Log Likelihood Penalty') ax2.set_ylim(numpy.log(ymin), numpy.log(ymax)) # add meta data and save figure -plot_title = f'{ifo} DQ Trigger Rates' +plot_title = f'{ifo}:{args.dq_label} DQ Trigger Rates' + +ax.set_title(plot_title) + plot_caption = 'The log likelihood correction \ during during each dq state for each template bin.' pycbc.results.save_fig_with_metadata( diff --git a/bin/workflows/pycbc_make_offline_search_workflow b/bin/workflows/pycbc_make_offline_search_workflow index a7fc7daf5c8..ac96e5a9815 100755 --- a/bin/workflows/pycbc_make_offline_search_workflow +++ b/bin/workflows/pycbc_make_offline_search_workflow @@ -561,19 +561,25 @@ for key in final_bg_files: # DQ log likelihood plots for dqf, dql in zip(dqfiles, dqfile_labels): + ifo = dqf.ifo + outdir = rdir[f'data_quality/{dqf.ifo}_DQ_results'] + # plot rates when flag was on - outdir = rdir[f'data_quality/{dqf.ifo}_{dql}_trigger_rates'] wf.make_dq_flag_trigger_rate_plot(workflow, dqf, dql, outdir, tags=[dql]) -# DQ background binning plot -ifo_bins = workflow.cp.get_subsections('bin_templates') -for ifo in ifo_bins: + # make table of dq segment info + wf.make_dq_segment_table(workflow, dqf, outdir, tags=[dql]) + + # plot background bins background_bins = workflow.cp.get_opt_tags( 'bin_templates', 'background-bins', tags=[ifo]) - wf.make_template_plot(workflow, hdfbank, rdir['data_quality'], + wf.make_template_plot(workflow, hdfbank, outdir, bins=background_bins, tags=[ifo, 'dq_bins'] + bank_tags) + # make table of template bin info + wf.make_template_bin_table(workflow, dqf, outdir, tags=[ifo]) + ############################## Setup the injection runs ####################### splitbank_files_inj = wf.setup_splittable_workflow(workflow, [hdfbank], diff --git a/pycbc/workflow/plotting.py b/pycbc/workflow/plotting.py index 165786c5715..cdf71b4119a 100644 --- a/pycbc/workflow/plotting.py +++ b/pycbc/workflow/plotting.py @@ -534,7 +534,6 @@ def make_singles_plot(workflow, trig_files, bank_file, veto_file, veto_name, def make_dq_flag_trigger_rate_plot(workflow, dq_file, dq_label, out_dir, tags=None): tags = [] if tags is None else tags makedir(out_dir) - files = FileList([]) node = PlotExecutable(workflow.cp, 'plot_dq_flag_likelihood', ifos=dq_file.ifo, out_dir=out_dir, tags=tags).create_node() @@ -543,5 +542,29 @@ def make_dq_flag_trigger_rate_plot(workflow, dq_file, dq_label, out_dir, tags=No node.add_opt('--ifo', dq_file.ifo) node.new_output_file_opt(dq_file.segment, '.png', '--output-file') workflow += node - files += node.output_files - return files + return node.output_files[0] + + +def make_dq_segment_table(workflow, dq_file, out_dir, tags=None): + tags = [] if tags is None else tags + makedir(out_dir) + node = PlotExecutable(workflow.cp, 'page_dq_table', ifos=dq_file.ifo, + out_dir=out_dir, tags=tags).create_node() + node.add_input_opt('--dq-file', dq_file) + node.add_opt('--ifo', dq_file.ifo) + node.new_output_file_opt(dq_file.segment, '.html', '--output-file') + workflow += node + return node.output_files[0] + + +def make_template_bin_table(workflow, dq_file, out_dir, tags=None): + tags = [] if tags is None else tags + makedir(out_dir) + node = PlotExecutable(workflow.cp, 'page_template_bin_table', + ifos=dq_file.ifo, out_dir=out_dir, + tags=tags).create_node() + node.add_input_opt('--dq-file', dq_file) + node.add_opt('--ifo', dq_file.ifo) + node.new_output_file_opt(dq_file.segment, '.html', '--output-file') + workflow += node + return node.output_files[0]