From 144f1c36181e564e0b2d40cbd1c075bf9b0852bb Mon Sep 17 00:00:00 2001 From: Gareth S Cabourn Davies Date: Wed, 7 Aug 2024 13:58:18 +0100 Subject: [PATCH] Various improvements to minifollowup tables (#4839) * Use extra spij parameters in inj info * Use statistic in coincinfo as well, allow tables to run over multiple lines, provide row names * sngl-ranking first, warn if ranking-xstatistic is supplied * Dont fail if (un-necessary) statistic files are not given * some tidying * specify sngl-ranking for coincinfo * try to fix CC issues * Fix minor bug, allow scrollable tables iin results pages * Dont duplicate info in the case that no DQ summary links are used --- bin/minifollowups/pycbc_page_coincinfo | 56 ++++++++++--- bin/minifollowups/pycbc_page_injinfo | 42 ++++++---- bin/minifollowups/pycbc_page_snglinfo | 34 ++++---- examples/search/plotting.ini | 2 + pycbc/results/dq.py | 6 +- pycbc/results/static/css/pycbc/orange.css | 14 ++++ pycbc/results/table_utils.py | 96 +++++++++++++++++------ 7 files changed, 183 insertions(+), 67 deletions(-) diff --git a/bin/minifollowups/pycbc_page_coincinfo b/bin/minifollowups/pycbc_page_coincinfo index 10f08bc2a01..b2dc9e363b7 100644 --- a/bin/minifollowups/pycbc_page_coincinfo +++ b/bin/minifollowups/pycbc_page_coincinfo @@ -30,7 +30,7 @@ from pycbc import add_common_pycbc_options import pycbc.results import pycbc.pnutils from pycbc.io.hdf import HFile -from pycbc.events import ranking +from pycbc.events import ranking, stat as pystat from pycbc.results import followup parser = argparse.ArgumentParser() @@ -69,11 +69,22 @@ parser.add_argument('--include-summary-page-link', action='store_true', parser.add_argument('--include-gracedb-link', action='store_true', help="If given, will provide a link to search GraceDB for events " "within a 3s window around the coincidence time.") - +parser.add_argument('--max-columns', type=int, + help="Maximum number of columns allowed in the table (not including detector names)") +pystat.insert_statistic_option_group(parser, + default_ranking_statistic='single_ranking_only') args = parser.parse_args() pycbc.init_logging(args.verbose) +if args.ranking_statistic not in ['quadsum', 'single_ranking_only']: + logging.warning( + "For the coincident info table, we only use single ranking, not %s, " + "this option will be ignored", + args.ranking_statistic + ) + args.ranking_statistic = 'quadsum' + # Get the nth loudest trigger from the output of pycbc_coinc_statmap f = HFile(args.statmap_file, 'r') d = f[args.statmap_file_subspace_name] @@ -146,12 +157,16 @@ statmapfile = d # table. Each entry in data corresponds to each row in the final table and # should be a list of values. So data is will be a list of lists. data = [] +row_labels = [] +rank_method = pystat.get_statistic_from_opts(args, list(files.keys())) + for ifo in files.keys(): # ignore ifo if coinc didn't participate (only for multi-ifo workflow) if (statmapfile['%s/time' % ifo][n] == -1.0): continue + row_labels.append(ifo) d = files[ifo] i = idx[ifo] tid = d['template_id'][i] @@ -161,7 +176,12 @@ for ifo in files.keys(): time = d['end_time'][i] utc = lal.GPSToUTC(int(time))[0:6] - + trig_dict = { + k: numpy.array([d[k][i]]) + for k in d.keys() + if not k.endswith('_template') + and k not in ['gating', 'search', 'template_boundaries'] + } # Headers will store the headers that will appear in the table. headers = [] data.append([]) @@ -170,9 +190,6 @@ for ifo in files.keys(): if args.include_summary_page_link: data[-1].append(pycbc.results.dq.get_summary_page_link(ifo, utc)) headers.append("Detector status") - else: - data[-1].append(ifo) - headers.append("Ifo") # End times data[-1].append(str(datetime.datetime(*utc))) @@ -180,14 +197,28 @@ for ifo in files.keys(): headers.append("UTC End Time") headers.append("GPS End time") + #headers.append("Stat") + # Determine statistic naming + if args.sngl_ranking == "newsnr": + sngl_stat_name = "Reweighted SNR" + elif args.sngl_ranking == "newsnr_sgveto": + sngl_stat_name = "Reweighted SNR (+sgveto)" + elif args.sngl_ranking == "newsnr_sgveto_psdvar": + sngl_stat_name = "Reweighted SNR (+sgveto+psdvar)" + elif args.sngl_ranking == "snr": + sngl_stat_name = "SNR" + else: + sngl_stat_name = args.sngl_ranking + + stat = rank_method.get_sngl_ranking(trig_dict) + headers.append(sngl_stat_name) + data[-1].append('%5.2f' % stat[0]) + # SNR and phase (not showing any single-det stat here) data[-1].append('%5.2f' % d['snr'][i]) data[-1].append('%5.2f' % d['coa_phase'][i]) - #data[-1].append('%5.2f' % ranking.newsnr(d['snr'][i], rchisq)) headers.append("ρ") headers.append("Phase") - #headers.append("Stat") - # Signal-glitch discrimators data[-1].append('%5.2f' % rchisq) data[-1].append('%i' % d['chisq_dof'][i]) @@ -218,7 +249,12 @@ for ifo in files.keys(): headers.append("s2z") headers.append("Duration") -html += str(pycbc.results.static_table(data, headers)) +html += str(pycbc.results.static_table( + data, + headers, + columns_max=args.max_columns, + row_labels=row_labels +)) ############################################################################### pycbc.results.save_fig_with_metadata(html, args.output_file, {}, diff --git a/bin/minifollowups/pycbc_page_injinfo b/bin/minifollowups/pycbc_page_injinfo index a09292ecc6e..e4bf1cfe8e8 100644 --- a/bin/minifollowups/pycbc_page_injinfo +++ b/bin/minifollowups/pycbc_page_injinfo @@ -20,6 +20,7 @@ import sys import numpy import pycbc.results +from pycbc import conversions as conv import pycbc.pnutils from pycbc import init_logging, add_common_pycbc_options from pycbc.detector import Detector @@ -34,6 +35,8 @@ parser.add_argument('--injection-index', type=int, required=True, help="The index of the injection to print out. Required") parser.add_argument('--n-nearest', type=int, help="Optional, used in the title") +parser.add_argument('--max-columns', type=int, + help="Optional, maximum number of columns used for the table") args = parser.parse_args() @@ -65,13 +68,24 @@ labels = { 'spin1z': 's1z', 'spin2x': 's2x', 'spin2y': 's2y', - 'spin2z': 's2z' + 'spin2z': 's2z', + 'chieff': 'χeff', + 'chip': 'χp', } params += ['tc'] m1, m2 = f['injections']['mass1'][iidx], f['injections']['mass2'][iidx] -mchirp, eta = pycbc.pnutils.mass1_mass2_to_mchirp_eta(m1, m2) +s1x, s2x = f['injections']['spin1x'][iidx], f['injections']['spin2x'][iidx] +s1y, s2y = f['injections']['spin1y'][iidx], f['injections']['spin2y'][iidx] +s1z, s2z = f['injections']['spin1z'][iidx], f['injections']['spin2z'][iidx] + +derived = {} +derived['mchirp'], derived['eta'] = \ + pycbc.pnutils.mass1_mass2_to_mchirp_eta(m1, m2) +derived['mtotal'] = conv.mtotal_from_mass1_mass2(m1, m2) +derived['chieff'] = conv.chi_eff(m1, m2, s1z, s2z) +derived['chip'] = conv.chi_p(m1, m2, s1x, s1y, s2x, s2y) if 'optimal_snr' in ' '.join(list(f['injections'].keys())): ifolist = f.attrs['ifos'].split(' ') @@ -82,32 +96,30 @@ else: eff_dist = {} for ifo in ['H1', 'L1', 'V1']: eff_dist[ifo] = Detector(ifo).effective_distance( - f['injections/distance'][iidx], - f['injections/ra'][iidx], - f['injections/dec'][iidx], - f['injections/polarization'][iidx], - f['injections/tc'][iidx], - f['injections/inclination'][iidx]) - + f['injections/distance'][iidx], + f['injections/ra'][iidx], + f['injections/dec'][iidx], + f['injections/polarization'][iidx], + f['injections/tc'][iidx], + f['injections/inclination'][iidx] + ) params += ['dec_chirp_dist', 'eff_dist_h', 'eff_dist_l', 'eff_dist_v'] dec_dist = max(eff_dist['H1'], eff_dist['L1']) dec_chirp_dist = pycbc.pnutils.chirp_distance(dec_dist, mchirp) params += ['mass1', 'mass2', 'mchirp', 'eta', 'ra', 'dec', 'inclination', 'spin1x', 'spin1y', 'spin1z', 'spin2x', 'spin2y', - 'spin2z'] + 'spin2z', 'chieff', 'chip'] for p in params: if p in f['injections']: data += ["%.2f" % f['injections'][p][iidx]] + elif p in derived.keys(): + data += [f'{derived[p]:.2f}'] elif 'eff_dist' in p: ifo = '%s1' % p.split('_')[-1] data += ["%.2f" % eff_dist[ifo.upper()]] - elif p == 'mchirp': - data += ["%.2f" % mchirp] - elif p == 'eta': - data += ["%.2f" % eta] elif p == 'dec_chirp_dist': data += ["%.2f" % dec_chirp_dist] else: @@ -117,7 +129,7 @@ for p in params: headers += [labels[p]] table = numpy.array([data], dtype=str) -html = str(pycbc.results.static_table(table, headers)) +html = str(pycbc.results.static_table(table, headers, columns_max=args.max_columns)) tag = '' if args.n_nearest is not None: diff --git a/bin/minifollowups/pycbc_page_snglinfo b/bin/minifollowups/pycbc_page_snglinfo index 99f3a1629d2..2ce04edbcc1 100644 --- a/bin/minifollowups/pycbc_page_snglinfo +++ b/bin/minifollowups/pycbc_page_snglinfo @@ -68,6 +68,8 @@ parser.add_argument('--include-gracedb-link', action='store_true', parser.add_argument('--significance-file', help="If given, will search for this trigger's id in the file to see if " "stat and p_astro values exists for this trigger.") +parser.add_argument('--max-columns', type=int, + help="Optional. Set a maximum number of columns to be used in the output table") pystat.insert_statistic_option_group(parser, default_ranking_statistic='single_ranking_only') @@ -117,7 +119,7 @@ if args.n_loudest is not None: sngl_file.apply_mask(l[0]) # make a table for the single detector information ############################ -time = sngl_file.end_time +time = sngl_file.end_time[0] utc = lal.GPSToUTC(int(time))[0:6] # Headers here will contain the list of headers that will appear in the @@ -129,6 +131,8 @@ headers = [] # single list that will hold the values to go into the table. data = [[]] +row_labels = [args.instrument] + # DQ summary link if args.include_summary_page_link: data[0].append(pycbc.results.dq.get_summary_page_link(args.instrument, utc)) @@ -141,11 +145,10 @@ headers.append("UTC") headers.append("End time") # SNR and statistic -data[0].append('%5.2f' % sngl_file.snr) -data[0].append('%5.2f' % sngl_file.get_column('coa_phase')) -data[0].append('%5.2f' % stat) headers.append("ρ") +data[0].append('%5.2f' % sngl_file.snr[0]) headers.append("Phase") +data[0].append('%5.2f' % sngl_file.get_column('coa_phase')[0]) # Determine statistic naming if args.sngl_ranking == "newsnr": sngl_stat_name = "Reweighted SNR" @@ -169,30 +172,31 @@ else: ) headers.append(stat_name) +data[0].append('%5.2f' % stat[0]) # Signal-glitch discrimators -data[0].append('%5.2f' % sngl_file.rchisq) -data[0].append('%i' % sngl_file.get_column('chisq_dof')) +data[0].append('%5.2f' % sngl_file.rchisq[0]) +data[0].append('%i' % sngl_file.get_column('chisq_dof')[0]) headers.append("χ2r") headers.append("χ2 bins") try: - data[0].append('%5.2f' % sngl_file.sgchisq) + data[0].append('%5.2f' % sngl_file.sgchisq[0]) headers.append("sgχ2") except: pass try: - data[0].append('%5.2f' % sngl_file.psd_var_val) + data[0].append('%5.2f' % sngl_file.psd_var_val[0]) headers.append("PSD var") except: pass # Template parameters -data[0].append('%5.2f' % sngl_file.mass1) -data[0].append('%5.2f' % sngl_file.mass2) -data[0].append('%5.2f' % sngl_file.mchirp) -data[0].append('%5.2f' % sngl_file.spin1z) -data[0].append('%5.2f' % sngl_file.spin2z) -data[0].append('%5.2f' % sngl_file.template_duration) +data[0].append('%5.2f' % sngl_file.mass1[0]) +data[0].append('%5.2f' % sngl_file.mass2[0]) +data[0].append('%5.2f' % sngl_file.mchirp[0]) +data[0].append('%5.2f' % sngl_file.spin1z[0]) +data[0].append('%5.2f' % sngl_file.spin2z[0]) +data[0].append('%5.2f' % sngl_file.template_duration[0]) headers.append("m1") headers.append("m2") headers.append("Mc") @@ -221,7 +225,7 @@ if args.include_gracedb_link: data[0].append(gdb_search_link) html = pycbc.results.dq.redirect_javascript + \ - str(pycbc.results.static_table(data, headers)) + str(pycbc.results.static_table(data, headers, row_labels=row_labels, columns_max=args.max_columns)) ############################################################################### # Set up default titles and the captions for the file diff --git a/examples/search/plotting.ini b/examples/search/plotting.ini index 0b1ab2a2cbe..5a7d4f55837 100644 --- a/examples/search/plotting.ini +++ b/examples/search/plotting.ini @@ -61,6 +61,8 @@ window = 0.1 [html_snippet] [page_coincinfo] +sngl-ranking = newsnr_sgveto_psdvar + [page_coincinfo-background] statmap-file-subspace-name=background_exc diff --git a/pycbc/results/dq.py b/pycbc/results/dq.py index ce3aeb43b8f..7be7bb1d4ff 100644 --- a/pycbc/results/dq.py +++ b/pycbc/results/dq.py @@ -22,8 +22,7 @@ """ -data_h1_string = """H1 -  +data_h1_string = """ Summary   @@ -31,8 +30,7 @@ 'https://alog.ligo-wa.caltech.edu/aLOG/includes/search.php?adminType=search'); return true;">aLOG""" -data_l1_string="""L1 -  +data_l1_string=""" Summary   diff --git a/pycbc/results/static/css/pycbc/orange.css b/pycbc/results/static/css/pycbc/orange.css index 8b2b44b0aea..1674f8ae1c2 100644 --- a/pycbc/results/static/css/pycbc/orange.css +++ b/pycbc/results/static/css/pycbc/orange.css @@ -92,3 +92,17 @@ font-size:16px; a { color:#000000; } + +table { + display: block; + overflow-x: auto; + white-space: nowrap; +} + +td { + text-align: center; +} + +th { + text-align: center; +} diff --git a/pycbc/results/table_utils.py b/pycbc/results/table_utils.py index dfdaa7297d2..aec7c62ffdb 100644 --- a/pycbc/results/table_utils.py +++ b/pycbc/results/table_utils.py @@ -23,7 +23,10 @@ # """ This module provides functions to generate sortable html tables """ -import mako.template, uuid +import mako.template +import uuid +import copy +import numpy google_table_template = mako.template.Template(""" @@ -103,42 +106,89 @@ def html_table(columns, names, page_size=None, format_strings=None): static_table_template = mako.template.Template(""" - % if titles is not None: - - % for i in range(len(titles)): - - % endfor - - % endif - - % for i in range(len(data)): - - % for j in range(len(data[i])): - + % for row in range(n_rows): + % if titles is not None: + + % if row_labels is not None: + + % endif + % for i in range(n_columns): + + % endfor + + % endif + + % for i in range(len(data)): + + % if row_labels is not None: + + % endif + % for j in range(n_columns): + + % endfor + % endfor - % endfor
- ${titles[i]} -
- ${data[i][j]} -
+ + ${titles[row * n_columns + i]} +
+ ${row_labels[i]} + + ${data[i][row * n_columns + j]} +
""") -def static_table(data, titles=None): - """ Return an html tableo of this data +def static_table(data, titles=None, columns_max=None, row_labels=None): + """ Return an html table of this data Parameters ---------- - data : two-dimensional numpy string array + data : two-dimensional string array Array containing the cell values titles : numpy array - Vector str of titles + Vector str of titles, must be the same length as data + columns_max : integer or None + If given, will restrict the number of columns in the table + row_labels : list of strings + Optional list of row labels to be given as the first cell in + each data row. Does not count towards columns_max Returns ------- html_table : str A string containing the html table. """ - return static_table_template.render(data=data, titles=titles) + data = copy.deepcopy(data) + titles = copy.deepcopy(titles) + row_labels = copy.deepcopy(row_labels) + drows, dcols = numpy.array(data).shape + if titles is not None and not len(titles) == dcols: + raise ValueError("titles and data lengths do not match") + + if row_labels is not None and not len(row_labels) == drows: + raise ValueError( + "row_labels must be the same number of rows supplied to data" + ) + + if columns_max is not None: + n_rows = int(numpy.ceil(len(data[0]) / columns_max)) + n_columns = min(columns_max, len(data[0])) + if len(data[0]) < n_rows * n_columns: + # Pad the data and titles with empty strings + n_missing = int(n_rows * n_columns - len(data[0])) + data = numpy.hstack((data, numpy.zeros((len(data), n_missing), dtype='U1'))) + if titles is not None: + titles += [' '] * n_missing + else: + n_rows = 1 + n_columns = len(data[0]) + + return static_table_template.render( + data=data, + titles=titles, + n_columns=n_columns, + n_rows=n_rows, + row_labels=row_labels, + )