From a61f52726e3872e05025ff159324d581429cb1c5 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 4 Apr 2023 16:40:06 -0400 Subject: [PATCH 1/8] Remove yaml and bokeh code --- .../monitor_pages/monitor_readnoise_bokeh.py | 160 --------------- .../yaml/monitor_readnoise_interface.yaml | 189 ------------------ 2 files changed, 349 deletions(-) delete mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py delete mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py deleted file mode 100644 index 8c8be095e..000000000 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ /dev/null @@ -1,160 +0,0 @@ -"""This module contains code for the readnoise monitor Bokeh plots. - -Author ------- - - - Ben Sunnquist - -Use ---- - - This module can be used from the command line as such: - - :: - - from jwql.website.apps.jwql import monitor_pages - monitor_template = monitor_pages.ReadnoiseMonitor() - monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL') -""" - -from datetime import datetime, timedelta -import os - -import numpy as np - -from jwql.bokeh_templating import BokehTemplate -from jwql.database.database_interface import session -from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - - -class ReadnoiseMonitor(BokehTemplate): - - # Combine the input parameters into a single property because we - # do not want to invoke the setter unless all are updated - @property - def input_parameters(self): - return (self._instrument, self._aperture) - - @input_parameters.setter - def input_parameters(self, info): - self._instrument, self._aperture = info - self.pre_init() - self.post_init() - - def identify_tables(self): - """Determine which database tables to use for the given instrument""" - - mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] - self.stats_table = eval('{}ReadnoiseStats'.format(mixed_case_name)) - - def load_data(self): - """Query the database tables to get all of the relevant readnoise data""" - - # Determine which database tables are needed based on instrument - self.identify_tables() - - # Query database for all data in readnoise stats with a matching aperture, - # and sort the data by exposure start time. - self.query_results = session.query(self.stats_table) \ - .filter(self.stats_table.aperture == self._aperture) \ - .order_by(self.stats_table.expstart) \ - .all() - - session.close() - - def pre_init(self): - - # Start with default values for instrument and aperture because - # BokehTemplate's __init__ method does not allow input arguments - try: - dummy_instrument = self._instrument - dummy_aperture = self._aperture - except AttributeError: - self._instrument = 'NIRCam' - self._aperture = '' - - self._embed = True - self.format_string = None - self.interface_file = os.path.join(SCRIPT_DIR, 'yaml', 'monitor_readnoise_interface.yaml') - - def post_init(self): - - # Load the readnoise data - self.load_data() - - # Update the mean readnoise figures - self.update_mean_readnoise_figures() - - # Update the readnoise difference image and histogram - self.update_readnoise_diff_plots() - - def update_mean_readnoise_figures(self): - """Updates the mean readnoise bokeh plots""" - - # Get the dark exposures info - filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.query_results] - expstarts_iso = np.array([result.expstart for result in self.query_results]) - expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) - nints = [result.nints for result in self.query_results] - ngroups = [result.ngroups for result in self.query_results] - - # Update the mean readnoise figures for all amps - for amp in ['1', '2', '3', '4']: - readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results]) - self.refs['mean_readnoise_source_amp{}'.format(amp)].data = {'time': expstarts, - 'time_iso': expstarts_iso, - 'mean_rn': readnoise_vals, - 'filename': filenames, - 'nints': nints, - 'ngroups': ngroups} - self.refs['mean_readnoise_figure_amp{}'.format(amp)].title.text = 'Amp {}'.format(amp) - self.refs['mean_readnoise_figure_amp{}'.format(amp)].hover.tooltips = [('file', '@filename'), - ('time', '@time_iso'), - ('nints', '@nints'), - ('ngroups', '@ngroups'), - ('readnoise', '@mean_rn')] - - # Update plot limits if data exists - if len(readnoise_vals) != 0: - self.refs['mean_readnoise_xr_amp{}'.format(amp)].start = expstarts.min() - timedelta(days=3) - self.refs['mean_readnoise_xr_amp{}'.format(amp)].end = expstarts.max() + timedelta(days=3) - min_val, max_val = min(x for x in readnoise_vals if x is not None), max(x for x in readnoise_vals if x is not None) - if min_val == max_val: - self.refs['mean_readnoise_yr_amp{}'.format(amp)].start = min_val - 1 - self.refs['mean_readnoise_yr_amp{}'.format(amp)].end = max_val + 1 - else: - offset = (max_val - min_val) * .1 - self.refs['mean_readnoise_yr_amp{}'.format(amp)].start = min_val - offset - self.refs['mean_readnoise_yr_amp{}'.format(amp)].end = max_val + offset - - def update_readnoise_diff_plots(self): - """Updates the readnoise difference image and histogram""" - - # Update the readnoise difference image and histogram, if data exists - if len(self.query_results) != 0: - # Get the most recent data; the entries were sorted by time when - # loading the database, so the last entry will always be the most recent. - diff_image_png = self.query_results[-1].readnoise_diff_image - diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) - diff_image_n = np.array(self.query_results[-1].diff_image_n) - diff_image_bin_centers = np.array(self.query_results[-1].diff_image_bin_centers) - - # Update the readnoise difference image and histogram - self.refs['readnoise_diff_image'].image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") - self.refs['diff_hist_source'].data = {'n': diff_image_n, - 'bin_centers': diff_image_bin_centers} - self.refs['diff_hist_xr'].start = diff_image_bin_centers.min() - self.refs['diff_hist_xr'].end = diff_image_bin_centers.max() - self.refs['diff_hist_yr'].start = diff_image_n.min() - self.refs['diff_hist_yr'].end = diff_image_n.max() + diff_image_n.max() * 0.05 - - # Update the readnoise difference image style - self.refs['readnoise_diff_image'].xaxis.visible = False - self.refs['readnoise_diff_image'].yaxis.visible = False - self.refs['readnoise_diff_image'].xgrid.grid_line_color = None - self.refs['readnoise_diff_image'].ygrid.grid_line_color = None - self.refs['readnoise_diff_image'].title.text_font_size = '22px' - self.refs['readnoise_diff_image'].title.align = 'center' diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml deleted file mode 100644 index 4d2fd8a78..000000000 --- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml +++ /dev/null @@ -1,189 +0,0 @@ -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp1 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp1 - ref: "mean_readnoise_source_amp1" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp1 - ref: "mean_readnoise_xr_amp1" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp1 - ref: "mean_readnoise_yr_amp1" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp1 - ref: "mean_readnoise_figure_amp1" - title: "Amp 1" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp1 - y_range: *mean_readnoise_yr_amp1 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp1} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp2 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp2 - ref: "mean_readnoise_source_amp2" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp2 - ref: "mean_readnoise_xr_amp2" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp2 - ref: "mean_readnoise_yr_amp2" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp2 - ref: "mean_readnoise_figure_amp2" - title: "Amp 2" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp2 - y_range: *mean_readnoise_yr_amp2 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp2} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp3 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp3 - ref: "mean_readnoise_source_amp3" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp3 - ref: "mean_readnoise_xr_amp3" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp3 - ref: "mean_readnoise_yr_amp3" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp3 - ref: "mean_readnoise_figure_amp3" - title: "Amp 3" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp3 - y_range: *mean_readnoise_yr_amp3 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp3} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp4 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp4 - ref: "mean_readnoise_source_amp4" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp4 - ref: "mean_readnoise_xr_amp4" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp4 - ref: "mean_readnoise_yr_amp4" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp4 - ref: "mean_readnoise_figure_amp4" - title: "Amp 4" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp4 - y_range: *mean_readnoise_yr_amp4 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp4} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Readnoise Difference Image -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &diff_source - ref: "diff_source" - data: - dh: [1] - dw: [1] - image: [[[0,0], [0, 0]]] -- !Figure: &readnoise_diff_image - ref: "readnoise_diff_image" - title: 'Readnoise Difference (most recent dark - pipeline reffile)' - elements: - - {"kind": "image", "image": "image", "x": 0, "y": 0, "dh": 'dh', "dw": 'dh', "source": *diff_source} - tools: "" -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Readnoise Difference Histogram -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &diff_hist_source - ref: "diff_hist_source" - data: - n: [] - bin_centers: [] -- !Range1d: &diff_hist_xr - ref: "diff_hist_xr" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &diff_hist_yr - ref: "diff_hist_yr" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &readnoise_diff_hist - ref: "readnoise_diff_hist" - x_axis_label: "Readnoise Difference [DN]" - y_axis_label: "Number of Pixels" - x_range: *diff_hist_xr - y_range: *diff_hist_yr - height: 250 - width: 300 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'bin_centers', 'y': 'n', 'size': 4, 'source': *diff_hist_source} - -# Document structure -# - !Document: -# - !row: -# - *mean_readnoise_figure \ No newline at end of file From 9320346cc7c0b4c7c9d9e0ef1d5ee6bcb2e6dd37 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 6 Apr 2023 16:22:29 -0400 Subject: [PATCH 2/8] Adding bones to read noise monitor plotting --- .../monitor_pages/monitor_readnoise_bokeh.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py new file mode 100644 index 000000000..3a5d22a6a --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -0,0 +1,98 @@ +"""This module contains code for the readnoise monitor Bokeh plots. + +Author +------ + + - Ben Sunnquist + +Use +--- + + This module can be used from the command line as such: + + :: + + from jwql.website.apps.jwql import monitor_pages + monitor_template = monitor_pages.ReadnoiseMonitor() + monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL') +""" + +from datetime import datetime, timedelta +import os + +import numpy as np + +from jwql.bokeh_templating import BokehTemplate +from jwql.database.database_interface import session +from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE + + +class ReadnoiseMonitorData(): + """Class to hold bias data to be plotted + + Parameters + ---------- + + instrument : str + Instrument name (e.g. nircam) + + Attributes + ---------- + + instrument : str + Instrument name (e.g. nircam) + aperture : str + Aperture name (e.g. apername) + + """ + + def __init__(self, instrument, aperture): + self.instrument = instrument + self.aperture = aperture + self.load_data() + + + def identify_tables(self): + """Determine which database tables to use for the given instrument""" + + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument.lower()] + self.stats_table = eval('{}ReadnoiseStats'.format(mixed_case_name)) + + def load_data(self): + """Query the database tables to get all of the relevant readnoise data""" + + # Determine which database tables are needed based on instrument + self.identify_tables() + + # Query database for all data in readnoise stats with a matching aperture, + # and sort the data by exposure start time. + self.query_results = session.query(self.stats_table) \ + .filter(self.stats_table.aperture == self.aperture) \ + .order_by(self.stats_table.expstart) \ + .all() + + session.close() + +class ReadNoisePlots(): + """Class to make readnoise plots. + """ + def __init__(self, instrument, aperture): + self.instrument = instrument + self.aperture = aperture + + self.db = ReadnoiseMonitorData(self.instrument, self.aperture) + + + def read_noise_amp_plots(self): + """Make read noise monitor trending plots per amplifier + """ + print('make plot here') + # for amp in ['1', '2', '3', '4']: + # expstarts_iso = np.array([result.expstart for result in self.db]) + # readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results]) + + + + + From 1e6073ec1b0083ebd9366d7241bc4dc63b6e8516 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 2 Oct 2023 15:03:05 -0400 Subject: [PATCH 3/8] Removed all templating, adding new calls to monitor views to display proper html --- .../monitor_pages/monitor_readnoise_bokeh.py | 118 ++++++++++++++++-- jwql/website/apps/jwql/monitor_views.py | 4 +- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 3a5d22a6a..5e2b0c80f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -12,7 +12,7 @@ :: - from jwql.website.apps.jwql import monitor_pages + . monitor_template = monitor_pages.ReadnoiseMonitor() monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL') """ @@ -20,12 +20,17 @@ from datetime import datetime, timedelta import os +from bokeh.embed import components +from bokeh.layouts import column, row +from bokeh.models import Panel, Tabs # bokeh <= 3.0 +from bokeh.models import ColumnDataSource, HoverTool +# from bokeh.models import TabPanel, Tabs # bokeh >= 3.0 +from bokeh.plotting import figure import numpy as np -from jwql.bokeh_templating import BokehTemplate from jwql.database.database_interface import session from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE class ReadnoiseMonitorData(): @@ -74,8 +79,24 @@ def load_data(self): session.close() -class ReadNoisePlots(): - """Class to make readnoise plots. + +class ReadNoiseFigure(): + """Generate tabbed plot displayed in JWQL web application + """ + def __init__(self, instrument): + instrument_apertures = FULL_FRAME_APERTURES[instrument.upper()] + + self.tabs = [] + for aperture in instrument_apertures: + readnoise_tab = ReadNoisePlotTab(instrument, aperture) + self.tabs.append(readnoise_tab.tab) + + self.plot = Tabs(tabs=self.tabs) + self.tab_components = components(self.plot) + + +class ReadNoisePlotTab(): + """Class to make instrument/aperture panels """ def __init__(self, instrument, aperture): self.instrument = instrument @@ -83,16 +104,89 @@ def __init__(self, instrument, aperture): self.db = ReadnoiseMonitorData(self.instrument, self.aperture) + self.plot_readnoise_amplifers() + self.plot_readnoise_difference_image() + self.plot_readnoise_histogram() - def read_noise_amp_plots(self): - """Make read noise monitor trending plots per amplifier - """ - print('make plot here') - # for amp in ['1', '2', '3', '4']: - # expstarts_iso = np.array([result.expstart for result in self.db]) - # readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results]) + self.tab = Panel(child=column(row(*self.amp_plots), self.diff_image_plot, self.readnoise_histogram), title= self.aperture) + + def plot_readnoise_amplifers(self): + + self.amp_plots = [] + for amp in ['1', '2', '3', '4']: + + amp_plot = figure(width=350, height=350, x_axis_type='datetime') + + readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) + filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] + expstarts_iso = np.array([result.expstart for result in self.db.query_results]) + expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) + nints = [result.nints for result in self.db.query_results] + ngroups = [result.ngroups for result in self.db.query_results] + + source = ColumnDataSource(data=dict( + file=filenames, + expstarts=expstarts, + nints=nints, + ngroups=ngroups, + readnoise=readnoise_vals, + )) + + amp_plot.add_tools(HoverTool(tooltips=[("file", "@filenames"), + ("time", "@expstarts"), + ("nints", "@nints"), + ("ngroups", "@ngroups"), + ("readnoise", "@readnoise")])) + + amp_plot.circle(x='expstarts', y='readnoise', source=source) + + amp_plot.xaxis.axis_label = 'Date' + amp_plot.yaxis.axis_label = 'Mean Readnoise [DN]' + + self.amp_plots.append(amp_plot) + + def plot_readnoise_difference_image(self): + """Updates the readnoise difference image""" + + # Update the readnoise difference image and histogram, if data exists + + self.diff_image_plot = figure(height=700, width=700) + + if len(self.db.query_results) != 0: + diff_image_png = self.db.query_results[-1].readnoise_diff_image + self.diff_image_plot.image_url(url=[diff_image_png], x=1, y=1, w=2048, h=2048, anchor="bottom_left") + self.diff_image_plot.xaxis.visible = False + self.diff_image_plot.yaxis.visible = False + self.diff_image_plot.xgrid.grid_line_color = None + self.diff_image_plot.ygrid.grid_line_color = None + self.diff_image_plot.title.text_font_size = '22px' + self.diff_image_plot.title.align = 'center' + + def plot_readnoise_histogram(self): + """Updates the readnoise histogram""" + + diff_image_n = np.array(self.db.query_results[-1].diff_image_n) + diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers) + + hist_xr_start = diff_image_bin_centers.min() + hist_xr_end = diff_image_bin_centers.max() + hist_yr_start = diff_image_n.min() + hist_yr_end = diff_image_n.max() + diff_image_n.max() * 0.05 + + self.readnoise_histogram = figure(height=800, width=800, + x_range=(hist_xr_start, hist_xr_end), + y_range=(hist_yr_start, hist_yr_end)) + + source = ColumnDataSource(data=dict( + x=diff_image_bin_centers, + y=diff_image_n, + )) + self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"),])) + self.readnoise_histogram.circle(x='x', y='y', source=source) + self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]' + self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels' diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index b62f339de..09cf423f6 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -37,12 +37,14 @@ from . import bokeh_containers from jwql.website.apps.jwql import bokeh_containers +from jwql.website.apps.jwql.monitor_pages.monitor_readnoise_bokeh import ReadNoiseFigure from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_config, get_base_url from jwql.instrument_monitors.nirspec_monitors.ta_monitors import msata_monitor from jwql.instrument_monitors.nirspec_monitors.ta_monitors import wata_monitor from jwql.utils import monitor_utils + CONFIG = get_config() FILESYSTEM_DIR = os.path.join(CONFIG['jwql_dir'], 'filesystem') @@ -219,7 +221,7 @@ def readnoise_monitor(request, inst): inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] # Get the html and JS needed to render the readnoise tab plots - tabs_components = bokeh_containers.readnoise_monitor_tabs(inst) + tabs_components = ReadNoiseFigure(inst).tab_components template = "readnoise_monitor.html" From d4db60902588ceb844ac06218274c0cb6e466b45 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 2 Oct 2023 15:50:28 -0400 Subject: [PATCH 4/8] Remove import from __init__ and updating typos for plotting, updating source for readnoise scatter --- jwql/website/apps/jwql/monitor_pages/__init__.py | 3 +-- .../apps/jwql/monitor_pages/monitor_readnoise_bokeh.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 1405523c0..8a4054c76 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,2 +1 @@ -from .monitor_cosmic_rays_bokeh import CosmicRayMonitor -from .monitor_readnoise_bokeh import ReadnoiseMonitor +from .monitor_cosmic_rays_bokeh import CosmicRayMonitor \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 5e2b0c80f..c4e4a350f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -133,7 +133,7 @@ def plot_readnoise_amplifers(self): readnoise=readnoise_vals, )) - amp_plot.add_tools(HoverTool(tooltips=[("file", "@filenames"), + amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"), ("time", "@expstarts"), ("nints", "@nints"), ("ngroups", "@ngroups"), @@ -155,7 +155,7 @@ def plot_readnoise_difference_image(self): if len(self.db.query_results) != 0: diff_image_png = self.db.query_results[-1].readnoise_diff_image - self.diff_image_plot.image_url(url=[diff_image_png], x=1, y=1, w=2048, h=2048, anchor="bottom_left") + self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") self.diff_image_plot.xaxis.visible = False self.diff_image_plot.yaxis.visible = False self.diff_image_plot.xgrid.grid_line_color = None From 385ec3d6243ffba2399d0f66951169a91c99c249 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Fri, 13 Oct 2023 11:22:39 -0400 Subject: [PATCH 5/8] Adding string slicing for path to readnoise diff image --- .../monitor_pages/monitor_readnoise_bokeh.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index c4e4a350f..551623606 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -108,14 +108,18 @@ def __init__(self, instrument, aperture): self.plot_readnoise_difference_image() self.plot_readnoise_histogram() - self.tab = Panel(child=column(row(*self.amp_plots), self.diff_image_plot, self.readnoise_histogram), title= self.aperture) + self.tab = Panel(child=column(row(*self.amp_plots), + self.diff_image_plot, + self.readnoise_histogram), + title= self.aperture) def plot_readnoise_amplifers(self): self.amp_plots = [] for amp in ['1', '2', '3', '4']: - amp_plot = figure(width=350, height=350, x_axis_type='datetime') + amp_plot = figure(title='Amp {}'.format(amp), width=280, height=280, x_axis_type='datetime') + amp_plot.xaxis[0].ticker.desired_num_ticks = 4 readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) @@ -151,10 +155,12 @@ def plot_readnoise_difference_image(self): # Update the readnoise difference image and histogram, if data exists - self.diff_image_plot = figure(height=700, width=700) + self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', + height=500, width=500, sizing_mode='scale_width') if len(self.db.query_results) != 0: diff_image_png = self.db.query_results[-1].readnoise_diff_image + diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") self.diff_image_plot.xaxis.visible = False self.diff_image_plot.yaxis.visible = False @@ -174,9 +180,10 @@ def plot_readnoise_histogram(self): hist_yr_start = diff_image_n.min() hist_yr_end = diff_image_n.max() + diff_image_n.max() * 0.05 - self.readnoise_histogram = figure(height=800, width=800, + self.readnoise_histogram = figure(height=500, width=500, x_range=(hist_xr_start, hist_xr_end), - y_range=(hist_yr_start, hist_yr_end)) + y_range=(hist_yr_start, hist_yr_end), + sizing_mode='scale_width') source = ColumnDataSource(data=dict( @@ -190,3 +197,5 @@ def plot_readnoise_histogram(self): self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]' self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels' + self.readnoise_histogram.xaxis.axis_label_text_font_size = "15pt" + self.readnoise_histogram.yaxis.axis_label_text_font_size = "15pt" From 6cdb3f7527cb680514ae04f2862992ee8eab6d07 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Fri, 13 Oct 2023 13:11:23 -0400 Subject: [PATCH 6/8] pep8speaks --- .../apps/jwql/monitor_pages/__init__.py | 2 +- .../monitor_pages/monitor_readnoise_bokeh.py | 34 +++++++++---------- jwql/website/apps/jwql/monitor_views.py | 2 +- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 8a4054c76..ed184d7ff 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1 +1 @@ -from .monitor_cosmic_rays_bokeh import CosmicRayMonitor \ No newline at end of file +from .monitor_cosmic_rays_bokeh import CosmicRayMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 551623606..7657aeeed 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -35,21 +35,21 @@ class ReadnoiseMonitorData(): """Class to hold bias data to be plotted - + Parameters ---------- - + instrument : str Instrument name (e.g. nircam) - + Attributes ---------- - + instrument : str Instrument name (e.g. nircam) aperture : str Aperture name (e.g. apername) - + """ def __init__(self, instrument, aperture): @@ -57,7 +57,6 @@ def __init__(self, instrument, aperture): self.aperture = aperture self.load_data() - def identify_tables(self): """Determine which database tables to use for the given instrument""" @@ -96,7 +95,7 @@ def __init__(self, instrument): class ReadNoisePlotTab(): - """Class to make instrument/aperture panels + """Class to make instrument/aperture panels """ def __init__(self, instrument, aperture): self.instrument = instrument @@ -108,10 +107,10 @@ def __init__(self, instrument, aperture): self.plot_readnoise_difference_image() self.plot_readnoise_histogram() - self.tab = Panel(child=column(row(*self.amp_plots), - self.diff_image_plot, - self.readnoise_histogram), - title= self.aperture) + self.tab = Panel(child=column(row(*self.amp_plots), + self.diff_image_plot, + self.readnoise_histogram), + title=self.aperture) def plot_readnoise_amplifers(self): @@ -122,7 +121,7 @@ def plot_readnoise_amplifers(self): amp_plot.xaxis[0].ticker.desired_num_ticks = 4 readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) - + filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] expstarts_iso = np.array([result.expstart for result in self.db.query_results]) expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) @@ -138,10 +137,10 @@ def plot_readnoise_amplifers(self): )) amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"), - ("time", "@expstarts"), - ("nints", "@nints"), - ("ngroups", "@ngroups"), - ("readnoise", "@readnoise")])) + ("time", "@expstarts"), + ("nints", "@nints"), + ("ngroups", "@ngroups"), + ("readnoise", "@readnoise")])) amp_plot.circle(x='expstarts', y='readnoise', source=source) @@ -155,7 +154,7 @@ def plot_readnoise_difference_image(self): # Update the readnoise difference image and histogram, if data exists - self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', + self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', height=500, width=500, sizing_mode='scale_width') if len(self.db.query_results) != 0: @@ -185,7 +184,6 @@ def plot_readnoise_histogram(self): y_range=(hist_yr_start, hist_yr_end), sizing_mode='scale_width') - source = ColumnDataSource(data=dict( x=diff_image_bin_centers, y=diff_image_n, diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 09cf423f6..3835f744b 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -69,7 +69,7 @@ def bad_pixel_monitor(request, inst): context = { 'inst': inst, - } + } return render(request, template, context) From e6e43bef746b0fabb2e2fdcf4b78439fdd7332d8 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 19 Oct 2023 15:21:28 -0400 Subject: [PATCH 7/8] Pep8 fixes --- .../jwql/monitor_pages/monitor_readnoise_bokeh.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 7657aeeed..82f0fe914 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -129,12 +129,11 @@ def plot_readnoise_amplifers(self): ngroups = [result.ngroups for result in self.db.query_results] source = ColumnDataSource(data=dict( - file=filenames, - expstarts=expstarts, - nints=nints, - ngroups=ngroups, - readnoise=readnoise_vals, - )) + file=filenames, + expstarts=expstarts, + nints=nints, + ngroups=ngroups, + readnoise=readnoise_vals)) amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"), ("time", "@expstarts"), From 33c8ddb5e5af9b3afce9d9e0fa694840495abdda Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Fri, 20 Oct 2023 11:03:29 -0400 Subject: [PATCH 8/8] Addressing Bryans comments --- .../monitor_pages/monitor_readnoise_bokeh.py | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 82f0fe914..552cac71f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -41,6 +41,8 @@ class ReadnoiseMonitorData(): instrument : str Instrument name (e.g. nircam) + aperture : str + Aperture name (e.g. apername) Attributes ---------- @@ -49,7 +51,12 @@ class ReadnoiseMonitorData(): Instrument name (e.g. nircam) aperture : str Aperture name (e.g. apername) - + query_results : list + Results from read noise statistics table based on + instrument, aperture and exposure start time + stats_table : sqlalchemy.orm.decl_api.DeclarativeMeta + Statistics table object to query based on instrument + and aperture """ def __init__(self, instrument, aperture): @@ -113,14 +120,18 @@ def __init__(self, instrument, aperture): title=self.aperture) def plot_readnoise_amplifers(self): - + """Class to create readnoise scatter plots per amplifier. + """ self.amp_plots = [] for amp in ['1', '2', '3', '4']: amp_plot = figure(title='Amp {}'.format(amp), width=280, height=280, x_axis_type='datetime') amp_plot.xaxis[0].ticker.desired_num_ticks = 4 - readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) + if self.db.query_results: + readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) + else: + readnoise_vals = np.array(list()) filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] expstarts_iso = np.array([result.expstart for result in self.db.query_results]) @@ -160,18 +171,24 @@ def plot_readnoise_difference_image(self): diff_image_png = self.db.query_results[-1].readnoise_diff_image diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") - self.diff_image_plot.xaxis.visible = False - self.diff_image_plot.yaxis.visible = False - self.diff_image_plot.xgrid.grid_line_color = None - self.diff_image_plot.ygrid.grid_line_color = None - self.diff_image_plot.title.text_font_size = '22px' - self.diff_image_plot.title.align = 'center' + + self.diff_image_plot.xaxis.visible = False + self.diff_image_plot.yaxis.visible = False + self.diff_image_plot.xgrid.grid_line_color = None + self.diff_image_plot.ygrid.grid_line_color = None + self.diff_image_plot.title.text_font_size = '22px' + self.diff_image_plot.title.align = 'center' + def plot_readnoise_histogram(self): """Updates the readnoise histogram""" - diff_image_n = np.array(self.db.query_results[-1].diff_image_n) - diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers) + if len(self.db.query_results) != 0: + diff_image_n = np.array(self.db.query_results[-1].diff_image_n) + diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers) + else: + diff_image_n = np.array(list()) + diff_image_bin_centers = np.array(list()) hist_xr_start = diff_image_bin_centers.min() hist_xr_end = diff_image_bin_centers.max()