diff --git a/CHANGES.rst b/CHANGES.rst index 960bae0e5..41343e3b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,50 @@ +0.25.0 (2020-12-31) +=================== + +New Features +------------ + +Project & API Documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Added project citation information to ``README``, along with a Zenodo badge +- Added API Documentation for ``bokeh`` templating software + +Web Application +~~~~~~~~~~~~~~~ + +- Reorganized and made aesthetic improvements to instrument landing pages to be more user-friendly +- Enabled more dynamic search options and aesthetic improvements for anomaly query webpage +- Added web app view for displaying a particular table of the ``jwqldb`` database +- Added webpage for displaying Bias Monitor results with ``bokeh`` plots + +``jwql`` Repository +~~~~~~~~~~~~~~~~~~~ + +- Changed ``utils.credentials.py`` to always authenticate a MAST user with a user-identified token in the ``config.json`` file, instead of using a cached token, which was sometimes causes errors +- Updated software to support the latest versions of ``django`` and ``bokeh`` +- Removed ``affected_tables`` column of ``monitor`` database table, as it stored redundant information +- Updated the Readnoise Monitor to work for all JWST instruments + + +Bug Fixes +--------- + + +Web Application +~~~~~~~~~~~~~~~ + +- Fixed bug in Readnoise Monitor webpage that was causing the web app and ``jwql`` database to hang + +``jwql`` Repository +~~~~~~~~~~~~~~~~~~~ + +- Fixed bug that was causing the ``test_amplifier_info()`` test in ``test_instrument_properties.py`` to fail; truth values were updated to reflect a change in the format of the returned dictionaries from the ``amplifier_info()`` function +- Fixed bug in ``get_header_info()`` that was causing ``test_data_containers.py`` to fail; the function expected the filename without the FITS extension, and the returned header info is in a dictionary (not a string) +- Fixed bug in ``test_utils.py``, and changed ``utils.py`` to make it robust in matching upper and lowercase detector names +- Updated ``utils.instrument_properties`` fix MIRI amplifier bounds calculation when omitting reference pixels + + 0.24.0 (2020-10-20) =================== @@ -15,6 +62,8 @@ Web Application - Added webpage that describes how to use the JWQL web app API - Added webpage that enables a user to query the ``jwqldb`` database contents - Enabled more search options and aesthetic improvements for anomaly query webpage +- Added webpage for displaying Readnoise Monitor results with ``bokeh`` plots +- Added webpage for displaying Bad Pixel Monitor results with ``bokeh`` plots ``jwql`` Repository ~~~~~~~~~~~~~~~~~~~ @@ -24,8 +73,6 @@ Web Application - Added unit tests for Bias Monitor - Added unit tests for Bad Pixel Monitor - Added unit tests for ``bokeh`` templating library -- Added webpage for displaying Readnoise Monitor results with ``bokeh`` plots -- Added webpage for displaying Bad Pixel Monitor results with ``bokeh`` plots Bug Fixes diff --git a/README.md b/README.md index eae697c3d..245343693 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ [![Build Status](https://ssbjenkins.stsci.edu/job/STScI/job/jwql/job/master/badge/icon)](https://ssbjenkins.stsci.edu/job/STScI/job/jwql/job/master/) [![Documentation Status](https://readthedocs.org/projects/jwql/badge/?version=latest)](https://jwql.readthedocs.io/en/latest/?badge=latest) [![STScI](https://img.shields.io/badge/powered%20by-STScI-blue.svg?colorA=707170&colorB=3e8ddd&style=flat)](http://www.stsci.edu) +[![DOI](https://zenodo.org/badge/109727729.svg)](https://zenodo.org/badge/latestdoi/109727729) [![codecov](https://codecov.io/gh/spacetelescope/jwql/branch/develop/graph/badge.svg)](https://codecov.io/gh/spacetelescope/jwql) @@ -106,6 +107,16 @@ The package should now appear if you run `conda list jwql`. Much of the `jwql` software depends on the existence of a `config.json` file within the `utils` directory. This file contains data that may be unique to users and/or contain sensitive information. Please see the [Config File wiki page](https://github.com/spacetelescope/jwql/wiki/Config-file) for instructions on how to provide this file. +## Citation + +If you use `JWQL` for work/research presented in a publication (whether directly, +or as a dependency to another package), we recommend and encourage the following acknowledgment: + + This research made use of the open source Python package JWQL (Bourque et al, 2020). + +where (Bourque et al, 2020) is a citation of the Zenodo record available using the DOI badge above. By using the `Export` box in the lower right corner of the Zenodo page, you can export the citation in the format most convenient for you. + + ## Software Contributions There are two current pages to review before you begin contributing to the `jwql` development. The first is our [style guide](https://github.com/spacetelescope/jwql/blob/master/style_guide/README.md) and the second is our [suggested git workflow page](https://github.com/spacetelescope/jwql/wiki/git-&-GitHub-workflow-for-contributing), which contains an in-depth explanation of the workflow. diff --git a/docs/source/bokeh_templating.rst b/docs/source/bokeh_templating.rst new file mode 100644 index 000000000..c62f0389a --- /dev/null +++ b/docs/source/bokeh_templating.rst @@ -0,0 +1,21 @@ +**************** +bokeh_templating +**************** + +template.py +----------------- +.. automodule:: jwql.bokeh_templating.template + :members: + :undoc-members: + +keyword_map.py +----------------- +.. automodule:: jwql.bokeh_templating.keyword_map + :members: + :undoc-members: + +factory.py +----------------- +.. automodule:: jwql.bokeh_templating.factory + :members: + :undoc-members: \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 00e9c25c3..378b3fd31 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,6 +23,7 @@ API documentation :maxdepth: 1 :caption: Contents: + bokeh_templating.rst common_monitors.rst database.rst edb.rst diff --git a/environment_python_3_6.yml b/environment_python_3_6.yml index 36d68e578..a327fe9ad 100644 --- a/environment_python_3_6.yml +++ b/environment_python_3_6.yml @@ -3,9 +3,9 @@ channels: - http://ssb.stsci.edu/astroconda dependencies: - astroquery=0.4 -- authlib=0.15.1 -- bokeh>=1.0,<1.4 -- django=2.2.5 +- authlib=0.15.2 +- bokeh=2.2.3 +- django=3.1.4 - drizzle=1.13 - flake8=3.7.9 - inflection=0.3.1 @@ -29,12 +29,13 @@ dependencies: - sphinx=3.0.3 - sqlalchemy=1.3.17 - twine=2.0.0 +- wtforms=2.3.3 - pip: - asdf==2.7.1 - - astropy==4.0.1 - - codecov==2.1.9 - - crds==10.0.0 - - cryptography==3.1.1 + - astropy==4.1 + - codecov==2.1.10 + - crds==10.1.0 + - cryptography==3.2.1 - jwedb>=0.0.3 - jwst==0.17.1 - pysqlite3==0.2.2 diff --git a/environment_python_3_7.yml b/environment_python_3_7.yml index 377685857..bbc0de909 100644 --- a/environment_python_3_7.yml +++ b/environment_python_3_7.yml @@ -3,9 +3,9 @@ channels: - http://ssb.stsci.edu/astroconda dependencies: - astroquery=0.4 -- authlib=0.15.1 -- bokeh>=1.0,<1.4 -- django=3.0.3 +- authlib=0.15.2 +- bokeh=2.2.3 +- django=3.1.4 - flake8=3.8.3 - inflection=0.3.1 - ipython=7.16.1 @@ -28,12 +28,13 @@ dependencies: - sphinx=3.1.2 - sqlalchemy=1.3.18 - twine=2.0.0 +- wtforms=2.3.3 - pip: - asdf==2.7.1 - - astropy==4.0.1.post1 - - codecov==2.1.9 - - crds==10.0.0 - - cryptography==3.1.1 + - astropy==4.1 + - codecov==2.1.10 + - crds==10.1.0 + - cryptography==3.2.1 - jwedb==0.0.6 - jwst==0.17.1 - pysqlite3==0.4.3 diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index db8bba479..ae849b19d 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -2,8 +2,9 @@ channels: - defaults - http://ssb.stsci.edu/astroconda dependencies: -- authlib=0.15.1 -- django=3.0.3 +- authlib=0.15.2 +- bokeh=2.2.3 +- django=3.1.4 - flake8=3.8.3 - inflection=0.3.1 - ipython=7.16.1 @@ -24,14 +25,14 @@ dependencies: - setuptools=49.6.0 - sphinx=3.2.1 - twine=2.0.0 +- wtforms=2.3.3 - pip: - asdf==2.7.1 - - astropy==4.0.1.post1 + - astropy==4.1 - astroquery==0.4.1 - - bokeh>=1.0,<1.4 - - codecov==2.1.9 - - crds==10.0.0 - - cryptography==3.1.1 + - codecov==2.1.10 + - crds==10.1.0 + - cryptography==3.2.1 - jwedb==0.0.6 - jwst==0.17.1 - pysiaf==0.9.0 diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 855a5c67e..63e156683 100755 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -219,7 +219,6 @@ class Monitor(base): start_time = Column(DateTime, nullable=False) end_time = Column(DateTime, nullable=True) status = Column(Enum('SUCCESS', 'FAILURE', name='monitor_status'), nullable=True) - affected_tables = Column(ARRAY(String, dimensions=1), nullable=True) log_file = Column(String(), nullable=False) @@ -423,6 +422,12 @@ class : obj NIRCamReadnoiseStats = monitor_orm_factory('nircam_readnoise_stats') NIRISSReadnoiseQueryHistory = monitor_orm_factory('niriss_readnoise_query_history') NIRISSReadnoiseStats = monitor_orm_factory('niriss_readnoise_stats') +NIRSpecReadnoiseQueryHistory = monitor_orm_factory('nirspec_readnoise_query_history') +NIRSpecReadnoiseStats = monitor_orm_factory('nirspec_readnoise_stats') +MIRIReadnoiseQueryHistory = monitor_orm_factory('miri_readnoise_query_history') +MIRIReadnoiseStats = monitor_orm_factory('miri_readnoise_stats') +FGSReadnoiseQueryHistory = monitor_orm_factory('fgs_readnoise_query_history') +FGSReadnoiseStats = monitor_orm_factory('fgs_readnoise_stats') if __name__ == '__main__': diff --git a/jwql/database/monitor_table_definitions/fgs/fgs_readnoise_query_history.txt b/jwql/database/monitor_table_definitions/fgs/fgs_readnoise_query_history.txt new file mode 100644 index 000000000..c6deea152 --- /dev/null +++ b/jwql/database/monitor_table_definitions/fgs/fgs_readnoise_query_history.txt @@ -0,0 +1,8 @@ +INSTRUMENT, string +APERTURE, string +START_TIME_MJD, float +END_TIME_MJD, float +ENTRIES_FOUND, integer +FILES_FOUND, integer +RUN_MONITOR, bool +ENTRY_DATE, datetime \ No newline at end of file diff --git a/jwql/database/monitor_table_definitions/fgs/fgs_readnoise_stats.txt b/jwql/database/monitor_table_definitions/fgs/fgs_readnoise_stats.txt new file mode 100644 index 000000000..d6cebf4c2 --- /dev/null +++ b/jwql/database/monitor_table_definitions/fgs/fgs_readnoise_stats.txt @@ -0,0 +1,35 @@ +UNCAL_FILENAME, string +APERTURE, string +DETECTOR, string +SUBARRAY, string +READ_PATTERN, string +NINTS, string +NGROUPS, string +EXPSTART, string +READNOISE_FILENAME, string +FULL_IMAGE_MEAN, float +FULL_IMAGE_STDDEV, float +FULL_IMAGE_N, float_array_1d +FULL_IMAGE_BIN_CENTERS, float_array_1d +READNOISE_DIFF_IMAGE, string +DIFF_IMAGE_MEAN, float +DIFF_IMAGE_STDDEV, float +DIFF_IMAGE_N, float_array_1d +DIFF_IMAGE_BIN_CENTERS, float_array_1d +ENTRY_DATE, datetime +AMP1_MEAN, float +AMP1_STDDEV, float +AMP1_N, float_array_1d +AMP1_BIN_CENTERS, float_array_1d +AMP2_MEAN, float +AMP2_STDDEV, float +AMP2_N, float_array_1d +AMP2_BIN_CENTERS, float_array_1d +AMP3_MEAN, float +AMP3_STDDEV, float +AMP3_N, float_array_1d +AMP3_BIN_CENTERS, float_array_1d +AMP4_MEAN, float +AMP4_STDDEV, float +AMP4_N, float_array_1d +AMP4_BIN_CENTERS, float_array_1d \ No newline at end of file diff --git a/jwql/database/monitor_table_definitions/miri/miri_readnoise_query_history.txt b/jwql/database/monitor_table_definitions/miri/miri_readnoise_query_history.txt new file mode 100644 index 000000000..c6deea152 --- /dev/null +++ b/jwql/database/monitor_table_definitions/miri/miri_readnoise_query_history.txt @@ -0,0 +1,8 @@ +INSTRUMENT, string +APERTURE, string +START_TIME_MJD, float +END_TIME_MJD, float +ENTRIES_FOUND, integer +FILES_FOUND, integer +RUN_MONITOR, bool +ENTRY_DATE, datetime \ No newline at end of file diff --git a/jwql/database/monitor_table_definitions/miri/miri_readnoise_stats.txt b/jwql/database/monitor_table_definitions/miri/miri_readnoise_stats.txt new file mode 100644 index 000000000..d6cebf4c2 --- /dev/null +++ b/jwql/database/monitor_table_definitions/miri/miri_readnoise_stats.txt @@ -0,0 +1,35 @@ +UNCAL_FILENAME, string +APERTURE, string +DETECTOR, string +SUBARRAY, string +READ_PATTERN, string +NINTS, string +NGROUPS, string +EXPSTART, string +READNOISE_FILENAME, string +FULL_IMAGE_MEAN, float +FULL_IMAGE_STDDEV, float +FULL_IMAGE_N, float_array_1d +FULL_IMAGE_BIN_CENTERS, float_array_1d +READNOISE_DIFF_IMAGE, string +DIFF_IMAGE_MEAN, float +DIFF_IMAGE_STDDEV, float +DIFF_IMAGE_N, float_array_1d +DIFF_IMAGE_BIN_CENTERS, float_array_1d +ENTRY_DATE, datetime +AMP1_MEAN, float +AMP1_STDDEV, float +AMP1_N, float_array_1d +AMP1_BIN_CENTERS, float_array_1d +AMP2_MEAN, float +AMP2_STDDEV, float +AMP2_N, float_array_1d +AMP2_BIN_CENTERS, float_array_1d +AMP3_MEAN, float +AMP3_STDDEV, float +AMP3_N, float_array_1d +AMP3_BIN_CENTERS, float_array_1d +AMP4_MEAN, float +AMP4_STDDEV, float +AMP4_N, float_array_1d +AMP4_BIN_CENTERS, float_array_1d \ No newline at end of file diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_readnoise_query_history.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_readnoise_query_history.txt new file mode 100644 index 000000000..c6deea152 --- /dev/null +++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_readnoise_query_history.txt @@ -0,0 +1,8 @@ +INSTRUMENT, string +APERTURE, string +START_TIME_MJD, float +END_TIME_MJD, float +ENTRIES_FOUND, integer +FILES_FOUND, integer +RUN_MONITOR, bool +ENTRY_DATE, datetime \ No newline at end of file diff --git a/jwql/database/monitor_table_definitions/nirspec/nirspec_readnoise_stats.txt b/jwql/database/monitor_table_definitions/nirspec/nirspec_readnoise_stats.txt new file mode 100644 index 000000000..d6cebf4c2 --- /dev/null +++ b/jwql/database/monitor_table_definitions/nirspec/nirspec_readnoise_stats.txt @@ -0,0 +1,35 @@ +UNCAL_FILENAME, string +APERTURE, string +DETECTOR, string +SUBARRAY, string +READ_PATTERN, string +NINTS, string +NGROUPS, string +EXPSTART, string +READNOISE_FILENAME, string +FULL_IMAGE_MEAN, float +FULL_IMAGE_STDDEV, float +FULL_IMAGE_N, float_array_1d +FULL_IMAGE_BIN_CENTERS, float_array_1d +READNOISE_DIFF_IMAGE, string +DIFF_IMAGE_MEAN, float +DIFF_IMAGE_STDDEV, float +DIFF_IMAGE_N, float_array_1d +DIFF_IMAGE_BIN_CENTERS, float_array_1d +ENTRY_DATE, datetime +AMP1_MEAN, float +AMP1_STDDEV, float +AMP1_N, float_array_1d +AMP1_BIN_CENTERS, float_array_1d +AMP2_MEAN, float +AMP2_STDDEV, float +AMP2_N, float_array_1d +AMP2_BIN_CENTERS, float_array_1d +AMP3_MEAN, float +AMP3_STDDEV, float +AMP3_N, float_array_1d +AMP3_BIN_CENTERS, float_array_1d +AMP4_MEAN, float +AMP4_STDDEV, float +AMP4_N, float_array_1d +AMP4_BIN_CENTERS, float_array_1d \ No newline at end of file diff --git a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py index 31076126a..e89d6bd54 100755 --- a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py +++ b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py @@ -53,16 +53,21 @@ from pysiaf import Siaf from sqlalchemy.sql.expression import and_ +from jwql.database.database_interface import FGSReadnoiseQueryHistory, FGSReadnoiseStats +from jwql.database.database_interface import MIRIReadnoiseQueryHistory, MIRIReadnoiseStats +from jwql.database.database_interface import NIRCamReadnoiseQueryHistory, NIRCamReadnoiseStats +from jwql.database.database_interface import NIRISSReadnoiseQueryHistory, NIRISSReadnoiseStats +from jwql.database.database_interface import NIRSpecReadnoiseQueryHistory, NIRSpecReadnoiseStats from jwql.database.database_interface import session -from jwql.database.database_interface import NIRCamReadnoiseQueryHistory, NIRCamReadnoiseStats, NIRISSReadnoiseQueryHistory, NIRISSReadnoiseStats from jwql.instrument_monitors import pipeline_tools from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks from jwql.utils import instrument_properties -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, initialize_instrument_monitor, update_monitor_table + class Readnoise(): """Class for executing the readnoise monitor. @@ -257,7 +262,7 @@ def image_to_png(self, image, outname): vmin, vmax = zscale.get_limits(image) # Plot the image - plt.figure(figsize=(12,12)) + plt.figure(figsize=(12, 12)) im = plt.imshow(image, cmap='gray', origin='lower', vmin=vmin, vmax=vmax) plt.colorbar(im, label='Readnoise Difference (most recent dark - reffile) [DN]') plt.title('{}'.format(outname)) @@ -376,8 +381,8 @@ def most_recent_search(self): where the readnoise monitor was run. """ - query = session.query(self.query_table).filter(and_(self.query_table.aperture==self.aperture, - self.query_table.run_monitor==True)).order_by(self.query_table.end_time_mjd).all() + query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture, + self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all() if len(query) == 0: query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 @@ -453,7 +458,8 @@ def process(self, file_list): # Find the difference between the current readnoise image and the pipeline readnoise reffile, and record image stats. # Sometimes, the pipeline readnoise reffile needs to be cutout to match the subarray. - pipeline_readnoise = pipeline_readnoise[self.substrt2-1:self.substrt2+self.subsize2-1, self.substrt1-1:self.substrt1+self.subsize1-1] + if readnoise.shape != pipeline_readnoise.shape: + pipeline_readnoise = pipeline_readnoise[self.substrt2 - 1:self.substrt2 + self.subsize2 - 1, self.substrt1 - 1:self.substrt1 + self.subsize1 - 1] readnoise_diff = readnoise - pipeline_readnoise clipped = sigma_clip(readnoise_diff, sigma=3.0, maxiters=5) diff_image_mean, diff_image_stddev = np.nanmean(clipped), np.nanstd(clipped) @@ -518,7 +524,7 @@ def run(self): self.query_end = Time.now().mjd # Loop over all instruments - for instrument in ['nircam', 'niriss']: + for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use @@ -575,7 +581,7 @@ def run(self): logging.info('\t{} does not exist in JWQL filesystem, even though {} does'.format(uncal_filename, filename)) else: num_groups = fits.getheader(uncal_filename)['NGROUPS'] - if num_groups > 1: # skip processing if the file doesnt have enough groups to calculate the readnoise; TODO change to 10 before incorporating MIRI + if num_groups > 10: # skip processing if the file doesnt have enough groups to calculate the readnoise shutil.copy(uncal_filename, self.data_dir) logging.info('\tCopied {} to {}'.format(uncal_filename, output_filename)) set_permissions(output_filename) @@ -607,6 +613,7 @@ def run(self): logging.info('Readnoise Monitor completed successfully.') + if __name__ == '__main__': module = os.path.basename(__file__).strip('.py') diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index e97667100..07fd45380 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -101,8 +101,8 @@ def test_get_filenames_by_rootname(): def test_get_header_info(): """Tests the ``get_header_info`` function.""" - header = data_containers.get_header_info('jw86600008001_02101_00007_guider2_uncal.fits') - assert isinstance(header, str) + header = data_containers.get_header_info('jw86600008001_02101_00007_guider2_uncal') + assert isinstance(header, dict) assert len(header) > 0 diff --git a/jwql/tests/test_instrument_properties.py b/jwql/tests/test_instrument_properties.py index 7f18e6cbd..dc074ead2 100644 --- a/jwql/tests/test_instrument_properties.py +++ b/jwql/tests/test_instrument_properties.py @@ -38,32 +38,34 @@ def test_amplifier_info(): data_dir = os.path.join(get_config()['test_dir'], 'dark_monitor') fullframe = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_ff.fits')) - fullframe_truth = (4, {'1': [(4, 4), (512, 2044)], - '2': [(512, 4), (1024, 2044)], - '3': [(1024, 4), (1536, 2044)], - '4': [(1536, 4), (2044, 2044)]}) + fullframe_truth = (4, {'1': [(4, 512, 1), (4, 2044, 1)], + '2': [(512, 1024, 1), (4, 2044, 1)], + '3': [(1024, 1536, 1), (4, 2044, 1)], + '4': [(1536, 2044, 1), (4, 2044, 1)]}) + assert fullframe == fullframe_truth fullframe = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_ff.fits'), omit_reference_pixels=False) - fullframe_truth = (4, {'1': [(0, 0), (512, 2048)], - '2': [(512, 0), (1024, 2048)], - '3': [(1024, 0), (1536, 2048)], - '4': [(1536, 0), (2048, 2048)]}) + fullframe_truth = (4, {'1': [(0, 512, 1), (0, 2048, 1)], + '2': [(512, 1024, 1), (0, 2048, 1)], + '3': [(1024, 1536, 1), (0, 2048, 1)], + '4': [(1536, 2048, 1), (0, 2048, 1)]}) + assert fullframe == fullframe_truth subarray = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_1.fits')) - subarray_truth = (1, {'1': [(0, 0), (10, 10)]}) + subarray_truth = (1, {'1': [(0, 10, 1), (0, 10, 1)]}) assert subarray == subarray_truth subarray_one = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_grismstripe_one_amp.fits')) - subarray_one_truth = (1, {'1': [(4, 4), (2044, 64)]}) + subarray_one_truth = (1, {'1': [(4, 2044, 1), (4, 64, 1)]}) assert subarray_one == subarray_one_truth subarray_four = instrument_properties.amplifier_info(os.path.join(data_dir, 'test_image_grismstripe_four_amp.fits')) - subarray_four_truth = (4, {'1': [(4, 4), (512, 64)], - '2': [(512, 4), (1024, 64)], - '3': [(1024, 4), (1536, 64)], - '4': [(1536, 4), (2044, 64)]}) + subarray_four_truth = (4, {'1': [(4, 512, 1), (4, 64, 1)], + '2': [(512, 1024, 1), (4, 64, 1)], + '3': [(1024, 1536, 1), (4, 64, 1)], + '4': [(1536, 2044, 1), (4, 64, 1)]}) assert subarray_four == subarray_four_truth diff --git a/jwql/utils/anomaly_query_config.py b/jwql/utils/anomaly_query_config.py index ed8f2ce94..67377cb23 100644 --- a/jwql/utils/anomaly_query_config.py +++ b/jwql/utils/anomaly_query_config.py @@ -26,20 +26,29 @@ # Default is all anomalies common to all instruments CURRENT_ANOMALIES = {} +# Observing modes selected by user in anomaly_query +DETECTORS_CHOSEN = {} + # Maximum exposure time selected by user in anomaly_query. Corresponds to EFFEXPTM in MAST. EXPTIME_MAX = ['999999999999999'] # select all as default # Minimum exposure time selected by user in anomaly_query. Corresponds to EFFEXPTM in MAST. EXPTIME_MIN = ['0'] # select all as default +# Exposure types selected by user in anomaly_query +EXPTYPES_CHOSEN = {} + # Filters selected by user in anomaly_query FILTERS_CHOSEN = {} +# Gratings selected by user in anomaly_query +GRATINGS_CHOSEN = {} + # Instruments selected by user in anomaly_query INSTRUMENTS_CHOSEN = [] -# Observing modes selected by user in anomaly_query -OBSERVING_MODES_CHOSEN = {} +# Read patterns selected by user in anomaly_query +READPATTS_CHOSEN = {} # Thumbnails selected by user in anomaly_query THUMBNAILS = [] diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 28abec2d3..eae73282f 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -59,7 +59,7 @@ 'excessive_saturation': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'], 'guidestar_failure': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'], 'persistence': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'], - #anomalies affecting multiple instruments: + # anomalies affecting multiple instruments: 'crosstalk': ['fgs', 'nircam', 'niriss', 'nirspec'], 'data_transfer_error': ['fgs', 'nircam', 'niriss', 'nirspec'], 'ghost': ['fgs', 'nircam', 'niriss', 'nirspec'], @@ -75,34 +75,44 @@ # additional anomalies: 'other': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec']} -# Defines the possible anomalies (with rendered name) to flag through the web app +# Defines the possible anomalies to flag through the web app ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) if anomaly != "dominant_msa_leakage" - else (anomaly, "Dominant MSA Leakage") - for anomaly in ANOMALIES_PER_INSTRUMENT] + else (anomaly, "Dominant MSA Leakage") + for anomaly in ANOMALIES_PER_INSTRUMENT] ANOMALY_CHOICES_FGS = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT - if 'fgs' in ANOMALIES_PER_INSTRUMENT[anomaly]] + if 'fgs' in ANOMALIES_PER_INSTRUMENT[anomaly]] ANOMALY_CHOICES_MIRI = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT - if 'miri' in ANOMALIES_PER_INSTRUMENT[anomaly]] + if 'miri' in ANOMALIES_PER_INSTRUMENT[anomaly]] ANOMALY_CHOICES_NIRCAM = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT - if 'nircam' in ANOMALIES_PER_INSTRUMENT[anomaly]] + if 'nircam' in ANOMALIES_PER_INSTRUMENT[anomaly]] ANOMALY_CHOICES_NIRISS = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT - if 'niriss' in ANOMALIES_PER_INSTRUMENT[anomaly]] + if 'niriss' in ANOMALIES_PER_INSTRUMENT[anomaly]] ANOMALY_CHOICES_NIRSPEC = [(anomaly, inflection.titleize(anomaly)) if anomaly != "dominant_msa_leakage" - else (anomaly, "Dominant MSA Leakage") - for anomaly in ANOMALIES_PER_INSTRUMENT - if 'nirspec' in ANOMALIES_PER_INSTRUMENT[anomaly]] + else (anomaly, "Dominant MSA Leakage") + for anomaly in ANOMALIES_PER_INSTRUMENT + if 'nirspec' in ANOMALIES_PER_INSTRUMENT[anomaly]] ANOMALY_CHOICES_PER_INSTRUMENT = {'fgs': ANOMALY_CHOICES_FGS, 'miri': ANOMALY_CHOICES_MIRI, 'nircam': ANOMALY_CHOICES_NIRCAM, 'niriss': ANOMALY_CHOICES_NIRISS, 'nirspec': ANOMALY_CHOICES_NIRSPEC - } + } + +APERTURES_PER_INSTRUMENT = {'NIRCAM': ['NRCA1_FULL', 'NRCA2_FULL', 'NRCA3_FULL', 'NRCA4_FULL', + 'NRCA5_FULL', 'NRCB1_FULL', 'NRCB2_FULL', 'NRCB3_FULL', + 'NRCB4_FULL', 'NRCB5_FULL'], + 'NIRISS': ['NIS_CEN', 'NIS_SOSSFULL', 'NIS_AMIFULL', 'NIS_SOSSTA', 'NIS_AMI1', + 'NIS_SUBSTRIP256', 'NIS_SUBSTRIP96'], + 'NIRSPEC': ['NRS1_FULL', 'NRS2_FULL', 'NRS_FULL_MSA', 'NRS_FULL_IFU', + 'NRS_S200A1_SLIT', 'NRS_S200A2_SLIT', 'NRS_S1600A1_SLIT'], + 'MIRI': ['MIRIM_FULL', 'MIRIM_MASKLYOT', 'MIRIM_SLITLESSPRISM', 'MIRIM_SUB256', + 'MIRIM_SUB128', 'MIRIM_SLIT']} # Bad pixel types by the type of data used to find them BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL'] @@ -116,10 +126,39 @@ 'nirspec': ['NRS_DARK'], 'fgs': ['FGS_DARK']} +# Dictionary of observing modes available for each instrument +DETECTOR_PER_INSTRUMENT = {'fgs': ['FGS_DARK', 'FGS_FOCUS', 'FGS_IMAGE', + 'FGS_INTFLAT', 'FGS_SKYFLAT'], + 'miri': ['MIRIFULONG', 'MIRIFUSHORT', 'MIRIMAGE'], + 'nircam': ['NRCB4', 'NRCA4', 'NRCA2', 'NRCALONG', + 'NRCBLONG', 'NRCB2', 'NRCB3', 'NRCA1', + 'NRCA3', 'NRCB1'], + 'niriss': ['NIS'], + 'nirspec': ['NRS1', 'NRS2']} + +EXP_TYPE_PER_INSTRUMENT = {'fgs': [], + 'miri': ['MIR_FLATMRS', 'MIR_MRS', 'MIR_FLATIMAGE', + 'MIR_DARK', 'MIR_LYOT', 'MIR_IMAGE', + 'MIR_LRS-FIXEDSLIT', 'MIR_LRS-SLITLESS', + 'MIR_CORONCAL', 'MIR_4QPM', 'MIR_FLATIMAGE-EXT', + 'MIR_TACQ', 'MIR_DARKMRS', + 'MIR_DARKIMG', 'MIR_FLATMRS-EXT', 'MIR_TACONFIRM'], + 'nircam': ['NRC_LED', 'NRC_DARK', 'NRC_CORON', + 'NRC_IMAGE', 'NRC_FOCUS', 'NRC_TSGRISM', + 'NRC_TSIMAGE', 'NRC_WFSS', 'NRC_TACQ', + 'NRC_TACONFIRM', 'NRC_FLAT', 'NRC_GRISM'], + 'niriss': ['NIS_IMAGE', 'NIS_FOCUS', 'NIS_SOSS', + 'NIS_AMI', 'NIS_LAMP', 'NIS_WFSS', 'NIS_DARK', + 'NIS_EXTCAL', 'NIS_TACONFIRM', 'NIS_TACQ'], + 'nirspec': ['NRS_IFU', 'NRS_MSASPEC', 'NRS_BRIGHTOBJ', 'NRS_DARK', + 'NRS_AUTOWAVE', 'NRS_LAMP', 'NRS_AUTOFLAT', 'NRS_IMAGE', + 'NRS_CONFIRM', 'NRS_FIXEDSLIT', 'NRS_MIMF', 'NRS_FOCUS', + 'NRS_TACONFIRM', 'NRS_WATA', 'NRS_MSATA']} + EXPTYPES = {"nircam": {"imaging": "NRC_IMAGE", "ts_imaging": "NRC_TSIMAGE", "wfss": "NRC_WFSS", "ts_grism": "NRC_TSGRISM"}, - "niriss": {"imaging": "NIS_IMAGE", "ami": "NIS_IMAGE", "pom": "NIS_IMAGE", - "wfss": "NIS_WFSS"}, + "niriss": {"imaging": "NIS_IMAGE", "ami": "NIS_IMAGE", + "pom": "NIS_IMAGE", "wfss": "NIS_WFSS"}, "fgs": {"imaging": "FGS_IMAGE"}} FLAT_EXP_TYPES = {'nircam': ['NRC_FLAT'], @@ -128,16 +167,16 @@ 'nirspec': ['NRS_AUTOFLAT', 'NRS_LAMP'], 'fgs': ['FGS_INTFLAT']} -FILTERS_PER_INSTRUMENT = {'miri': ['F560W', 'F770W', 'F1000W', 'F1065C', 'F1130W', 'F1140C', 'F1280W', - 'F1500W', 'F1550C', 'F1800W', 'F2100W', 'F2300C', 'F2550W'], - 'nircam': ['F070W', 'F090W', 'F115W', 'F140M', 'F150W', 'F150W2', 'F162M', - 'F164N', 'F182M', 'F187N', 'F200W', 'F210M', 'F212N', 'F250M', - 'F277W', 'F300M', 'F322W2', 'F323N', 'F335M', 'F356W', 'F360M', - 'F405N', 'F410M', 'F430M', 'F444W', 'F460M', 'F466N', 'F470N', - 'F480M'], - 'niriss': ['F090W', 'F115W', 'F140M', 'F150W', 'F185M', 'F200W', 'F227W', - 'F356W', 'F380M', 'F430M', 'F444W', 'F480M'], - 'nirspec': ['CLEAR', 'F070LP', 'F100LP', 'F170LP', 'F290LP']} +FILTERS_PER_INSTRUMENT = {'miri': ['F1000W', 'F1130W', 'F1280W', 'OPAQUE', 'F2300C', 'F560W', 'P750L', + 'F1500W', 'F2550W', 'F770W', 'FLENS', 'FND', 'F2100W', 'F1800W', + 'F1550C', 'F1140C', 'F2550WR', 'F1065C'], + 'nircam': ['F070W', 'F090W', 'F115W', 'F140M', 'F150W', 'F150W2', 'F182M', + 'F187N', 'F200W', 'F210M', 'F212N', 'F250M', 'F277W', 'F300M', + 'F322W2', 'F335M', 'F356W', 'F360M', 'F410M', 'F430M', 'F444W', + 'F460M', 'F480M'], + 'niriss': ['CLEAR', 'F380M', 'F480M', 'GR150R', 'F430M', 'GR150C', 'F444W', + 'F356W', 'F277W'], + 'nirspec': ['F290LP', 'F170LP', 'OPAQUE', 'F100LP', 'F070LP', 'F140X', 'CLEAR', 'F110W']} FOUR_AMP_SUBARRAYS = ['WFSS128R', 'WFSS64R'] @@ -156,6 +195,14 @@ 'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints', 'crf', 'ramp', 'fitopt', 'bsubints', 'bsub', 'cat'] +# Gratings available for each instrument +GRATING_PER_INSTRUMENT = {'miri': [], + 'nircam': [], + 'niriss': [], + 'nirspec': ['G140M', 'G235M', 'G395M', 'G140H', + 'G235H', 'G395H', 'PRISM'] + } + # Possible suffix types for guider exposures GUIDER_SUFFIX_TYPES = ['stream', 'stacked_uncal', 'image_uncal', 'stacked_cal', 'image_cal'] @@ -195,17 +242,19 @@ # Available monitor names and their location for each JWST instrument MONITORS = { - 'fgs': [('Bad Pixel Monitor', '/fgs/bad_pixel_monitor')], + 'fgs': [('Bad Pixel Monitor', '/fgs/bad_pixel_monitor'), + ('Readnoise Monitor', '/fgs/readnoise_monitor')], 'miri': [('Dark Current Monitor', '#'), ('Data Trending', '/miri/miri_data_trending'), ('Bad Pixel Monitor', '/miri/bad_pixel_monitor'), + ('Readnoise Monitor', '/miri/readnoise_monitor'), ('Cosmic Ray Monitor', '#'), ('Photometry Monitor', '#'), ('TA Failure Monitor', '#'), ('Blind Pointing Accuracy Monitor', '#'), ('Filter and Calibration Lamp Monitor', '#'), ('Thermal Emission Monitor', '#')], - 'nircam': [('Bias Monitor', '#'), + 'nircam': [('Bias Monitor', '/nircam/bias_monitor'), ('Readnoise Monitor', '/nircam/readnoise_monitor'), ('Gain Level Monitor', '#'), ('Mean Dark Current Rate Monitor', '/nircam/dark_monitor'), @@ -217,6 +266,7 @@ ('TSO RMS Monitor', '#')], 'nirspec': [('Optical Short Monitor', '#'), ('Bad Pixel Monitor', '/nirspec/bad_pixel_monitor'), + ('Readnoise Monitor', '/nirspec/readnoise_monitor'), ('Target Acquisition Monitor', '#'), ('Data Trending', '/nirspec/nirspec_data_trending'), ('Detector Health Monitor', '#'), @@ -244,20 +294,12 @@ # Possible suffix types for AMI files NIRISS_AMI_SUFFIX_TYPES = ['amiavg', 'aminorm', 'ami'] -# Dictionary of observing modes available for each instrument -OBSERVING_MODE_PER_INSTRUMENT = {'fgs': ['FGS_DARK', 'FGS_FOCUS', 'FGS_IMAGE', 'FGS_INTFLAT', 'FGS_SKYFLAT'], - 'miri': ['MIR_IMAGE', 'MIR_TACQ', 'MIR_LYOT', 'MIR_4QPM', 'MIR_LRS-FIXEDSLIT', - 'MIR_LRS-SLITLESS', 'MIR_MRS', 'MIR_DARKIMG', 'MIR_DARKMRS', 'MIR_DARKALL', - 'MIR_FLATIMAGE', 'MIR_FLATMRS', 'MIR_CORONCAL'], - 'nircam': ['NRC_IMAGE', 'NRC_WFSS', 'NRC_TACQ', 'NRC_CORON', 'NRC_FOCUS', 'NRC_DARK', - 'NRC_FLAT', 'NRC_GRISM', 'NRC_LED', 'NRC_TSIMAGE', 'NRC_TSGRISM', - 'NRC_TACONFIRM', 'NRC_WFSC'], - 'niriss': ['NIS_AMI', 'NIS_DARK', 'NIS_EXTCAL', 'NIS_FOCUS', 'NIS_IMAGE', 'NIS_LAMP', - 'NIS_SOSS', 'NIS_WFSS', 'NIS_TACQ', 'NIS_TACONFIRM'], - 'nirspec': ['NRS_AUTOFLAT', 'NRS_AUTOWAVE', 'NRS_BRIGHTOBJ', 'NRS_CONFIRM', - 'NRS_DARK', 'NRS_FIXEDSLIT', 'NRS_FOCUS', 'NRS_IFU', 'NRS_IMAGE', - 'NRS_LAMP', 'NRS_MIMF', 'NRS_MSASPEC', 'NRS_TACONFIRM', 'NRS_TACQ', - 'NRS_TASLIT']} +READPATT_PER_INSTRUMENT = {'fgs': [], + 'miri': ['FAST', 'SLOW', 'FASTGRPAVG'], + 'nircam': ['RAPID', 'SHALLOW2', 'BRIGHT2', 'MEDIUM2', 'SHALLOW4', + 'MEDIUM8', 'BRIGHT1', 'DEEP2', 'DEEP8'], + 'niriss': ['NISRAPID', 'NIS'], + 'nirspec': ['NRS', 'NRSRAPID', 'NRSRAPIDD2', 'NRSRAPIDD6']} SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] diff --git a/jwql/utils/credentials.py b/jwql/utils/credentials.py index 3677cf429..6ae7e631e 100644 --- a/jwql/utils/credentials.py +++ b/jwql/utils/credentials.py @@ -41,11 +41,6 @@ def get_mast_token(request=None): print('Authenticated with Astroquery MAST magic') return None else: - if request is not None: - token = str(request.POST.get('access_token')) - if token != 'None': - print('Authenticated with cached MAST token.') - return token try: # check if token is available via config file check_config_for_key('mast_token') diff --git a/jwql/utils/instrument_properties.py b/jwql/utils/instrument_properties.py index 083bb6dae..918991b70 100644 --- a/jwql/utils/instrument_properties.py +++ b/jwql/utils/instrument_properties.py @@ -140,12 +140,16 @@ def amplifier_info(filename, omit_reference_pixels=True): # Adjust the minimum and maximum x and y values if they are within # the reference pixels - for key in amp_bounds: + for i,key in enumerate(amp_bounds): bounds = amp_bounds[key] prev_xmin, prev_xmax, prev_xstep = bounds[0] prev_ymin, prev_ymax, prev_ystep = bounds[1] if prev_xmin < xmin: - new_xmin = xmin + # Ensure the x start values remain interleaved for MIRI + if instrument.lower() == 'miri': + new_xmin = xmin + i + else: + new_xmin = xmin else: new_xmin = prev_xmin if prev_ymin < ymin: diff --git a/jwql/utils/monitor_utils.py b/jwql/utils/monitor_utils.py index 6f4126430..fa118bc64 100644 --- a/jwql/utils/monitor_utils.py +++ b/jwql/utils/monitor_utils.py @@ -64,7 +64,6 @@ def update_monitor_table(module, start_time, log_file): new_entry['start_time'] = start_time new_entry['end_time'] = datetime.datetime.now() new_entry['status'] = get_log_status(log_file) - new_entry['affected_tables'] = INSTRUMENT_MONITOR_DATABASE_TABLES[module] new_entry['log_file'] = os.path.basename(log_file) Monitor.__table__.insert().execute(new_entry) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index af8da56e0..fcdf0f355 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -311,7 +311,7 @@ def filename_parser(filename): filename_dict['instrument'] = 'fgs' elif 'detector' in filename_dict.keys(): filename_dict['instrument'] = JWST_INSTRUMENT_NAMES_SHORTHAND[ - filename_dict['detector'][:3] + filename_dict['detector'][:3].lower() ] # Raise error if unable to parse the filename diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py index fd2e85fa7..5283d2d80 100644 --- a/jwql/website/apps/jwql/bokeh_containers.py +++ b/jwql/website/apps/jwql/bokeh_containers.py @@ -110,6 +110,68 @@ def bad_pixel_monitor_tabs(instrument): return div, script +def bias_monitor_tabs(instrument): + """Creates the various tabs of the bias monitor results page. + + Parameters + ---------- + instrument : str + The JWST instrument of interest (e.g. ``nircam``). + + Returns + ------- + div : str + The HTML div to render bias monitor plots + script : str + The JS script to render bias monitor plots + """ + + # Make a separate tab for each aperture + tabs = [] + for aperture in FULL_FRAME_APERTURES[instrument.upper()]: + monitor_template = monitor_pages.BiasMonitor() + monitor_template.input_parameters = (instrument, aperture) + + # Add the mean bias vs time plots for each amp and odd/even columns + plots = [] + for amp in ['1', '2', '3', '4']: + for kind in ['even', 'odd']: + bias_plot = monitor_template.refs['mean_bias_figure_amp{}_{}'.format(amp, kind)] + bias_plot.sizing_mode = 'scale_width' # Make sure the sizing is adjustable + plots.append(bias_plot) + + # Add the calibrated 0th group image + calibrated_image = monitor_template.refs['cal_image'] + calibrated_image.sizing_mode = 'scale_width' + plots.append(calibrated_image) + + # Add the collapsed row/column plots + for direction in ['rows', 'columns']: + collapsed_plot = monitor_template.refs['collapsed_{}_figure'.format(direction)] + collapsed_plot.sizing_mode = 'scale_width' + plots.append(collapsed_plot) + + # Put the mean bias plots on the top 2 rows, the calibrated image on the + # third row, and the collapsed row/column plots on the bottom row. + bias_layout = layout( + plots[0:8][::2], + plots[0:8][1::2], + plots[8:9], + plots[9:11] + ) + bias_layout.sizing_mode = 'scale_width' + bias_tab = Panel(child=bias_layout, title=aperture) + tabs.append(bias_tab) + + # Build tabs + tabs = Tabs(tabs=tabs) + + # Return tab HTML and JavaScript to web app + script, div = components(tabs) + + return div, script + + def dark_monitor_tabs(instrument): """Creates the various tabs of the dark monitor results page. @@ -262,7 +324,7 @@ def readnoise_monitor_tabs(instrument): # histogram on the second row. readnoise_layout = layout( plots[0:4], - plots[4:6], + plots[4:6] ) readnoise_layout.sizing_mode = 'scale_width' # Make sure the sizing is adjustable readnoise_tab = Panel(child=readnoise_layout, title=aperture) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c235bba3e..19095bfd3 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -556,13 +556,13 @@ def get_header_info(filename): Parameters ---------- filename : str - The name of the file of interest (e.g. - ``'jw86600008001_02101_00007_guider2_uncal.fits'``). + The name of the file of interest, without the extension + (e.g. ``'jw86600008001_02101_00007_guider2_uncal'``). Returns ------- - header : str - The primary FITS header for the given ``file``. + header_info : dict + The FITS headers of the extensions in the given ``file``. """ # Initialize dictionary to store header information @@ -881,7 +881,7 @@ def get_thumbnails_all_instruments(parameters): instruments apertures filters - observing_modes + detector effexptm_min effexptm_max anomalies @@ -893,14 +893,13 @@ def get_thumbnails_all_instruments(parameters): given instrument. """ - filters = parameters['filters'] - effexptm_min = parameters['exposure_time_min'] - effexptm_max = parameters['exposure_time_max'] anomalies = parameters['anomalies'] thumbnail_list = [] filenames = [] + if parameters['instruments'] is None: + thumbnails = [] for inst in parameters['instruments']: print("Retrieving thumbnails for", inst) # Make sure instruments are of the proper format (e.g. "Nircam") @@ -911,90 +910,28 @@ def get_thumbnails_all_instruments(parameters): params = {"columns": "*", "filters": [{"paramName": "apername", - "values": [parameters['apertures'][inst.lower()]]}]} + "values": parameters['apertures'][inst.lower()] + }, + {"paramName": "detector", + "values": parameters['detectors'][inst.lower()] + }, + {"paramName": "filter", + "values": parameters['filters'][inst.lower()] + }, + {"paramName": "exp_type", + "values": parameters['exposure_types'][inst.lower()] + }, + {"paramName": "readpatt", + "values": parameters['read_patterns'][inst.lower()] + } + ]} response = Mast.service_request_async(service, params) results = response[0].json()['data'] - # Further filter results and parse to get rootnames for result in results: - if parameters['observing_modes'][inst.lower()]: - if result['exp_type'] in parameters['observing_modes'][inst.lower()]: - if effexptm_max: - if result['effexptm'] < int(effexptm_max): - if effexptm_min: - if result['effexptm'] > int(effexptm_min): - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - if effexptm_min: - if result['effexptm'] > int(effexptm_min): - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - if effexptm_max: - if result['effexptm'] < int(effexptm_max): - if effexptm_min: - if result['effexptm'] > int(effexptm_min): - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - if effexptm_min: - if result['effexptm'] > int(effexptm_min): - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - if filters: - if result['filter'] in filters[inst.lower()]: - filename = result['filename'].split('.')[0] - filenames.append(filename) - else: - filename = result['filename'].split('.')[0] - filenames.append(filename) + filename = result['filename'].split('.')[0] + filenames.append(filename) # Get list of all thumbnails thumbnails = glob.glob(os.path.join(THUMBNAIL_FILESYSTEM, '*', '*.thumb')) @@ -1031,6 +968,7 @@ def get_thumbnails_all_instruments(parameters): final_subset.append(thumbnail) except KeyError: print("Error with thumbnail: ", thumbnail) + if not final_subset: print("No images matched anomaly selection") final_subset = thumbnails_subset @@ -1312,7 +1250,14 @@ def thumbnails_query_ajax(rootnames, insts): # Extract information for sorting with dropdown menus detectors = [data_dict['file_data'][rootname]['filename_dict']['detector'] for rootname in list(data_dict['file_data'].keys())] - dropdown_menus = {'detector': detectors} + instruments = [data_dict['file_data'][rootname]['inst'].lower() for + rootname in list(data_dict['file_data'].keys())] + proposals = [data_dict['file_data'][rootname]['filename_dict']['program_id'] for + rootname in list(data_dict['file_data'].keys())] + + dropdown_menus = {'instrument': instruments, + 'detector': detectors, + 'proposal': proposals} data_dict['tools'] = MONITORS data_dict['dropdown_menus'] = dropdown_menus diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index 30bfc33be..be4e68fd5 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -53,14 +53,17 @@ def view_function(request): from jwql.database import database_interface as di from jwql.utils.constants import ANOMALY_CHOICES +from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT from jwql.utils.constants import ANOMALIES_PER_INSTRUMENT +from jwql.utils.constants import APERTURES_PER_INSTRUMENT +from jwql.utils.constants import DETECTOR_PER_INSTRUMENT +from jwql.utils.constants import EXP_TYPE_PER_INSTRUMENT from jwql.utils.constants import FILTERS_PER_INSTRUMENT -from jwql.utils.constants import FULL_FRAME_APERTURES from jwql.utils.constants import GENERIC_SUFFIX_TYPES +from jwql.utils.constants import GRATING_PER_INSTRUMENT from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.constants import JWST_INSTRUMENT_NAMES_SHORTHAND -from jwql.utils.constants import OBSERVING_MODE_PER_INSTRUMENT -from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT +from jwql.utils.constants import READPATT_PER_INSTRUMENT from jwql.utils.utils import get_config, filename_parser from jwql.utils.utils import query_format @@ -85,137 +88,87 @@ class AnomalyQueryForm(BaseForm): # Form submits calculate_submit = SubmitField() - # Generate dynamic lists of apertures to use in forms - aperture_list = [] - for instrument in FULL_FRAME_APERTURES.keys(): - for aperture in FULL_FRAME_APERTURES[instrument]: - item = [query_format(aperture), query_format(aperture)] - aperture_list.append(item) - - miri_aperture_list = [] - for aperture in FULL_FRAME_APERTURES['MIRI']: - miri_aperture_list.append([query_format(aperture), query_format(aperture)]) - - nirspec_aperture_list = [] - for aperture in FULL_FRAME_APERTURES['NIRSPEC']: - nirspec_aperture_list.append([query_format(aperture), query_format(aperture)]) - - nircam_aperture_list = [] - for aperture in FULL_FRAME_APERTURES['NIRCAM']: - nircam_aperture_list.append([query_format(aperture), query_format(aperture)]) - - niriss_aperture_list = [] - for aperture in FULL_FRAME_APERTURES['NIRISS']: - niriss_aperture_list.append([query_format(aperture), query_format(aperture)]) - - # Generate dynamic lists of filters to use in forms - filter_list = [] - for instrument in FILTERS_PER_INSTRUMENT.keys(): - # # if instrument in anomaly_query_config.INSTRUMENTS_CHOSEN: # eg ['nirspec']: selects relevant filters, but not specific to chosen instruments - filters_per_inst = FILTERS_PER_INSTRUMENT[instrument] - for filt in filters_per_inst: + # Generate lists of form options for each instrument + params = {} + for instrument in ['miri', 'niriss', 'nircam', 'nirspec']: + params[instrument] = {} + params[instrument]['aperture_list'] = [] + params[instrument]['filter_list'] = [] + params[instrument]['detector_list'] = [] + params[instrument]['readpatt_list'] = [] + params[instrument]['exptype_list'] = [] + params[instrument]['grating_list'] = [] + params[instrument]['anomalies_list'] = [] + # Generate dynamic lists of apertures to use in forms + for aperture in APERTURES_PER_INSTRUMENT[instrument.upper()]: + params[instrument]['aperture_list'].append([query_format(aperture), query_format(aperture)]) + # Generate dynamic lists of filters to use in forms + for filt in FILTERS_PER_INSTRUMENT[instrument]: filt = query_format(filt) - filter_list.append([filt, filt]) if [filt, filt] not in filter_list else filter_list - - miri_filter_list = [] - for filt in FILTERS_PER_INSTRUMENT['miri']: - filt = query_format(filt) - miri_filter_list.append([filt, filt]) - - nirspec_filter_list = [] - for filt in FILTERS_PER_INSTRUMENT['nirspec']: - filt = query_format(filt) - nirspec_filter_list.append([filt, filt]) - - niriss_filter_list = [] - for filt in FILTERS_PER_INSTRUMENT['niriss']: - filt = query_format(filt) - niriss_filter_list.append([filt, filt]) - - nircam_filter_list = [] - for filt in FILTERS_PER_INSTRUMENT['nircam']: - filt = query_format(filt) - nircam_filter_list.append([filt, filt]) - - # Generate dynamic lists of observing modes to use in forms - miri_obsmode_list = [] - for obsmode in OBSERVING_MODE_PER_INSTRUMENT['miri']: - obsmode = query_format(obsmode) - miri_obsmode_list.append([obsmode, obsmode]) - - niriss_obsmode_list = [] - for obsmode in OBSERVING_MODE_PER_INSTRUMENT['niriss']: - obsmode = query_format(obsmode) - niriss_obsmode_list.append([obsmode, obsmode]) - - nircam_obsmode_list = [] - for obsmode in OBSERVING_MODE_PER_INSTRUMENT['nircam']: - obsmode = query_format(obsmode) - nircam_obsmode_list.append([obsmode, obsmode]) - - nirspec_obsmode_list = [] - for obsmode in OBSERVING_MODE_PER_INSTRUMENT['nirspec']: - obsmode = query_format(obsmode) - nirspec_obsmode_list.append([obsmode, obsmode]) - - # Generate dynamic lists of anomalies to use in forms - miri_anomalies_list = [] - for anomaly in ANOMALIES_PER_INSTRUMENT.keys(): - if 'miri' in ANOMALIES_PER_INSTRUMENT[anomaly]: - item = [query_format(anomaly), query_format(anomaly)] - miri_anomalies_list.append(item) - - nircam_anomalies_list = [] - for anomaly in ANOMALIES_PER_INSTRUMENT.keys(): - if 'nircam' in ANOMALIES_PER_INSTRUMENT[anomaly]: - item = [query_format(anomaly), query_format(anomaly)] - nircam_anomalies_list.append(item) - - niriss_anomalies_list = [] - for anomaly in ANOMALIES_PER_INSTRUMENT.keys(): - if 'niriss' in ANOMALIES_PER_INSTRUMENT[anomaly]: - item = [query_format(anomaly), query_format(anomaly)] - niriss_anomalies_list.append(item) - - nirspec_anomalies_list = [] - for anomaly in ANOMALIES_PER_INSTRUMENT.keys(): - if 'nirspec' in ANOMALIES_PER_INSTRUMENT[anomaly]: - item = [query_format(anomaly), query_format(anomaly)] - nirspec_anomalies_list.append(item) + params[instrument]['filter_list'].append([filt, filt]) + # Generate dynamic lists of detectors to use in forms + for detector in DETECTOR_PER_INSTRUMENT[instrument]: + detector = query_format(detector) + params[instrument]['detector_list'].append([detector, detector]) + # Generate dynamic lists of read patterns to use in forms + for readpatt in READPATT_PER_INSTRUMENT[instrument]: + readpatt = query_format(readpatt) + params[instrument]['readpatt_list'].append([readpatt, readpatt]) + # Generate dynamic lists of exposure types to use in forms + for exptype in EXP_TYPE_PER_INSTRUMENT[instrument]: + exptype = query_format(exptype) + params[instrument]['exptype_list'].append([exptype, exptype]) + # Generate dynamic lists of grating options to use in forms + for grating in GRATING_PER_INSTRUMENT[instrument]: + grating = query_format(grating) + params[instrument]['grating_list'].append([grating, grating]) + # Generate dynamic lists of anomalies to use in forms + for anomaly in ANOMALIES_PER_INSTRUMENT.keys(): + if instrument in ANOMALIES_PER_INSTRUMENT[anomaly]: + item = [query_format(anomaly), query_format(anomaly)] + params[instrument]['anomalies_list'].append(item) # Anomaly Parameters instrument = forms.MultipleChoiceField(required=False, choices=[(inst, JWST_INSTRUMENT_NAMES_MIXEDCASE[inst]) for inst in JWST_INSTRUMENT_NAMES_MIXEDCASE], widget=forms.CheckboxSelectMultiple) - aperture = forms.MultipleChoiceField(required=False, choices=aperture_list, widget=forms.CheckboxSelectMultiple) - filt = forms.MultipleChoiceField(required=False, choices=filter_list, widget=forms.CheckboxSelectMultiple) - early_date = forms.DateField(required=False, initial="eg, 2021-10-02 12:04:39 or 2021-10-02") - late_date = forms.DateField(required=False, initial="eg, 2021-11-25 14:30:59 or 2021-11-25") exp_time_max = forms.DecimalField(required=False, initial="685") exp_time_min = forms.DecimalField(required=False, initial="680") - miri_aper = forms.MultipleChoiceField(required=False, choices=miri_aperture_list, widget=forms.CheckboxSelectMultiple) - nirspec_aper = forms.MultipleChoiceField(required=False, choices=nirspec_aperture_list, widget=forms.CheckboxSelectMultiple) - niriss_aper = forms.MultipleChoiceField(required=False, choices=niriss_aperture_list, widget=forms.CheckboxSelectMultiple) - nircam_aper = forms.MultipleChoiceField(required=False, choices=nircam_aperture_list, widget=forms.CheckboxSelectMultiple) - - # should use something like 'nirpsec_filt', choices=[...] in order to choose particular series to show up - miri_filt = forms.MultipleChoiceField(required=False, choices=miri_filter_list, widget=forms.CheckboxSelectMultiple) #choices=[('lrs', 'LRS')]) - nirspec_filt = forms.MultipleChoiceField(required=False, choices=nirspec_filter_list, widget=forms.CheckboxSelectMultiple) #choices=[('f070lp_g140h', 'F070LP/G140H'), ('f100lp_g140h', 'F100LP/G140H'), ('f070lp_g140m', 'F070LP/G140M'), ('f100lp_g140m', 'F100LP/G140M'), ('f170lp_g235h', 'F170LP/G235H'), ('f170lp_g235m', 'F170LP/G235M'), ('f290lp_g395h', 'F290LP/G395H'), ('f290lp_g395m', 'F290LP/G395M')]) - niriss_filt = forms.MultipleChoiceField(required=False, choices=niriss_filter_list, widget=forms.CheckboxSelectMultiple) #choices=[('soss', 'SOSS')]) - nircam_filt = forms.MultipleChoiceField(required=False, choices=nircam_filter_list, widget=forms.CheckboxSelectMultiple) #choices=[('f322w2', 'F322W2'), ('f444w', 'F444W'), ('f277w', 'F277W')]) - - miri_obsmode= forms.MultipleChoiceField(required=False, choices=miri_obsmode_list, widget=forms.CheckboxSelectMultiple) - nirspec_obsmode = forms.MultipleChoiceField(required=False, choices=nirspec_obsmode_list, widget=forms.CheckboxSelectMultiple) - niriss_obsmode = forms.MultipleChoiceField(required=False, choices=niriss_obsmode_list, widget=forms.CheckboxSelectMultiple) - nircam_obsmode = forms.MultipleChoiceField(required=False, choices=nircam_obsmode_list, widget=forms.CheckboxSelectMultiple) - - miri_anomalies= forms.MultipleChoiceField(required=False, choices=miri_anomalies_list, widget=forms.CheckboxSelectMultiple) - nirspec_anomalies = forms.MultipleChoiceField(required=False, choices=nirspec_anomalies_list, widget=forms.CheckboxSelectMultiple) - niriss_anomalies = forms.MultipleChoiceField(required=False, choices=niriss_anomalies_list, widget=forms.CheckboxSelectMultiple) - nircam_anomalies = forms.MultipleChoiceField(required=False, choices=nircam_anomalies_list, widget=forms.CheckboxSelectMultiple) - - anomalies = forms.MultipleChoiceField(required=False, choices=ANOMALY_CHOICES, widget=forms.CheckboxSelectMultiple()) + miri_aper = forms.MultipleChoiceField(required=False, choices=params['miri']['aperture_list'], widget=forms.CheckboxSelectMultiple) + nirspec_aper = forms.MultipleChoiceField(required=False, choices=params['nirspec']['aperture_list'], widget=forms.CheckboxSelectMultiple) + niriss_aper = forms.MultipleChoiceField(required=False, choices=params['niriss']['aperture_list'], widget=forms.CheckboxSelectMultiple) + nircam_aper = forms.MultipleChoiceField(required=False, choices=params['nircam']['aperture_list'], widget=forms.CheckboxSelectMultiple) + + miri_filt = forms.MultipleChoiceField(required=False, choices=params['miri']['filter_list'], widget=forms.CheckboxSelectMultiple) + nirspec_filt = forms.MultipleChoiceField(required=False, choices=params['nirspec']['filter_list'], widget=forms.CheckboxSelectMultiple) + niriss_filt = forms.MultipleChoiceField(required=False, choices=params['niriss']['filter_list'], widget=forms.CheckboxSelectMultiple) + nircam_filt = forms.MultipleChoiceField(required=False, choices=params['nircam']['filter_list'], widget=forms.CheckboxSelectMultiple) + + miri_detector = forms.MultipleChoiceField(required=False, choices=params['miri']['detector_list'], widget=forms.CheckboxSelectMultiple) + nirspec_detector = forms.MultipleChoiceField(required=False, choices=params['nirspec']['detector_list'], widget=forms.CheckboxSelectMultiple) + niriss_detector = forms.MultipleChoiceField(required=False, choices=params['niriss']['detector_list'], widget=forms.CheckboxSelectMultiple) + nircam_detector = forms.MultipleChoiceField(required=False, choices=params['nircam']['detector_list'], widget=forms.CheckboxSelectMultiple) + + miri_anomalies = forms.MultipleChoiceField(required=False, choices=params['miri']['anomalies_list'], widget=forms.CheckboxSelectMultiple) + nirspec_anomalies = forms.MultipleChoiceField(required=False, choices=params['nirspec']['anomalies_list'], widget=forms.CheckboxSelectMultiple) + niriss_anomalies = forms.MultipleChoiceField(required=False, choices=params['niriss']['anomalies_list'], widget=forms.CheckboxSelectMultiple) + nircam_anomalies = forms.MultipleChoiceField(required=False, choices=params['nircam']['anomalies_list'], widget=forms.CheckboxSelectMultiple) + + miri_readpatt = forms.MultipleChoiceField(required=False, choices=params['miri']['readpatt_list'], widget=forms.CheckboxSelectMultiple) + nirspec_readpatt = forms.MultipleChoiceField(required=False, choices=params['nirspec']['readpatt_list'], widget=forms.CheckboxSelectMultiple) + niriss_readpatt = forms.MultipleChoiceField(required=False, choices=params['niriss']['readpatt_list'], widget=forms.CheckboxSelectMultiple) + nircam_readpatt = forms.MultipleChoiceField(required=False, choices=params['nircam']['readpatt_list'], widget=forms.CheckboxSelectMultiple) + + miri_exptype = forms.MultipleChoiceField(required=False, choices=params['miri']['exptype_list'], widget=forms.CheckboxSelectMultiple) + nirspec_exptype = forms.MultipleChoiceField(required=False, choices=params['nirspec']['exptype_list'], widget=forms.CheckboxSelectMultiple) + niriss_exptype = forms.MultipleChoiceField(required=False, choices=params['niriss']['exptype_list'], widget=forms.CheckboxSelectMultiple) + nircam_exptype = forms.MultipleChoiceField(required=False, choices=params['nircam']['exptype_list'], widget=forms.CheckboxSelectMultiple) + + miri_grating = forms.MultipleChoiceField(required=False, choices=params['miri']['grating_list'], widget=forms.CheckboxSelectMultiple) + nirspec_grating = forms.MultipleChoiceField(required=False, choices=params['nirspec']['grating_list'], widget=forms.CheckboxSelectMultiple) + niriss_grating = forms.MultipleChoiceField(required=False, choices=params['niriss']['grating_list'], widget=forms.CheckboxSelectMultiple) + nircam_grating = forms.MultipleChoiceField(required=False, choices=params['nircam']['grating_list'], widget=forms.CheckboxSelectMultiple) def clean_inst(self): @@ -224,24 +177,14 @@ def clean_inst(self): return inst -class AnomalyForm(forms.Form): - """Creates a ``AnomalyForm`` object that allows for anomaly input - in a form field.""" - query = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, widget=forms.CheckboxSelectMultiple()) # Update depending on chosen instruments - - def clean_anomalies(self): - - anomalies = self.cleaned_data['query'] - - return anomalies - - -class AnomalySubmitForm(forms.Form): +class InstrumentAnomalySubmitForm(forms.Form): """A multiple choice field for specifying flagged anomalies.""" - # Define anomaly choice field - anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, - widget=forms.CheckboxSelectMultiple()) + def __init__(self, *args, **kwargs): + instrument = kwargs.pop('instrument') + super(InstrumentAnomalySubmitForm, self).__init__(*args, **kwargs) + self.fields['anomaly_choices'] = forms.MultipleChoiceField(choices=ANOMALY_CHOICES_PER_INSTRUMENT[instrument], widget=forms.CheckboxSelectMultiple()) + self.instrument = instrument def update_anomaly_table(self, rootname, user, anomaly_choices): """Updated the ``anomaly`` table of the database with flagged @@ -264,218 +207,18 @@ def update_anomaly_table(self, rootname, user, anomaly_choices): data_dict['rootname'] = rootname data_dict['flag_date'] = datetime.datetime.now() data_dict['user'] = user - for choice in anomaly_choices: data_dict[choice] = True - if 'guider' in rootname: + if self.instrument == 'fgs': di.engine.execute(di.FGSAnomaly.__table__.insert(), data_dict) - elif "nrs" in rootname: + elif self.instrument == 'nirspec': di.engine.execute(di.NIRSpecAnomaly.__table__.insert(), data_dict) - elif "miri" in rootname: + elif self.instrument == 'miri': di.engine.execute(di.MIRIAnomaly.__table__.insert(), data_dict) - elif "nis" in rootname: + elif self.instrument == 'niriss': di.engine.execute(di.NIRISSAnomaly.__table__.insert(), data_dict) - elif "nrc" in rootname: + elif self.instrument == 'nircam': di.engine.execute(di.NIRCamAnomaly.__table__.insert(), data_dict) - else: - print("cannot determine instrument anomaly corresponds to") - # '{}Anomaly'.format(JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] - # no attribute 'Anomaly' - - def clean_anomalies(self): - - anomalies = self.cleaned_data['anomaly_choices'] - - return anomalies - - -class FGSAnomalySubmitForm(forms.Form): - """A multiple choice field for specifying flagged anomalies.""" - - # Define anomaly choice field - anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES_PER_INSTRUMENT['fgs'], - widget=forms.CheckboxSelectMultiple()) - - def update_anomaly_table(self, rootname, user, anomaly_choices): - """Updated the ``anomaly`` table of the database with flagged - anomaly information - - Parameters - ---------- - rootname : str - The rootname of the image to flag (e.g. - ``jw86600008001_02101_00001_guider2``) - user : str - The ``ezid`` of the authenticated user that is flagging the - anomaly - anomaly_choices : list - A list of anomalies that are to be flagged (e.g. - ``['snowball', 'crosstalk']``) - """ - - data_dict = {} - data_dict['rootname'] = rootname - data_dict['flag_date'] = datetime.datetime.now() - data_dict['user'] = user - for choice in anomaly_choices: - data_dict[choice] = True - di.engine.execute(di.FGSAnomaly.__table__.insert(), data_dict) - - def clean_anomalies(self): - - anomalies = self.cleaned_data['anomaly_choices'] - - return anomalies - - -class MIRIAnomalySubmitForm(forms.Form): - """A multiple choice field for specifying flagged anomalies.""" - - # Define anomaly choice field - anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES_PER_INSTRUMENT['miri'], - widget=forms.CheckboxSelectMultiple()) - - def update_anomaly_table(self, rootname, user, anomaly_choices): - """Updated the ``anomaly`` table of the database with flagged - anomaly information - - Parameters - ---------- - rootname : str - The rootname of the image to flag (e.g. - ``jw86600008001_02101_00001_guider2``) - user : str - The ``ezid`` of the authenticated user that is flagging the - anomaly - anomaly_choices : list - A list of anomalies that are to be flagged (e.g. - ``['snowball', 'crosstalk']``) - """ - - data_dict = {} - data_dict['rootname'] = rootname - data_dict['flag_date'] = datetime.datetime.now() - data_dict['user'] = user - for choice in anomaly_choices: - data_dict[choice] = True - di.engine.execute(di.MIRIAnomaly.__table__.insert(), data_dict) - - def clean_anomalies(self): - - anomalies = self.cleaned_data['anomaly_choices'] - - return anomalies - - -class NIRCamAnomalySubmitForm(forms.Form): - """A multiple choice field for specifying flagged anomalies.""" - - # Define anomaly choice field - anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES_PER_INSTRUMENT['nircam'], - widget=forms.CheckboxSelectMultiple()) - - def update_anomaly_table(self, rootname, user, anomaly_choices): - """Updated the ``anomaly`` table of the database with flagged - anomaly information - - Parameters - ---------- - rootname : str - The rootname of the image to flag (e.g. - ``jw86600008001_02101_00001_guider2``) - user : str - The ``ezid`` of the authenticated user that is flagging the - anomaly - anomaly_choices : list - A list of anomalies that are to be flagged (e.g. - ``['snowball', 'crosstalk']``) - """ - - data_dict = {} - data_dict['rootname'] = rootname - data_dict['flag_date'] = datetime.datetime.now() - data_dict['user'] = user - for choice in anomaly_choices: - data_dict[choice] = True - di.engine.execute(di.NIRCamAnomaly.__table__.insert(), data_dict) - - def clean_anomalies(self): - - anomalies = self.cleaned_data['anomaly_choices'] - - return anomalies - - -class NIRISSAnomalySubmitForm(forms.Form): - """A multiple choice field for specifying flagged anomalies.""" - - # Define anomaly choice field - anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES_PER_INSTRUMENT['niriss'], - widget=forms.CheckboxSelectMultiple()) - - def update_anomaly_table(self, rootname, user, anomaly_choices): - """Updated the ``anomaly`` table of the database with flagged - anomaly information - - Parameters - ---------- - rootname : str - The rootname of the image to flag (e.g. - ``jw86600008001_02101_00001_guider2``) - user : str - The ``ezid`` of the authenticated user that is flagging the - anomaly - anomaly_choices : list - A list of anomalies that are to be flagged (e.g. - ``['snowball', 'crosstalk']``) - """ - - data_dict = {} - data_dict['rootname'] = rootname - data_dict['flag_date'] = datetime.datetime.now() - data_dict['user'] = user - for choice in anomaly_choices: - data_dict[choice] = True - di.engine.execute(di.NIRISSAnomaly.__table__.insert(), data_dict) - - def clean_anomalies(self): - - anomalies = self.cleaned_data['anomaly_choices'] - - return anomalies - - -class NIRSpecAnomalySubmitForm(forms.Form): - """A multiple choice field for specifying flagged anomalies.""" - - # Define anomaly choice field - anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES_PER_INSTRUMENT['nirspec'], - widget=forms.CheckboxSelectMultiple()) - - def update_anomaly_table(self, rootname, user, anomaly_choices): - """Updated the ``anomaly`` table of the database with flagged - anomaly information - - Parameters - ---------- - rootname : str - The rootname of the image to flag (e.g. - ``jw86600008001_02101_00001_guider2``) - user : str - The ``ezid`` of the authenticated user that is flagging the - anomaly - anomaly_choices : list - A list of anomalies that are to be flagged (e.g. - ``['snowball', 'crosstalk']``) - """ - - data_dict = {} - data_dict['rootname'] = rootname - data_dict['flag_date'] = datetime.datetime.now() - data_dict['user'] = user - for choice in anomaly_choices: - data_dict[choice] = True - di.engine.execute(di.NIRSpecAnomaly.__table__.insert(), data_dict) def clean_anomalies(self): @@ -544,8 +287,7 @@ def clean_search(self): # If they searched for a fileroot... elif self.search_type == 'fileroot': - # See if there are any matching fileroots and, if so, what - # instrument they are for + # See if there are any matching fileroots and, if so, what instrument they are for search_string = os.path.join(FILESYSTEM_DIR, search[:7], '{}*.fits'.format(search)) all_files = glob.glob(search_string) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 19b14e5e4..ed0aaf2fd 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,4 +1,5 @@ from .monitor_bad_pixel_bokeh import BadPixelMonitor +from .monitor_bias_bokeh import BiasMonitor from .monitor_dark_bokeh import DarkMonitor from .monitor_filesystem_bokeh import MonitorFilesystem from .monitor_mast_bokeh import MastMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py new file mode 100644 index 000000000..caac079de --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py @@ -0,0 +1,162 @@ +"""This module contains code for the bias monitor Bokeh plots. + +Author +------ + + - Ben Sunnquist + +Use +--- + + This module can be used from the command line as such: + + :: + + from jwql.website.apps.jwql import monitor_pages + monitor_template = monitor_pages.BiasMonitor() + monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL') +""" + +from datetime import datetime, timedelta +import os + +from astropy.stats import sigma_clip +import numpy as np + +from jwql.bokeh_templating import BokehTemplate +from jwql.database.database_interface import session, NIRCamBiasStats +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class BiasMonitor(BokehTemplate): + + # Combine the input parameters into a single property because we + # do not want to invoke the setter unless all are updated + @property + def input_parameters(self): + return (self._instrument, self._aperture) + + @input_parameters.setter + def input_parameters(self, info): + self._instrument, self._aperture = info + self.pre_init() + self.post_init() + + def identify_tables(self): + """Determine which database tables to use for the given instrument""" + + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] + self.stats_table = eval('{}BiasStats'.format(mixed_case_name)) + + def load_data(self): + """Query the database tables to get all of the relevant bias data""" + + # Determine which database tables are needed based on instrument + self.identify_tables() + + # Query database for all data in bias stats with a matching aperture, + # and sort the data by exposure start time. + self.query_results = session.query(self.stats_table) \ + .filter(self.stats_table.aperture == self._aperture) \ + .order_by(self.stats_table.expstart) \ + .all() + + def pre_init(self): + + # Start with default values for instrument and aperture because + # BokehTemplate's __init__ method does not allow input arguments + try: + dummy_instrument = self._instrument + dummy_aperture = self._aperture + except AttributeError: + self._instrument = 'NIRCam' + self._aperture = '' + + self._embed = True + self.format_string = None + self.interface_file = os.path.join(SCRIPT_DIR, 'yaml', 'monitor_bias_interface.yaml') + + def post_init(self): + + # Load the bias data + self.load_data() + + # Update the mean bias over time figures + self.update_mean_bias_figures() + + # Update the calibrated 0th group image + self.update_calibrated_image() + + # Update the calibrated collapsed values figures + self.update_collapsed_vals_figures() + + def update_calibrated_image(self): + """Updates the calibrated 0th group image""" + + if len(self.query_results) != 0: + # Get the most recent data; the entries were sorted by time when + # loading the database, so the last entry will always be the most recent. + cal_image_png = self.query_results[-1].cal_image + cal_image_png = os.path.join('/static', '/'.join(cal_image_png.split('/')[-6:])) + + # Update the image source for the figure + self.refs['cal_image'].image_url(url=[cal_image_png], x=0, y=0, w=2000, h=2000, anchor="bottom_left") + + # Update the calibrated image style + self.refs['cal_image'].title.align = 'center' + self.refs['cal_image'].xaxis.visible = False + self.refs['cal_image'].yaxis.visible = False + self.refs['cal_image'].xgrid.grid_line_color = None + self.refs['cal_image'].ygrid.grid_line_color = None + self.refs['cal_image'].title.text_font_size = '30px' + + def update_collapsed_vals_figures(self): + """Updates the calibrated median-collapsed row and column figures""" + + if len(self.query_results) != 0: + for direction in ['rows', 'columns']: + # Get the most recent data; the entries were sorted by time when + # loading the database, so the last entry will always be the most recent. + vals = np.array(self.query_results[-1].__dict__['collapsed_{}'.format(direction)]) + pixels = np.arange(len(vals)) + self.refs['collapsed_{}_source'.format(direction)].data = {'pixel': pixels, + 'signal': vals} + + # Update the signal and pixel limits + clipped = sigma_clip(vals) + mean, stddev = np.nanmean(clipped), np.nanstd(clipped) + thresh_low, thresh_high = mean - 5 * stddev, mean + 5 * stddev + self.refs['collapsed_{}_pixel_range'.format(direction)].start = pixels.min() - 10 + self.refs['collapsed_{}_pixel_range'.format(direction)].end = pixels.max() + 10 + self.refs['collapsed_{}_signal_range'.format(direction)].start = thresh_low + self.refs['collapsed_{}_signal_range'.format(direction)].end = thresh_high + + def update_mean_bias_figures(self): + """Updates the mean bias over time bokeh plots""" + + # Get the dark exposures and their starts times + filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.query_results] + expstarts_iso = np.array([result.expstart for result in self.query_results]) + expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) + + # Update the mean bias figures for all amps and odd/even columns + for amp in ['1', '2', '3', '4']: + for kind in ['odd', 'even']: + bias_vals = np.array([getattr(result, 'amp{}_{}_med'.format(amp, kind)) for result in self.query_results]) + self.refs['mean_bias_source_amp{}_{}'.format(amp, kind)].data = {'time': expstarts, + 'time_iso': expstarts_iso, + 'mean_bias': bias_vals, + 'filename': filenames} + self.refs['mean_bias_figure_amp{}_{}'.format(amp, kind)].title.text = 'Amp {} {}'.format(amp, kind.capitalize()) + self.refs['mean_bias_figure_amp{}_{}'.format(amp, kind)].hover.tooltips = [('file', '@filename'), + ('time', '@time_iso'), + ('bias level', '@mean_bias')] + + # Update plot limits if data exists + if len(bias_vals) != 0: + self.refs['mean_bias_xr_amp{}_{}'.format(amp, kind)].start = expstarts.min() - timedelta(days=3) + self.refs['mean_bias_xr_amp{}_{}'.format(amp, kind)].end = expstarts.max() + timedelta(days=3) + self.refs['mean_bias_yr_amp{}_{}'.format(amp, kind)].start = bias_vals.min() - 10 + self.refs['mean_bias_yr_amp{}_{}'.format(amp, kind)].end = bias_vals.max() + 10 diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 1fe355ff3..8ea72449f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -23,7 +23,8 @@ import numpy as np from jwql.bokeh_templating import BokehTemplate -from jwql.database.database_interface import session, NIRCamReadnoiseStats, NIRISSReadnoiseStats +from jwql.database.database_interface import session +from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -72,8 +73,8 @@ def pre_init(self): dummy_amp = self._amp except AttributeError: self._instrument = 'NIRCam' - self._aperture = 'NRCA1_FULL' - self._amp = '1' + self._aperture = '' + self._amp = '' self._embed = True self.format_string = None @@ -108,10 +109,6 @@ def update_mean_readnoise_figure(self): 'filename': self.filenames, 'nints': self.nints, 'ngroups': self.ngroups} - self.refs['mean_readnoise_xr'].start = self.expstarts.min() - timedelta(days=3) - self.refs['mean_readnoise_xr'].end = self.expstarts.max() + timedelta(days=3) - self.refs['mean_readnoise_yr'].start = self.readnoise_vals.min() - 1 - self.refs['mean_readnoise_yr'].end = self.readnoise_vals.max() + 1 self.refs['mean_readnoise_figure'].title.text = 'Amp {}'.format(self._amp) self.refs['mean_readnoise_figure'].hover.tooltips = [('file', '@filename'), ('time', '@time_iso'), @@ -120,30 +117,37 @@ def update_mean_readnoise_figure(self): ('readnoise', '@mean_rn') ] + # Update plot limits if data exists + if len(self.query_results) != 0: + self.refs['mean_readnoise_xr'].start = self.expstarts.min() - timedelta(days=3) + self.refs['mean_readnoise_xr'].end = self.expstarts.max() + timedelta(days=3) + self.refs['mean_readnoise_yr'].start = self.readnoise_vals.min() - 1 + self.refs['mean_readnoise_yr'].end = self.readnoise_vals.max() + 1 + def update_readnoise_diff_plots(self): """Updates the readnoise difference image and histogram""" - # Get the most recent data; the entries were sorted by time when - # loading the database, so the last entry will always be the most recent. - diff_image_png = self.query_results[-1].readnoise_diff_image - diff_image_n = np.array(self.query_results[-1].diff_image_n) - diff_image_bin_centers = np.array(self.query_results[-1].diff_image_bin_centers) - - # Update the readnoise difference image - self.refs['readnoise_diff_image'].image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") + # Update the readnoise difference image and histogram, if data exists + if len(self.query_results) != 0: + # Get the most recent data; the entries were sorted by time when + # loading the database, so the last entry will always be the most recent. + diff_image_png = self.query_results[-1].readnoise_diff_image + diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) + diff_image_n = np.array(self.query_results[-1].diff_image_n) + diff_image_bin_centers = np.array(self.query_results[-1].diff_image_bin_centers) + + # Update the readnoise difference image and histogram + self.refs['readnoise_diff_image'].image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") + self.refs['diff_hist_source'].data = {'n': diff_image_n, + 'bin_centers': diff_image_bin_centers} + self.refs['diff_hist_xr'].start = diff_image_bin_centers.min() + self.refs['diff_hist_xr'].end = diff_image_bin_centers.max() + self.refs['diff_hist_yr'].start = diff_image_n.min() + self.refs['diff_hist_yr'].end = diff_image_n.max() + diff_image_n.max() * 0.05 + + # Update the readnoise difference image style self.refs['readnoise_diff_image'].xaxis.visible = False self.refs['readnoise_diff_image'].yaxis.visible = False self.refs['readnoise_diff_image'].xgrid.grid_line_color = None self.refs['readnoise_diff_image'].ygrid.grid_line_color = None self.refs['readnoise_diff_image'].title.text_font_size = '30px' - - # Update the readnoise difference histogram - self.refs['diff_hist_source'].data = {'n': diff_image_n, - 'bin_centers': diff_image_bin_centers} - self.refs['diff_hist_xr'].start = diff_image_bin_centers.min() - self.refs['diff_hist_xr'].end = diff_image_bin_centers.max() - self.refs['diff_hist_yr'].start = diff_image_n.min() - self.refs['diff_hist_yr'].end = diff_image_n.max() + diff_image_n.max() * 0.05 - - -ReadnoiseMonitor() diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_bias_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_bias_interface.yaml new file mode 100644 index 000000000..9f2ca9e00 --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_bias_interface.yaml @@ -0,0 +1,336 @@ +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp1 Even +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp1_even + ref: "mean_bias_source_amp1_even" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp1_even + ref: "mean_bias_xr_amp1_even" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp1_even + ref: "mean_bias_yr_amp1_even" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp1_even + ref: "mean_bias_figure_amp1_even" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp1_even + y_range: *mean_bias_yr_amp1_even + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp1_even} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp1 Odd +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp1_odd + ref: "mean_bias_source_amp1_odd" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp1_odd + ref: "mean_bias_xr_amp1_odd" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp1_odd + ref: "mean_bias_yr_amp1_odd" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp1_odd + ref: "mean_bias_figure_amp1_odd" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp1_odd + y_range: *mean_bias_yr_amp1_odd + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp1_odd} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp2 Even +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp2_even + ref: "mean_bias_source_amp2_even" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp2_even + ref: "mean_bias_xr_amp2_even" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp2_even + ref: "mean_bias_yr_amp2_even" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp2_even + ref: "mean_bias_figure_amp2_even" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp2_even + y_range: *mean_bias_yr_amp2_even + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp2_even} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp2 Odd +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp2_odd + ref: "mean_bias_source_amp2_odd" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp2_odd + ref: "mean_bias_xr_amp2_odd" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp2_odd + ref: "mean_bias_yr_amp2_odd" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp2_odd + ref: "mean_bias_figure_amp2_odd" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp2_odd + y_range: *mean_bias_yr_amp2_odd + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp2_odd} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp3 Even +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp3_even + ref: "mean_bias_source_amp3_even" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp3_even + ref: "mean_bias_xr_amp3_even" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp3_even + ref: "mean_bias_yr_amp3_even" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp3_even + ref: "mean_bias_figure_amp3_even" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp3_even + y_range: *mean_bias_yr_amp3_even + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp3_even} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp3 Odd +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp3_odd + ref: "mean_bias_source_amp3_odd" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp3_odd + ref: "mean_bias_xr_amp3_odd" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp3_odd + ref: "mean_bias_yr_amp3_odd" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp3_odd + ref: "mean_bias_figure_amp3_odd" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp3_odd + y_range: *mean_bias_yr_amp3_odd + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp3_odd} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp4 Even +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp4_even + ref: "mean_bias_source_amp4_even" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp4_even + ref: "mean_bias_xr_amp4_even" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp4_even + ref: "mean_bias_yr_amp4_even" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp4_even + ref: "mean_bias_figure_amp4_even" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp4_even + y_range: *mean_bias_yr_amp4_even + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp4_even} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Bias vs Time Figures Amp4 Odd +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_bias_source_amp4_odd + ref: "mean_bias_source_amp4_odd" + data: + time: [] + time_iso: [] + mean_bias: [] + filename: [] +- !Range1d: &mean_bias_xr_amp4_odd + ref: "mean_bias_xr_amp4_odd" + start: 0 + end: 1 + bounds: 'auto' +- !Range1d: &mean_bias_yr_amp4_odd + ref: "mean_bias_yr_amp4_odd" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &mean_bias_figure_amp4_odd + ref: "mean_bias_figure_amp4_odd" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Bias Level [DN]" + x_range: *mean_bias_xr_amp4_odd + y_range: *mean_bias_yr_amp4_odd + height: 800 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'mean_bias', 'size': 6, 'source': *mean_bias_source_amp4_odd} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Calibrated 0th Group Image +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &cal_data + ref: "cal_data" + data: + dh: [0] + dw: [0] + image: [[[0,0], [0, 0]]] +- !Figure: &cal_image + ref: "cal_image" + title: 'Calibrated Zeroth Group of Most Recent Dark' + height: 800 + width: 800 + elements: + - {"kind": "image", "image": "image", "x": 0, "y": 0, "dh": 'dh', "dw": 'dh', "source": *cal_data} + tools: "" +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Collapsed Columns Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &collapsed_columns_source + ref: "collapsed_columns_source" + data: + pixel: [] + signal: [] +- !Range1d: &collapsed_columns_pixel_range + ref: "collapsed_columns_pixel_range" + start: 0 + end: 10 + bounds: 'auto' +- !Range1d: &collapsed_columns_signal_range + ref: "collapsed_columns_signal_range" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &collapsed_columns_figure + ref: "collapsed_columns_figure" + x_axis_label: "Column #" + y_axis_label: "Median Signal [DN]" + x_range: *collapsed_columns_pixel_range + y_range: *collapsed_columns_signal_range + height: 500 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'pixel', 'y': 'signal', 'size': 3, 'source': *collapsed_columns_source} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Collapsed Rows Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &collapsed_rows_source + ref: "collapsed_rows_source" + data: + pixel: [] + signal: [] +- !Range1d: &collapsed_rows_pixel_range + ref: "collapsed_rows_pixel_range" + start: 0 + end: 10 + bounds: 'auto' +- !Range1d: &collapsed_rows_signal_range + ref: "collapsed_rows_signal_range" + start: 0 + end: 10 + bounds: 'auto' +- !Figure: &collapsed_rows_figure + ref: "collapsed_rows_figure" + x_axis_label: "Row #" + y_axis_label: "Median Signal [DN]" + x_range: *collapsed_rows_pixel_range + y_range: *collapsed_rows_signal_range + height: 500 + width: 800 + tools: "hover, wheel_zoom, pan, reset" + elements: + - {'kind': 'circle', 'x': 'pixel', 'y': 'signal', 'size': 3, 'source': *collapsed_rows_source} + +# Document structure +# - !Document: +# - !row: +# - *mean_bias_figure \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml index 7ccbac7c1..29a2d016a 100644 --- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml +++ b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml @@ -41,7 +41,7 @@ data: dh: [1] dw: [1] - image: [[[1,0], [0, 1]]] + image: [[[0,0], [0, 0]]] - !Figure: &readnoise_diff_image ref: "readnoise_diff_image" title: 'Readnoise difference with pipeline reference file' diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index d0a81dda6..1e1803b62 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -70,6 +70,39 @@ def bad_pixel_monitor(request, inst): return render(request, template, context) +def bias_monitor(request, inst): + """Generate the bias monitor page for a given instrument + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage + inst : str + Name of JWST instrument + + Returns + ------- + HttpResponse object + Outgoing response sent to the webpage + """ + + # Ensure the instrument is correctly capitalized + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + + # Get the html and JS needed to render the bias tab plots + tabs_components = bokeh_containers.bias_monitor_tabs(inst) + + template = "bias_monitor.html" + + context = { + 'inst': inst, + 'tabs_components': tabs_components, + } + + # Return a HTTP response with the template and dictionary of variables + return render(request, template, context) + + def dark_monitor(request, inst): """Generate the dark monitor page for a given instrument diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index f93ff0765..b46ebd9c9 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -374,9 +374,8 @@ function update_archive_page(inst, base_url) { * @param {Object} data - The data returned by the update_thumbnails_page AJAX method */ function update_filter_options(data) { - + content = 'Filter by:' for (var i = 0; i < Object.keys(data.dropdown_menus).length; i++) { - // Parse out useful variables filter_type = Object.keys(data.dropdown_menus)[i]; filter_options = Array.from(new Set(data.dropdown_menus[filter_type])); @@ -384,8 +383,8 @@ function update_filter_options(data) { dropdown_key_list = Object.keys(data.dropdown_menus); // Build div content - content = '
'; - content += 'Show only ' + filter_type + ':'; + content += '
'; + content += '
'; content += '
'; content += ''; content += '
'; @@ -395,6 +394,7 @@ function update_filter_options(data) { content += '' + filter_options[j] + ''; }; + content += '
'; content += '
'; }; @@ -472,13 +472,13 @@ function update_thumbnail_array(data) { filename_dict = file.filename_dict; // Build div content - content = '
'; if (data.inst!="all") { + content = '
'; content += ''; } else { - content += ''; + content = '
'; + content += ''; } - content += ''; content += ''; content += '
'; content += '
'; diff --git a/jwql/website/apps/jwql/templates/anomaly_query.html b/jwql/website/apps/jwql/templates/anomaly_query.html index d7c646be8..5c779940d 100644 --- a/jwql/website/apps/jwql/templates/anomaly_query.html +++ b/jwql/website/apps/jwql/templates/anomaly_query.html @@ -9,49 +9,67 @@ if (document.getElementById("id_instrument_1").checked){ $("#miri_filt").show(); $("#miri_aper").show(); - $("#miri_obsmode").show(); + $("#miri_detector").show(); + $("#miri_readpatt").show(); + $("#miri_exptype").show(); $("#miri_anomalies").show(); } if (document.getElementById("id_instrument_2").checked){ $("#nircam_filt").show(); $("#nircam_aper").show(); - $("#nircam_obsmode").show(); + $("#nircam_detector").show(); + $("#nircam_readpatt").show(); + $("#nircam_exptype").show(); $("#nircam_anomalies").show(); } if (document.getElementById("id_instrument_3").checked){ $("#niriss_filt").show(); $("#niriss_aper").show(); - $("#niriss_obsmode").show(); + $("#niriss_detector").show(); + $("#niriss_readpatt").show(); + $("#niriss_exptype").show(); $("#niriss_anomalies").show(); } if (document.getElementById("id_instrument_4").checked){ $("#nirspec_filt").show(); $("#nirspec_aper").show(); - $("#nirspec_obsmode").show(); + $("#nirspec_detector").show(); + $("#nirspec_readpatt").show(); + $("#nirspec_exptype").show(); + $("#nirspec_grating").hide(); $("#nirspec_anomalies").show(); } if (document.getElementById("id_instrument_1").checked==false){ $("#miri_filt").hide(); $("#miri_aper").hide(); - $("#miri_obsmode").hide(); + $("#miri_detector").hide(); + $("#miri_readpatt").hide(); + $("#miri_exptype").hide(); $("#miri_anomalies").hide(); } if (document.getElementById("id_instrument_2").checked==false){ $("#nircam_filt").hide(); $("#nircam_aper").hide(); - $("#nircam_obsmode").hide(); + $("#nircam_detector").hide(); + $("#nircam_readpatt").hide(); + $("#nircam_exptype").hide(); $("#nircam_anomalies").hide(); } if (document.getElementById("id_instrument_3").checked==false){ $("#niriss_filt").hide(); $("#niriss_aper").hide(); - $("#niriss_obsmode").hide(); + $("#niriss_detector").hide(); + $("#niriss_readpatt").hide(); + $("#niriss_exptype").hide(); $("#niriss_anomalies").hide(); } if (document.getElementById("id_instrument_4").checked==false){ $("#nirspec_filt").hide(); $("#nirspec_aper").hide(); - $("#nirspec_obsmode").hide(); + $("#nirspec_detector").hide(); + $("#nirspec_readpatt").hide(); + $("#nirspec_exptype").hide(); + $("#nirspec_grating").hide(); $("#nirspec_anomalies").hide(); } }); @@ -71,7 +89,7 @@

Dynamic Anomaly Form

-
+
{% for option in form.instrument %} @@ -81,276 +99,294 @@

Dynamic Anomaly Form

{% endfor %}
+
-
- -
-
- {{ form.exp_time_min }} +
+
+
+
+ MIRI Aperture(s) + {% for field in form.miri_aper %} +
+ {{ field }} +
+ {% endfor %}
-
- {% for error in form.exp_time_min.errors %} -

{{ error }}

- {% endfor %} -
-
-
- -
-
- {{ form.exp_time_max }} + +
+
+ MIRI Filter(s) + {% for field in form.miri_filt %} +
+ {{ field }} +
+ {% endfor %} +
-
- {% for error in form.exp_time_max.errors %} -

{{ error }}

- {% endfor %} -
-
-
-
-
- MIRI Aperture(s) - {% for field in form.miri_aper %} -
- {{ field }} -
- {% endfor %} -
-
- {% for error in form.miri_aper.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRCam Aperture(s) - {% for field in form.nircam_aper %} -
- {{ field }} -
- {% endfor %} -
-
- {% for error in form.nircam_aper.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRISS Aperture(s) - {% for field in form.niriss_aper %} -
- {{ field }} -
- {% endfor %} -
-
- {% for error in form.niriss_aper.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRSpec Aperture(s) - {% for field in form.nirspec_aper %} -
- {{ field }} -
- {% endfor %} -
-
- {% for error in form.nirspec_aper.errors %} -

{{ error }}

- {% endfor %} -
- -
-
-
- -
- -
- + +
+
+
+ NIRCam Aperture(s) + {% for field in form.nircam_aper %} +
+ {{ field }} +
+ {% endfor %} +
-
- - {% for option in form.niriss_filt %} -
- {{ option }} +
+
+
+ NIRCam Filter(s) + {% for field in form.nircam_filt %} +
+ {{ field }}
{% endfor %} +
-
-
- +
+
+ +
+
+ NIRCam Exposure Type + {% for field in form.nircam_exptype %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRCam Anomalies + {% for field in form.nircam_anomalies %} +
+ {{ field }} +
+ {% endfor %} +
+
+
- - -
-
-
- MIRI Observing Modes - {% for field in form.miri_obsmode %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.miri_obsmode.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRCam Observing Modes - {% for field in form.nircam_obsmode %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.nircam_obsmode.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRISS Observing Modes - {% for field in form.niriss_obsmode %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.niriss_obsmode.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRSpec Observing Modes - {% for field in form.nirspec_obsmode %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.nirspec_obsmode.errors %} -

{{ error }}

- {% endfor %} -
- - - - - - -
-
-
- MIRI Anomalies - {% for field in form.miri_anomalies %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.miri_anomalies.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRCam Anomalies - {% for field in form.nircam_anomalies %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.nircam_anomalies.errors %} -

{{ error }}

- {% endfor %} -
-
-
- NIRISS Anomalies - {% for field in form.niriss_anomalies %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.niriss_anomalies.errors %} -

{{ error }}

- {% endfor %} +
+
+
+ NIRISS Aperture(s) + {% for field in form.niriss_aper %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRISS Filter(s) + {% for field in form.niriss_filt %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRISS Detector + {% for field in form.niriss_detector %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRISS Read Pattern + {% for field in form.niriss_readpatt %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRISS Exposure Type + {% for field in form.niriss_exptype %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRISS Anomalies + {% for field in form.niriss_anomalies %} +
+ {{ field }} +
+ {% endfor %} +
+
+
-
-
- NIRSpec Anomalies - {% for field in form.nirspec_anomalies %} -
- {{ field }} -
- {% endfor %} -
-
-
- {% for error in form.nirspec_anomalies.errors %} -

{{ error }}

- {% endfor %} + +
+
+
+ NIRSpec Aperture(s) + {% for field in form.nirspec_aper %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRSpec Filter(s) + {% for field in form.nirspec_filt %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRSpec Detector + {% for field in form.nirspec_detector %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRSpec Read Pattern + {% for field in form.nirspec_readpatt %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRSpec Exposure Type + {% for field in form.nirspec_exptype %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRSpec Anomalies + {% for field in form.nirspec_anomalies %} +
+ {{ field }} +
+ {% endfor %} +
+
+
+
+
+ NIRSpec Grating + {% for field in form.nirspec_grating %} +
+ {{ field }} +
+ {% endfor %} +
+
+
-
diff --git a/jwql/website/apps/jwql/templates/bad_pixel_monitor.html b/jwql/website/apps/jwql/templates/bad_pixel_monitor.html index 0ba5a1274..d2d2bd51a 100644 --- a/jwql/website/apps/jwql/templates/bad_pixel_monitor.html +++ b/jwql/website/apps/jwql/templates/bad_pixel_monitor.html @@ -21,6 +21,10 @@

{{ inst }} Bad Pixel Monitor

{{ tabs_components[1] | safe }} +

{{ inst }} Bad Pixel Stats Table

+
+ + Bad Pixel Stats {% endblock %} diff --git a/jwql/website/apps/jwql/templates/base.html b/jwql/website/apps/jwql/templates/base.html index 328a64290..214d5d95b 100644 --- a/jwql/website/apps/jwql/templates/base.html +++ b/jwql/website/apps/jwql/templates/base.html @@ -93,6 +93,22 @@
  • Dashboard(current) +
    • + +
  • + JWQLDB(current) +
    • + +
  • + EDB(current) +
    • + +
  • + API(current) +
    • + +
  • + Documentation(current)
    • {% for inst_key, inst_tools in tools.items() %}
  • @@ -129,18 +145,6 @@
    • {% endfor %} -
  • - EDB(current) -
    • -
  • - JWQLDB(current) -
    • -
  • - API(current) -
    • -
  • - Documentation(current) -
    • @@ -235,7 +239,7 @@ href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/10.1.2/styles/default.min.css"> - + diff --git a/jwql/website/apps/jwql/templates/bias_monitor.html b/jwql/website/apps/jwql/templates/bias_monitor.html new file mode 100644 index 000000000..b3fe218ce --- /dev/null +++ b/jwql/website/apps/jwql/templates/bias_monitor.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block preamble %} + + {{ inst }} Bias Monitor - JWQL + +{% endblock %} + +{% block content %} + +
    + +

    {{ inst }} Bias Monitor

    +
    + +
    + + {{ tabs_components[0] | safe }} +
    + + + {{ tabs_components[1] | safe }} + +
    + +{% endblock %} \ No newline at end of file diff --git a/jwql/website/apps/jwql/templates/instrument.html b/jwql/website/apps/jwql/templates/instrument.html index 88d800166..a969246e2 100644 --- a/jwql/website/apps/jwql/templates/instrument.html +++ b/jwql/website/apps/jwql/templates/instrument.html @@ -1,5 +1,4 @@ {% extends "base.html" %} - {% block preamble %} {{ inst }} Home - JWQL @@ -8,6 +7,15 @@ {% block content %} + +

    -
    - -

    Images

    - Unlooked Images -
    - Image Anomalies -
    - Archived Images -
    +
    +
    +

    Archive

    + +
    +

    Access available images for {{ inst }} here.

    +
    + +
    + Unlooked Images (Currently Unavailable) + Image Anomalies (Currently Unavailable) + Archived Images +
    +
    +
    -
    - -

    Monitors

    - - {% for key, inst_tools in tools.items() %} - {% if key == inst.lower() %} - {% for monitor_name, monitor_url in inst_tools %} - {% if monitor_url == '#' %} - {{ monitor_name }} -
    - {% else %} - {{ monitor_name }} -
    - {% endif %} - {% endfor %} - {% endif %} - {% endfor %} +
    +
    + +

    Monitors

    + +
    +

    Explore the {{ inst }} instrument monitoring suite.

    +
    + +
    + {% for key, inst_tools in tools.items() %} + {% if key == inst.lower() %} + {% for monitor_name, monitor_url in inst_tools %} + {% if monitor_url == '#' %} + {{ monitor_name }} (Currently Unavailable) + {% else %} + {{ monitor_name }} + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} +
    +
    +
    - -
    - -

    Documentation

    - - {{ inst }} Documentation - -
    + +
    +
    + +

    Documentation

    + +
    +

    View {{ inst }} JWST Documentation (JDOX) here.

    + + {{ inst }} Documentation + +
    +
    +

    diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 241b7fcde..1bf83a2a3 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -71,22 +71,23 @@ # Common monitor views re_path(r'^(?P({}))/dark_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), re_path(r'^(?P({}))/bad_pixel_monitor/$'.format(instruments), monitor_views.bad_pixel_monitor, name='bad_pixel_monitor'), - re_path(r'^(?P({}))/+readnoise_monitor/$'.format(instruments), monitor_views.readnoise_monitor, name='readnoise_monitor'), + re_path(r'^(?P({}))/bias_monitor/$'.format(instruments), monitor_views.bias_monitor, name='bias_monitor'), + re_path(r'^(?P({}))/readnoise_monitor/$'.format(instruments), monitor_views.readnoise_monitor, name='readnoise_monitor'), # Main site views path('about/', views.about, name='about'), + path('anomaly_query/', views.anomaly_query, name='anomaly_query'), path('api/', views.api_landing, name='api'), path('dashboard/', views.dashboard, name='dashboard'), - path('edb/', views.engineering_database, name='edb'), path('download_table/', views.export, name='download_table'), + path('edb/', views.engineering_database, name='edb'), + path('jwqldb/', views.jwqldb_table_viewer, name='jwqldb'), + path('jwqldb/', views.jwqldb_table_viewer, name='jwqldb_table_viewer'), path('query_submit/', views.query_submit, name='query_submit'), - path('anomaly_query/', views.anomaly_query, name='anomaly_query'), - path('table_viewer', views.jwqldb_table_viewer, name='table_viewer'), re_path(r'^(?P({}))/$'.format(instruments), views.instrument, name='instrument'), re_path(r'^(?P({}))/archive/$'.format(instruments), views.archived_proposals, name='archive'), re_path(r'^(?P({}))/unlooked/$'.format(instruments), views.unlooked_images, name='unlooked'), re_path(r'^(?P({}))/(?P[\w]+)/$'.format(instruments), views.view_image, name='view_image'), - re_path(r'^(?P[\w]+)/$/all', views.view_all_images, name='view_all_images'), re_path(r'^(?P({}))/(?P.+)/header/$'.format(instruments), views.view_header, name='view_header'), re_path(r'^(?P({}))/archive/(?P[\d]{{1,5}})/$'.format(instruments), views.archive_thumbnails, name='archive_thumb'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 8cb262f51..79a993ca2 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -42,16 +42,20 @@ from django.http import JsonResponse from django.http import HttpRequest as request +from django.http import HttpResponse from django.shortcuts import render from django.shortcuts import redirect +import pandas as pd from jwql.database.database_interface import load_connection +from jwql.utils import anomaly_query_config from jwql.utils.constants import MONITORS from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.constants import JWST_INSTRUMENT_NAMES_SHORTHAND from jwql.utils.utils import get_base_url from jwql.utils.utils import get_config from jwql.utils.utils import query_unformat +from jwql.website.apps.jwql.data_containers import get_jwqldb_table_view_components from .data_containers import data_trending from .data_containers import get_acknowledgements @@ -68,18 +72,12 @@ from .data_containers import get_jwqldb_table_view_components from .data_containers import thumbnails_ajax from .data_containers import thumbnails_query_ajax -from .forms import FGSAnomalySubmitForm -from .forms import MIRIAnomalySubmitForm -from .forms import NIRCamAnomalySubmitForm -from .forms import NIRISSAnomalySubmitForm -from .forms import NIRSpecAnomalySubmitForm +from .forms import InstrumentAnomalySubmitForm from .forms import AnomalyQueryForm from .data_containers import build_table -from .forms import AnomalyForm from .forms import FileSearchForm from .oauth import auth_info, auth_required -from jwql.utils import anomaly_query_config FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') @@ -91,49 +89,26 @@ def anomaly_query(request): if request.method == 'POST': if form.is_valid(): - miri_filters = [query_unformat(i) for i in form.cleaned_data['miri_filt']] - miri_apers = [query_unformat(i) for i in form.cleaned_data['miri_aper']] - miri_obsmode = [query_unformat(i) for i in form.cleaned_data['miri_obsmode']] - miri_anomalies = [query_unformat(i) for i in form.cleaned_data['miri_anomalies']] - - nirspec_filters = [query_unformat(i) for i in form.cleaned_data['nirspec_filt']] - nirspec_apers = [query_unformat(i) for i in form.cleaned_data['nirspec_aper']] - nirspec_obsmode = [query_unformat(i) for i in form.cleaned_data['nirspec_obsmode']] - nirspec_anomalies = [query_unformat(i) for i in form.cleaned_data['nirspec_anomalies']] - - niriss_filters = [query_unformat(i) for i in form.cleaned_data['niriss_filt']] - niriss_apers = [query_unformat(i) for i in form.cleaned_data['niriss_aper']] - niriss_obsmode = [query_unformat(i) for i in form.cleaned_data['niriss_obsmode']] - niriss_anomalies = [query_unformat(i) for i in form.cleaned_data['niriss_anomalies']] - - nircam_filters = [query_unformat(i) for i in form.cleaned_data['nircam_filt']] - nircam_apers = [query_unformat(i) for i in form.cleaned_data['nircam_aper']] - nircam_obsmode = [query_unformat(i) for i in form.cleaned_data['nircam_obsmode']] - nircam_anomalies = [query_unformat(i) for i in form.cleaned_data['nircam_anomalies']] - - all_filters = {} - all_filters['miri'] = miri_filters - all_filters['nirspec'] = nirspec_filters - all_filters['niriss'] = niriss_filters - all_filters['nircam'] = nircam_filters - - all_apers = {} - all_apers['miri'] = miri_apers - all_apers['nirspec'] = nirspec_apers - all_apers['niriss'] = niriss_apers - all_apers['nircam'] = nircam_apers - - all_obsmodes = {} - all_obsmodes['miri'] = miri_obsmode - all_obsmodes['nirspec'] = nirspec_obsmode - all_obsmodes['niriss'] = niriss_obsmode - all_obsmodes['nircam'] = nircam_obsmode - - all_anomalies = {} - all_anomalies['miri'] = miri_anomalies - all_anomalies['nirspec'] = nirspec_anomalies - all_anomalies['niriss'] = niriss_anomalies - all_anomalies['nircam'] = nircam_anomalies + query_configs = {} + for instrument in ['miri', 'nirspec', 'niriss', 'nircam']: + query_configs[instrument] = {} + query_configs[instrument]['filters'] = [query_unformat(i) for i in form.cleaned_data['{}_filt'.format(instrument)]] + query_configs[instrument]['apertures'] = [query_unformat(i) for i in form.cleaned_data['{}_aper'.format(instrument)]] + query_configs[instrument]['detectors'] = [query_unformat(i) for i in form.cleaned_data['{}_detector'.format(instrument)]] + query_configs[instrument]['exptypes'] = [query_unformat(i) for i in form.cleaned_data['{}_exptype'.format(instrument)]] + query_configs[instrument]['readpatts'] = [query_unformat(i) for i in form.cleaned_data['{}_readpatt'.format(instrument)]] + query_configs[instrument]['gratings'] = [query_unformat(i) for i in form.cleaned_data['{}_grating'.format(instrument)]] + query_configs[instrument]['anomalies'] = [query_unformat(i) for i in form.cleaned_data['{}_anomalies'.format(instrument)]] + + all_filters, all_apers, all_detectors, all_exptypes, all_readpatts, all_gratings, all_anomalies = {}, {}, {}, {}, {}, {}, {} + for instrument in query_configs: + all_filters[instrument] = query_configs[instrument]['filters'] + all_apers[instrument] = query_configs[instrument]['apertures'] + all_detectors[instrument] = query_configs[instrument]['detectors'] + all_exptypes[instrument] = query_configs[instrument]['exptypes'] + all_readpatts[instrument] = query_configs[instrument]['readpatts'] + all_gratings[instrument] = query_configs[instrument]['gratings'] + all_anomalies[instrument] = query_configs[instrument]['anomalies'] anomaly_query_config.INSTRUMENTS_CHOSEN = form.cleaned_data['instrument'] anomaly_query_config.ANOMALIES_CHOSEN_FROM_CURRENT_ANOMALIES = all_anomalies @@ -141,7 +116,10 @@ def anomaly_query(request): anomaly_query_config.FILTERS_CHOSEN = all_filters anomaly_query_config.EXPTIME_MIN = str(form.cleaned_data['exp_time_min']) anomaly_query_config.EXPTIME_MAX = str(form.cleaned_data['exp_time_max']) - anomaly_query_config.OBSERVING_MODES_CHOSEN = all_obsmodes + anomaly_query_config.DETECTORS_CHOSEN = all_detectors + anomaly_query_config.EXPTYPES_CHOSEN = all_exptypes + anomaly_query_config.READPATTS_CHOSEN = all_readpatts + anomaly_query_config.GRATINGS_CHOSEN = all_gratings return redirect('/query_submit') @@ -396,6 +374,7 @@ def archive_thumbnails_query_ajax(request, user): HttpResponse object Outgoing response sent to the webpage """ + # Ensure the instrument is correctly capitalized instruments_list = [] for instrument in anomaly_query_config.INSTRUMENTS_CHOSEN: @@ -403,6 +382,7 @@ def archive_thumbnails_query_ajax(request, user): instruments_list.append(instrument) rootnames = anomaly_query_config.THUMBNAILS + data = thumbnails_query_ajax(rootnames, instruments_list) return JsonResponse(data, json_dumps_params={'indent': 2}) @@ -463,6 +443,35 @@ def engineering_database(request, user): return render(request, template, context) +def export(request, tablename): + """Function to export and download data from JWQLDB Table Viewer + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage + + tablename : str + Name of table to download + + Returns + ------- + response : HttpResponse object + Outgoing response sent to the webpage + """ + table_meta = build_table(tablename) + + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(tablename) + + writer = csv.writer(response) + writer.writerow(table_meta.columns.values) + for _, row in table_meta.iterrows(): + writer.writerow(row.values) + + return response + + def home(request): """Generate the home page @@ -528,7 +537,7 @@ def instrument(request, inst): return render(request, template, context) -def jwqldb_table_viewer(request): +def jwqldb_table_viewer(request, tablename_param=None): """Generate the JWQL Table Viewer view. Parameters @@ -536,8 +545,8 @@ def jwqldb_table_viewer(request): request : HttpRequest object Incoming request from the webpage - user : dict - A dictionary of user credentials. + tablename_param : str + Table name parameter from URL Returns ------- @@ -545,7 +554,12 @@ def jwqldb_table_viewer(request): Outgoing response sent to the webpage """ - table_meta, tablename = get_jwqldb_table_view_components(request) + if tablename_param is None: + table_meta, tablename = get_jwqldb_table_view_components(request) + else: + table_meta = build_table(tablename_param) + tablename = tablename_param + _, _, engine, _ = load_connection(get_config()['connection_string']) all_jwql_tables = engine.table_names() @@ -588,24 +602,6 @@ def jwqldb_table_viewer(request): return render(request, template, context) -def export(request, tablename): - import pandas as pd - import csv - from jwql.website.apps.jwql.data_containers import get_jwqldb_table_view_components - - table_meta = build_table(tablename) - - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(tablename) - - writer = csv.writer(response) - writer.writerow(table_meta.columns.values) - for _, row in table_meta.iterrows(): - writer.writerow(row.values) - - return response - - def not_found(request, *kwargs): """Generate a ``not_found`` page @@ -644,21 +640,15 @@ def query_submit(request): template = 'query_submit.html' - instruments = anomaly_query_config.INSTRUMENTS_CHOSEN - apertures = anomaly_query_config.APERTURES_CHOSEN - filters = anomaly_query_config.FILTERS_CHOSEN - exposure_time_min = anomaly_query_config.EXPTIME_MIN - exposure_time_max = anomaly_query_config.EXPTIME_MAX - observing_modes = anomaly_query_config.OBSERVING_MODES_CHOSEN - anomalies = anomaly_query_config.ANOMALIES_CHOSEN_FROM_CURRENT_ANOMALIES parameters = {} - parameters['instruments'] = instruments - parameters['apertures'] = apertures - parameters['filters'] = filters - parameters['observing_modes'] = observing_modes - parameters['exposure_time_min'] = exposure_time_min - parameters['exposure_time_max'] = exposure_time_max - parameters['anomalies'] = anomalies + parameters['instruments'] = anomaly_query_config.INSTRUMENTS_CHOSEN + parameters['apertures'] = anomaly_query_config.APERTURES_CHOSEN + parameters['filters'] = anomaly_query_config.FILTERS_CHOSEN + parameters['detectors'] = anomaly_query_config.DETECTORS_CHOSEN + parameters['exposure_types'] = anomaly_query_config.EXPTYPES_CHOSEN + parameters['read_patterns'] = anomaly_query_config.READPATTS_CHOSEN + parameters['gratings'] = anomaly_query_config.GRATINGS_CHOSEN + parameters['anomalies'] = anomaly_query_config.ANOMALIES_CHOSEN_FROM_CURRENT_ANOMALIES thumbnails = get_thumbnails_all_instruments(parameters) anomaly_query_config.THUMBNAILS = thumbnails @@ -666,12 +656,11 @@ def query_submit(request): proposal_info = get_proposal_info(thumbnails) context = {'inst': '', - 'anomalies_chosen_from_current_anomalies': - anomaly_query_config.ANOMALIES_CHOSEN_FROM_CURRENT_ANOMALIES, - 'apertures_chosen': apertures, - 'filters_chosen': filters, - 'inst_list_chosen': instruments, - 'observing_modes_chosen': observing_modes, + 'anomalies_chosen_from_current_anomalies': anomaly_query_config.ANOMALIES_CHOSEN_FROM_CURRENT_ANOMALIES, + 'apertures_chosen': anomaly_query_config.APERTURES_CHOSEN, + 'filters_chosen': anomaly_query_config.FILTERS_CHOSEN, + 'inst_list_chosen': anomaly_query_config.INSTRUMENTS_CHOSEN, + 'detectors_chosen': anomaly_query_config.DETECTORS_CHOSEN, 'thumbnails': thumbnails, 'base_url': get_base_url(), 'rootnames': thumbnails, @@ -679,9 +668,9 @@ def query_submit(request): 'all_filenames': thumbnails, 'num_proposals': proposal_info['num_proposals'], 'thumbnails': {'proposals': proposal_info['proposals'], - 'thumbnail_paths': proposal_info['thumbnail_paths'], - 'num_files': proposal_info['num_files']}} - } + 'thumbnail_paths': proposal_info['thumbnail_paths'], + 'num_files': proposal_info['num_files']}} + } return render(request, template, context) @@ -769,80 +758,8 @@ def view_image(request, user, inst, file_root, rewrite=False): current_anomalies = get_current_flagged_anomalies(file_root, inst) # Create a form instance - if inst == 'FGS': - form = FGSAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'MIRI': - form = MIRIAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'NIRCam': - form = NIRCamAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'NIRISS': - form = NIRISSAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'NIRSpec': - form = NIRSpecAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - # If this is a POST request, process the form data - if request.method == 'POST': - anomaly_choices = dict(request.POST)['anomaly_choices'] - if form.is_valid(): - form.update_anomaly_table(file_root, user['ezid'], anomaly_choices) - - # Build the context - context = {'inst': inst, - 'prop_id': file_root[2:7], - 'file_root': file_root, - 'jpg_files': image_info['all_jpegs'], - 'fits_files': image_info['all_files'], - 'suffixes': image_info['suffixes'], - 'num_ints': image_info['num_ints'], - 'form': form} + form = InstrumentAnomalySubmitForm(request.POST or None, instrument=inst.lower(), initial={'anomaly_choices': current_anomalies}) - return render(request, template, context) - - -@auth_required -def view_all_images(request, user, file_root, rewrite=False): - """Generate the image view page - - Parameters - ---------- - request : HttpRequest object - Incoming request from the webpage - user : dict - A dictionary of user credentials. - inst : str - Name of JWST instrument - file_root : str - FITS filename of selected image in filesystem - rewrite : bool, optional - Regenerate the jpg preview of `file` if it already exists? - - Returns - ------- - HttpResponse object - Outgoing response sent to the webpage - """ - - inst = JWST_INSTRUMENT_NAMES_SHORTHAND[file_root.split("_")[-1][:3]] - - # Ensure the instrument is correctly capitalized - inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - - template = 'view_image.html' - image_info = get_image_info(file_root, rewrite) - - # Determine current flagged anomalies - current_anomalies = get_current_flagged_anomalies(file_root, inst) - - # Create a form instance - if inst == 'FGS': - form = FGSAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'MIRI': - form = MIRIAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'NIRCam': - form = NIRCamAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'NIRISS': - form = NIRISSAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) - if inst == 'NIRSpec': - form = NIRSpecAnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) # If this is a POST request, process the form data if request.method == 'POST': anomaly_choices = dict(request.POST)['anomaly_choices'] diff --git a/requirements.txt b/requirements.txt index 33df6ca07..fa1620f6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,31 +1,32 @@ asdf==2.7.1 -astropy==4.0.1.post1 +astropy==4.1 astroquery==0.4.1 -authlib==0.15.1 -bokeh==1.3.4 -codecov==2.1.9 -crds==10.0.0 -cryptography==3.1.1 -django==2.2.5 -flake8==3.8.3 +authlib==0.15.2 +bokeh==2.2.3 +codecov==2.1.10 +crds==10.1.0 +cryptography==3.2.1 +django==3.1.4 +flake8==3.8.4 inflection==0.5.1 -ipython==7.18.1 +ipython==7.19.0 jinja2==2.11.2 jsonschema==3.2.0 jwedb>=0.0.6 jwst==0.17.1 -matplotlib==3.3.2 +matplotlib==3.3.3 nodejs==10.13.0 -numpy==1.19.2 +numpy==1.19.4 numpydoc==1.1.0 -pandas==1.1.2 +pandas==1.1.4 psycopg2==2.8.6 pysiaf==0.10.0 -pytest==6.1.0 +pytest==6.1.2 pytest-cov==2.10.1 -scipy==1.5.2 -sphinx==3.2.1 -sqlalchemy==1.3.19 +scipy==1.5.4 +sphinx==3.3.1 +sqlalchemy==1.3.20 stsci_rtd_theme==0.0.2 twine==3.2.0 +wtforms==2.3.3 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles diff --git a/rtd_requirements.txt b/rtd_requirements.txt index c28f7ce81..3e28ae3e9 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,3 +1,3 @@ -pygments==2.7.1 -pytest==6.1.0 +pygments==2.7.2 +pytest==6.1.2 git+https://github.com/spacetelescope/jwst@stable \ No newline at end of file diff --git a/setup.py b/setup.py index cdbc5788c..f79eb1c19 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -VERSION = '0.24.0' +VERSION = '0.25.0' AUTHORS = 'Matthew Bourque, Lauren Chambers, Misty Cracraft, Mike Engesser, Mees Fix, Joe Filippazzo, Bryan Hilbert, ' AUTHORS += 'Graham Kanarek, Teagan King, Catherine Martlin, Maria Pena-Guerrero, Johannes Sahlmann, Ben Sunnquist' @@ -16,11 +16,11 @@ 'astropy>=3.2.1', 'astroquery>=0.3.9', 'authlib', - 'bokeh>=1.0,<1.4', + 'bokeh', 'codecov', 'crds', 'cryptography', - 'django>=2.0,<3.0', + 'django', 'flake8', 'inflection', 'ipython', @@ -41,7 +41,8 @@ 'sphinx', 'sqlalchemy', 'stsci_rtd_theme', - 'twine' + 'twine', + 'wtforms' ] setup(