Merge branch 'develop' into change-update-monitor-table-to-django

.github/workflows/build.yml

@@ -43,9 +43,8 @@ jobs:
       - run: pip install .
       - run: pip list
       - id: version
-        uses: mtkennerly/dunamai-action@v1
-        with:
-          args: --strict --pattern "default-unprefixed" --style semver
+        name: retrieve package version
+        run: echo version=$(python -c "import jwql; print(jwql.__version__)") >> $GITHUB_OUTPUT
       - id: filename
         run: echo "filename=jwql_${{ steps.version.outputs.version }}_conda_${{ runner.os }}_${{ runner.arch }}_py${{ matrix.python-version }}.yml" >> $GITHUB_OUTPUT
       - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.filename.outputs.filename }}

CHANGES.rst

@@ -1,5 +1,23 @@
 ## What's Changed
 
+1.3.0 (2024-12-19)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+* Exclude source-specific WFSS files from observation page by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1651
+* Switch URL for prog info scraping to use the OPO site by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1662
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added logging configuration to config file, and use it when opening logging by @york-stsci in https://github.com/spacetelescope/jwql/pull/1635
+* Fix bad parens in dark monitor model definitions by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1644
+* Add radius keyword to bokeh.figure.circle calls by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1643
+* Remove bokeh templating code by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1647
+* Update Bad Pixel Monitor to use Django DB Models by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1497
+* Update Bias Monitor to use Django DB Models by @bsunnquist in https://github.com/spacetelescope/jwql/pull/1503
+
+
 1.2.11 (2024-08-26)
 ===================
 

jwql/__init__.py

@@ -1,4 +1,6 @@
+import warnings
 from importlib.metadata import version
+
 from jwql.utils import utils
 
 __version__ = version('jwql')
@@ -10,4 +12,4 @@
               f"while JWQL is using {__version__}")
 
 except FileNotFoundError:
-    print('Could not determine jwql config version')
+    warnings.warn('Could not determine jwql config version')

jwql/jwql_monitors/generate_preview_images.py

@@ -36,8 +36,11 @@
 import numpy as np
 
 from jwql.utils import permissions
-from jwql.utils.constants import IGNORED_SUFFIXES, JWST_INSTRUMENT_NAMES_SHORTHAND, NIRCAM_LONGWAVE_DETECTORS, \
-    NIRCAM_SHORTWAVE_DETECTORS, PREVIEW_IMAGE_LISTFILE, THUMBNAIL_LISTFILE
+from jwql.utils.constants import (IGNORED_SUFFIXES,
+                                  JWST_INSTRUMENT_NAMES_SHORTHAND,
+                                  NIRCAM_LONGWAVE_DETECTORS,
+                                  NIRCAM_SHORTWAVE_DETECTORS
+                                  )
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.protect_module import lock_module
 from jwql.utils.preview_image import PreviewImage
@@ -616,22 +619,6 @@ def generate_preview_images(overwrite, programs=None):
         full_preview_files.extend(r[0])
         full_thumbnail_files.extend(r[1])
 
-    # Filter the preview and thumbnail images by instrument and update the listfiles.
-    # We do this by looking for instrument abbreviations in the filenames. But will
-    # this work for level 3 files?? If an instrument abbreviation is not in the filename,
-    # then the preview/thubnail images won't be caught and added here.
-    for abbrev, inst_name in JWST_INSTRUMENT_NAMES_SHORTHAND.items():
-        inst_previews = [ele for ele in full_preview_files if re.search(abbrev, ele, re.IGNORECASE)]
-        inst_thumbs = [ele for ele in full_thumbnail_files if abbrev in ele]
-
-        # Read in the preview image listfile and the thumbnail image list file
-        # and add these new files to each
-        preview_image_listfile = os.path.join(SETTINGS['preview_image_filesystem'], f"{PREVIEW_IMAGE_LISTFILE}_{inst_name}.txt")
-        update_listfile(preview_image_listfile, inst_previews, 'preview')
-
-        thumbnail_image_listfile = os.path.join(SETTINGS['thumbnail_filesystem'], f"{THUMBNAIL_LISTFILE}_{inst_name}.txt")
-        update_listfile(thumbnail_image_listfile, inst_thumbs, 'thumbnail')
-
     # Complete logging:
     logging.info("Completed.")
 
@@ -842,43 +829,6 @@ def process_program(program, overwrite):
     return preview_image_files, thumbnail_files
 
 
-def update_listfile(filename, file_list, filetype):
-    """Add a list of files to a text file. Designed to add new files to the
-    file containing the list of all preview images and the file containing the
-    list of all thumbnail images.
-
-    Parameters
-    ----------
-    filename : str
-        Name, including path, of the file to be amended/created
-
-    file_list : list
-        List of filenames to be added to filename
-
-    filetype : str
-        Descriptor of the contents of the file being amended. Used only for
-        the logging statement
-    """
-    if len(file_list) > 0:
-        if not os.path.isfile(filename):
-            logging.warning(f"{filetype} image listfile not found!! Expected to be at {filename}. Creating a new file.")
-
-        with open(filename, 'a+') as fobj:
-            # Move read cursor to the start of file.
-            fobj.seek(0)
-
-            # If file is not empty then append '\n'
-            data = fobj.read(100)
-            if len(data) > 0:
-                fobj.write("\n")
-
-            # Append file_list at the end of file
-            for file_to_add in file_list:
-                fobj.write(f'{file_to_add}\n')
-
-        logging.info(f"{filetype} image listfile {filename} updated with new entries.")
-
-
 @lock_module
 def protected_code(overwrite, programs):
     """Protected code ensures only 1 instance of module will run at any given time

jwql/jwql_monitors/monitor_filesystem.py

@@ -49,22 +49,27 @@
 import numpy as np
 from sqlalchemy.exc import DataError
 
-from jwql.database.database_interface import engine
-from jwql.database.database_interface import session
-from jwql.database.database_interface import FilesystemCharacteristics
-from jwql.database.database_interface import FilesystemGeneral
-from jwql.database.database_interface import FilesystemInstrument
-from jwql.database.database_interface import CentralStore
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
 from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS
 from jwql.utils.utils import filename_parser
 from jwql.utils.utils import get_config
 from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
 from jwql.utils.protect_module import lock_module
 from jwql.website.apps.jwql.data_containers import get_instrument_proposals
 
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # Need to set up django apps before we can access the models
+    import django  # noqa: E402 (module level import not at top of file)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    django.setup()
+
+    # Import * is okay here because this module specifically only contains database models
+    # for this monitor
+    from jwql.website.apps.jwql.monitor_models.common import *  # noqa: E402 (module level import not at top of file)
+
 SETTINGS = get_config()
 FILESYSTEM = SETTINGS['filesystem']
 PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary')
@@ -74,6 +79,7 @@
 PREVIEW_IMAGES = SETTINGS['preview_image_filesystem']
 THUMBNAILS = SETTINGS['thumbnail_filesystem']
 LOGS = SETTINGS['log_dir']
+WORKING = SETTINGS['working']
 
 
 def files_per_filter():
@@ -232,7 +238,8 @@ def get_area_stats(central_storage_dict):
              'logs': LOGS,
              'preview_images': PREVIEW_IMAGES,
              'thumbnails': THUMBNAILS,
-             'all': CENTRAL}
+             'all': CENTRAL,
+             'working':WORKING}
 
     counteddirs = []
 
@@ -368,7 +375,7 @@ def initialize_results_dicts():
         A dictionary for the ``central_storage`` database table
     """
 
-    now = datetime.datetime.now()
+    now = datetime.datetime.now(datetime.timezone.utc)
 
     general_results_dict = {}
     general_results_dict['date'] = now
@@ -430,9 +437,9 @@ def update_central_store_database(central_storage_dict):
         new_record['size'] = central_storage_dict[area]['size']
         new_record['used'] = central_storage_dict[area]['used']
         new_record['available'] = central_storage_dict[area]['available']
-        with engine.begin() as connection:
-            connection.execute(CentralStore.__table__.insert(), new_record)
-    session.close()
+
+        entry = CentralStorage(**new_record)
+        entry.save()
 
 
 def update_characteristics_database(char_info):
@@ -447,7 +454,7 @@ def update_characteristics_database(char_info):
         using that filter/pupil.
     """
     logging.info('\tUpdating the characteristics database')
-    now = datetime.datetime.now()
+    now = datetime.datetime.now(datetime.timezone.utc)
 
     # Add data to filesystem_instrument table
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
@@ -458,11 +465,9 @@ def update_characteristics_database(char_info):
         new_record['instrument'] = instrument
         new_record['filter_pupil'] = optics
         new_record['obs_per_filter_pupil'] = values
-        with engine.begin() as connection:
-            connection.execute(
-                FilesystemCharacteristics.__table__.insert(), new_record)
 
-    session.close()
+        entry = FilesystemCharacteristics(**new_record)
+        entry.save()
 
 
 def update_database(general_results_dict, instrument_results_dict, central_storage_dict):
@@ -478,8 +483,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
     """
     logging.info('\tUpdating the database')
 
-    with engine.begin() as connection:
-        connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict)
+    fs_general_entry = FilesystemGeneral(**general_results_dict)
+    fs_general_entry.save()
 
     # Add data to filesystem_instrument table
     for instrument in JWST_INSTRUMENT_NAMES:
@@ -493,13 +498,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
 
             # Protect against updated enum options that have not been propagated to
             # the table definition
-            try:
-                with engine.begin() as connection:
-                    connection.execute(FilesystemInstrument.__table__.insert(), new_record)
-            except DataError as e:
-                logging.error(e)
-
-    session.close()
+            fs_instrument_entry = FilesystemInstrument(**new_record)
+            fs_instrument_entry.save()
 
 
 @lock_module

jwql/shared_tasks/shared_tasks.py

@@ -184,7 +184,7 @@ def _caller(*args, **kwargs):
 
 
 def create_task_log_handler(logger, propagate):
-    log_file_name = configure_logging('shared_tasks')
+    log_file_name = configure_logging('shared_tasks', include_time=False)
     working_dir = os.path.join(get_config()['working'], 'calibrated_data')
     ensure_dir_exists(working_dir)
     celery_log_file_handler = FileHandler(log_file_name)

jwql/tests/test_archive_database_update.py

@@ -0,0 +1,38 @@
+#! /usr/bin/env python
+
+"""Tests for the ``archive_database_update`` module.
+
+Authors
+-------
+
+    - Bryan Hilbert
+
+Use
+---
+
+    These tests can be run via the command line (omit the ``-s`` to
+    suppress verbose output to stdout):
+    ::
+
+        pytest -s test_archive_database_update.py
+"""
+
+
+import pytest
+
+from jwql.website.apps.jwql import archive_database_update
+
+
+def test_filter_rootnames():
+    """Test the filtering of source-based level 2 files
+    """
+    files = ['jw06434-c1021_s000001510_nircam_f444w-grismr.fits',
+             'jw01068004001_02102_00001_nrcb4_rate.fits',
+             'jw06434-c1021_t000_nircam_clear-f090w_segm.fits',
+             'jw06434-o001_t000_nircam_clear-f090w_segm.fits',
+             'jw02183117001_03103_00001-seg001_nrca1_rate.fits']
+
+    filtered = archive_database_update.filter_rootnames(files)
+    expected = ['jw01068004001_02102_00001_nrcb4_rate.fits',
+                'jw02183117001_03103_00001-seg001_nrca1_rate.fits']
+    assert filtered == expected