From 79c8f36a43b5d98c48b51121ef9156e25d069481 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Wed, 21 Feb 2024 16:58:37 -0800 Subject: [PATCH 01/31] Update filesystem monitor to use django db --- jwql/jwql_monitors/monitor_filesystem.py | 42 +++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 2c1f4d379..bcb531458 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -49,22 +49,27 @@ import numpy as np from sqlalchemy.exc import DataError -from jwql.database.database_interface import engine -from jwql.database.database_interface import session -from jwql.database.database_interface import FilesystemCharacteristics -from jwql.database.database_interface import FilesystemGeneral -from jwql.database.database_interface import FilesystemInstrument -from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table from jwql.utils.protect_module import lock_module from jwql.website.apps.jwql.data_containers import get_instrument_proposals +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + # Need to set up django apps before we can access the models + import django # noqa: E402 (module level import not at top of file) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + django.setup() + + # Import * is okay here because this module specifically only contains database models + # for this monitor + from jwql.website.apps.jwql.monitor_models.common import * # noqa: E402 (module level import not at top of file) + SETTINGS = get_config() FILESYSTEM = SETTINGS['filesystem'] PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary') @@ -426,9 +431,9 @@ def update_central_store_database(central_storage_dict): new_record['size'] = central_storage_dict[area]['size'] new_record['used'] = central_storage_dict[area]['used'] new_record['available'] = central_storage_dict[area]['available'] - with engine.begin() as connection: - connection.execute(CentralStore.__table__.insert(), new_record) - session.close() + + entry = CentralStorage(**new_record) + entry.save() def update_characteristics_database(char_info): @@ -454,11 +459,9 @@ def update_characteristics_database(char_info): new_record['instrument'] = instrument new_record['filter_pupil'] = optics new_record['obs_per_filter_pupil'] = values - with engine.begin() as connection: - connection.execute( - FilesystemCharacteristics.__table__.insert(), new_record) - session.close() + entry = FilesystemCharacteristics(**new_record) + entry.save() def update_database(general_results_dict, instrument_results_dict, central_storage_dict): @@ -474,8 +477,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora """ logging.info('\tUpdating the database') - with engine.begin() as connection: - connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict) + fs_general_entry = FilesystemGeneral(**general_results_dict) + fs_general_entry.save() # Add data to filesystem_instrument table for instrument in JWST_INSTRUMENT_NAMES: @@ -489,13 +492,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora # Protect against updated enum options that have not been propagated to # the table definition - try: - with engine.begin() as connection: - connection.execute(FilesystemInstrument.__table__.insert(), new_record) - except DataError as e: - logging.error(e) - - session.close() + fs_instrument_entry = FilesystemInstrument(**new_record) + fs_instrument_entry.save() @lock_module From 7607712b342d191faa46f8258e3ec2cc4cedc2bd Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Mon, 14 Oct 2024 11:54:00 -0400 Subject: [PATCH 02/31] explicitly install and run dunamai within environment to retrieve version --- .github/workflows/build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39515b10f..52f8cce1a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,12 +40,10 @@ jobs: conda init-shell: none generate-run-shell: true + - id: version + run: echo version=$(dunamai from any --strict --pattern "default-unprefixed" --style semver) >> $GITHUB_OUTPUT - run: pip install . - run: pip list - - id: version - uses: mtkennerly/dunamai-action@v1 - with: - args: --strict --pattern "default-unprefixed" --style semver - id: filename run: echo "filename=jwql_${{ steps.version.outputs.version }}_conda_${{ runner.os }}_${{ runner.arch }}_py${{ matrix.python-version }}.yml" >> $GITHUB_OUTPUT - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.filename.outputs.filename }} From e930d84eaf89c04f334b512e164d5da57c189012 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Mon, 14 Oct 2024 11:59:23 -0400 Subject: [PATCH 03/31] attempt to install / remove dunamai with micromamba --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52f8cce1a..b2ce03d9b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,11 @@ jobs: init-shell: none generate-run-shell: true - id: version - run: echo version=$(dunamai from any --strict --pattern "default-unprefixed" --style semver) >> $GITHUB_OUTPUT + name: retrieve package version with dunamai + run: | + micromamba install -y dunamai + echo version=$(dunamai from any --strict --pattern "default-unprefixed" --style semver) >> $GITHUB_OUTPUT + micromamba remove -y dunamai - run: pip install . - run: pip list - id: filename From b0ac819b9a25c552990f349cb90551509477e9ea Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Mon, 14 Oct 2024 12:06:19 -0400 Subject: [PATCH 04/31] just use __version__ --- .github/workflows/build.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2ce03d9b..d6fe8f42b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,14 +40,11 @@ jobs: conda init-shell: none generate-run-shell: true - - id: version - name: retrieve package version with dunamai - run: | - micromamba install -y dunamai - echo version=$(dunamai from any --strict --pattern "default-unprefixed" --style semver) >> $GITHUB_OUTPUT - micromamba remove -y dunamai - run: pip install . - run: pip list + - id: version + name: retrieve package version + run: echo version=$(python -c "import jwql; print(jwql.__version__)") >> $GITHUB_OUTPUT - id: filename run: echo "filename=jwql_${{ steps.version.outputs.version }}_conda_${{ runner.os }}_${{ runner.arch }}_py${{ matrix.python-version }}.yml" >> $GITHUB_OUTPUT - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.filename.outputs.filename }} From a21a01f177235a8a9dc768544c8b27b2f3f7acdc Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Mon, 14 Oct 2024 12:10:36 -0400 Subject: [PATCH 05/31] use warning instead --- jwql/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwql/__init__.py b/jwql/__init__.py index d1949b6ee..630d20207 100644 --- a/jwql/__init__.py +++ b/jwql/__init__.py @@ -1,4 +1,6 @@ +import warnings from importlib.metadata import version + from jwql.utils import utils __version__ = version('jwql') @@ -10,4 +12,4 @@ f"while JWQL is using {__version__}") except FileNotFoundError: - print('Could not determine jwql config version') + warnings.warn('Could not determine jwql config version') From f288a55f218661fbe8bcb0b7a6fad5ed13c5ec1b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 30 Oct 2024 14:11:25 -0400 Subject: [PATCH 06/31] Filter rootnames/filenames before calling the filename_parser --- jwql/utils/constants.py | 1 + jwql/utils/utils.py | 13 +++- .../apps/jwql/archive_database_update.py | 72 ++++++++++++++++++- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 37d390a74..37ee98757 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -381,6 +381,7 @@ FILE_PROG_ID_LEN = 5 FILE_SEG_LEN = 3 FILE_SOURCE_ID_LEN = 5 +FILE_SOURCE_ID_LONG_LEN = 9 FILE_TARG_ID_LEN = 3 FILE_VISIT_GRP_LEN = 2 FILE_VISIT_LEN = 3 diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index d96388d9a..49d060422 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -52,7 +52,7 @@ from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ - FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ + FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SOURCE_ID_LONG_LEN, FILE_SUFFIX_TYPES, \ FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ JWST_INSTRUMENT_NAMES_SHORTHAND, ON_GITHUB_ACTIONS __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -426,6 +426,17 @@ def filename_parser(filename): r"(?P\d{" + f"{FILE_VISIT_LEN}" + "})"\ r"(_.._msa.fits)" + # Stage 2 WFSS source-based files + # e.g. jw06434-c1021_s000001510_nircam_f444w-grismr + #stage_2_source = \ + # r"jw" \ + # r"(?P\d{" + f"{FILE_PROG_ID_LEN}" + "})"\ + # r"-(?P(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"\ + # r"_(?P(s)\d{" + f"{FILE_SOURCE_ID_LONG_LEN}" + "})"\ + # r"_(?P(nircam|niriss|nirspec|miri|fgs))"\ + # r"_(?P((?!_)[\w-])+)"\ + # r"-" + # Stage 3 filenames with target ID # e.g. "jw80600-o009_t001_miri_f1130w_i2d.fits" stage_3_target_id = \ diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index 748cde5dc..674991f8b 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -43,13 +43,20 @@ import logging import os import argparse +import re import numpy as np import django from django.apps import apps from jwql.utils.protect_module import lock_module -from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD +from jwql.utils.constants import (DEFAULT_MODEL_CHARFIELD, + FILE_PROG_ID_LEN, + FILE_AC_O_ID_LEN, + FILE_AC_CAR_ID_LEN, + FILE_SOURCE_ID_LONG_LEN, + FILE_TARG_ID_LEN + ) # These lines are needed in order to use the Django models in a standalone # script (as opposed to code run as a result of a webpage request). If these @@ -113,6 +120,11 @@ def get_updates(update_database): # Get set of unique rootnames all_rootnames = set(['_'.join(f.split('/')[-1].split('_')[:-1]) for f in filenames]) + + # Filter source-based level 2 files out of the rootnames and filenames + all_rootnames = filter_rootnames(all_rootnames) + filenames = filter_filenames(filenames, all_rootnames) + rootnames = [] for rootname in all_rootnames: filename_dict = filename_parser(rootname) @@ -510,6 +522,64 @@ def fill_empty_rootfileinfo(rootfileinfo_set): logging.info(f'\tSaved {saved_rootfileinfos} Root File Infos') +def filter_filenames(fnames, roots): + """Filter out filenames from ``fnames`` that don't match the names in ``roots`` + + Parameters + ---------- + fnames : list + List of filenames + + roots : list + List of rootnames + + Returns + ------- + filtered_fnames : list + Filtered list of filenames + """ + filtered_fnames = [] + for fname in fnames: + for root in roots: + if root in fname: + filtered_fnames.append(fname) + break + return filtered_fnames + + +def filter_rootnames(rootnames): + """Filter out rootnames that we know can't be parsed by the filename_parser. We use this + custom filter here rather than within the filename parser itself because in archive_database_update + we can end up providing thousands of unrecognized filenames (e.g. source-based WFSS files) to + the filename parser, which would result in thousands of logging statments and massive log files. + This way, we filter out the rootnames that obviously won't be parsed before calling the + filename_parser with the rest. jw06434-c1021_s000001510_nircam_f444w-grismr + jw06434-c1021_t000_nircam_clear-f090w_segm.fits + + Parameters + ---------- + rootnames : list + List of rootnames + + Returns + ------- + good_rootnames : list + List of rootnames that do not match the filters + """ + stage_2_source = \ + r"jw" \ + r"(?P\d{" + f"{FILE_PROG_ID_LEN}" + "})"\ + r"-(?P(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"\ + r"_(?P(s\d{" + f"{FILE_SOURCE_ID_LONG_LEN}" + r"}|(t)\d{" + f"{FILE_TARG_ID_LEN}" + "}))"\ + r"_(?P(nircam|niriss|miri))"\ + r"_(?P((?!_)[\w-])+)"\ + r"-" + + elements = re.compile(stage_2_source) + good_rootnames = [e for e in rootnames if elements.match(e) is None] + return good_rootnames + + @lock_module def protected_code(update_database, fill_empty_list): """Protected code ensures only 1 instance of module will run at any given time From b074376716e8bd3c12f785e2e0898016327adf87 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 30 Oct 2024 14:19:58 -0400 Subject: [PATCH 07/31] Remove commented out regex --- jwql/utils/utils.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 49d060422..349b5fe0d 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -426,17 +426,6 @@ def filename_parser(filename): r"(?P\d{" + f"{FILE_VISIT_LEN}" + "})"\ r"(_.._msa.fits)" - # Stage 2 WFSS source-based files - # e.g. jw06434-c1021_s000001510_nircam_f444w-grismr - #stage_2_source = \ - # r"jw" \ - # r"(?P\d{" + f"{FILE_PROG_ID_LEN}" + "})"\ - # r"-(?P(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"\ - # r"_(?P(s)\d{" + f"{FILE_SOURCE_ID_LONG_LEN}" + "})"\ - # r"_(?P(nircam|niriss|nirspec|miri|fgs))"\ - # r"_(?P((?!_)[\w-])+)"\ - # r"-" - # Stage 3 filenames with target ID # e.g. "jw80600-o009_t001_miri_f1130w_i2d.fits" stage_3_target_id = \ From dbad33779f18b2ae29af3b1c64a5b617ce0cf161 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 30 Oct 2024 14:20:53 -0400 Subject: [PATCH 08/31] Remove unused constant --- jwql/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 349b5fe0d..d96388d9a 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -52,7 +52,7 @@ from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ - FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SOURCE_ID_LONG_LEN, FILE_SUFFIX_TYPES, \ + FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ JWST_INSTRUMENT_NAMES_SHORTHAND, ON_GITHUB_ACTIONS __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) From c023f81f57db153d1386ec78a296ce3ecfd44897 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 12 Nov 2024 20:36:08 -0500 Subject: [PATCH 09/31] Updating fields to be array fields with proper fields and defaults --- .../apps/jwql/monitor_models/common.py | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index ebcf7c82a..96e6938c7 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -172,11 +172,14 @@ class Meta: For more information please see: ```https://docs.djangoproject.com/en/2.0/topics/db/models/``` """ + # This is an auto-generated Django model module. # Feel free to rename the models, but don't rename db_table values or field names. from django.db import models from django.contrib.postgres.fields import ArrayField +from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER + class Monitor(models.Model): monitor_name = models.CharField() @@ -187,7 +190,7 @@ class Monitor(models.Model): class Meta: managed = True - db_table = 'monitor' + db_table = "monitor" class CentralStorage(models.Model): @@ -199,18 +202,34 @@ class CentralStorage(models.Model): class Meta: managed = True - db_table = 'central_storage' + db_table = "central_storage" class FilesystemCharacteristics(models.Model): date = models.DateTimeField() instrument = models.TextField() # This field type is a guess. - filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. - obs_per_filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. + filter_pupil = ArrayField( + models.CharField( + max_length=MAX_LEN_FILTER, + help_text="source file names", + default=DEFAULT_MODEL_CHARFIELD, + ), + blank=True, + null=True, + ) + obs_per_filter_pupil = ArrayField( + models.CharField( + max_length=MAX_LEN_FILTER, + help_text="source file names", + default=DEFAULT_MODEL_CHARFIELD, + ), + blank=True, + null=True, + ) class Meta: managed = True - db_table = 'filesystem_characteristics' + db_table = "filesystem_characteristics" class FilesystemGeneral(models.Model): @@ -224,7 +243,7 @@ class FilesystemGeneral(models.Model): class Meta: managed = True - db_table = 'filesystem_general' + db_table = "filesystem_general" class FilesystemInstrument(models.Model): @@ -236,8 +255,8 @@ class FilesystemInstrument(models.Model): class Meta: managed = True - db_table = 'filesystem_instrument' - unique_together = (('date', 'instrument', 'filetype'),) + db_table = "filesystem_instrument" + unique_together = (("date", "instrument", "filetype"),) class FgsAnomaly(models.Model): @@ -257,7 +276,7 @@ class FgsAnomaly(models.Model): class Meta: managed = True - db_table = 'fgs_anomaly' + db_table = "fgs_anomaly" class MiriAnomaly(models.Model): @@ -274,15 +293,19 @@ class MiriAnomaly(models.Model): row_pull_down = models.BooleanField() other = models.BooleanField() column_pull_down = models.BooleanField() - mrs_glow = models.BooleanField(db_column='MRS_Glow') # Field name made lowercase. - mrs_zipper = models.BooleanField(db_column='MRS_Zipper') # Field name made lowercase. + mrs_glow = models.BooleanField(db_column="MRS_Glow") # Field name made lowercase. + mrs_zipper = models.BooleanField( + db_column="MRS_Zipper" + ) # Field name made lowercase. row_pull_up = models.BooleanField() - lrs_contamination = models.BooleanField(db_column='LRS_Contamination') # Field name made lowercase. + lrs_contamination = models.BooleanField( + db_column="LRS_Contamination" + ) # Field name made lowercase. tree_rings = models.BooleanField() class Meta: managed = True - db_table = 'miri_anomaly' + db_table = "miri_anomaly" class NircamAnomaly(models.Model): @@ -307,7 +330,7 @@ class NircamAnomaly(models.Model): class Meta: managed = True - db_table = 'nircam_anomaly' + db_table = "nircam_anomaly" class NirissAnomaly(models.Model): @@ -329,7 +352,7 @@ class NirissAnomaly(models.Model): class Meta: managed = True - db_table = 'niriss_anomaly' + db_table = "niriss_anomaly" class NirspecAnomaly(models.Model): @@ -345,10 +368,12 @@ class NirspecAnomaly(models.Model): data_transfer_error = models.BooleanField() ghost = models.BooleanField() snowball = models.BooleanField() - dominant_msa_leakage = models.BooleanField(db_column='Dominant_MSA_Leakage') # Field name made lowercase. + dominant_msa_leakage = models.BooleanField( + db_column="Dominant_MSA_Leakage" + ) # Field name made lowercase. optical_short = models.BooleanField() other = models.BooleanField() class Meta: managed = True - db_table = 'nirspec_anomaly' + db_table = "nirspec_anomaly" From 69680686437f478cef6016ff944c056d0dac158e Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 12 Nov 2024 21:25:25 -0500 Subject: [PATCH 10/31] Updating field from guess of char to int --- jwql/website/apps/jwql/monitor_models/common.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 96e6938c7..527130c25 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -211,18 +211,14 @@ class FilesystemCharacteristics(models.Model): filter_pupil = ArrayField( models.CharField( max_length=MAX_LEN_FILTER, - help_text="source file names", + help_text="filter and/or pupil name", default=DEFAULT_MODEL_CHARFIELD, ), blank=True, null=True, ) obs_per_filter_pupil = ArrayField( - models.CharField( - max_length=MAX_LEN_FILTER, - help_text="source file names", - default=DEFAULT_MODEL_CHARFIELD, - ), + models.IntegerField(), blank=True, null=True, ) From 1e83d51afb127caae72e99ca49b6ef6e0d2b6c8b Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 14 Nov 2024 11:23:41 -0500 Subject: [PATCH 11/31] Adding timezone info --- jwql/jwql_monitors/monitor_filesystem.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 0809a2e69..295d2ea01 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -79,6 +79,7 @@ PREVIEW_IMAGES = SETTINGS['preview_image_filesystem'] THUMBNAILS = SETTINGS['thumbnail_filesystem'] LOGS = SETTINGS['log_dir'] +WORKING = SETTINGS['working'] def files_per_filter(): @@ -237,7 +238,8 @@ def get_area_stats(central_storage_dict): 'logs': LOGS, 'preview_images': PREVIEW_IMAGES, 'thumbnails': THUMBNAILS, - 'all': CENTRAL} + 'all': CENTRAL, + 'working':WORKING} counteddirs = [] @@ -373,7 +375,7 @@ def initialize_results_dicts(): A dictionary for the ``central_storage`` database table """ - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) general_results_dict = {} general_results_dict['date'] = now @@ -452,7 +454,7 @@ def update_characteristics_database(char_info): using that filter/pupil. """ logging.info('\tUpdating the characteristics database') - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) # Add data to filesystem_instrument table for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: From 7c5481ba9f04b35d5ddc007b133b126d51e6a561 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 14 Nov 2024 11:24:11 -0500 Subject: [PATCH 12/31] Adding migration file --- ...emcharacteristics_filter_pupil_and_more.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py diff --git a/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py new file mode 100644 index 000000000..2e4e04a7d --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1 on 2024-11-13 14:56 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0027_alter_fgsbadpixelstats_source_files_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='filesystemcharacteristics', + name='filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='filesystemcharacteristics', + name='obs_per_filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None), + ), + ] From d4d44e25a2eb1721098477ec53dcb211de9d63c9 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 18 Nov 2024 17:25:05 -0500 Subject: [PATCH 13/31] Generating filetype and count to be EnumFields --- jwql/website/apps/jwql/monitor_models/common.py | 11 +++++++---- pyproject.toml | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 527130c25..8c523d9df 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -177,8 +177,12 @@ class Meta: # Feel free to rename the models, but don't rename db_table values or field names. from django.db import models from django.contrib.postgres.fields import ArrayField +from django_enum import EnumField +from enum import Enum -from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER +from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER, FILE_SUFFIX_TYPES + +FILE_SUFFIX_CLASS = Enum('FILE_SUFFIX_CLASS', FILE_SUFFIX_TYPES) class Monitor(models.Model): @@ -244,9 +248,8 @@ class Meta: class FilesystemInstrument(models.Model): date = models.DateTimeField() - instrument = models.TextField() # This field type is a guess. - filetype = models.TextField() # This field type is a guess. - count = models.IntegerField() + filetype = EnumField(FILE_SUFFIX_CLASS) # This field type is a guess. + count = EnumField(FILE_SUFFIX_CLASS) size = models.FloatField() class Meta: diff --git a/pyproject.toml b/pyproject.toml index 51a66c40b..306d3c631 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "celery>=5.3.6,<6", "crds>=11.17.19,<12", "django>=5.0.3,<6", + "django-enum>=2.0.2,<3", "gunicorn>=22.0.0,<23.0.0", "inflection>=0.5.1,<0.6", "jsonschema>=4.21.1,<5", From 5b36c373ceb6d4fd7ba7aeefb145b5647e53e30b Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Wed, 20 Nov 2024 13:34:15 -0500 Subject: [PATCH 14/31] Re-adding instrument and updating field types --- jwql/website/apps/jwql/monitor_models/common.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 8c523d9df..4d53e7390 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -184,6 +184,12 @@ class Meta: FILE_SUFFIX_CLASS = Enum('FILE_SUFFIX_CLASS', FILE_SUFFIX_TYPES) +class InstrumentEnum(Enum): + FGS_TYPE = "fgs" + MIRI_TYPE = "miri" + NIRCAM_TYPE = "nircam" + NIRISS_TYPE = "niriss" + NIRSPEC_TYPE = "nirspec" class Monitor(models.Model): monitor_name = models.CharField() @@ -248,8 +254,9 @@ class Meta: class FilesystemInstrument(models.Model): date = models.DateTimeField() + instrument = EnumField(InstrumentEnum) # This field type is a guess. filetype = EnumField(FILE_SUFFIX_CLASS) # This field type is a guess. - count = EnumField(FILE_SUFFIX_CLASS) + count = models.IntegerField() size = models.FloatField() class Meta: From f573bef52e3a70354e0a46542d157557eb9e54da Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 25 Nov 2024 10:18:00 -0500 Subject: [PATCH 15/31] Adding strenum in place of enum --- jwql/website/apps/jwql/monitor_models/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 4d53e7390..47c01f8b9 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -178,13 +178,13 @@ class Meta: from django.db import models from django.contrib.postgres.fields import ArrayField from django_enum import EnumField -from enum import Enum +from enum import StrEnum from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER, FILE_SUFFIX_TYPES -FILE_SUFFIX_CLASS = Enum('FILE_SUFFIX_CLASS', FILE_SUFFIX_TYPES) +FILE_SUFFIX_CLASS = StrEnum('FILE_SUFFIX_CLASS', FILE_SUFFIX_TYPES) -class InstrumentEnum(Enum): +class InstrumentEnum(StrEnum): FGS_TYPE = "fgs" MIRI_TYPE = "miri" NIRCAM_TYPE = "nircam" @@ -217,7 +217,7 @@ class Meta: class FilesystemCharacteristics(models.Model): date = models.DateTimeField() - instrument = models.TextField() # This field type is a guess. + instrument = EnumField(InstrumentEnum) # This field type is a guess. filter_pupil = ArrayField( models.CharField( max_length=MAX_LEN_FILTER, From d133cf84ad507637c12bb5e2e1e9b48dca901b27 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 18 Dec 2024 09:57:51 -0500 Subject: [PATCH 16/31] Switch URL for prog info scraping to use the OPO site --- jwql/website/apps/jwql/data_containers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 9819282fa..2eb0a00fd 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -2057,12 +2057,15 @@ def text_scrape(prop_id): program_meta : dict Dictionary containing information about program """ + # Ensure prop_id is a 5-digit string + prop_id = str(prop_id).zfill(5) # Generate url - url = 'http://www.stsci.edu/cgi-bin/get-proposal-info?id=' + str(prop_id) + '&submit=Go&observatory=JWST' + url = f'https://www.stsci.edu/jwst-program-info/program/?program={prop_id}' html = BeautifulSoup(requests.get(url).text, 'lxml') not_available = "not available via this interface" in html.text not_available |= "temporarily unable" in html.text + not_available |= "internal error" in html.text program_meta = {} program_meta['prop_id'] = prop_id From 0e3c5ed72798cef20eae2cf178bd054f79cd4271 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 19 Dec 2024 17:42:53 -0500 Subject: [PATCH 17/31] Changes for 1.3.0 --- CHANGES.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index b29b79840..5cdd209b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,23 @@ ## What's Changed +1.3.0 (2024-12-19) +================== + +Web Application +~~~~~~~~~~~~~~~ +* Exclude source-specific WFSS files from observation page by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1651 +* Switch URL for prog info scraping to use the OPO site by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1662 + +Project & API Documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Added logging configuration to config file, and use it when opening logging by @york-stsci in https://github.com/spacetelescope/jwql/pull/1635 +* Fix bad parens in dark monitor model definitions by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1644 +* Add radius keyword to bokeh.figure.circle calls by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1643 +* Remove bokeh templating code by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1647 +* Update Bad Pixel Monitor to use Django DB Models by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1497 +* Update Bias Monitor to use Django DB Models by @bsunnquist in https://github.com/spacetelescope/jwql/pull/1503 + + 1.2.11 (2024-08-26) =================== From 5e87c79ad7156a142240a1fdf060c170623c7268 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 20 Dec 2024 09:00:40 -0500 Subject: [PATCH 18/31] Fix program type info from new link --- jwql/website/apps/jwql/data_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 2eb0a00fd..c8bb57fff 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -2084,7 +2084,7 @@ def text_scrape(prop_id): links = html.findAll('a') - proposal_type = links[0].contents[0] + proposal_type = links[3].contents[0] program_meta['prop_type'] = proposal_type From e125c9813710d3cdb558035c6dddcc179c03e543 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 15 Jan 2025 10:56:44 -0500 Subject: [PATCH 19/31] Add test for new function --- jwql/tests/test_archive_database_update.py | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 jwql/tests/test_archive_database_update.py diff --git a/jwql/tests/test_archive_database_update.py b/jwql/tests/test_archive_database_update.py new file mode 100644 index 000000000..74568f829 --- /dev/null +++ b/jwql/tests/test_archive_database_update.py @@ -0,0 +1,38 @@ +#! /usr/bin/env python + +"""Tests for the ``archive_database_update`` module. + +Authors +------- + + - Bryan Hilbert + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s test_archive_database_update.py +""" + + +import pytest + +from jwql.website.apps.jwql import archive_database_update + + +def test_filter_rootnames(): + """Test the filtering of source-based level 2 files + """ + files = ['jw06434-c1021_s000001510_nircam_f444w-grismr.fits', + 'jw01068004001_02102_00001_nrcb4_rate.fits', + 'jw06434-c1021_t000_nircam_clear-f090w_segm.fits', + 'jw06434-o001_t000_nircam_clear-f090w_segm.fits', + 'jw02183117001_03103_00001-seg001_nrca1_rate.fits'] + + filtered = archive_database_update.filter_rootnames(files) + expected = ['jw01068004001_02102_00001_nrcb4_rate.fits', + 'jw02183117001_03103_00001-seg001_nrca1_rate.fits'] + assert filtered == expected From 79f835a2774082699f851c71b9e66cbde68ba07b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 15 Jan 2025 12:02:18 -0500 Subject: [PATCH 20/31] Only define FILESYSTEM if not on github actions --- .../apps/jwql/archive_database_update.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index 674991f8b..93d50cee2 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -55,27 +55,30 @@ FILE_AC_O_ID_LEN, FILE_AC_CAR_ID_LEN, FILE_SOURCE_ID_LONG_LEN, - FILE_TARG_ID_LEN + FILE_TARG_ID_LEN, + ON_GITHUB_ACTIONS, + ON_READTHEDOCS ) -# These lines are needed in order to use the Django models in a standalone -# script (as opposed to code run as a result of a webpage request). If these -# lines are not run, the script will crash when attempting to import the -# Django models in the line below. -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") -django.setup() - -from jwql.website.apps.jwql.models import Archive, Observation, Proposal, RootFileInfo # noqa -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE # noqa -from jwql.utils.logging_functions import log_info, log_fail # noqa -from jwql.utils.monitor_utils import initialize_instrument_monitor # noqa -from jwql.utils.constants import MAST_QUERY_LIMIT # noqa -from jwql.utils.utils import filename_parser, filesystem_path, get_config # noqa -from jwql.website.apps.jwql.data_containers import create_archived_proposals_context # noqa -from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument # noqa -from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument, mast_query_by_rootname # noqa - -FILESYSTEM = get_config()['filesystem'] +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + # These lines are needed in order to use the Django models in a standalone + # script (as opposed to code run as a result of a webpage request). If these + # lines are not run, the script will crash when attempting to import the + # Django models in the line below. + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + django.setup() + + from jwql.website.apps.jwql.models import Archive, Observation, Proposal, RootFileInfo # noqa + from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE # noqa + from jwql.utils.logging_functions import log_info, log_fail # noqa + from jwql.utils.monitor_utils import initialize_instrument_monitor # noqa + from jwql.utils.constants import MAST_QUERY_LIMIT # noqa + from jwql.utils.utils import filename_parser, filesystem_path, get_config # noqa + from jwql.website.apps.jwql.data_containers import create_archived_proposals_context # noqa + from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument # noqa + from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument, mast_query_by_rootname # noqa + + FILESYSTEM = get_config()['filesystem'] @log_info From 365506d0f5179d7197150d568ac1f0801c276d48 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 15 Jan 2025 12:11:14 -0500 Subject: [PATCH 21/31] Fix import order and location --- .../apps/jwql/archive_database_update.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index 93d50cee2..b9b2db181 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -56,9 +56,18 @@ FILE_AC_CAR_ID_LEN, FILE_SOURCE_ID_LONG_LEN, FILE_TARG_ID_LEN, + JWST_INSTRUMENT_NAMES_MIXEDCASE, + MAST_QUERY_LIMIT, ON_GITHUB_ACTIONS, ON_READTHEDOCS ) +from jwql.utils.logging_functions import log_info, log_fail +from jwql.utils.monitor_utils import initialize_instrument_monitor +from jwql.utils.utils import filename_parser, filesystem_path, get_config +from jwql.website.apps.jwql.data_containers import create_archived_proposals_context # noqa +from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument # noqa +from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument, mast_query_by_rootname # noqa + if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: # These lines are needed in order to use the Django models in a standalone @@ -69,15 +78,6 @@ django.setup() from jwql.website.apps.jwql.models import Archive, Observation, Proposal, RootFileInfo # noqa - from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE # noqa - from jwql.utils.logging_functions import log_info, log_fail # noqa - from jwql.utils.monitor_utils import initialize_instrument_monitor # noqa - from jwql.utils.constants import MAST_QUERY_LIMIT # noqa - from jwql.utils.utils import filename_parser, filesystem_path, get_config # noqa - from jwql.website.apps.jwql.data_containers import create_archived_proposals_context # noqa - from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument # noqa - from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument, mast_query_by_rootname # noqa - FILESYSTEM = get_config()['filesystem'] From bcc92d1eddee2f09239d5c83f5eef7dc171efa11 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 15 Jan 2025 12:12:12 -0500 Subject: [PATCH 22/31] make imports easier to read --- jwql/website/apps/jwql/archive_database_update.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index b9b2db181..6a028563f 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -64,9 +64,12 @@ from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.monitor_utils import initialize_instrument_monitor from jwql.utils.utils import filename_parser, filesystem_path, get_config -from jwql.website.apps.jwql.data_containers import create_archived_proposals_context # noqa -from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument # noqa -from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument, mast_query_by_rootname # noqa +from jwql.website.apps.jwql.data_containers import create_archived_proposals_context +from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument +from jwql.website.apps.jwql.data_containers import (get_proposal_info, + mast_query_filenames_by_instrument, + mast_query_by_rootname + ) if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: From 8b2441c40b64253de59a12f01784f949f1477d87 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 15 Jan 2025 12:45:34 -0500 Subject: [PATCH 23/31] Remove code supporting preview image and thumbnail listfiles, which are not used --- ...initial_preview_and_thumbnail_listfiles.py | 63 ------------------- jwql/jwql_monitors/generate_preview_images.py | 53 ---------------- jwql/utils/constants.py | 8 --- jwql/website/apps/jwql/data_containers.py | 38 ----------- 4 files changed, 162 deletions(-) delete mode 100644 jwql/jwql_monitors/create_initial_preview_and_thumbnail_listfiles.py diff --git a/jwql/jwql_monitors/create_initial_preview_and_thumbnail_listfiles.py b/jwql/jwql_monitors/create_initial_preview_and_thumbnail_listfiles.py deleted file mode 100644 index 46cf7cdb5..000000000 --- a/jwql/jwql_monitors/create_initial_preview_and_thumbnail_listfiles.py +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env python - -"""This script can be used to create a new set of preview and thumbnail image listfiles -to be used by generate_preview_image.py. The listfiles will be saved into the same -directories as the preview images and thumbnail images themselves, as defined in -config.json - -Author: B. Hilbert -""" -from glob import glob -import os - -from jwql.utils.protect_module import lock_module -from jwql.utils.utils import get_config - - -# Lock module makes create_files() protected code, ensures only one instance of module will run -@lock_module -def create_files(): - """Create a new set of listfiles""" - inst_strings = ['guider', 'nrc', 'miri', 'nis', 'nrs'] - - config = get_config() - prev_img_dir = config["preview_image_filesystem"] - thumb_img_dir = config["thumbnail_filesystem"] - - # Preview images - for inst_abbrev in inst_strings: - - # Instrument abbreviation to use in output filename - file_abbrev = inst_abbrev - if file_abbrev == 'guider': - file_abbrev = 'fgs' - - # Get list of preview images for each instrument and save - preview_files = sorted(glob(os.path.join(prev_img_dir, f'j*/j*{inst_abbrev}*jpg'))) - prev_listfile = os.path.join(prev_img_dir, f'preview_image_inventory_{file_abbrev}.txt') - write_file(preview_files, prev_listfile) - - # Get list of thumbnail images for each instrument and save - thumb_files = sorted(glob(os.path.join(thumb_img_dir, f'j*/j*{inst_abbrev}*thumb'))) - thumb_listfile = os.path.join(thumb_img_dir, f'thumbnail_inventory_{file_abbrev}.txt') - write_file(thumb_files, thumb_listfile) - - -def write_file(filelist, output_file): - """Write a list of filenames to an ascii file - - Parameters - ---------- - filelist : list - List of strings - - output_file : str - Filename to write strings to - """ - with open(output_file, 'w') as fobj: - for filename in filelist: - fobj.write(f'{filename}\n') - - -if __name__ == '__main__': - create_files() diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py index cb897368e..04c18a9a5 100755 --- a/jwql/jwql_monitors/generate_preview_images.py +++ b/jwql/jwql_monitors/generate_preview_images.py @@ -616,22 +616,6 @@ def generate_preview_images(overwrite, programs=None): full_preview_files.extend(r[0]) full_thumbnail_files.extend(r[1]) - # Filter the preview and thumbnail images by instrument and update the listfiles. - # We do this by looking for instrument abbreviations in the filenames. But will - # this work for level 3 files?? If an instrument abbreviation is not in the filename, - # then the preview/thubnail images won't be caught and added here. - for abbrev, inst_name in JWST_INSTRUMENT_NAMES_SHORTHAND.items(): - inst_previews = [ele for ele in full_preview_files if re.search(abbrev, ele, re.IGNORECASE)] - inst_thumbs = [ele for ele in full_thumbnail_files if abbrev in ele] - - # Read in the preview image listfile and the thumbnail image list file - # and add these new files to each - preview_image_listfile = os.path.join(SETTINGS['preview_image_filesystem'], f"{PREVIEW_IMAGE_LISTFILE}_{inst_name}.txt") - update_listfile(preview_image_listfile, inst_previews, 'preview') - - thumbnail_image_listfile = os.path.join(SETTINGS['thumbnail_filesystem'], f"{THUMBNAIL_LISTFILE}_{inst_name}.txt") - update_listfile(thumbnail_image_listfile, inst_thumbs, 'thumbnail') - # Complete logging: logging.info("Completed.") @@ -842,43 +826,6 @@ def process_program(program, overwrite): return preview_image_files, thumbnail_files -def update_listfile(filename, file_list, filetype): - """Add a list of files to a text file. Designed to add new files to the - file containing the list of all preview images and the file containing the - list of all thumbnail images. - - Parameters - ---------- - filename : str - Name, including path, of the file to be amended/created - - file_list : list - List of filenames to be added to filename - - filetype : str - Descriptor of the contents of the file being amended. Used only for - the logging statement - """ - if len(file_list) > 0: - if not os.path.isfile(filename): - logging.warning(f"{filetype} image listfile not found!! Expected to be at {filename}. Creating a new file.") - - with open(filename, 'a+') as fobj: - # Move read cursor to the start of file. - fobj.seek(0) - - # If file is not empty then append '\n' - data = fobj.read(100) - if len(data) > 0: - fobj.write("\n") - - # Append file_list at the end of file - for file_to_add in file_list: - fobj.write(f'{file_to_add}\n') - - logging.info(f"{filetype} image listfile {filename} updated with new entries.") - - @lock_module def protected_code(overwrite, programs): """Protected code ensures only 1 instance of module will run at any given time diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 37ee98757..07d0d72fd 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -795,10 +795,6 @@ # Determine if the code is being run as part of a Readthedocs build ON_READTHEDOCS = os.environ.get('READTHEDOCS', False) -# Base name for the file listing the preview images for a given instrument. -# The complete name will have "_{instrument.lower}.txt" added to the end of this. -PREVIEW_IMAGE_LISTFILE = "preview_image_inventory" - # All possible proposal categories PROPOSAL_CATEGORIES = ["AR", "CAL", "COM", "DD", "ENG", "GO", "GTO", "NASA", "SURVEY"] @@ -1008,10 +1004,6 @@ class QueryConfigKeys: # boolean accessed according to a viewed flag THUMBNAIL_FILTER_LOOK = ["New", "Viewed"] -# Base name for the file listing the thumbnail images for a given instrument. -# The complete name will have "_{instrument.lower}.txt" added to the end of this. -THUMBNAIL_LISTFILE = "thumbnail_inventory" - # Possible suffix types for time-series exposures TIME_SERIES_SUFFIX_TYPES = ["phot", "whtlt"] diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c8bb57fff..cdb171e25 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -69,7 +69,6 @@ SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, THUMBNAIL_FILTER_LOOK, - THUMBNAIL_LISTFILE, QueryConfigKeys, ) from jwql.utils.credentials import get_mast_token @@ -1873,43 +1872,6 @@ def get_rootnames_from_query(parameters): return filtered_rootnames -def get_thumbnails_by_instrument(inst): - """Return a list of thumbnails available in the filesystem for the - given instrument. - - Parameters - ---------- - inst : str - The instrument of interest (e.g. ``NIRCam``). - - Returns - ------- - preview_images : list - A list of thumbnails available in the filesystem for the - given instrument. - """ - # Get list of all thumbnails - thumb_inventory = f'{THUMBNAIL_LISTFILE}_{inst.lower()}.txt' - all_thumbnails = retrieve_filelist(os.path.join(THUMBNAIL_FILESYSTEM, thumb_inventory)) - - thumbnails = [] - all_proposals = get_instrument_proposals(inst) - for proposal in all_proposals: - results = mast_query_filenames_by_instrument(inst, proposal) - - # Parse the results to get the rootnames - filenames = [result['filename'].split('.')[0] for result in results] - - if len(filenames) > 0: - # Get subset of preview images that match the filenames - prop_thumbnails = [os.path.basename(item) for item in all_thumbnails if - os.path.basename(item).split('_integ')[0] in filenames] - - thumbnails.extend(prop_thumbnails) - - return thumbnails - - def get_thumbnails_by_proposal(proposal): """Return a list of thumbnails available in the filesystem for the given ``proposal``. From 03802350b762bc78bfb8c51fe149814830eb01fe Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 15 Jan 2025 12:55:37 -0500 Subject: [PATCH 24/31] Remove imports for deleted constants --- jwql/jwql_monitors/generate_preview_images.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py index 04c18a9a5..b6510a2ab 100755 --- a/jwql/jwql_monitors/generate_preview_images.py +++ b/jwql/jwql_monitors/generate_preview_images.py @@ -36,8 +36,11 @@ import numpy as np from jwql.utils import permissions -from jwql.utils.constants import IGNORED_SUFFIXES, JWST_INSTRUMENT_NAMES_SHORTHAND, NIRCAM_LONGWAVE_DETECTORS, \ - NIRCAM_SHORTWAVE_DETECTORS, PREVIEW_IMAGE_LISTFILE, THUMBNAIL_LISTFILE +from jwql.utils.constants import (IGNORED_SUFFIXES, + JWST_INSTRUMENT_NAMES_SHORTHAND, + NIRCAM_LONGWAVE_DETECTORS, + NIRCAM_SHORTWAVE_DETECTORS + ) from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.protect_module import lock_module from jwql.utils.preview_image import PreviewImage From 3e192cd338a1651a28d61cc9be86ba2775544b79 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 16 Jan 2025 10:24:26 -0500 Subject: [PATCH 25/31] Switch build_table() to use Django models --- jwql/tests/test_bokeh_dashboard.py | 43 ++++++++++++ jwql/tests/test_data_containers.py | 2 +- jwql/website/apps/jwql/apps.py | 30 ++++++++ jwql/website/apps/jwql/bokeh_dashboard.py | 86 +++++++++++------------ jwql/website/apps/jwql/data_containers.py | 57 ++++++++------- 5 files changed, 145 insertions(+), 73 deletions(-) create mode 100644 jwql/tests/test_bokeh_dashboard.py create mode 100644 jwql/website/apps/jwql/apps.py diff --git a/jwql/tests/test_bokeh_dashboard.py b/jwql/tests/test_bokeh_dashboard.py new file mode 100644 index 000000000..ab672c7e7 --- /dev/null +++ b/jwql/tests/test_bokeh_dashboard.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +"""Tests for the ``bokeh_dashboard`` module in the ``jwql`` web +application. + +Authors +------- + + - Bryan Hilbert + +Use +--- + + These tests can be run via the command line (omit the -s to + suppress verbose output to stdout): + + :: + + pytest -s test_bokeh_dashboard.py +""" + +import os + +from django import setup +import pandas as pd +import pytest + +from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, ON_GITHUB_ACTIONS, ON_READTHEDOCS + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + +# Skip testing this module if on Github Actions +from jwql.website.apps.jwql import bokeh_dashboard # noqa: E402 (module level import not at top of file) + +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + setup() + + +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') +def test_build_table_latest_entry(): + tab = bokeh_dashboard.build_table('FilesystemCharacteristics') + assert isinstance(tab, pd.DataFrame) + assert len(tab['date']) > 0 diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 4b1f1c4ba..275b7a86d 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -48,7 +48,7 @@ @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') def test_build_table(): - tab = data_containers.build_table('filesystem_general') + tab = data_containers.build_table('FilesystemGeneral') assert isinstance(tab, pd.DataFrame) assert len(tab['date']) > 0 diff --git a/jwql/website/apps/jwql/apps.py b/jwql/website/apps/jwql/apps.py new file mode 100644 index 000000000..d8c347d31 --- /dev/null +++ b/jwql/website/apps/jwql/apps.py @@ -0,0 +1,30 @@ +#! /usr/bin/env python + +""" +apps.py is the standard and recommended way to configure application-specific settings +in Django, including tasks like importing additional modules during initialization. + +Author +------ + +B. Hilbert +""" + +from django.apps import AppConfig + + +class JwqlAppConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'jwql' + + def ready(self): + # Import models not defined in models.py here + # By importing these models here, they will be available + # to the build_table() function. + import jwql.website.apps.jwql.monitor_models.bad_pixel + import jwql.website.apps.jwql.monitor_models.bias + import jwql.website.apps.jwql.monitor_models.claw + import jwql.website.apps.jwql.monitor_models.common + import jwql.website.apps.jwql.monitor_models.dark_current + import jwql.website.apps.jwql.monitor_models.readnoise + import jwql.website.apps.jwql.monitor_models.ta diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index a4469be82..14880c721 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -30,6 +30,7 @@ placed in the ``jwql`` directory. """ +from collections import defaultdict from datetime import datetime as dt from math import pi from operator import itemgetter @@ -40,17 +41,30 @@ from bokeh.models.layouts import TabPanel, Tabs from bokeh.plotting import figure from bokeh.transform import cumsum +from django import setup +from django.db.models import OuterRef, Subquery import numpy as np import pandas as pd from sqlalchemy import func, and_ import jwql.database.database_interface as di from jwql.database.database_interface import CentralStore -from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, + FILTERS_PER_INSTRUMENT, + JWST_INSTRUMENT_NAMES_MIXEDCASE, + ON_GITHUB_ACTIONS, + ON_READTHEDOCS + ) from jwql.utils.utils import get_base_url, get_config -from jwql.website.apps.jwql.data_containers import build_table +from jwql.website.apps.jwql.data_containers import build_table, import_all_models from jwql.website.apps.jwql.models import Anomalies +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + setup() + + from jwql.website.apps.jwql.models import get_model_column_names + def build_table_latest_entry(tablename): """Create Pandas dataframe from the most recent entry of a JWQLDB table. @@ -65,46 +79,29 @@ def build_table_latest_entry(tablename): table_meta_data : pandas.DataFrame Pandas data frame version of JWQL database table. """ - # Make dictionary of tablename : class object - # This matches what the user selects in the select element - # in the webform to the python object on the backend. - tables_of_interest = {} - for item in di.__dict__.keys(): - table = getattr(di, item) - if hasattr(table, '__tablename__'): - tables_of_interest[table.__tablename__] = table - - session, _, _, _ = di.load_connection(get_config()['connection_string']) - table_object = tables_of_interest[tablename] # Select table object - - subq = session.query(table_object.instrument, - func.max(table_object.date).label('maxdate') - ).group_by(table_object.instrument).subquery('t2') - - result = session.query(table_object).join( - subq, - and_( - table_object.instrument == subq.c.instrument, - table_object.date == subq.c.maxdate - ) - ) - - # Turn query result into list of dicts - result_dict = [row.__dict__ for row in result.all()] - column_names = table_object.__table__.columns.keys() - - # Build list of column data based on column name. - data = [] - for column in column_names: - column_data = list(map(itemgetter(column), result_dict)) - data.append(column_data) - - data = dict(zip(column_names, data)) + all_models = import_all_models() + table_object = all_models.get(tablename) + column_names = get_model_column_names(table_object) + + if 'instrument' not in column_names: + raise ValueError(f"No 'instrument' column name in {tablename}. Unable to get latest entry by instrument.") + + # Create a subquery to get the latest date for each instrument + subquery = table_object.objects.filter(instrument=OuterRef('instrument')).order_by('-date').values('date')[:1] + + # Query the model with the subquery + most_recent_entries = table_object.objects.filter(date=Subquery(subquery)) + + # Convert the QuerySet into a dictionary + rows = most_recent_entries.values() + data = defaultdict(list) + + for row in rows: + for key, value in row.items(): + data[key].append(value) # Build table. table_meta_data = pd.DataFrame(data) - - session.close() return table_meta_data @@ -360,7 +357,7 @@ def dashboard_filetype_bar_chart(self): # Make Pandas DF for filesystem_instrument # If time delta exists, filter data based on that. - data = build_table('filesystem_instrument') + data = build_table('FilesystemInstrument') # Keep only the rows containing the most recent timestamp data = data[data['date'] == data['date'].max()] @@ -390,8 +387,7 @@ def dashboard_instrument_pie_chart(self): plot : bokeh.plotting.figure Pie chart figure """ - # Replace with jwql.website.apps.jwql.data_containers.build_table - data = build_table('filesystem_instrument') + data = build_table('FilesystemInstrument') # Keep only the rows containing the most recent timestamp data = data[data['date'] == data['date'].max()] @@ -439,7 +435,7 @@ def dashboard_files_per_day(self): A figure with tabs for each instrument. """ - source = build_table('filesystem_general') + source = build_table('FilesystemGeneral') if not pd.isnull(self.delta_t): source = source[(source['date'] >= self.date - self.delta_t) & (source['date'] <= self.date)] @@ -495,7 +491,7 @@ def dashboard_monitor_tracking(self): Numpy array of column values from monitor table. """ - data = build_table('monitor') + data = build_table('Monitor') if not pd.isnull(self.delta_t): data = data[(data['start_time'] >= self.date - self.delta_t) & (data['start_time'] <= self.date)] @@ -551,7 +547,7 @@ def dashboard_exposure_count_by_filter(self): """ # build_table_latest_query will return only the database entries with the latest date. This should # correspond to one row/entry per instrument - data = build_table_latest_entry('filesystem_characteristics') + data = build_table_latest_entry('FilesystemCharacteristics') # Sort by instrument name so that the order of the tabs will always be the same data = data.sort_values('instrument') diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c8bb57fff..46707069b 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -33,7 +33,7 @@ import os import re import tempfile -from collections import OrderedDict +from collections import defaultdict, OrderedDict from datetime import datetime from operator import getitem, itemgetter @@ -46,13 +46,12 @@ from astroquery.mast import Mast from bs4 import BeautifulSoup from django import forms, setup +from django.apps import apps from django.conf import settings from django.contrib import messages from django.core.exceptions import ObjectDoesNotExist from django.db.models.query import QuerySet -from jwql.database import database_interface as di -from jwql.database.database_interface import load_connection from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info, mnemonic_inventory from jwql.utils.constants import ( DEFAULT_MODEL_COMMENT, @@ -83,6 +82,7 @@ get_rootnames_for_instrument_proposal, ) + # Increase the limit on the number of entries that can be returned by # a MAST query. Mast._portal_api_connection.PAGESIZE = MAST_QUERY_LIMIT @@ -96,7 +96,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") setup() - from jwql.website.apps.jwql.models import Anomalies, Observation, Proposal, RootFileInfo + from jwql.website.apps.jwql.models import Anomalies, get_model_column_names, Observation, Proposal, RootFileInfo from .forms import ( InstrumentAnomalySubmitForm, @@ -139,36 +139,23 @@ def build_table(tablename): table_meta_data : pandas.DataFrame Pandas data frame version of JWQL database table. """ - # Make dictionary of tablename : class object - # This matches what the user selects in the select element - # in the webform to the python object on the backend. - tables_of_interest = {} - for item in di.__dict__.keys(): - table = getattr(di, item) - if hasattr(table, '__tablename__'): - tables_of_interest[table.__tablename__] = table - - session, _, _, _ = load_connection(get_config()['connection_string']) - table_object = tables_of_interest[tablename] # Select table object - - result = session.query(table_object) + all_models = import_all_models() + table_object = all_models.get(tablename) - # Turn query result into list of dicts - result_dict = [row.__dict__ for row in result.all()] - column_names = table_object.__table__.columns.keys() + result = table_object.objects.all() + column_names = get_model_column_names(table_object) - # Build list of column data based on column name. - data = [] - for column in column_names: - column_data = list(map(itemgetter(column), result_dict)) - data.append(column_data) + # Convert the QuerySet into a dictionary + rows = result.values() + data = defaultdict(list) - data = dict(zip(column_names, data)) + for row in rows: + for key, value in row.items(): + data[key].append(value) # Build table. table_meta_data = pd.DataFrame(data) - session.close() return table_meta_data @@ -1970,6 +1957,22 @@ def get_thumbnail_by_rootname(rootname): return thumbnail_basename +def import_all_models(): + """ + Dynamically import and return all Django models as a dictionary. + Keys are model names (as strings), and values are model classes. + + Returns + ------- + models : dict + Keys are model names, values are model classes + """ + models = {} + for model in apps.get_app_config('jwql').get_models(): + models[model.__name__] = model + return models + + def log_into_mast(request): """Login via astroquery.mast if user authenticated in web app. From e91b1db3bd11dfbfdd570d9b09fdbd6e14d8ba63 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 16 Jan 2025 11:25:37 -0500 Subject: [PATCH 26/31] Replace bokeh_dashboard sqlalchemy queries with model queries --- jwql/website/apps/jwql/bokeh_dashboard.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index 14880c721..2266055a1 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -49,6 +49,7 @@ import jwql.database.database_interface as di from jwql.database.database_interface import CentralStore +from jwql.website.apps.jwql.monitor_models.common import CentralStorage from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT, JWST_INSTRUMENT_NAMES_MIXEDCASE, @@ -174,16 +175,12 @@ def dashboard_disk_usage(self): # server disk information. config = get_config() - log_data = di.session.query(CentralStore.date, CentralStore.size, CentralStore.available) \ - .filter(CentralStore.area == 'logs') \ - .all() + log_data = list(CentralStorage.objects.filter(area='logs').values('date', 'size', 'available')) # Convert to dataframe log_data = pd.DataFrame(log_data) - preview_data = di.session.query(CentralStore.date, CentralStore.size, CentralStore.available) \ - .filter(CentralStore.area == 'preview_images') \ - .all() + preview_data = list(CentralStorage.objects.filter(area='preview_images').values('date', 'size', 'available')) # Convert to dataframe preview_data = pd.DataFrame(preview_data) @@ -244,7 +241,6 @@ def dashboard_disk_usage(self): tabs = Tabs(tabs=tabs) - di.session.close() return tabs def dashboard_central_store_data_volume(self): @@ -274,7 +270,7 @@ def dashboard_central_store_data_volume(self): for area, color in zip(arealist, colors): # Query for used sizes - results = di.session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area).all() + results = list(CentralStorage.objects.filter(area=area).values('date', 'used')) if results: # Convert to dataframe @@ -311,7 +307,7 @@ def dashboard_central_store_data_volume(self): x_axis_label='Date', y_axis_label='Disk Space (TB)') - cen_store_results = di.session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == 'all').all() + cen_store_results = list(CentralStorage.objects.filter(area='all').values('date', 'used')) # Group by date if cen_store_results: @@ -343,7 +339,6 @@ def dashboard_central_store_data_volume(self): hover_tool.formatters = {'@date': 'datetime'} cen_store_plot.tools.append(hover_tool) - di.session.close() return plot, cen_store_plot def dashboard_filetype_bar_chart(self): From 1ad2d84b8971c7f2c477840cf9c012063f7f2e50 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 16 Jan 2025 11:34:13 -0500 Subject: [PATCH 27/31] PEP8 --- jwql/website/apps/jwql/bokeh_dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index 2266055a1..9261a9f6c 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -219,7 +219,7 @@ def dashboard_disk_usage(self): y_axis_label='Disk Space (TB)') plots[data['shortname']].line(x='date', y='available', source=source, legend_label='Available', line_dash='dashed', line_color='#C85108', line_width=3) - plots[data['shortname']].circle(x='date', y='available', source=source,color='#C85108', radius=5, radius_dimension='y', radius_units='screen') + plots[data['shortname']].circle(x='date', y='available', source=source, color='#C85108', radius=5, radius_dimension='y', radius_units='screen') plots[data['shortname']].line(x='date', y='used', source=source, legend_label='Used', line_dash='dashed', line_color='#355C7D', line_width=3) plots[data['shortname']].circle(x='date', y='used', source=source, color='#355C7D', radius=5, radius_dimension='y', radius_units='screen') From bd3175c6b432c5a1ad4300a427ace093bf7f26e7 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 16 Jan 2025 11:41:53 -0500 Subject: [PATCH 28/31] Skip test if on github --- jwql/tests/test_bokeh_dashboard.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/tests/test_bokeh_dashboard.py b/jwql/tests/test_bokeh_dashboard.py index ab672c7e7..e2d87d043 100644 --- a/jwql/tests/test_bokeh_dashboard.py +++ b/jwql/tests/test_bokeh_dashboard.py @@ -27,13 +27,11 @@ from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, ON_GITHUB_ACTIONS, ON_READTHEDOCS -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") - # Skip testing this module if on Github Actions -from jwql.website.apps.jwql import bokeh_dashboard # noqa: E402 (module level import not at top of file) - if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") setup() + from jwql.website.apps.jwql import bokeh_dashboard # noqa: E402 (module level import not at top of file) @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') From 561ea0189e214951e7db1309083bd5831dd1ba50 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 16 Jan 2025 16:43:41 -0500 Subject: [PATCH 29/31] Updating types for filesystem monitor migrations --- ...emcharacteristics_filter_pupil_and_more.py | 17 ++++++++++- .../apps/jwql/monitor_models/common.py | 28 +++++++++---------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py index 2e4e04a7d..7e6fb3dd3 100644 --- a/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py +++ b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1 on 2024-11-13 14:56 +# Generated by Django 5.1.4 on 2025-01-16 21:35 import django.contrib.postgres.fields from django.db import migrations, models @@ -16,9 +16,24 @@ class Migration(migrations.Migration): name='filter_pupil', field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None), ), + migrations.AlterField( + model_name='filesystemcharacteristics', + name='instrument', + field=models.CharField(), + ), migrations.AlterField( model_name='filesystemcharacteristics', name='obs_per_filter_pupil', field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None), ), + migrations.AlterField( + model_name='filesysteminstrument', + name='filetype', + field=models.CharField(), + ), + migrations.AlterField( + model_name='filesysteminstrument', + name='instrument', + field=models.CharField(default='empty', help_text='JWST instrument name', max_length=7), + ), ] diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 47c01f8b9..ee6da326e 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -177,25 +177,19 @@ class Meta: # Feel free to rename the models, but don't rename db_table values or field names. from django.db import models from django.contrib.postgres.fields import ArrayField -from django_enum import EnumField -from enum import StrEnum -from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER, FILE_SUFFIX_TYPES +from jwql.utils.constants import ( + DEFAULT_MODEL_CHARFIELD, + MAX_LEN_FILTER, + MAX_LEN_INSTRUMENT, +) -FILE_SUFFIX_CLASS = StrEnum('FILE_SUFFIX_CLASS', FILE_SUFFIX_TYPES) - -class InstrumentEnum(StrEnum): - FGS_TYPE = "fgs" - MIRI_TYPE = "miri" - NIRCAM_TYPE = "nircam" - NIRISS_TYPE = "niriss" - NIRSPEC_TYPE = "nirspec" class Monitor(models.Model): monitor_name = models.CharField() start_time = models.DateTimeField() end_time = models.DateTimeField(blank=True, null=True) - status = models.TextField(blank=True, null=True) # This field type is a guess. + status = models.TextField(blank=True, null=True) log_file = models.CharField() class Meta: @@ -217,7 +211,7 @@ class Meta: class FilesystemCharacteristics(models.Model): date = models.DateTimeField() - instrument = EnumField(InstrumentEnum) # This field type is a guess. + instrument = models.CharField() filter_pupil = ArrayField( models.CharField( max_length=MAX_LEN_FILTER, @@ -254,8 +248,12 @@ class Meta: class FilesystemInstrument(models.Model): date = models.DateTimeField() - instrument = EnumField(InstrumentEnum) # This field type is a guess. - filetype = EnumField(FILE_SUFFIX_CLASS) # This field type is a guess. + instrument = models.CharField( + max_length=MAX_LEN_INSTRUMENT, + help_text="JWST instrument name", + default=DEFAULT_MODEL_CHARFIELD, + ) + filetype = models.CharField() count = models.IntegerField() size = models.FloatField() From d77a9fc1cf5e20f332f8381c41356f61c7df94b0 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Fri, 17 Jan 2025 09:10:05 -0500 Subject: [PATCH 30/31] Removing django-enum as dep --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 306d3c631..51a66c40b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ dependencies = [ "celery>=5.3.6,<6", "crds>=11.17.19,<12", "django>=5.0.3,<6", - "django-enum>=2.0.2,<3", "gunicorn>=22.0.0,<23.0.0", "inflection>=0.5.1,<0.6", "jsonschema>=4.21.1,<5", From 4d1e69763fa2a2f95c66d9c9b8f964ad63c68f11 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 17 Jan 2025 12:21:17 -0500 Subject: [PATCH 31/31] Changing celery worker logs to be daily rather than per-task --- jwql/shared_tasks/shared_tasks.py | 2 +- jwql/utils/logging_functions.py | 26 +++++++++++--------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 8e3be0a5c..7696f26c7 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -184,7 +184,7 @@ def _caller(*args, **kwargs): def create_task_log_handler(logger, propagate): - log_file_name = configure_logging('shared_tasks') + log_file_name = configure_logging('shared_tasks', include_time=False) working_dir = os.path.join(get_config()['working'], 'calibrated_data') ensure_dir_exists(working_dir) celery_log_file_handler = FileHandler(log_file_name) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index c20bfce3d..55980a1ee 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -102,7 +102,7 @@ def filter(record): return filter -def configure_logging(module): +def configure_logging(module, include_time=True): """ Configure the log file with a standard logging format. The format in question is set up as follows: @@ -116,11 +116,8 @@ def configure_logging(module): ---------- module : str The name of the module being logged. - production_mode : bool - Whether or not the output should be written to the production - environement. - path : str - Where to write the log if user-supplied path; default to working dir. + include_time : bool, default True + Whether or not the file name should include the time as well as the date. Returns ------- @@ -129,7 +126,7 @@ def configure_logging(module): """ # Determine log file location - log_file = make_log_file(module) + log_file = make_log_file(module, include_time=include_time) # Get the logging configuration dictionary logging_config = get_config()['logging'] @@ -171,7 +168,7 @@ def get_log_status(log_file): return 'FAILURE' -def make_log_file(module): +def make_log_file(module, include_time=True): """Create the log file name based on the module name. The name of the ``log_file`` is a combination of the name of the @@ -181,12 +178,8 @@ def make_log_file(module): ---------- module : str The name of the module being logged. - production_mode : bool - Whether or not the output should be written to the production - environment. - path : str - Where to write the log if user-supplied path; default to - working dir. + include_time : bool, default True + Whether or not the file name should include the time as well as the date. Returns ------- @@ -195,7 +188,10 @@ def make_log_file(module): """ # Build filename - timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M') + if include_time: + timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M') + else: + timestamp = datetime.datetime.now().strftime('%Y-%m-%d') hostname = socket.gethostname() filename = '{0}_{1}_{2}.log'.format(module, hostname, timestamp)