diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 27364f888..295d2ea01 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -49,22 +49,27 @@ import numpy as np from sqlalchemy.exc import DataError -from jwql.database.database_interface import engine -from jwql.database.database_interface import session -from jwql.database.database_interface import FilesystemCharacteristics -from jwql.database.database_interface import FilesystemGeneral -from jwql.database.database_interface import FilesystemInstrument -from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table from jwql.utils.protect_module import lock_module from jwql.website.apps.jwql.data_containers import get_instrument_proposals +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + # Need to set up django apps before we can access the models + import django # noqa: E402 (module level import not at top of file) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + django.setup() + + # Import * is okay here because this module specifically only contains database models + # for this monitor + from jwql.website.apps.jwql.monitor_models.common import * # noqa: E402 (module level import not at top of file) + SETTINGS = get_config() FILESYSTEM = SETTINGS['filesystem'] PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary') @@ -74,6 +79,7 @@ PREVIEW_IMAGES = SETTINGS['preview_image_filesystem'] THUMBNAILS = SETTINGS['thumbnail_filesystem'] LOGS = SETTINGS['log_dir'] +WORKING = SETTINGS['working'] def files_per_filter(): @@ -232,7 +238,8 @@ def get_area_stats(central_storage_dict): 'logs': LOGS, 'preview_images': PREVIEW_IMAGES, 'thumbnails': THUMBNAILS, - 'all': CENTRAL} + 'all': CENTRAL, + 'working':WORKING} counteddirs = [] @@ -368,7 +375,7 @@ def initialize_results_dicts(): A dictionary for the ``central_storage`` database table """ - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) general_results_dict = {} general_results_dict['date'] = now @@ -430,9 +437,9 @@ def update_central_store_database(central_storage_dict): new_record['size'] = central_storage_dict[area]['size'] new_record['used'] = central_storage_dict[area]['used'] new_record['available'] = central_storage_dict[area]['available'] - with engine.begin() as connection: - connection.execute(CentralStore.__table__.insert(), new_record) - session.close() + + entry = CentralStorage(**new_record) + entry.save() def update_characteristics_database(char_info): @@ -447,7 +454,7 @@ def update_characteristics_database(char_info): using that filter/pupil. """ logging.info('\tUpdating the characteristics database') - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) # Add data to filesystem_instrument table for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: @@ -458,11 +465,9 @@ def update_characteristics_database(char_info): new_record['instrument'] = instrument new_record['filter_pupil'] = optics new_record['obs_per_filter_pupil'] = values - with engine.begin() as connection: - connection.execute( - FilesystemCharacteristics.__table__.insert(), new_record) - session.close() + entry = FilesystemCharacteristics(**new_record) + entry.save() def update_database(general_results_dict, instrument_results_dict, central_storage_dict): @@ -478,8 +483,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora """ logging.info('\tUpdating the database') - with engine.begin() as connection: - connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict) + fs_general_entry = FilesystemGeneral(**general_results_dict) + fs_general_entry.save() # Add data to filesystem_instrument table for instrument in JWST_INSTRUMENT_NAMES: @@ -493,13 +498,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora # Protect against updated enum options that have not been propagated to # the table definition - try: - with engine.begin() as connection: - connection.execute(FilesystemInstrument.__table__.insert(), new_record) - except DataError as e: - logging.error(e) - - session.close() + fs_instrument_entry = FilesystemInstrument(**new_record) + fs_instrument_entry.save() @lock_module diff --git a/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py new file mode 100644 index 000000000..7e6fb3dd3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 5.1.4 on 2025-01-16 21:35 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0027_alter_fgsbadpixelstats_source_files_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='filesystemcharacteristics', + name='filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='filesystemcharacteristics', + name='instrument', + field=models.CharField(), + ), + migrations.AlterField( + model_name='filesystemcharacteristics', + name='obs_per_filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='filesysteminstrument', + name='filetype', + field=models.CharField(), + ), + migrations.AlterField( + model_name='filesysteminstrument', + name='instrument', + field=models.CharField(default='empty', help_text='JWST instrument name', max_length=7), + ), + ] diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index ebcf7c82a..ee6da326e 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -172,22 +172,29 @@ class Meta: For more information please see: ```https://docs.djangoproject.com/en/2.0/topics/db/models/``` """ + # This is an auto-generated Django model module. # Feel free to rename the models, but don't rename db_table values or field names. from django.db import models from django.contrib.postgres.fields import ArrayField +from jwql.utils.constants import ( + DEFAULT_MODEL_CHARFIELD, + MAX_LEN_FILTER, + MAX_LEN_INSTRUMENT, +) + class Monitor(models.Model): monitor_name = models.CharField() start_time = models.DateTimeField() end_time = models.DateTimeField(blank=True, null=True) - status = models.TextField(blank=True, null=True) # This field type is a guess. + status = models.TextField(blank=True, null=True) log_file = models.CharField() class Meta: managed = True - db_table = 'monitor' + db_table = "monitor" class CentralStorage(models.Model): @@ -199,18 +206,30 @@ class CentralStorage(models.Model): class Meta: managed = True - db_table = 'central_storage' + db_table = "central_storage" class FilesystemCharacteristics(models.Model): date = models.DateTimeField() - instrument = models.TextField() # This field type is a guess. - filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. - obs_per_filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. + instrument = models.CharField() + filter_pupil = ArrayField( + models.CharField( + max_length=MAX_LEN_FILTER, + help_text="filter and/or pupil name", + default=DEFAULT_MODEL_CHARFIELD, + ), + blank=True, + null=True, + ) + obs_per_filter_pupil = ArrayField( + models.IntegerField(), + blank=True, + null=True, + ) class Meta: managed = True - db_table = 'filesystem_characteristics' + db_table = "filesystem_characteristics" class FilesystemGeneral(models.Model): @@ -224,20 +243,24 @@ class FilesystemGeneral(models.Model): class Meta: managed = True - db_table = 'filesystem_general' + db_table = "filesystem_general" class FilesystemInstrument(models.Model): date = models.DateTimeField() - instrument = models.TextField() # This field type is a guess. - filetype = models.TextField() # This field type is a guess. + instrument = models.CharField( + max_length=MAX_LEN_INSTRUMENT, + help_text="JWST instrument name", + default=DEFAULT_MODEL_CHARFIELD, + ) + filetype = models.CharField() count = models.IntegerField() size = models.FloatField() class Meta: managed = True - db_table = 'filesystem_instrument' - unique_together = (('date', 'instrument', 'filetype'),) + db_table = "filesystem_instrument" + unique_together = (("date", "instrument", "filetype"),) class FgsAnomaly(models.Model): @@ -257,7 +280,7 @@ class FgsAnomaly(models.Model): class Meta: managed = True - db_table = 'fgs_anomaly' + db_table = "fgs_anomaly" class MiriAnomaly(models.Model): @@ -274,15 +297,19 @@ class MiriAnomaly(models.Model): row_pull_down = models.BooleanField() other = models.BooleanField() column_pull_down = models.BooleanField() - mrs_glow = models.BooleanField(db_column='MRS_Glow') # Field name made lowercase. - mrs_zipper = models.BooleanField(db_column='MRS_Zipper') # Field name made lowercase. + mrs_glow = models.BooleanField(db_column="MRS_Glow") # Field name made lowercase. + mrs_zipper = models.BooleanField( + db_column="MRS_Zipper" + ) # Field name made lowercase. row_pull_up = models.BooleanField() - lrs_contamination = models.BooleanField(db_column='LRS_Contamination') # Field name made lowercase. + lrs_contamination = models.BooleanField( + db_column="LRS_Contamination" + ) # Field name made lowercase. tree_rings = models.BooleanField() class Meta: managed = True - db_table = 'miri_anomaly' + db_table = "miri_anomaly" class NircamAnomaly(models.Model): @@ -307,7 +334,7 @@ class NircamAnomaly(models.Model): class Meta: managed = True - db_table = 'nircam_anomaly' + db_table = "nircam_anomaly" class NirissAnomaly(models.Model): @@ -329,7 +356,7 @@ class NirissAnomaly(models.Model): class Meta: managed = True - db_table = 'niriss_anomaly' + db_table = "niriss_anomaly" class NirspecAnomaly(models.Model): @@ -345,10 +372,12 @@ class NirspecAnomaly(models.Model): data_transfer_error = models.BooleanField() ghost = models.BooleanField() snowball = models.BooleanField() - dominant_msa_leakage = models.BooleanField(db_column='Dominant_MSA_Leakage') # Field name made lowercase. + dominant_msa_leakage = models.BooleanField( + db_column="Dominant_MSA_Leakage" + ) # Field name made lowercase. optical_short = models.BooleanField() other = models.BooleanField() class Meta: managed = True - db_table = 'nirspec_anomaly' + db_table = "nirspec_anomaly"