diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 27364f888..295d2ea01 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -49,22 +49,27 @@ import numpy as np from sqlalchemy.exc import DataError -from jwql.database.database_interface import engine -from jwql.database.database_interface import session -from jwql.database.database_interface import FilesystemCharacteristics -from jwql.database.database_interface import FilesystemGeneral -from jwql.database.database_interface import FilesystemInstrument -from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table from jwql.utils.protect_module import lock_module from jwql.website.apps.jwql.data_containers import get_instrument_proposals +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + # Need to set up django apps before we can access the models + import django # noqa: E402 (module level import not at top of file) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + django.setup() + + # Import * is okay here because this module specifically only contains database models + # for this monitor + from jwql.website.apps.jwql.monitor_models.common import * # noqa: E402 (module level import not at top of file) + SETTINGS = get_config() FILESYSTEM = SETTINGS['filesystem'] PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary') @@ -74,6 +79,7 @@ PREVIEW_IMAGES = SETTINGS['preview_image_filesystem'] THUMBNAILS = SETTINGS['thumbnail_filesystem'] LOGS = SETTINGS['log_dir'] +WORKING = SETTINGS['working'] def files_per_filter(): @@ -232,7 +238,8 @@ def get_area_stats(central_storage_dict): 'logs': LOGS, 'preview_images': PREVIEW_IMAGES, 'thumbnails': THUMBNAILS, - 'all': CENTRAL} + 'all': CENTRAL, + 'working':WORKING} counteddirs = [] @@ -368,7 +375,7 @@ def initialize_results_dicts(): A dictionary for the ``central_storage`` database table """ - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) general_results_dict = {} general_results_dict['date'] = now @@ -430,9 +437,9 @@ def update_central_store_database(central_storage_dict): new_record['size'] = central_storage_dict[area]['size'] new_record['used'] = central_storage_dict[area]['used'] new_record['available'] = central_storage_dict[area]['available'] - with engine.begin() as connection: - connection.execute(CentralStore.__table__.insert(), new_record) - session.close() + + entry = CentralStorage(**new_record) + entry.save() def update_characteristics_database(char_info): @@ -447,7 +454,7 @@ def update_characteristics_database(char_info): using that filter/pupil. """ logging.info('\tUpdating the characteristics database') - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) # Add data to filesystem_instrument table for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: @@ -458,11 +465,9 @@ def update_characteristics_database(char_info): new_record['instrument'] = instrument new_record['filter_pupil'] = optics new_record['obs_per_filter_pupil'] = values - with engine.begin() as connection: - connection.execute( - FilesystemCharacteristics.__table__.insert(), new_record) - session.close() + entry = FilesystemCharacteristics(**new_record) + entry.save() def update_database(general_results_dict, instrument_results_dict, central_storage_dict): @@ -478,8 +483,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora """ logging.info('\tUpdating the database') - with engine.begin() as connection: - connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict) + fs_general_entry = FilesystemGeneral(**general_results_dict) + fs_general_entry.save() # Add data to filesystem_instrument table for instrument in JWST_INSTRUMENT_NAMES: @@ -493,13 +498,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora # Protect against updated enum options that have not been propagated to # the table definition - try: - with engine.begin() as connection: - connection.execute(FilesystemInstrument.__table__.insert(), new_record) - except DataError as e: - logging.error(e) - - session.close() + fs_instrument_entry = FilesystemInstrument(**new_record) + fs_instrument_entry.save() @lock_module diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 8e3be0a5c..7696f26c7 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -184,7 +184,7 @@ def _caller(*args, **kwargs): def create_task_log_handler(logger, propagate): - log_file_name = configure_logging('shared_tasks') + log_file_name = configure_logging('shared_tasks', include_time=False) working_dir = os.path.join(get_config()['working'], 'calibrated_data') ensure_dir_exists(working_dir) celery_log_file_handler = FileHandler(log_file_name) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index c20bfce3d..55980a1ee 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -102,7 +102,7 @@ def filter(record): return filter -def configure_logging(module): +def configure_logging(module, include_time=True): """ Configure the log file with a standard logging format. The format in question is set up as follows: @@ -116,11 +116,8 @@ def configure_logging(module): ---------- module : str The name of the module being logged. - production_mode : bool - Whether or not the output should be written to the production - environement. - path : str - Where to write the log if user-supplied path; default to working dir. + include_time : bool, default True + Whether or not the file name should include the time as well as the date. Returns ------- @@ -129,7 +126,7 @@ def configure_logging(module): """ # Determine log file location - log_file = make_log_file(module) + log_file = make_log_file(module, include_time=include_time) # Get the logging configuration dictionary logging_config = get_config()['logging'] @@ -171,7 +168,7 @@ def get_log_status(log_file): return 'FAILURE' -def make_log_file(module): +def make_log_file(module, include_time=True): """Create the log file name based on the module name. The name of the ``log_file`` is a combination of the name of the @@ -181,12 +178,8 @@ def make_log_file(module): ---------- module : str The name of the module being logged. - production_mode : bool - Whether or not the output should be written to the production - environment. - path : str - Where to write the log if user-supplied path; default to - working dir. + include_time : bool, default True + Whether or not the file name should include the time as well as the date. Returns ------- @@ -195,7 +188,10 @@ def make_log_file(module): """ # Build filename - timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M') + if include_time: + timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M') + else: + timestamp = datetime.datetime.now().strftime('%Y-%m-%d') hostname = socket.gethostname() filename = '{0}_{1}_{2}.log'.format(module, hostname, timestamp) diff --git a/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py new file mode 100644 index 000000000..7e6fb3dd3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 5.1.4 on 2025-01-16 21:35 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0027_alter_fgsbadpixelstats_source_files_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='filesystemcharacteristics', + name='filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='filesystemcharacteristics', + name='instrument', + field=models.CharField(), + ), + migrations.AlterField( + model_name='filesystemcharacteristics', + name='obs_per_filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='filesysteminstrument', + name='filetype', + field=models.CharField(), + ), + migrations.AlterField( + model_name='filesysteminstrument', + name='instrument', + field=models.CharField(default='empty', help_text='JWST instrument name', max_length=7), + ), + ] diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index ebcf7c82a..ee6da326e 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -172,22 +172,29 @@ class Meta: For more information please see: ```https://docs.djangoproject.com/en/2.0/topics/db/models/``` """ + # This is an auto-generated Django model module. # Feel free to rename the models, but don't rename db_table values or field names. from django.db import models from django.contrib.postgres.fields import ArrayField +from jwql.utils.constants import ( + DEFAULT_MODEL_CHARFIELD, + MAX_LEN_FILTER, + MAX_LEN_INSTRUMENT, +) + class Monitor(models.Model): monitor_name = models.CharField() start_time = models.DateTimeField() end_time = models.DateTimeField(blank=True, null=True) - status = models.TextField(blank=True, null=True) # This field type is a guess. + status = models.TextField(blank=True, null=True) log_file = models.CharField() class Meta: managed = True - db_table = 'monitor' + db_table = "monitor" class CentralStorage(models.Model): @@ -199,18 +206,30 @@ class CentralStorage(models.Model): class Meta: managed = True - db_table = 'central_storage' + db_table = "central_storage" class FilesystemCharacteristics(models.Model): date = models.DateTimeField() - instrument = models.TextField() # This field type is a guess. - filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. - obs_per_filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. + instrument = models.CharField() + filter_pupil = ArrayField( + models.CharField( + max_length=MAX_LEN_FILTER, + help_text="filter and/or pupil name", + default=DEFAULT_MODEL_CHARFIELD, + ), + blank=True, + null=True, + ) + obs_per_filter_pupil = ArrayField( + models.IntegerField(), + blank=True, + null=True, + ) class Meta: managed = True - db_table = 'filesystem_characteristics' + db_table = "filesystem_characteristics" class FilesystemGeneral(models.Model): @@ -224,20 +243,24 @@ class FilesystemGeneral(models.Model): class Meta: managed = True - db_table = 'filesystem_general' + db_table = "filesystem_general" class FilesystemInstrument(models.Model): date = models.DateTimeField() - instrument = models.TextField() # This field type is a guess. - filetype = models.TextField() # This field type is a guess. + instrument = models.CharField( + max_length=MAX_LEN_INSTRUMENT, + help_text="JWST instrument name", + default=DEFAULT_MODEL_CHARFIELD, + ) + filetype = models.CharField() count = models.IntegerField() size = models.FloatField() class Meta: managed = True - db_table = 'filesystem_instrument' - unique_together = (('date', 'instrument', 'filetype'),) + db_table = "filesystem_instrument" + unique_together = (("date", "instrument", "filetype"),) class FgsAnomaly(models.Model): @@ -257,7 +280,7 @@ class FgsAnomaly(models.Model): class Meta: managed = True - db_table = 'fgs_anomaly' + db_table = "fgs_anomaly" class MiriAnomaly(models.Model): @@ -274,15 +297,19 @@ class MiriAnomaly(models.Model): row_pull_down = models.BooleanField() other = models.BooleanField() column_pull_down = models.BooleanField() - mrs_glow = models.BooleanField(db_column='MRS_Glow') # Field name made lowercase. - mrs_zipper = models.BooleanField(db_column='MRS_Zipper') # Field name made lowercase. + mrs_glow = models.BooleanField(db_column="MRS_Glow") # Field name made lowercase. + mrs_zipper = models.BooleanField( + db_column="MRS_Zipper" + ) # Field name made lowercase. row_pull_up = models.BooleanField() - lrs_contamination = models.BooleanField(db_column='LRS_Contamination') # Field name made lowercase. + lrs_contamination = models.BooleanField( + db_column="LRS_Contamination" + ) # Field name made lowercase. tree_rings = models.BooleanField() class Meta: managed = True - db_table = 'miri_anomaly' + db_table = "miri_anomaly" class NircamAnomaly(models.Model): @@ -307,7 +334,7 @@ class NircamAnomaly(models.Model): class Meta: managed = True - db_table = 'nircam_anomaly' + db_table = "nircam_anomaly" class NirissAnomaly(models.Model): @@ -329,7 +356,7 @@ class NirissAnomaly(models.Model): class Meta: managed = True - db_table = 'niriss_anomaly' + db_table = "niriss_anomaly" class NirspecAnomaly(models.Model): @@ -345,10 +372,12 @@ class NirspecAnomaly(models.Model): data_transfer_error = models.BooleanField() ghost = models.BooleanField() snowball = models.BooleanField() - dominant_msa_leakage = models.BooleanField(db_column='Dominant_MSA_Leakage') # Field name made lowercase. + dominant_msa_leakage = models.BooleanField( + db_column="Dominant_MSA_Leakage" + ) # Field name made lowercase. optical_short = models.BooleanField() other = models.BooleanField() class Meta: managed = True - db_table = 'nirspec_anomaly' + db_table = "nirspec_anomaly"