diff --git a/README.md b/README.md index 70b4e173b..377005a11 100644 --- a/README.md +++ b/README.md @@ -83,16 +83,16 @@ source activate base/root **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step. -Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): +Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): ``` -conda env create -f environment_python_3_9.yml +conda env create -f environment_python_3.9.yml ``` or ``` -conda env create -f environment_python_3_10.yml +conda env create -f environment_python_3.10.yml ``` ### Configuration File diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f61973680..9f2ed409b 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.4 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.6 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - psycopg2=2.9.7 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.10.12 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - pyyaml=6.0.1 + - redis=5.0.0 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.3 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index d3724b1d8..4f4159244 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.3 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.5 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - psycopg2=2.9.7 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.9.17 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - pyyaml=6.0.1 + - redis=5.0.0 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.3 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index c1c2ecc4c..c73dd3543 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -84,7 +84,6 @@ from astropy.modeling import models from astropy.stats import sigma_clipped_stats from astropy.time import Time -from bokeh.io import export_png from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend from bokeh.models import LinearColorMapper from bokeh.plotting import figure @@ -106,7 +105,7 @@ from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt') @@ -276,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no # Create figure start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y") - end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") + end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='') # tools='pan,box_zoom,reset,wheel_zoom,save') self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0 # Create the color mapper that will be used to scale the image - mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image and add color bar imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -292,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec') self.plot.add_layout(color_bar, 'right') - # Add hover tool for all pixel values - #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'), - # ('value:', '@image') - # ], - # renderers=[imgplot]) - #self.plot.tools.append(hover_tool) - if (('FULL' in self.aperture) or ('_CEN' in self.aperture)): if hotxy is not None: @@ -338,22 +330,21 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no base_start = Time(float(base_parts[3]), format='mjd').tt.datetime base_end = Time(float(base_parts[5]), format='mjd').tt.datetime base_start_time = base_start.strftime("%m/%d/%Y") - base_end_time = base_end.strftime("%m/%d/%Y") + base_end_time = base_end.strftime("%m/%d/%Y") legend_title = f'Compared to dark from {base_start_time} to {base_end_time}' else: legend_title = 'Compared to previous mean dark' legend = Legend(items=[hot_legend, dead_legend, noisy_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') # Save the plot in a png - export_png(self.plot, filename=output_filename) + save_png(self.plot, filename=output_filename) set_permissions(output_filename) - def get_metadata(self, filename): """Collect basic metadata from a fits file @@ -650,7 +641,7 @@ def overplot_bad_pix(self, pix_type, coords, values): sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], pixels_y=coords[1] - ) + ) ) # Overplot the bad pixel locations @@ -658,7 +649,7 @@ def overplot_bad_pix(self, pix_type, coords, values): source=sources[pix_type], color=colors[pix_type]) # Add to the legend - if numpix > 0: + if numpix > 0: if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT: text = f"{numpix} pix {adjective[pix_type]} than baseline" else: diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index c2f6795c1..cd50c6a01 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -22,7 +22,11 @@ from pathlib import Path import pytest -from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, _validate_config +from bokeh.models import LinearColorMapper +from bokeh.plotting import figure +import numpy as np + +from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, save_png, _validate_config # Determine if tests are being run on Github Actions @@ -479,6 +483,17 @@ def test_filesystem_path(): assert check == location +def test_save_png(): + """Test that we can create a png file""" + plot = figure(title='test', tools='') + image = np.zeros((200, 200)) + image[100:105, 100:105] = 1 + ny, nx = image.shape + mapper = LinearColorMapper(palette='Viridis256', low=0, high=1.1) + imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") + save_png(plot, filename='test.png') + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_validate_config(): """Test that the config validator works.""" diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 33d0c272c..113dd66c9 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -45,14 +45,15 @@ from bokeh.plotting import figure import numpy as np from PIL import Image +from selenium import webdriver from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ - FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ - FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ - FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ - FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ - JWST_INSTRUMENT_NAMES_SHORTHAND + FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ + FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ + FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ + FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ + JWST_INSTRUMENT_NAMES_SHORTHAND __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -157,8 +158,7 @@ def create_png_from_fits(filename, outdir): plot.ygrid.visible = False # Create the color mapper that will be used to scale the image - #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -169,8 +169,8 @@ def create_png_from_fits(filename, outdir): plot.yaxis.visible = False # Save the plot in a png - output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) - export_png(plot, filename=output_filename) + output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png')) + save_png(plot, filename=output_filename) permissions.set_permissions(output_filename) return output_filename else: @@ -749,13 +749,34 @@ def read_png(filename): # Copy the RGBA image into view, flipping it so it comes right-side up # with a lower-left origin - view[:,:,:] = np.flipud(np.asarray(rgba_img)) + view[:, :, :] = np.flipud(np.asarray(rgba_img)) else: view = None # Return the 2D version return img +def save_png(fig, filename=''): + """Starting with selenium version 4.10.0, our testing has shown that on the JWQL + servers, we need to specify an instance of a web driver when exporting a Bokeh + figure as a png. This is a wrapper function that creates the web driver instance + and calls Bokeh's export_png function. + + Parameters + ---------- + fig : bokeh.plotting.figure + Bokeh figure to be saved as a png + + filename : str + Filename to use for the png file + """ + options = webdriver.FirefoxOptions() + options.add_argument('-headless') + driver = webdriver.Firefox(options=options) + export_png(fig, filename=filename, webdriver=driver) + driver.quit() + + def grouper(iterable, chunksize): """ Take a list of items (iterable), and group it into chunks of chunksize, with the diff --git a/jwql/website/apps/jwql/migrations/0001_initial.py b/jwql/website/apps/jwql/migrations/0001_initial.py new file mode 100644 index 000000000..9de5ca6a9 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 3.1.7 on 2022-09-09 17:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ImageData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('inst', models.CharField(choices=[('FGS', 'FGS'), ('MIRI', 'MIRI'), ('NIRCam', 'NIRCam'), ('NIRISS', 'NIRISS'), ('NIRSpec', 'NIRSpec')], default=None, max_length=7, verbose_name='instrument')), + ('pub_date', models.DateTimeField(verbose_name='date published')), + ('filepath', models.FilePathField(path='/user/lchambers/jwql/')), + ], + options={ + 'verbose_name_plural': 'image data', + 'db_table': 'imagedata', + }, + ), + migrations.CreateModel( + name='InstrumentFilterHandler', + fields=[ + ('instrument', models.CharField(max_length=10, primary_key=True, serialize=False)), + ], + ), + migrations.CreateModel( + name='ThumbnailFilterInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('observation', models.PositiveIntegerField()), + ('proposal', models.PositiveIntegerField()), + ('root_name', models.CharField(max_length=300)), + ('marked_viewed', models.BooleanField(default=False)), + ('inst_handler', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.instrumentfilterhandler')), + ], + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py new file mode 100644 index 000000000..049901811 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py @@ -0,0 +1,27 @@ +# Generated by Django 3.1.7 on 2022-09-13 20:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='id', + ), + migrations.AlterField( + model_name='instrumentfilterhandler', + name='instrument', + field=models.TextField(max_length=10, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='thumbnailfilterinfo', + name='root_name', + field=models.TextField(max_length=300, primary_key=True, serialize=False), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py new file mode 100644 index 000000000..df7bc6965 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py @@ -0,0 +1,58 @@ +# Generated by Django 3.1.7 on 2022-09-21 14:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0002_auto_20220913_1525'), + ] + + operations = [ + migrations.CreateModel( + name='Archive', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)), + ], + options={ + 'ordering': ['instrument'], + }, + ), + migrations.CreateModel( + name='Observation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)), + ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')), + ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')), + ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')), + ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)), + ], + options={ + 'ordering': ['-obsnum'], + }, + ), + migrations.CreateModel( + name='Proposal', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)), + ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')), + ], + options={ + 'ordering': ['-prop_id'], + 'unique_together': {('prop_id', 'archive')}, + }, + ), + migrations.DeleteModel( + name='ImageData', + ), + migrations.AddField( + model_name='observation', + name='proposal', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py new file mode 100644 index 000000000..998dd2ce4 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py @@ -0,0 +1,33 @@ +# Generated by Django 3.1.7 on 2022-09-22 14:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0003_auto_20220921_0955'), + ] + + operations = [ + migrations.AddField( + model_name='thumbnailfilterinfo', + name='obsnum', + field=models.CharField(default=11, help_text='Observation number, as a 3 digit string', max_length=3), + preserve_default=False, + ), + migrations.CreateModel( + name='RootFileInfo', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7)), + ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)), + ('viewed', models.BooleanField(default=False)), + ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')), + ], + options={ + 'ordering': ['-root_name'], + }, + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py new file mode 100644 index 000000000..3c51cbe23 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2022-09-22 19:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0004_auto_20220922_0911'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='inst_handler', + ), + migrations.DeleteModel( + name='InstrumentFilterHandler', + ), + migrations.DeleteModel( + name='ThumbnailFilterInfo', + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py new file mode 100644 index 000000000..493608df3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py @@ -0,0 +1,95 @@ +# Generated by Django 3.1.7 on 2023-02-14 21:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0005_auto_20220922_1422'), + ] + + operations = [ + migrations.CreateModel( + name='Anomalies', + fields=[ + ('root_file_info', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='jwql.rootfileinfo')), + ('cosmic_ray_shower', models.BooleanField(default=False)), + ('diffraction_spike', models.BooleanField(default=False)), + ('excessive_saturation', models.BooleanField(default=False)), + ('guidestar_failure', models.BooleanField(default=False)), + ('persistence', models.BooleanField(default=False)), + ('crosstalk', models.BooleanField(default=False)), + ('data_transfer_error', models.BooleanField(default=False)), + ('ghost', models.BooleanField(default=False)), + ('snowball', models.BooleanField(default=False)), + ('column_pull_up', models.BooleanField(default=False)), + ('column_pull_down', models.BooleanField(default=False)), + ('dominant_msa_leakage', models.BooleanField(default=False)), + ('dragons_breath', models.BooleanField(default=False)), + ('mrs_glow', models.BooleanField(default=False)), + ('mrs_zipper', models.BooleanField(default=False)), + ('internal_reflection', models.BooleanField(default=False)), + ('optical_short', models.BooleanField(default=False)), + ('row_pull_up', models.BooleanField(default=False)), + ('row_pull_down', models.BooleanField(default=False)), + ('lrs_contamination', models.BooleanField(default=False)), + ('tree_rings', models.BooleanField(default=False)), + ('scattered_light', models.BooleanField(default=False)), + ('claws', models.BooleanField(default=False)), + ('wisps', models.BooleanField(default=False)), + ('tilt_event', models.BooleanField(default=False)), + ('light_saber', models.BooleanField(default=False)), + ('other', models.BooleanField(default=False)), + ], + options={ + 'ordering': ['-root_file_info'], + }, + ), + migrations.AddField( + model_name='proposal', + name='cat_type', + field=models.CharField(default='', help_text='Category Type', max_length=10), + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperature', + field=models.CharField(default='', help_text='Aperature', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(default='', help_text='Detector', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(default='', help_text='Instrument name', max_length=7), + ), + migrations.AddField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(default='', help_text='Grating', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(default='', help_text='Pupil', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(default='', help_text='Read Pattern', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt_num', + field=models.IntegerField(default=0, help_text='Read Pattern Number'), + ), + migrations.AddField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(default='', help_text='Subarray', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py new file mode 100644 index 000000000..2d4ffdba3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.7 on 2023-02-22 16:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0006_auto_20230214_1624'), + ] + + operations = [ + migrations.RemoveField( + model_name='rootfileinfo', + name='aperature', + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(default='', help_text='Aperture', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py new file mode 100644 index 000000000..2c67cd955 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.7 on 2023-02-22 17:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0007_auto_20230222_1157'), + ] + + operations = [ + migrations.AddField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(default='', help_text='Exposure Type', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py new file mode 100644 index 000000000..d0dbafc76 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py @@ -0,0 +1,63 @@ +# Generated by Django 3.1.7 on 2023-03-03 14:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0008_rootfileinfo_exp_type'), + ] + + operations = [ + migrations.RenameField( + model_name='proposal', + old_name='cat_type', + new_name='category', + ), + migrations.AddField( + model_name='rootfileinfo', + name='expstart', + field=models.FloatField(default=0.0, help_text='Exposure Start Time'), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(blank=True, default='', help_text='Aperture', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(blank=True, default='', help_text='Detector', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(blank=True, default='', help_text='Exposure Type', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(blank=True, default='', help_text='Instrument name', max_length=7, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(blank=True, default='', help_text='Grating', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(blank=True, default='', help_text='Pupil', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(blank=True, default='', help_text='Read Pattern', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(blank=True, default='', help_text='Subarray', max_length=40, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py new file mode 100644 index 000000000..3d80e95fe --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2023-03-13 15:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0009_auto_20230303_0930'), + ] + + operations = [ + migrations.AddField( + model_name='anomalies', + name='flag_date', + field=models.DateTimeField(blank=True, help_text='flag date', null=True), + ), + migrations.AddField( + model_name='anomalies', + name='user', + field=models.CharField(blank=True, default='', help_text='user', max_length=50, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/__init__.py b/jwql/website/apps/jwql/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 48c47e216..5e522f7cd 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -22,7 +22,7 @@ from astropy.stats import sigma_clipped_stats from astropy.time import Time from bokeh.embed import components, file_html -from bokeh.io import export_png, show +from bokeh.io import show from bokeh.layouts import layout from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title from bokeh.plotting import figure @@ -40,7 +40,7 @@ from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.permissions import set_permissions -from jwql.utils.utils import filesystem_path, get_config, read_png +from jwql.utils.utils import filesystem_path, get_config, read_png, save_png SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) OUTPUT_DIR = get_config()['outputs'] @@ -74,8 +74,6 @@ def __init__(self, instrument): # Get the relevant database tables self.identify_tables() - #self.detectors = get_unique_values_per_column(self.pixel_table, 'detector') - self.detectors = sorted(DETECTOR_PER_INSTRUMENT[self.instrument]) if self.instrument == 'miri': self.detectors = ['MIRIMAGE'] @@ -128,7 +126,6 @@ def modify_bokeh_saved_html(self): self._html = "".join(newlines) - def run(self): # Right now, the aperture name in the query history table is used as the title of the @@ -163,7 +160,7 @@ def run(self): # Insert into our html template and save template_dir = os.path.join(os.path.dirname(__file__), '../templates') template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html') - temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} + temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div': div} self._html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) # Modify the html such that our Django-related lines are kept in place, @@ -264,7 +261,6 @@ def __init__(self, pixel_table, instrument, detector): for badtype in self.badtypes: self.get_trending_data(badtype) - def get_most_recent_entry(self): """Get all nedded data from the database tables. """ @@ -312,8 +308,8 @@ def get_trending_data(self, badpix_type): # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ - .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ - .all() + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .all() # Organize the results num_pix = [] @@ -463,7 +459,7 @@ def create_plot(self): self.plot = figure(title=title_text, tools=tools, x_axis_label="Pixel Number", y_axis_label="Pixel Number") else: - self.plot = figure(tools='') #, x_axis_label="Pixel Number", y_axis_label="Pixel Number") + self.plot = figure(tools='') self.plot.toolbar.logo = None self.plot.toolbar_location = None self.plot.min_border = 0 @@ -475,18 +471,9 @@ def create_plot(self): # Plot image if image is not None: ny, nx = image.shape - #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") - - - # Shift the figure title slightly right in this case to get it # to align with the axes - #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*, - # tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen, alpha=0.5) - - - else: # If the background image is not present, manually set the x and y range self.plot.x_range.start = 0 @@ -511,11 +498,10 @@ def create_plot(self): legend = Legend(items=[plot_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') - def overplot_bad_pix(self): """Add a scatter plot of potential new bad pixels to the plot @@ -537,7 +523,7 @@ def overplot_bad_pix(self): # If there are no new bad pixels, write text within the figure mentioning that txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2], text=[f'No new {self.badpix_type} pixels found'])) - glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) + glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value': '20px'}) self.plot.add_glyph(txt_source, glyph) # Insert a fake one, in order to get the plot to be made @@ -593,7 +579,7 @@ def switch_to_png(self, filename, title): Title to add to the Figure """ # Save the figure as a png - export_png(self.plot, filename=filename) + save_png(self.plot, filename=filename) set_permissions(filename) # Read in the png and insert into a replacement figure @@ -668,7 +654,7 @@ def create_plot(self): string_time=string_times, value=[self.badpix_type] * len(self.num_pix) ) - ) + ) self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") @@ -696,7 +682,7 @@ def create_plot(self): time_pad = datetime.timedelta(days=1) self.plot.x_range.start = min(self.time) - time_pad self.plot.x_range.end = max(self.time) + time_pad - self.plot.grid.grid_line_color="white" + self.plot.grid.grid_line_color = "white" self.plot.xaxis.axis_label = 'Date' self.plot.yaxis.axis_label = f'Number of {self.badpix_type} pixels' diff --git a/pyproject.toml b/pyproject.toml index c17d3dc28..24f701717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ file = "LICENSE" content-type = "text/plain" [build-system] -requires = ["setuptools>=61.2,<=65.5.0", "numpy", "wheel", "setuptools_scm"] +requires = ["setuptools>=68.0.0", "numpy", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools] diff --git a/requirements.txt b/requirements.txt index 4391be8b3..6a7f670cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,38 +1,38 @@ -astropy==5.3.2 +astropy==5.3.3 astroquery==0.4.6 bandit==1.7.5 beautifulsoup4==4.12.2 bokeh==2.4.3 -celery==5.3.1 -cryptography==41.0.3 -django==4.2.3 +celery==5.3.4 +cryptography==41.0.4 +django==4.2.5 inflection==0.5.1 -ipython==8.14.0 +ipython==8.16.1 jinja2==3.1.2 -jsonschema==4.19.0 -jwst==1.11.4 -matplotlib==3.7.2 -nodejs==20.5.1 +jsonschema==4.19.1 +jwst==1.12.3 +matplotlib==3.8.0 +nodejs==20.8.0 numpy==1.25.2 numpydoc==1.5.0 -pandas==2.0.3 +pandas==2.1.1 psycopg2-binary==2.9.7 pysiaf==0.20.0 -pysqlite3==0.5.1 -pytest==7.4.0 +pysqlite3==0.5.2 +pytest==7.4.2 pytest-cov==4.1.0 pytest-mock==3.11.1 pyvo==1.4.2 -pyyaml==6.0 +pyyaml==6.0.1 redis==5.0.0 -ruff==0.0.285 +ruff==0.0.292 scipy==1.9.3 -selenium==4.11.2 -setuptools==68.0.0 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 -sqlalchemy==2.0.20 -stdatamodels==1.7.2 +selenium==4.13.0 +setuptools==68.2.2 +sphinx==7.2.6 +sphinx_rtd_theme==1.3.0 +sqlalchemy==2.0.21 +stdatamodels==1.8.3 stsci_rtd_theme==1.0.0 twine==4.0.2 vine==5.0.0 diff --git a/rtd_requirements.txt b/rtd_requirements.txt index aa1047b5e..abac6b128 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,15 +1,15 @@ -sphinx_automodapi==0.15.0 +sphinx_automodapi>=0.15.0 bokeh==2.4.3 -celery==5.3.1 -cython==3.0.0 -django==4.2.3 -docutils==0.18.1 -jwst==1.11.4 +celery==5.3.4 +cython>=3.0.0 +django==4.2.5 +docutils>=0.18.1 +jwst==1.12.3 pygments==2.16.1 -pytest==7.4.0 +pytest==7.4.2 redis==5.0.0 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 +selenium==4.13.0 +sphinx==7.2.6 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles