From e6c000f8e20f2fb8f5b373a92729e8e42d7362d5 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 10 Nov 2020 13:43:19 -0500 Subject: [PATCH 001/341] testing --- jwql/website/apps/jwql/data_containers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 8d364ff3f..082106d52 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -580,7 +580,8 @@ def get_header_info(filename): header_info[ext] = {} # Get header - header = fits.getheader(fits_filepath, ext=ext) + #header = fits.getheader(fits_filepath, ext=ext) + header = hdulist[ext].header # Determine the extension name if ext == 0: @@ -599,7 +600,9 @@ def get_header_info(filename): hdulist.close() # Build tables - for ext in header_info: + #for ext in header_info: + for ext in [0]: + print('Working on ext: {}'.format(ext)) table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) with open(temp_path_for_html, 'w') as f: From d2d89bcd2f2999ec677e34ae07eeb496411200e6 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 12 Nov 2020 16:22:44 -0500 Subject: [PATCH 002/341] Try cleaning up temp files --- jwql/website/apps/jwql/data_containers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 7cce56167..27d0b7432 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -599,14 +599,13 @@ def get_header_info(filename): hdulist.close() # Build tables - #for ext in header_info: - for ext in [0]: - print('Working on ext: {}'.format(ext)) + for ext in header_info: table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) with open(temp_path_for_html, 'w') as f: table.write(f, format='jsviewer', jskwargs={'display_length': 20}) header_info[ext]['table'] = open(temp_path_for_html, 'r').read() + os.remove(temp_path_for_html) return header_info From 45afc84be653bca3b0663604059fcd4ebbd1aa60 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 12 Nov 2020 16:45:25 -0500 Subject: [PATCH 003/341] Check to see if server sees changes --- jwql/website/apps/jwql/data_containers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 27d0b7432..1a77db50d 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -595,6 +595,14 @@ def get_header_info(filename): for key in header_info[ext]['keywords']: header_info[ext]['values'].append(hdulist[ext].header[key]) + + + #XXXXXXTest to see if server can see my code changes + header_info[0]['values'][7] = 'my_file.fits' + + + + # Close the file hdulist.close() From 3e8a69a6a056beee756047373fa74995f4295fba Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 09:31:19 -0500 Subject: [PATCH 004/341] Try only the primary header --- jwql/website/apps/jwql/data_containers.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 1a77db50d..7efa91d21 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -573,13 +573,13 @@ def get_header_info(filename): hdulist = fits.open(fits_filepath) # Extract header information from file - for ext in range(0, len(hdulist)): + #for ext in range(0, len(hdulist)): + for ext in range(0, 1): # Initialize dictionary to store header information for particular extension header_info[ext] = {} # Get header - #header = fits.getheader(fits_filepath, ext=ext) header = hdulist[ext].header # Determine the extension name @@ -595,14 +595,6 @@ def get_header_info(filename): for key in header_info[ext]['keywords']: header_info[ext]['values'].append(hdulist[ext].header[key]) - - - #XXXXXXTest to see if server can see my code changes - header_info[0]['values'][7] = 'my_file.fits' - - - - # Close the file hdulist.close() From db581cf00a4b124ef22bd21f641a20ddc55aaccb Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 09:58:39 -0500 Subject: [PATCH 005/341] print name of temp file --- jwql/website/apps/jwql/data_containers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 7efa91d21..f0d8b2f0f 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -602,6 +602,13 @@ def get_header_info(filename): for ext in header_info: table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) + + + # Find where these temporary files are being placed. I think we have a pile of temporary + # files that need to be deleted in order to solve this issue + header_info[ext]['values'][7] = temp_path_for_html + + with open(temp_path_for_html, 'w') as f: table.write(f, format='jsviewer', jskwargs={'display_length': 20}) header_info[ext]['table'] = open(temp_path_for_html, 'r').read() From e7e1086e42223932502c2c1f9f91b0738a6d0c61 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 10:24:19 -0500 Subject: [PATCH 006/341] Put temp filename in dictionary --- jwql/website/apps/jwql/data_containers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index f0d8b2f0f..9f009640a 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -606,7 +606,8 @@ def get_header_info(filename): # Find where these temporary files are being placed. I think we have a pile of temporary # files that need to be deleted in order to solve this issue - header_info[ext]['values'][7] = temp_path_for_html + header_info[ext]['keywords'][0] = 'MY_TEST' + header_info[ext]['values'][0] = temp_path_for_html with open(temp_path_for_html, 'w') as f: From eb458ba898b1d728ce5c42db04791c4d5b5bffa7 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 10:32:24 -0500 Subject: [PATCH 007/341] Put temp filename in dictionary before making table --- jwql/website/apps/jwql/data_containers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 9f009640a..94edfacaf 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -600,8 +600,7 @@ def get_header_info(filename): # Build tables for ext in header_info: - table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) - temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) + # Find where these temporary files are being placed. I think we have a pile of temporary @@ -610,6 +609,12 @@ def get_header_info(filename): header_info[ext]['values'][0] = temp_path_for_html + + + + table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) + temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) + with open(temp_path_for_html, 'w') as f: table.write(f, format='jsviewer', jskwargs={'display_length': 20}) header_info[ext]['table'] = open(temp_path_for_html, 'r').read() From 6fef49ae31103377830198790cef5de74f02a470 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 10:49:11 -0500 Subject: [PATCH 008/341] Put temp filename in dictionary before making table --- jwql/website/apps/jwql/data_containers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 94edfacaf..a7eeb413a 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -600,6 +600,8 @@ def get_header_info(filename): # Build tables for ext in header_info: + #table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) + temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) @@ -607,13 +609,12 @@ def get_header_info(filename): # files that need to be deleted in order to solve this issue header_info[ext]['keywords'][0] = 'MY_TEST' header_info[ext]['values'][0] = temp_path_for_html + table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) - table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) - temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) with open(temp_path_for_html, 'w') as f: table.write(f, format='jsviewer', jskwargs={'display_length': 20}) From 7de61e903973da8081810a9ee1acdd01b101ee63 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 11:04:15 -0500 Subject: [PATCH 009/341] Use an exception to print name of temp file --- jwql/website/apps/jwql/data_containers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index a7eeb413a..95130c221 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -603,7 +603,8 @@ def get_header_info(filename): #table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) - + print(temp_path_for_html) + stop # Find where these temporary files are being placed. I think we have a pile of temporary # files that need to be deleted in order to solve this issue From 74413ea29fbec6eade96c127361c7e7c489a54e0 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 11:11:07 -0500 Subject: [PATCH 010/341] Make code functional. Temp files deleted by hand --- jwql/website/apps/jwql/data_containers.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 95130c221..826529b1f 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -600,22 +600,8 @@ def get_header_info(filename): # Build tables for ext in header_info: - #table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) - temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) - - print(temp_path_for_html) - stop - - # Find where these temporary files are being placed. I think we have a pile of temporary - # files that need to be deleted in order to solve this issue - header_info[ext]['keywords'][0] = 'MY_TEST' - header_info[ext]['values'][0] = temp_path_for_html table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) - - - - - + temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) with open(temp_path_for_html, 'w') as f: table.write(f, format='jsviewer', jskwargs={'display_length': 20}) From 2555e0c055112c5c4adbd2153d889ee1f7ef6ef4 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 12:04:56 -0500 Subject: [PATCH 011/341] Read all extension headers --- jwql/website/apps/jwql/data_containers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 826529b1f..06252cc05 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -573,8 +573,7 @@ def get_header_info(filename): hdulist = fits.open(fits_filepath) # Extract header information from file - #for ext in range(0, len(hdulist)): - for ext in range(0, 1): + for ext in range(0, len(hdulist)): # Initialize dictionary to store header information for particular extension header_info[ext] = {} From 7f92d9817efa15613a198eb8420ea454a8ca7289 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 13 Nov 2020 21:18:34 -0500 Subject: [PATCH 012/341] Try switching to new table viewer for headers --- jwql/website/apps/jwql/data_containers.py | 27 +++++-- .../apps/jwql/templates/view_header.html | 76 +++++++++++++++++-- jwql/website/apps/jwql/views.py | 35 +++++---- 3 files changed, 111 insertions(+), 27 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 06252cc05..e1eef586a 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -574,6 +574,7 @@ def get_header_info(filename): # Extract header information from file for ext in range(0, len(hdulist)): + #for ext in range(0, 1): # Initialize dictionary to store header information for particular extension header_info[ext] = {} @@ -599,13 +600,27 @@ def get_header_info(filename): # Build tables for ext in header_info: - table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) - temp_path_for_html = os.path.join(tempfile.mkdtemp(), '{}_table.html'.format(header_info[ext]['EXTNAME'])) + #table = Table([header_info[ext]['keywords'], header_info[ext]['values']], names=('Key', 'Value')) + + # Better use of temporary files + #with tempfile.TemporaryFile(mode='r+') as f: + # table.write(f, format='jsviewer', jskwargs={'display_length': 20}) + # f.seek(0) + # header_info[ext]['table'] = f.read() + + + + + # Try the new table viewer + data_dict = {} + data_dict['Keyword'] = header_info[ext]['keywords'] + data_dict['Value'] = header_info[ext]['values'] + header_info[ext]['table'] = pd.DataFrame(data_dict) + header_info[ext]['table_rows'] = header_info[ext]['table'].values + header_info[ext]['table_columns'] = header_info[ext]['table'].columns.values + + - with open(temp_path_for_html, 'w') as f: - table.write(f, format='jsviewer', jskwargs={'display_length': 20}) - header_info[ext]['table'] = open(temp_path_for_html, 'r').read() - os.remove(temp_path_for_html) return header_info diff --git a/jwql/website/apps/jwql/templates/view_header.html b/jwql/website/apps/jwql/templates/view_header.html index da9ee76d8..653425474 100644 --- a/jwql/website/apps/jwql/templates/view_header.html +++ b/jwql/website/apps/jwql/templates/view_header.html @@ -33,13 +33,67 @@

{{ filename }}

{% for extension in header_info %} {% set extname = header_info[extension]['EXTNAME'] %} {% set header_table = header_info[extension]['table'] %} - {% if extension == 0 %} -

Extension: {{ header_info[0]['EXTNAME'] }}

-
{{header_info[0]['table']|safe}}
- {% else %} -

Extension: {{ header_info[extension]['EXTNAME'] }}

-
{{header_info[extension]['table']|safe}}
- {% endif %} + {% set table_columns = header_info[extension]['table_columns'] %} + {% set table_rows = header_info[extension]['table_rows'] %} + {% if extension == 0 %} +

Extension: {{ header_info[extension]['EXTNAME'] }}

+
+ + + + {% for column in table_columns %} + + {% endfor %} + + + + {% for row in table_rows %} + + {% for value in row %} + + {% endfor %} + + {% endfor %} + + + + {% for column in table_columns %} + + {% endfor %} + + +
{{ column }}
{{ value }}
{{ column }}
+
+ {% else %} +

Extension: {{ header_info[extension]['EXTNAME'] }}

+
+ + + + {% for column in table_columns %} + + {% endfor %} + + + + {% for row in table_rows %} + + {% for value in row %} + + {% endfor %} + + {% endfor %} + + + + {% for column in table_columns %} + + {% endfor %} + + +
{{ column }}
{{ value }}
{{ column }}
+
+ {% endif %} {% endfor %}
@@ -51,4 +105,12 @@

Extension: {{ + + + + {% endblock %} \ No newline at end of file diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 8cb262f51..5666468cd 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -258,8 +258,9 @@ def api_landing(request): return render(request, template, context) -@auth_required -def archived_proposals(request, user, inst): +#@auth_required +#def archived_proposals(request, user, inst): +def archived_proposals(request, inst): """Generate the page listing all archived proposals in the database Parameters @@ -285,8 +286,9 @@ def archived_proposals(request, user, inst): return render(request, template, context) -@auth_required -def archived_proposals_ajax(request, user, inst): +#@auth_required +#def archived_proposals_ajax(request, user, inst): +def archived_proposals_ajax(request, inst): """Generate the page listing all archived proposals in the database Parameters @@ -320,8 +322,9 @@ def archived_proposals_ajax(request, user, inst): return JsonResponse(context, json_dumps_params={'indent': 2}) -@auth_required -def archive_thumbnails(request, user, inst, proposal): +#@auth_required +#def archive_thumbnails(request, user, inst, proposal): +def archive_thumbnails(request, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -350,8 +353,9 @@ def archive_thumbnails(request, user, inst, proposal): return render(request, template, context) -@auth_required -def archive_thumbnails_ajax(request, user, inst, proposal): +#@auth_required +#def archive_thumbnails_ajax(request, user, inst, proposal): +def archive_thumbnails_ajax(request, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -377,8 +381,9 @@ def archive_thumbnails_ajax(request, user, inst, proposal): return JsonResponse(data, json_dumps_params={'indent': 2}) -@auth_required -def archive_thumbnails_query_ajax(request, user): +#@auth_required +#def archive_thumbnails_query_ajax(request, user): +def archive_thumbnails_query_ajax(request): """Generate the page listing all archived images in the database for a certain proposal @@ -736,8 +741,9 @@ def view_header(request, inst, filename): return render(request, template, context) -@auth_required -def view_image(request, user, inst, file_root, rewrite=False): +#@auth_required +#def view_image(request, user, inst, file_root, rewrite=False): +def view_image(request, inst, file_root, rewrite=False): """Generate the image view page Parameters @@ -798,8 +804,9 @@ def view_image(request, user, inst, file_root, rewrite=False): return render(request, template, context) -@auth_required -def view_all_images(request, user, file_root, rewrite=False): +#@auth_required +#def view_all_images(request, user, file_root, rewrite=False): +def view_all_images(request, file_root, rewrite=False): """Generate the image view page Parameters From 4d4644037909de43a7c30662918721145f068cba Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 16 Nov 2020 16:40:37 -0500 Subject: [PATCH 013/341] Turn off alphabetizing of header keywords --- jwql/website/apps/jwql/templates/view_header.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/view_header.html b/jwql/website/apps/jwql/templates/view_header.html index 653425474..83a465a58 100644 --- a/jwql/website/apps/jwql/templates/view_header.html +++ b/jwql/website/apps/jwql/templates/view_header.html @@ -109,7 +109,9 @@

Extension: {{ From 599ffdf9273914667d1d0169b952dcc4c36b10ea Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 18 Nov 2020 16:10:33 -0500 Subject: [PATCH 014/341] Remove table footer --- jwql/website/apps/jwql/templates/view_header.html | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_header.html b/jwql/website/apps/jwql/templates/view_header.html index 83a465a58..c99827308 100644 --- a/jwql/website/apps/jwql/templates/view_header.html +++ b/jwql/website/apps/jwql/templates/view_header.html @@ -55,13 +55,6 @@

Extension: {{ header_info[ex {% endfor %} - - - {% for column in table_columns %} - {{ column }} - {% endfor %} - - {% else %} @@ -84,13 +77,6 @@

Extension: {{ {% endfor %} - - - {% for column in table_columns %} - {{ column }} - {% endfor %} - - {% endif %} From 4d2193ea5de65d758dd94d3762435816d11c2456 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 18 Nov 2020 17:34:09 -0500 Subject: [PATCH 015/341] Display only primary header initially --- jwql/website/apps/jwql/static/js/jwql.js | 2 +- jwql/website/apps/jwql/templates/view_header.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index f93ff0765..e1267f689 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -81,7 +81,7 @@ function change_int(file_root, num_ints, method, direction = 'right') { var integration = Number(suffix[suffix.length - 1][5]); var suffix = suffix[suffix.length - 2]; var program = file_root.slice(0,7); - + // Find the total number of integrations for the current image var num_ints = num_ints.replace(/'/g, '"'); var num_ints = JSON.parse(num_ints)[suffix]; diff --git a/jwql/website/apps/jwql/templates/view_header.html b/jwql/website/apps/jwql/templates/view_header.html index c99827308..3b30e23ce 100644 --- a/jwql/website/apps/jwql/templates/view_header.html +++ b/jwql/website/apps/jwql/templates/view_header.html @@ -58,8 +58,8 @@

Extension: {{ header_info[ex {% else %} -

Extension: {{ header_info[extension]['EXTNAME'] }}

-
+ +