From 10db68644b4d3fd373fed6d00518f8a037fee70c Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Thu, 20 Jul 2023 12:13:13 -0400 Subject: [PATCH 01/10] Allow Explicit request for a number of cores For some users on large machines even using 'quarter' leads to them using too many cores. This change allows string integer values to be used. --- src/stcal/jump/jump.py | 29 +++++++++++-------- src/stcal/ramp_fitting/ols_fit.py | 4 ++- src/stcal/ramp_fitting/utils.py | 46 +++++++++++++++---------------- tests/test_jump.py | 26 ++++++++++++++++- tests/test_ramp_fitting.py | 19 ++++++++++++- 5 files changed, 86 insertions(+), 38 deletions(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index 078d58b0..79e0d3a6 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -236,18 +236,8 @@ def detect_jumps(frames_per_group, data, gdq, pdq, err, row_below_gdq = np.zeros((n_ints, n_groups, n_cols), dtype=np.uint8) # figure out how many slices to make based on 'max_cores' - max_available = multiprocessing.cpu_count() - if max_cores.lower() == 'none': - n_slices = 1 - elif max_cores == 'quarter': - n_slices = max_available // 4 or 1 - elif max_cores == 'half': - n_slices = max_available // 2 or 1 - elif max_cores == 'all': - n_slices = max_available - # Make sure we don't have more slices than rows. - n_slices = min(n_rows, n_slices) + n_slices = calc_num_slices(n_rows, max_cores, max_available) if n_slices == 1: gdq, row_below_dq, row_above_dq, total_primary_crs, stddev = \ twopt.find_crs(data, gdq, readnoise_2d, rejection_thresh, @@ -820,3 +810,20 @@ def find_faint_extended(indata, gdq, readnoise_2d, nframes, minimum_sigclip_grou num_grps_masked=num_grps_masked, max_extended_radius=max_extended_radius) return gdq, len(all_ellipses) + + +def calc_num_slices(n_rows, max_cores, max_available): + n_slices = 1 + if max_cores.isnumeric(): + n_slices = int(max_cores) + elif max_cores.lower() == "none" or max_cores.lower() == 'one': + n_slices = 1 + elif max_cores == 'quarter': + n_slices = max_available // 4 or 1 + elif max_cores == 'half': + n_slices = max_available // 2 or 1 + elif max_cores == 'all': + n_slices = max_available + # Make sure we don't have more slices than rows or available cores. + n_slices = min([n_rows, n_slices, max_available]) + return n_slices diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index b3cc4bfe..db7cb1b9 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -2,6 +2,7 @@ import logging from multiprocessing.pool import Pool as Pool +from multiprocessing import cpu_count as cpu_count import numpy as np import time @@ -73,7 +74,8 @@ def ols_ramp_fit_multi( # Determine number of slices to use for multi-processor computations nrows = ramp_data.data.shape[2] - number_slices = utils.compute_slices(max_cores, nrows) + num_available_cores = cpu_count() + number_slices = utils.compute_num_slices(max_cores, nrows, num_available_cores) # For MIRI datasets having >1 group, if all pixels in the final group are # flagged as DO_NOT_USE, resize the input model arrays to exclude the diff --git a/src/stcal/ramp_fitting/utils.py b/src/stcal/ramp_fitting/utils.py index 172e00d2..2dcfdbba 100644 --- a/src/stcal/ramp_fitting/utils.py +++ b/src/stcal/ramp_fitting/utils.py @@ -2,7 +2,6 @@ # # utils.py: utility functions import logging -import multiprocessing import numpy as np import warnings @@ -1270,7 +1269,7 @@ def log_stats(c_rates): % (c_rates.min(), c_rates.mean(), c_rates.max(), c_rates.std())) -def compute_slices(max_cores, nrows): +def compute_num_slices(max_cores, nrows, max_available): """ Computes the number of slices to be created for multiprocessing. @@ -1279,35 +1278,34 @@ def compute_slices(max_cores, nrows): max_cores : str Number of cores to use for multiprocessing. If set to 'none' (the default), then no multiprocessing will be done. The other allowable values are 'quarter', - 'half', and 'all'. This is the fraction of cores to use for multi-proc. The - total number of cores includes the SMT cores (Hyper Threading for Intel). + 'half', and 'all' and string integers. This is the fraction of cores + to use for multi-proc. + nrows : int + The number of rows that will be used across all process. This is the + maximum number of slices to make sure that each process has some data. + max_available: int + This is the total number of cores available. The total number of cores + includes the SMT cores (Hyper Threading for Intel). Returns ------- number_slices : int The number of slices for multiprocessing. """ - if max_cores == 'none': - number_slices = 1 - else: - num_cores = multiprocessing.cpu_count() - log.debug(f'Found {num_cores} possible cores to use for ramp fitting') - if max_cores == 'quarter': - number_slices = num_cores // 4 or 1 - elif max_cores == 'half': - number_slices = num_cores // 2 or 1 - elif max_cores == 'all': - number_slices = num_cores - else: - number_slices = 1 - - # Make sure the number of slices created isn't more than the available - # number of rows. If so, this would cause empty datasets to be run - # through ramp fitting with dimensions (nints, ngroups, 0, ncols), - # which would cause a crash. - if number_slices > nrows: - number_slices = nrows + number_slices = 1 + if max_cores.isnumeric(): + number_slices = int(max_cores) + elif max_cores.lower() == "none" or max_cores.lower() == 'one': + number_slices = 1 + elif max_cores == 'quarter': + number_slices = max_available // 4 or 1 + elif max_cores == 'half': + number_slices = max_available // 2 or 1 + elif max_cores == 'all': + number_slices = max_available + # Make sure we don't have more slices than rows or available cores. + number_slices = min([nrows, number_slices, max_available]) return number_slices diff --git a/tests/test_jump.py b/tests/test_jump.py index 49d65ba0..dabe21c0 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -3,7 +3,7 @@ from astropy.io import fits from stcal.jump.jump import flag_large_events, find_ellipses, extend_saturation, \ - point_inside_ellipse, find_faint_extended + point_inside_ellipse, find_faint_extended, calc_num_slices DQFLAGS = {'JUMP_DET': 4, 'SATURATED': 2, 'DO_NOT_USE': 1, 'GOOD': 0, 'NO_GAIN_VALUE': 8} @@ -286,3 +286,27 @@ def test_inputjump_sat_star2(): expand_factor=2.0, use_ellipses=False, sat_required_snowball=True, min_sat_radius_extend=2.5, sat_expand=2) fits.writeto("outgdq_satstar.fits", testcube, overwrite=True) + +def test_calc_num_slices(): + n_rows = 20 + max_available_cores = 10 + assert(calc_num_slices(n_rows, 'none', max_available_cores) == 1) + assert (calc_num_slices(n_rows, 'half', max_available_cores) == 5) + assert (calc_num_slices(n_rows, '3', max_available_cores) == 3) + assert (calc_num_slices(n_rows, '7', max_available_cores) == 7) + assert (calc_num_slices(n_rows, '21', max_available_cores) == 10) + assert (calc_num_slices(n_rows, 'quarter', max_available_cores) == 2) + assert (calc_num_slices(n_rows, '7.5', max_available_cores) == 1) + assert (calc_num_slices(n_rows, 'one', max_available_cores) == 1) + assert (calc_num_slices(n_rows, '-5', max_available_cores) == 1) + assert (calc_num_slices(n_rows, 'all', max_available_cores) == 10) + assert (calc_num_slices(n_rows, '3/4', max_available_cores) == 1) + n_rows = 9 + assert (calc_num_slices(n_rows, '21', max_available_cores) == 9) + + + + + + + diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 8762705d..521afbc0 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -1,7 +1,7 @@ import numpy as np from stcal.ramp_fitting.ramp_fit import ramp_fit_data from stcal.ramp_fitting.ramp_fit_class import RampData - +from stcal.ramp_fitting.utils import compute_num_slices DELIM = "-" * 70 @@ -1567,3 +1567,20 @@ def print_all_info(slopes, cube, optional): print_slopes(slopes) print_integ(cube) print_optional(optional) + +def test_compute_num_slices(): + n_rows = 20 + max_available_cores = 10 + assert(compute_num_slices('none', n_rows, max_available_cores) == 1) + assert (compute_num_slices('half', n_rows, max_available_cores) == 5) + assert (compute_num_slices('3', n_rows, max_available_cores) == 3) + assert (compute_num_slices('7', n_rows, max_available_cores) == 7) + assert (compute_num_slices('21', n_rows, max_available_cores) == 10) + assert (compute_num_slices('quarter', n_rows,max_available_cores) == 2) + assert (compute_num_slices('7.5', n_rows, max_available_cores) == 1) + assert (compute_num_slices('one', n_rows, max_available_cores) == 1) + assert (compute_num_slices('-5', n_rows, max_available_cores) == 1) + assert (compute_num_slices('all', n_rows, max_available_cores) == 10) + assert (compute_num_slices('3/4', n_rows, max_available_cores) == 1) + n_rows = 9 + assert (compute_num_slices('21', n_rows, max_available_cores) == 9) From 0b30e2147d72ef7af3beac675f3e26a79e2dd6f4 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 25 Jul 2023 11:50:56 -0400 Subject: [PATCH 02/10] Update test_ramp_fitting.py --- tests/test_ramp_fitting.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 521afbc0..77f3e366 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -1052,9 +1052,10 @@ def test_multi_more_cores_than_rows(): var = rnval, gval tm = frame_time, nframes, groupgap - from stcal.ramp_fitting.utils import compute_slices - requested_slices = 8 - requested_slices = compute_slices(requested_slices, nrows) + from stcal.ramp_fitting.utils import compute_num_slices + requested_slices = '8' + max_available_cores = 10 + requested_slices = compute_num_slices(requested_slices, nrows, max_available_cores) assert requested_slices == 1 """ From dcef89eddd11ba5839525a10c83c23aeb3ee3f25 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 25 Jul 2023 11:52:40 -0400 Subject: [PATCH 03/10] Update test_jump.py --- tests/test_jump.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_jump.py b/tests/test_jump.py index dabe21c0..7dc61c2b 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -303,10 +303,3 @@ def test_calc_num_slices(): assert (calc_num_slices(n_rows, '3/4', max_available_cores) == 1) n_rows = 9 assert (calc_num_slices(n_rows, '21', max_available_cores) == 9) - - - - - - - From 246a4fb5cbd47f384838aa44a62e8e0ba85bf8df Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 25 Jul 2023 12:29:01 -0400 Subject: [PATCH 04/10] Update CHANGES.rst --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 4a817988..c0b7e698 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,13 @@ Bug Fixes jump ~~~~ +Added more allowable selections for the number of cores to use for +multiprocessing [#183]. + +Ramp Fitting +~~~~~~~~~~~~ +Added more allowable selections for the number of cores to use for +multiprocessing [#183]. - From 7419ae00582a2291324b5332661dfddbb64a95d4 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Tue, 25 Jul 2023 13:07:05 -0400 Subject: [PATCH 05/10] Update test_ramp_fitting.py --- tests/test_ramp_fitting.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py index 77f3e366..78e7962a 100644 --- a/tests/test_ramp_fitting.py +++ b/tests/test_ramp_fitting.py @@ -1357,6 +1357,24 @@ def create_blank_ramp_data(dims, var, tm): return ramp_data, gain, rnoise +def test_compute_num_slices(): + n_rows = 20 + max_available_cores = 10 + assert(compute_num_slices('none', n_rows, max_available_cores) == 1) + assert (compute_num_slices('half', n_rows, max_available_cores) == 5) + assert (compute_num_slices('3', n_rows, max_available_cores) == 3) + assert (compute_num_slices('7', n_rows, max_available_cores) == 7) + assert (compute_num_slices('21', n_rows, max_available_cores) == 10) + assert (compute_num_slices('quarter', n_rows,max_available_cores) == 2) + assert (compute_num_slices('7.5', n_rows, max_available_cores) == 1) + assert (compute_num_slices('one', n_rows, max_available_cores) == 1) + assert (compute_num_slices('-5', n_rows, max_available_cores) == 1) + assert (compute_num_slices('all', n_rows, max_available_cores) == 10) + assert (compute_num_slices('3/4', n_rows, max_available_cores) == 1) + n_rows = 9 + assert (compute_num_slices('21', n_rows, max_available_cores) == 9) + + # ----------------------------------------------------------------------------- # Set up functions @@ -1568,20 +1586,3 @@ def print_all_info(slopes, cube, optional): print_slopes(slopes) print_integ(cube) print_optional(optional) - -def test_compute_num_slices(): - n_rows = 20 - max_available_cores = 10 - assert(compute_num_slices('none', n_rows, max_available_cores) == 1) - assert (compute_num_slices('half', n_rows, max_available_cores) == 5) - assert (compute_num_slices('3', n_rows, max_available_cores) == 3) - assert (compute_num_slices('7', n_rows, max_available_cores) == 7) - assert (compute_num_slices('21', n_rows, max_available_cores) == 10) - assert (compute_num_slices('quarter', n_rows,max_available_cores) == 2) - assert (compute_num_slices('7.5', n_rows, max_available_cores) == 1) - assert (compute_num_slices('one', n_rows, max_available_cores) == 1) - assert (compute_num_slices('-5', n_rows, max_available_cores) == 1) - assert (compute_num_slices('all', n_rows, max_available_cores) == 10) - assert (compute_num_slices('3/4', n_rows, max_available_cores) == 1) - n_rows = 9 - assert (compute_num_slices('21', n_rows, max_available_cores) == 9) From 0f1d97a109360a513f5e88f496da6206b369af62 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Fri, 28 Jul 2023 08:55:14 -0400 Subject: [PATCH 06/10] Update CHANGES.rst --- CHANGES.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c0b7e698..f274cf1a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,20 +1,21 @@ 1.4.3 (unreleased) ================== -Bug Fixes ---------- +Other +----- jump ~~~~ -Added more allowable selections for the number of cores to use for -multiprocessing [#183]. -Ramp Fitting +- Added more allowable selections for the number of cores to use for + multiprocessing [#183]. + +ramp_fitting ~~~~~~~~~~~~ -Added more allowable selections for the number of cores to use for -multiprocessing [#183]. -- +- Added more allowable selections for the number of cores to use for + multiprocessing [#183]. + 1.4.2 (2023-07-11) ================== From bc631ce27d5a0a8c371632832d7932e55519fc35 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Wed, 9 Aug 2023 11:25:44 -0400 Subject: [PATCH 07/10] Update gls_fit.py --- src/stcal/ramp_fitting/gls_fit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stcal/ramp_fitting/gls_fit.py b/src/stcal/ramp_fitting/gls_fit.py index a9fef91a..4139f513 100644 --- a/src/stcal/ramp_fitting/gls_fit.py +++ b/src/stcal/ramp_fitting/gls_fit.py @@ -12,6 +12,7 @@ import logging from multiprocessing.pool import Pool as Pool +from multiprocessing import cpu_count as cpu_count import numpy as np import numpy.linalg as la import time @@ -101,7 +102,8 @@ def gls_ramp_fit(ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, max_cores Tuple of optional product ndarrays computed for GLS ramp fitting. """ nrows = ramp_data.data.shape[2] - number_slices = utils.compute_slices(max_cores, nrows) + num_available_cores = cpu_count() + number_slices = utils.compute_num_slices(max_cores, nrows, num_available_cores) log.info(f"Number of data slices: {number_slices}") From 0e9feb5b37cd7b7f109d61d1c5bbfac362e45020 Mon Sep 17 00:00:00 2001 From: Mike Regan Date: Wed, 9 Aug 2023 15:12:19 -0400 Subject: [PATCH 08/10] remove local testing tests --- tests/test_jump.py | 36 --------- tests/test_twopoint_difference.py | 122 ------------------------------ 2 files changed, 158 deletions(-) diff --git a/tests/test_jump.py b/tests/test_jump.py index 7dc61c2b..1f5c8963 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -251,42 +251,6 @@ def test_inside_ellipes5(): assert result -@pytest.mark.skip("Fails in CI") -def test_inputjumpall(): - testcube = fits.getdata('data/large_event_input_dq_cube2.fits') - flag_large_events(testcube, DQFLAGS['JUMP_DET'], DQFLAGS['SATURATED'], min_sat_area=1, - min_jump_area=6, - expand_factor=2.0, - sat_required_snowball=True, min_sat_radius_extend=2.5, sat_expand=2) - snowball_1 = testcube[0, 1, 1900:1934, 1710:1746] - correct_snowball_1 = fits.getdata('data/snowball1.fits') - snowball_diff = snowball_1 - correct_snowball_1 - assert (np.all(snowball_diff == 0)) - - -@pytest.mark.skip("Used for local testing") -def test_inputjump_sat_star(): - testcube = fits.getdata('data/input_gdq_flarge.fits') - num_extended_events = flag_large_events(testcube, DQFLAGS['JUMP_DET'], DQFLAGS['SATURATED'], - min_sat_area=1, - min_jump_area=6, - expand_factor=2.0, - sat_required_snowball=True, - min_sat_radius_extend=2.5, - sat_expand=2) - assert(num_extended_events == 312) - fits.writeto("outgdq2.fits", testcube, overwrite=True) - - -@pytest.mark.skip("Used for local testing") -def test_inputjump_sat_star2(): - testcube = fits.getdata('input_gdq_satstar.fits') - flag_large_events(testcube, DQFLAGS['JUMP_DET'], DQFLAGS['SATURATED'], min_sat_area=1, - min_jump_area=6, - expand_factor=2.0, use_ellipses=False, - sat_required_snowball=True, min_sat_radius_extend=2.5, sat_expand=2) - fits.writeto("outgdq_satstar.fits", testcube, overwrite=True) - def test_calc_num_slices(): n_rows = 20 max_available_cores = 10 diff --git a/tests/test_twopoint_difference.py b/tests/test_twopoint_difference.py index 303c425d..d82d1f46 100644 --- a/tests/test_twopoint_difference.py +++ b/tests/test_twopoint_difference.py @@ -1019,125 +1019,3 @@ def test_median_func(): arr = np.zeros(4 * 2 * 2).reshape(4, 2, 2) arr[:, 0, 0] = np.array([-1., -2., np.nan, np.nan]) assert calc_med_first_diffs(arr)[0, 0] == -1 -@pytest.mark.skip("Used for local testing") -def test_sigma_clip(): -# hdul = fits.open('TSOjump_sc__refpix.fits') - hdul = fits.open('lrs_TSOjump_sigmaclip5_00_refpix.fits') - data = hdul['SCI'].data * 4.0 - gdq = hdul['GROUPDQ'].data - indata = data[:53, :, :, :] - ingdq = gdq[:53, :, :, :] - read_noise = np.ones(shape=(indata.shape[2], indata.shape[3]), dtype=np.float32) * 5.9 * 4.0 - hdul.close() - gdq, row_below_gdq, row_above_gdq, total_primary_crs, stddev = find_crs(indata, ingdq, read_noise, 3, - 4, 5, 1, - False, 1000, - 10, DQFLAGS, - after_jump_flag_e1=0.0, - after_jump_flag_n1=0, - after_jump_flag_e2=0.0, - after_jump_flag_n2=0, - copy_arrs=True, minimum_groups=3, minimum_sigclip_groups=50,) - fits.writeto("outgdq.fits", gdq, overwrite=True) -@pytest.mark.skip("Used for local testing") -def test_first_grp_flag_issue(): - nints = 8 - nrows = 2 - ncols = 2 - ngroups = 10 - readnoise = 2 - data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32) - data = np.random.normal(0, readnoise, size=(nints, ngroups, nrows, ncols)) - read_noise = np.full((nrows, ncols), readnoise, dtype=np.float32) - gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint32) - - gdq[:, 0, :, :] = DQFLAGS['DO_NOT_USE'] - gdq[1:, 1, :, :] = DQFLAGS['DO_NOT_USE'] - gdq[:, -1, :, :] = DQFLAGS['DO_NOT_USE'] - gdq, row_below_gdq, row_above_gdq, total_total_crs, stddev = \ - find_crs(data, gdq, read_noise, 3, 4, 5, 1, False, 1000, 10, DQFLAGS, - after_jump_flag_e1=0.0, after_jump_flag_n1=0, - after_jump_flag_e2=0.0, after_jump_flag_n2=0, - copy_arrs=True, minimum_groups=3, minimum_sigclip_groups=50) - fits.writeto("outgdq.fits",gdq, overwrite=True) -@pytest.mark.skip("Used for local testing") -def test_5grp_TSO(): - nints=20 - nrows = 2 - ncols = 2 - ngroups = 5 - readnoise = 25 - data = np.random.normal(0, 0.1 * readnoise, size=(nints, ngroups, nrows, ncols)) - read_noise = np.full((nrows, ncols), readnoise, dtype=np.float32) - gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint32) - np.expand_dims(gdq, axis=0) - np.expand_dims(data, axis=0) - gdq[:, 0, :, :] = DQFLAGS['DO_NOT_USE'] -# gdq[1:, 1, :, :] = DQFLAGS['DO_NOT_USE'] - gdq[:, -1, :, :] = DQFLAGS['DO_NOT_USE'] - data[0, :, 0, 0] = [21500, 37600, 52082, 65068, 58627] - data[0, :, 0, 1] = [21500, 37600, 52082, 65068, 58627] - gdq, row_below_gdq, row_above_gdq, total_primary_crs, stddev = \ - find_crs(data, gdq, read_noise, 3, 4, 5, 1, False, 1000, 10, DQFLAGS, - after_jump_flag_e1=0.0, after_jump_flag_n1=0, - after_jump_flag_e2=0.0, after_jump_flag_n2=0, - copy_arrs=True, minimum_groups=3, minimum_sigclip_groups=5000) - fits.writeto("new_gdq.fits", gdq, overwrite=True) -@pytest.mark.skip("Used for local testing") -def test_5grp_realTSO(): - hdul = fits.open("obs2508_cutout_jump.fits") - gdq = hdul['groupdq'].data - data = hdul['sci'].data - readnoise = 25 - read_noise = np.full((3, 3), readnoise, dtype=np.float32) - - gdq, row_below_gdq, row_above_gdq, total_total_crs, stddev = \ - find_crs(data, gdq, read_noise, 3, 4, 5, 1, False, 1000, 10, DQFLAGS, - after_jump_flag_e1=0.0, after_jump_flag_n1=0, - after_jump_flag_e2=0.0, after_jump_flag_n2=0, - copy_arrs=True, minimum_groups=3, minimum_sigclip_groups=15000) - fits.writeto("new_gdq_cutout.fits", gdq, overwrite=True) -@pytest.mark.skip("Used for local testing") -def test_5grp_allTSO(): - hdul = fits.open("obs2508_noshower_sigclip_base_00_dark_current.fits") - gdq = hdul['groupdq'].data - data = hdul['sci'].data * 5.5 - readnoise = 5.5 * 6 - read_noise = np.full((gdq.shape[2], gdq.shape[3]), readnoise, dtype=np.float32) - - gdq, row_below_gdq, row_above_gdq, total_total_crs, stddev = \ - find_crs(data, gdq, read_noise, 5, 4, 5, 1, False, 1000, 10, DQFLAGS, - after_jump_flag_e1=0.0, after_jump_flag_n1=0, - after_jump_flag_e2=0.0, after_jump_flag_n2=0, - copy_arrs=True, minimum_groups=3, minimum_sigclip_groups=15000) - fits.writeto("new_no_sigma_clip_gdq.fits", gdq, overwrite=True) - -@pytest.mark.skip("Used for local testing") -def test_1059(): - hdul = fits.open("data/nircam_1059_00_dark_current.fits") - gdq = hdul['groupdq'].data - data = hdul['sci'].data * 5.5 - readnoise = 5.5 * 6 - read_noise = np.full((gdq.shape[2], gdq.shape[3]), readnoise, dtype=np.float32) - - gdq, row_below_gdq, row_above_gdq, total_total_crs, stddev = \ - find_crs(data, gdq, read_noise, 5, 4, 5, 1, False, 1000, 10, DQFLAGS, - after_jump_flag_e1=0.0, after_jump_flag_n1=0, - after_jump_flag_e2=0.0, after_jump_flag_n2=0, - copy_arrs=True, minimum_groups=3, minimum_sigclip_groups=15000) - fits.writeto("new_no_sigma_clip_gdq.fits", gdq, overwrite=True) - -@pytest.mark.skip("Used for local testing") -def test_1952(): - hdul = fits.open("data/obs1952_sc_shower_00_dark_current.fits") - gdq = hdul['groupdq'].data - data = hdul['sci'].data * 5.5 - readnoise = 5.5 * 6 - read_noise = np.full((gdq.shape[2], gdq.shape[3]), readnoise, dtype=np.float32) - gain = np.full((gdq.shape[2], gdq.shape[3]), 5.5, dtype=np.float32) - gdq, row_below_gdq, row_above_gdq, total_total_crs, stddev = \ - find_crs(data, gdq, read_noise, 5, 4, 5, 1, True, 1000, 10, DQFLAGS, - after_jump_flag_e1=100.0 * gain, after_jump_flag_n1=20/2.77, - after_jump_flag_e2=3000.0 * gain, after_jump_flag_n2=1000/2.77, - copy_arrs=True, minimum_groups=3, minimum_sigclip_groups=150) - fits.writeto("new_obs1952_gdq.fits", gdq, overwrite=True) From e4f02806937ee2d4a1f4c6a0a2d3e7c49ad558f8 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Fri, 11 Aug 2023 10:55:51 -0400 Subject: [PATCH 09/10] Update test_jump.py remove unused import --- tests/test_jump.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_jump.py b/tests/test_jump.py index 1f5c8963..3c831e87 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from astropy.io import fits from stcal.jump.jump import flag_large_events, find_ellipses, extend_saturation, \ point_inside_ellipse, find_faint_extended, calc_num_slices From 72a7e8339634e59935d0ec868cd0a7d9f9c24017 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Fri, 11 Aug 2023 10:57:13 -0400 Subject: [PATCH 10/10] Update test_twopoint_difference.py remove unused import --- tests/test_twopoint_difference.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_twopoint_difference.py b/tests/test_twopoint_difference.py index d82d1f46..cb08ac6a 100644 --- a/tests/test_twopoint_difference.py +++ b/tests/test_twopoint_difference.py @@ -1,6 +1,5 @@ import pytest import numpy as np -from astropy.io import fits from stcal.jump.twopoint_difference import find_crs, calc_med_first_diffs