From 5f1c5eceee1b1416cab242e9ac0927fbc06b9453 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 11:47:21 -0500 Subject: [PATCH 01/31] Refactor metadata computation for OLS CAS22 ramp fitting --- pyproject.toml | 1 - src/stcal/ramp_fitting/ols_cas22/_fit.pyx | 10 +-- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 14 ++-- src/stcal/ramp_fitting/ols_cas22/_ramp.pyx | 84 +++------------------- tests/test_jump_cas22.py | 25 ++++--- 5 files changed, 38 insertions(+), 96 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a945de6e..1f6184e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,7 +146,6 @@ extend-select = [ 'FLY', # flynt (f-string conversion where possible) 'NPY', # NumPy-specific checks (recommendations from NumPy) 'PERF', # Perflint (performance linting) - 'LOG', 'RUF', # ruff specific checks ] ignore = [ diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx index dbe3c536..423d214f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx @@ -43,7 +43,7 @@ from stcal.ramp_fitting.ols_cas22._jump cimport ( n_fixed_offsets, n_pixel_offsets, ) -from stcal.ramp_fitting.ols_cas22._ramp cimport ReadPattern, from_read_pattern +from stcal.ramp_fitting.ols_cas22._ramp cimport _fill_metadata from typing import NamedTuple @@ -150,10 +150,10 @@ def fit_ramps(float[:, :] resultants, f'match number of resultants {n_resultants}') # Compute the main metadata from the read pattern and cast it to memory views - cdef ReadPattern metadata = from_read_pattern(read_pattern, read_time, n_resultants) - cdef float[:] t_bar = metadata.t_bar - cdef float[:] tau = metadata.tau - cdef int[:] n_reads = metadata.n_reads + cdef float[:] t_bar = np.empty(n_resultants, dtype=np.float32) + cdef float[:] tau = np.empty(n_resultants, dtype=np.float32) + cdef int[:] n_reads = np.empty(n_resultants, dtype=np.int32) + _fill_metadata(read_pattern, read_time, t_bar, tau, n_reads) # Setup pre-compute arrays for jump detection cdef float[:, :] fixed diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index de31cd6c..785fb57c 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -17,19 +17,15 @@ cdef struct RampFit: ctypedef vector[RampIndex] RampQueue -cdef class ReadPattern: - cdef float[::1] t_bar - cdef float[::1] tau - cdef int[::1] n_reads - - cpdef RampQueue init_ramps(int[:] dq, int n_resultants) -cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, - float read_time, - int n_resultants) +cpdef _fill_metadata(list[list[int]] read_pattern, + float read_time, + float[:] t_bar, + float[:] tau, + int[:] n_reads) cdef RampFit fit_ramp(float[:] resultants_, diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx index cf9b9336..33308eb5 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx @@ -56,91 +56,29 @@ from cython cimport boundscheck, cdivision, cpow, wraparound from libc.math cimport INFINITY, NAN, fabs, fmaxf, sqrt from libcpp.vector cimport vector -from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampIndex, RampQueue, ReadPattern +from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampIndex, RampQueue # Initialize numpy for cython use in this module cnp.import_array() -cdef class ReadPattern: - """ - Class to contain the read pattern derived metadata - This exists only to allow us to output multiple memory views at the same time - from the same cython function. This is needed because neither structs nor unions - can contain memory views. - - In the case of this code memory views are the fastest "safe" array data structure. - This class will immediately be unpacked into raw memory views, so that we avoid - any further overhead of switching between python and cython. - - Attributes: - ---------- - t_bar : np.ndarray[float_t, ndim=1] - The mean time of each resultant - tau : np.ndarray[float_t, ndim=1] - The variance in time of each resultant - n_reads : np.ndarray[cnp.int32_t, ndim=1] - The number of reads in each resultant - """ - - def _to_dict(ReadPattern self): - """ - This is a private method to convert the ReadPattern object to a dictionary, - so that attributes can be directly accessed in python. Note that this - is needed because class attributes cannot be accessed on cython classes - directly in python. Instead they need to be accessed or set using a - python compatible method. This method is a pure puthon method bound - to to the cython class and should not be used by any cython code, and - only exists for testing purposes. - """ - return dict(t_bar=np.array(self.t_bar, dtype=np.float32), - tau=np.array(self.tau, dtype=np.float32), - n_reads=np.array(self.n_reads, dtype=np.int32)) - - @boundscheck(False) @wraparound(False) @cdivision(True) -cpdef ReadPattern from_read_pattern(list[list[int]] read_pattern, float read_time, int n_resultants): - """ - Derive the input data from the the read pattern - This is faster than using __init__ or __cinit__ to construct the object with - these calls. - - Parameters - ---------- - read pattern: list[list[int]] - read pattern for the image - read_time : float - Time to perform a readout. - n_resultants : int - Number of resultants in the image +cpdef _fill_metadata(list[list[int]] read_pattern, + float read_time, + float[:] t_bar, + float[:] tau, + int[:] n_reads): - Returns - ------- - ReadPattern - Contains: - - t_bar - - tau - - n_reads - """ - - cdef ReadPattern data = ReadPattern() - data.t_bar = np.empty(n_resultants, dtype=np.float32) - data.tau = np.empty(n_resultants, dtype=np.float32) - data.n_reads = np.empty(n_resultants, dtype=np.int32) - - cdef int index, n_reads + cdef int index, n_read cdef list[int] resultant for index, resultant in enumerate(read_pattern): - n_reads = len(resultant) - - data.n_reads[index] = n_reads - data.t_bar[index] = read_time * np.mean(resultant) - data.tau[index] = (np.sum((2 * (n_reads - np.arange(n_reads)) - 1) * resultant) * - read_time / n_reads**2) + n_read = len(resultant) - return data + n_reads[index] = n_read + t_bar[index] = read_time * np.mean(resultant) + tau[index] = np.sum((2 * (n_read - np.arange(n_read)) - 1) * resultant) * read_time / n_read**2 @boundscheck(False) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 18c19c96..0eb38c03 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -9,7 +9,7 @@ _fill_pixel_values, fill_fixed_values, ) -from stcal.ramp_fitting.ols_cas22._ramp import from_read_pattern, init_ramps +from stcal.ramp_fitting.ols_cas22._ramp import _fill_metadata, init_ramps # Purposefully set a fixed seed so that the tests in this module are deterministic RNG = np.random.default_rng(619) @@ -111,13 +111,18 @@ def read_pattern(): ] -def test_from_read_pattern(read_pattern): +def test__fill_metadata(read_pattern): """Test turning read_pattern into the time data""" - metadata = from_read_pattern(read_pattern, READ_TIME, len(read_pattern))._to_dict() # noqa: SLF001 - t_bar = metadata["t_bar"] - tau = metadata["tau"] - n_reads = metadata["n_reads"] + n_resultants = len(read_pattern) + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + _fill_metadata(read_pattern, READ_TIME, t_bar, tau, n_reads) + + assert t_bar.shape == (n_resultants,) + assert tau.shape == (n_resultants,) + assert n_reads.shape == (n_resultants,) # Check that the data is correct assert_allclose(t_bar, [7.6, 15.2, 21.279999, 41.040001, 60.799999, 88.159996]) @@ -142,9 +147,13 @@ def ramp_data(read_pattern): metadata : dict The metadata computed from the read pattern """ - data = from_read_pattern(read_pattern, READ_TIME, len(read_pattern))._to_dict() # noqa: SLF001 + n_resultants = len(read_pattern) + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + _fill_metadata(read_pattern, READ_TIME, t_bar, tau, n_reads) - return data["t_bar"], data["tau"], data["n_reads"], read_pattern + return t_bar, tau, n_reads, read_pattern def test_fill_fixed_values(ramp_data): From 8151490721cbfb880aebb9e9572d9100c0e017d8 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 11:57:03 -0500 Subject: [PATCH 02/31] Refactor jump detection pre-compute arrays --- src/stcal/ramp_fitting/ols_cas22/_fit.pyx | 35 +++-- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 35 +++-- src/stcal/ramp_fitting/ols_cas22/_jump.pyx | 147 ++++++++++----------- tests/test_jump_cas22.py | 83 ++++-------- 4 files changed, 135 insertions(+), 165 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx index 423d214f..9df3c1bd 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pyx @@ -38,7 +38,7 @@ from libcpp.list cimport list as cpp_list from stcal.ramp_fitting.ols_cas22._jump cimport ( JumpFits, Thresh, - fill_fixed_values, + _fill_fixed_values, fit_jumps, n_fixed_offsets, n_pixel_offsets, @@ -156,21 +156,34 @@ def fit_ramps(float[:, :] resultants, _fill_metadata(read_pattern, read_time, t_bar, tau, n_reads) # Setup pre-compute arrays for jump detection - cdef float[:, :] fixed - cdef float[:, :] pixel + cdef float[:, :] single_pixel + cdef float[:, :] double_pixel + cdef float[:, :] single_fixed + cdef float[:, :] double_fixed if use_jump: # Initialize arrays for the jump detection pre-computed values - fixed = np.empty((n_fixed_offsets, n_resultants - 1), dtype=np.float32) - pixel = np.empty((n_pixel_offsets, n_resultants - 1), dtype=np.float32) + single_pixel = np.empty((n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((n_pixel_offsets, n_resultants - 2), dtype=np.float32) + + single_fixed = np.empty((n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((n_fixed_offsets, n_resultants - 2), dtype=np.float32) # Pre-compute the values from the read pattern - fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) + _fill_fixed_values(single_fixed, + double_fixed, + t_bar, + tau, + n_reads, + n_resultants) else: # "Initialize" the arrays when not using jump detection, they need to be # initialized because they do get passed around, but they don't need # to actually have any entries - fixed = np.empty((0, 0), dtype=np.float32) - pixel = np.empty((0, 0), dtype=np.float32) + single_pixel = np.empty((0, 0), dtype=np.float32) + double_pixel = np.empty((0, 0), dtype=np.float32) + + single_fixed = np.empty((0, 0), dtype=np.float32) + double_fixed = np.empty((0, 0), dtype=np.float32) # Create a threshold struct cdef Thresh thresh = Thresh(intercept, constant) @@ -211,8 +224,10 @@ def fit_ramps(float[:, :] resultants, tau, n_reads, n_resultants, - fixed, - pixel, + single_pixel, + double_pixel, + single_fixed, + double_fixed, thresh, use_jump, include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 8693e791..3fd38852 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -7,22 +7,16 @@ from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampQueue cpdef enum FixedOffsets: - single_t_bar_diff - double_t_bar_diff - single_t_bar_diff_sqr - double_t_bar_diff_sqr - single_read_recip - double_read_recip - single_var_slope_val - double_var_slope_val + t_bar_diff + t_bar_diff_sqr + read_recip + var_slope_val n_fixed_offsets cpdef enum PixelOffsets: - single_local_slope - double_local_slope - single_var_read_noise - double_var_read_noise + local_slope + var_read_noise n_pixel_offsets @@ -41,11 +35,12 @@ cdef struct JumpFits: RampQueue index -cpdef float[:, :] fill_fixed_values(float[:, :] fixed, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants) +cpdef _fill_fixed_values(float[:, :] single_fixed, + float[:, :] double_fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants) cdef JumpFits fit_jumps(float[:] resultants, @@ -55,8 +50,10 @@ cdef JumpFits fit_jumps(float[:] resultants, float[:] tau, int[:] n_reads, int n_resultants, - float[:, :] fixed, - float[:, :] pixel, + float[:, :] single_pixel, + float[:, :] double_pixel, + float[:, :] single_fixed, + float[:, :] double_fixed, Thresh thresh, bool use_jump, bool include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx index 808482f3..a2e0160b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pyx @@ -65,11 +65,12 @@ from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampIndex, RampQueue, f @boundscheck(False) @wraparound(False) @cdivision(True) -cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants): +cpdef inline _fill_fixed_values(float[:, :] single_fixed, + float[:, :] double_fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants): """ Pre-compute all the values needed for jump detection which only depend on the read pattern. @@ -103,48 +104,38 @@ cpdef inline float[:, :] fill_fixed_values(float[:, :] fixed, """ # Cast the enum values into integers for indexing (otherwise compiler complains) # These will be optimized out - cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff - cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff - cdef int single_t_bar_diff_sqr = FixedOffsets.single_t_bar_diff_sqr - cdef int double_t_bar_diff_sqr = FixedOffsets.double_t_bar_diff_sqr - cdef int single_read_recip = FixedOffsets.single_read_recip - cdef int double_read_recip = FixedOffsets.double_read_recip - cdef int single_var_slope_val = FixedOffsets.single_var_slope_val - cdef int double_var_slope_val = FixedOffsets.double_var_slope_val + cdef int t_bar_diff = FixedOffsets.t_bar_diff + cdef int t_bar_diff_sqr = FixedOffsets.t_bar_diff_sqr + cdef int read_recip = FixedOffsets.read_recip + cdef int var_slope_val = FixedOffsets.var_slope_val # Coerce division to be using floats cdef float num = 1 cdef int i for i in range(n_resultants - 1): - fixed[single_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] - fixed[single_t_bar_diff_sqr, i] = fixed[single_t_bar_diff, i] ** 2 - fixed[single_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) - fixed[single_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) + single_fixed[t_bar_diff, i] = t_bar[i + 1] - t_bar[i] + single_fixed[t_bar_diff_sqr, i] = single_fixed[t_bar_diff, i] ** 2 + single_fixed[read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) + single_fixed[var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) if i < n_resultants - 2: - fixed[double_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] - fixed[double_t_bar_diff_sqr, i] = fixed[double_t_bar_diff, i] ** 2 - fixed[double_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) - fixed[double_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) - else: - # Last double difference is undefined - fixed[double_t_bar_diff, i] = NAN - fixed[double_t_bar_diff_sqr, i] = NAN - fixed[double_read_recip, i] = NAN - fixed[double_var_slope_val, i] = NAN - - return fixed + double_fixed[t_bar_diff, i] = t_bar[i + 2] - t_bar[i] + double_fixed[t_bar_diff_sqr, i] = double_fixed[t_bar_diff, i] ** 2 + double_fixed[read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) + double_fixed[var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) @boundscheck(False) @wraparound(False) @cdivision(True) -cpdef inline float[:, :] _fill_pixel_values(float[:, :] pixel, - float[:] resultants, - float[:, :] fixed, - float read_noise, - int n_resultants): +cpdef inline _fill_pixel_values(float[:, :] single_pixel, + float[:, :] double_pixel, + float[:, :] single_fixed, + float[:, :] double_fixed, + float[:] resultants, + float read_noise, + int n_resultants): """ Pre-compute all the values needed for jump detection which only depend on the a specific pixel (independent of the given ramp for a pixel). @@ -172,32 +163,22 @@ cpdef inline float[:, :] _fill_pixel_values(float[:, :] pixel, read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, ] """ - cdef int single_t_bar_diff = FixedOffsets.single_t_bar_diff - cdef int double_t_bar_diff = FixedOffsets.double_t_bar_diff - cdef int single_read_recip = FixedOffsets.single_read_recip - cdef int double_read_recip = FixedOffsets.double_read_recip + cdef int t_bar_diff = FixedOffsets.t_bar_diff + cdef int read_recip = FixedOffsets.read_recip - cdef int single_slope = PixelOffsets.single_local_slope - cdef int double_slope = PixelOffsets.double_local_slope - cdef int single_var = PixelOffsets.single_var_read_noise - cdef int double_var = PixelOffsets.double_var_read_noise + cdef int local_slope = PixelOffsets.local_slope + cdef int var_read_noise = PixelOffsets.var_read_noise cdef float read_noise_sqr = read_noise ** 2 cdef int i for i in range(n_resultants - 1): - pixel[single_slope, i] = (resultants[i + 1] - resultants[i]) / fixed[single_t_bar_diff, i] - pixel[single_var, i] = read_noise_sqr * fixed[single_read_recip, i] + single_pixel[local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[t_bar_diff, i] + single_pixel[var_read_noise, i] = read_noise_sqr * single_fixed[read_recip, i] if i < n_resultants - 2: - pixel[double_slope, i] = (resultants[i + 2] - resultants[i]) / fixed[double_t_bar_diff, i] - pixel[double_var, i] = read_noise_sqr * fixed[double_read_recip, i] - else: - # The last double difference is undefined - pixel[double_slope, i] = NAN - pixel[double_var, i] = NAN - - return pixel + double_pixel[local_slope, i] = (resultants[i + 2] - resultants[i]) / double_fixed[t_bar_diff, i] + double_pixel[var_read_noise, i] = read_noise_sqr * double_fixed[read_recip, i] cdef inline float _threshold(Thresh thresh, float slope): @@ -296,8 +277,10 @@ cdef inline float _statstic(float local_slope, @boundscheck(False) @wraparound(False) -cdef inline (int, float) _fit_statistic(float[:, :] pixel, - float[:, :] fixed, +cdef inline (int, float) _fit_statistic(float[:, :] single_pixel, + float[:, :] double_pixel, + float[:, :] single_fixed, + float[:, :] double_fixed, float[:] t_bar, float slope, RampIndex ramp): @@ -326,15 +309,11 @@ cdef inline (int, float) _fit_statistic(float[:, :] pixel, """ # Cast the enum values into integers for indexing (otherwise compiler complains) # These will be optimized out - cdef int single_local_slope = PixelOffsets.single_local_slope - cdef int double_local_slope = PixelOffsets.double_local_slope - cdef int single_var_read_noise = PixelOffsets.single_var_read_noise - cdef int double_var_read_noise = PixelOffsets.double_var_read_noise + cdef int local_slope = PixelOffsets.local_slope + cdef int var_read_noise = PixelOffsets.var_read_noise - cdef int single_t_bar_diff_sqr = FixedOffsets.single_t_bar_diff_sqr - cdef int double_t_bar_diff_sqr = FixedOffsets.double_t_bar_diff_sqr - cdef int single_var_slope_val = FixedOffsets.single_var_slope_val - cdef int double_var_slope_val = FixedOffsets.double_var_slope_val + cdef int t_bar_diff_sqr = FixedOffsets.t_bar_diff_sqr + cdef int var_slope_val = FixedOffsets.var_slope_val # Note that a ramp consisting of a single point is degenerate and has no # fit statistic so we bail out here @@ -353,10 +332,10 @@ cdef inline (int, float) _fit_statistic(float[:, :] pixel, # this makes it much easier to compute a "lazy" max. cdef int index = ramp.end - 1 cdef int argmax = ramp.end - ramp.start - 1 - cdef float max_stat = _statstic(pixel[single_local_slope, index], - pixel[single_var_read_noise, index], - fixed[single_t_bar_diff_sqr, index], - fixed[single_var_slope_val, index], + cdef float max_stat = _statstic(single_pixel[local_slope, index], + single_pixel[var_read_noise, index], + single_fixed[t_bar_diff_sqr, index], + single_fixed[var_slope_val, index], slope, correct) @@ -365,16 +344,16 @@ cdef inline (int, float) _fit_statistic(float[:, :] pixel, cdef int stat_index for stat_index, index in enumerate(range(ramp.start, ramp.end - 1)): # Compute max of single and double difference statistics - stat1 = _statstic(pixel[single_local_slope, index], - pixel[single_var_read_noise, index], - fixed[single_t_bar_diff_sqr, index], - fixed[single_var_slope_val, index], + stat1 = _statstic(single_pixel[local_slope, index], + single_pixel[var_read_noise, index], + single_fixed[t_bar_diff_sqr, index], + single_fixed[var_slope_val, index], slope, correct) - stat2 = _statstic(pixel[double_local_slope, index], - pixel[double_var_read_noise, index], - fixed[double_t_bar_diff_sqr, index], - fixed[double_var_slope_val, index], + stat2 = _statstic(double_pixel[local_slope, index], + double_pixel[var_read_noise, index], + double_fixed[t_bar_diff_sqr, index], + double_fixed[var_slope_val, index], slope, correct) stat = fmaxf(stat1, stat2) @@ -397,8 +376,10 @@ cdef inline JumpFits fit_jumps(float[:] resultants, float[:] tau, int[:] n_reads, int n_resultants, - float[:, :] fixed, - float[:, :] pixel, + float[:, :] single_pixel, + float[:, :] double_pixel, + float[:, :] single_fixed, + float[:, :] double_fixed, Thresh thresh, bool use_jump, bool include_diagnostic): @@ -461,7 +442,13 @@ cdef inline JumpFits fit_jumps(float[:] resultants, # Fill in the jump detection pre-compute values for a single pixel if use_jump: - pixel = _fill_pixel_values(pixel, resultants, fixed, read_noise, n_resultants) + _fill_pixel_values(single_pixel, + double_pixel, + single_fixed, + double_fixed, + resultants, + read_noise, + n_resultants) # Run while the Queue is non-empty while not ramps.empty(): @@ -479,8 +466,10 @@ cdef inline JumpFits fit_jumps(float[:] resultants, # Run jump detection if enabled if use_jump: - argmax, max_stat = _fit_statistic(pixel, - fixed, + argmax, max_stat = _fit_statistic(single_pixel, + double_pixel, + single_fixed, + double_fixed, t_bar, ramp_fit.slope, ramp) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 0eb38c03..9b2f6067 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -6,8 +6,8 @@ from stcal.ramp_fitting.ols_cas22._jump import ( FixedOffsets, PixelOffsets, + _fill_fixed_values, _fill_pixel_values, - fill_fixed_values, ) from stcal.ramp_fitting.ols_cas22._ramp import _fill_metadata, init_ramps @@ -161,43 +161,23 @@ def test_fill_fixed_values(ramp_data): t_bar, tau, n_reads, _ = ramp_data n_resultants = len(t_bar) - fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) - - # Sanity check that the shape of fixed is correct - assert fixed.shape == (2 * 4, n_resultants - 1) - - # Split into the different types of data - t_bar_diffs = fixed[FixedOffsets.single_t_bar_diff : FixedOffsets.double_t_bar_diff + 1, :] - t_bar_diff_sqrs = fixed[FixedOffsets.single_t_bar_diff_sqr : FixedOffsets.double_t_bar_diff_sqr + 1, :] - read_recip = fixed[FixedOffsets.single_read_recip : FixedOffsets.double_read_recip + 1, :] - var_slope_vals = fixed[FixedOffsets.single_var_slope_val : FixedOffsets.double_var_slope_val + 1, :] - - # Sanity check that these are all the right shape - assert t_bar_diffs.shape == (2, n_resultants - 1) - assert t_bar_diff_sqrs.shape == (2, n_resultants - 1) - assert read_recip.shape == (2, n_resultants - 1) - assert var_slope_vals.shape == (2, n_resultants - 1) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + _fill_fixed_values(single_fixed, double_fixed, t_bar, tau, n_reads, n_resultants) # Check the computed data # These are computed using loop in cython, here we check against numpy # Single diffs - assert np.all(t_bar_diffs[0] == t_bar[1:] - t_bar[:-1]) - assert np.all(t_bar_diff_sqrs[0] == (t_bar[1:] - t_bar[:-1]) ** 2) - assert np.all(read_recip[0] == np.float32(1 / n_reads[1:]) + np.float32(1 / n_reads[:-1])) - assert np.all(var_slope_vals[0] == (tau[1:] + tau[:-1] - 2 * np.minimum(t_bar[1:], t_bar[:-1]))) + assert np.all(single_fixed[0, :] == t_bar[1:] - t_bar[:-1]) + assert np.all(single_fixed[1, :] == (t_bar[1:] - t_bar[:-1]) ** 2) + assert np.all(single_fixed[2, :] == np.float32(1 / n_reads[1:]) + np.float32(1 / n_reads[:-1])) + assert np.all(single_fixed[3, :] == (tau[1:] + tau[:-1] - 2 * np.minimum(t_bar[1:], t_bar[:-1]))) # Double diffs - assert np.all(t_bar_diffs[1, :-1] == t_bar[2:] - t_bar[:-2]) - assert np.all(t_bar_diff_sqrs[1, :-1] == (t_bar[2:] - t_bar[:-2]) ** 2) - assert np.all(read_recip[1, :-1] == np.float32(1 / n_reads[2:]) + np.float32(1 / n_reads[:-2])) - assert np.all(var_slope_vals[1, :-1] == (tau[2:] + tau[:-2] - 2 * np.minimum(t_bar[2:], t_bar[:-2]))) - - # Last double diff should be NaN - assert np.isnan(t_bar_diffs[1, -1]) - assert np.isnan(t_bar_diff_sqrs[1, -1]) - assert np.isnan(read_recip[1, -1]) - assert np.isnan(var_slope_vals[1, -1]) + assert np.all(double_fixed[0, :] == t_bar[2:] - t_bar[:-2]) + assert np.all(double_fixed[1, :] == (t_bar[2:] - t_bar[:-2]) ** 2) + assert np.all(double_fixed[2, :] == np.float32(1 / n_reads[2:]) + np.float32(1 / n_reads[:-2])) + assert np.all(double_fixed[3, :] == (tau[2:] + tau[:-2] - 2 * np.minimum(t_bar[2:], t_bar[:-2]))) def _generate_resultants(read_pattern, n_pixels=1): @@ -260,53 +240,42 @@ def pixel_data(ramp_data): t_bar, tau, n_reads, read_pattern = ramp_data n_resultants = len(t_bar) - fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - fixed = fill_fixed_values(fixed, t_bar, tau, n_reads, n_resultants) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + _fill_fixed_values(single_fixed, double_fixed, t_bar, tau, n_reads, n_resultants) resultants = _generate_resultants(read_pattern) - return resultants, t_bar, tau, n_reads, fixed + return resultants, t_bar, n_reads, single_fixed, double_fixed def test__fill_pixel_values(pixel_data): """Test computing the initial pixel data""" - resultants, t_bar, tau, n_reads, fixed = pixel_data + resultants, t_bar, n_reads, single_fixed, double_fixed = pixel_data n_resultants = len(t_bar) - pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - pixel = _fill_pixel_values(pixel, resultants, fixed, READ_NOISE, n_resultants) - - # Sanity check that the shape of pixel is correct - assert pixel.shape == (2 * 2, n_resultants - 1) - - # Split into the different types of data - local_slopes = pixel[PixelOffsets.single_local_slope : PixelOffsets.double_local_slope + 1, :] - var_read_noise = pixel[PixelOffsets.single_var_read_noise : PixelOffsets.double_var_read_noise + 1, :] - - # Sanity check that these are all the right shape - assert local_slopes.shape == (2, n_resultants - 1) - assert var_read_noise.shape == (2, n_resultants - 1) + single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + _fill_pixel_values( + single_pixel, double_pixel, single_fixed, double_fixed, resultants, READ_NOISE, n_resultants + ) # Check the computed data # These are computed using loop in cython, here we check against numpy # Single diffs - assert np.all(local_slopes[0] == (resultants[1:] - resultants[:-1]) / (t_bar[1:] - t_bar[:-1])) + assert np.all(single_pixel[0, :] == (resultants[1:] - resultants[:-1]) / (t_bar[1:] - t_bar[:-1])) assert np.all( - var_read_noise[0] + single_pixel[1, :] == np.float32(READ_NOISE**2) * (np.float32(1 / n_reads[1:]) + np.float32(1 / n_reads[:-1])) ) # Double diffs - assert np.all(local_slopes[1, :-1] == (resultants[2:] - resultants[:-2]) / (t_bar[2:] - t_bar[:-2])) + assert np.all(double_pixel[0, :] == (resultants[2:] - resultants[:-2]) / (t_bar[2:] - t_bar[:-2])) assert np.all( - var_read_noise[1, :-1] + double_pixel[1, :] == np.float32(READ_NOISE**2) * (np.float32(1 / n_reads[2:]) + np.float32(1 / n_reads[:-2])) ) - # Last double diff should be NaN - assert np.isnan(local_slopes[1, -1]) - assert np.isnan(var_read_noise[1, -1]) - @pytest.fixture(scope="module") def detector_data(read_pattern): From 0e7c51b189aca60d3e91e6c4ac91d8bb242b150c Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 12:10:49 -0500 Subject: [PATCH 03/31] Update ramp fitting code to use .py file instead of .pyx file --- pyproject.toml | 1 + setup.py | 2 +- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 22 +-- .../ols_cas22/{_ramp.pyx => _ramp.py} | 170 ++++++++++-------- 4 files changed, 107 insertions(+), 88 deletions(-) rename src/stcal/ramp_fitting/ols_cas22/{_ramp.pyx => _ramp.py} (68%) diff --git a/pyproject.toml b/pyproject.toml index 1f6184e0..62dfdc56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ testpaths = [ "docs", ] norecursedirs = [ + 'src/stcal/ramp_fitting/ols_cas22', 'benchmarks', '.asv', '.eggs', diff --git a/setup.py b/setup.py index e176149e..58a83184 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ extensions = [ Extension( "stcal.ramp_fitting.ols_cas22._ramp", - ["src/stcal/ramp_fitting/ols_cas22/_ramp.pyx"], + ["src/stcal/ramp_fitting/ols_cas22/_ramp.py"], include_dirs=[np.get_include()], language="c++", ), diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 785fb57c..5741abc6 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -3,17 +3,18 @@ from libcpp.vector cimport vector +cpdef void _fill_metadata(list[list[int]] read_pattern, + float read_time, + float[:] t_bar, + float[:] tau, + int[:] n_reads) + + cdef struct RampIndex: int start int end -cdef struct RampFit: - float slope - float read_var - float poisson_var - - ctypedef vector[RampIndex] RampQueue @@ -21,11 +22,10 @@ cpdef RampQueue init_ramps(int[:] dq, int n_resultants) -cpdef _fill_metadata(list[list[int]] read_pattern, - float read_time, - float[:] t_bar, - float[:] tau, - int[:] n_reads) +cdef struct RampFit: + float slope + float read_var + float poisson_var cdef RampFit fit_ramp(float[:] resultants_, diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx b/src/stcal/ramp_fitting/ols_cas22/_ramp.py similarity index 68% rename from src/stcal/ramp_fitting/ols_cas22/_ramp.pyx rename to src/stcal/ramp_fitting/ols_cas22/_ramp.py index 33308eb5..c3ad8f97 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.py @@ -49,30 +49,32 @@ Implementation of running the Casertano+22 algorithm on a (sub)set of resultants listed for a single pixel """ +import cython import numpy as np - -cimport numpy as cnp -from cython cimport boundscheck, cdivision, cpow, wraparound -from libc.math cimport INFINITY, NAN, fabs, fmaxf, sqrt -from libcpp.vector cimport vector - -from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampIndex, RampQueue +from cython.cimports import numpy as cnp +from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, sqrt +from cython.cimports.libcpp.vector import vector +from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import RampFit, RampIndex, RampQueue # Initialize numpy for cython use in this module cnp.import_array() -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cpdef _fill_metadata(list[list[int]] read_pattern, - float read_time, - float[:] t_bar, - float[:] tau, - int[:] n_reads): - - cdef int index, n_read - cdef list[int] resultant +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.ccall +def _fill_metadata( + read_pattern: list[list[cython.int]], + read_time: cython.float, + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], +) -> cython.void: + index: cython.int + n_read: cython.int + resultant: list[cython.int] for index, resultant in enumerate(read_pattern): n_read = len(resultant) @@ -81,13 +83,15 @@ tau[index] = np.sum((2 * (n_read - np.arange(n_read)) - 1) * resultant) * read_time / n_read**2 -@boundscheck(False) -@wraparound(False) -cpdef inline RampQueue init_ramps(int[:] dq, int n_resultants): +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.ccall +def init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: """ Create the initial ramp "queue" for each pixel if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp - otherwise, the resultant is not in a ramp + otherwise, the resultant is not in a ramp. Parameters ---------- @@ -103,13 +107,13 @@ - vector with entry for each ramp found (last entry is last ramp found) - RampIndex with start and end indices of the ramp in the resultants """ - cdef RampQueue ramps = RampQueue() + ramps: RampQueue = RampQueue() # Note: if start/end are -1, then no value has been assigned # ramp.start == -1 means we have not started a ramp # dq[index_resultant, index_pixel] == 0 means resultant is in ramp - cdef RampIndex ramp = RampIndex(-1, -1) - cdef int index_resultant + ramp: RampIndex = RampIndex(-1, -1) + index_resultant: cython.int for index_resultant in range(n_resultants): if ramp.start == -1: # Looking for the start of a ramp @@ -119,12 +123,12 @@ else: # This is not the start of the ramp yet continue - else: + else: # noqa: PLR5501 (makes more logical sense than to use elif) # Looking for the end of a ramp if dq[index_resultant] == 0: # This pixel is in the ramp do nothing continue - else: + else: # noqa: RET507 (makes more logical sense than to remove else) # This pixel is not in the ramp # => index_resultant - 1 is the end of the ramp ramp.end = index_resultant - 1 @@ -141,19 +145,25 @@ return ramps -# Keeps the static type checker/highlighter happy this has no actual effect -ctypedef float[6] _row # Casertano+2022, Table 2 -cdef _row[2] _PTABLE = [[-INFINITY, 5, 10, 20, 50, 100], - [0, 0.4, 1, 3, 6, 10]] - - -@boundscheck(False) -@wraparound(False) -cdef inline float _get_power(float signal): +_P_TABLE = cython.declare( + cython.float[6][2], + [ + [-INFINITY, 5, 10, 20, 50, 100], + [0, 0.4, 1, 3, 6, 10], + ], +) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.cfunc +@cython.exceptval(check=False) +def _get_power(signal: cython.float) -> cython.float: """ - Return the power from Casertano+22, Table 2 + Return the power from Casertano+22, Table 2. Parameters ---------- @@ -164,23 +174,27 @@ ------- signal power from Table 2 """ - cdef int i + i: cython.int for i in range(6): - if signal < _PTABLE[0][i]: - return _PTABLE[1][i - 1] - - return _PTABLE[1][i] - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cdef inline RampFit fit_ramp(float[:] resultants_, - float[:] t_bar_, - float[:] tau_, - int[:] n_reads_, - float read_noise, - RampIndex ramp): + if signal < _P_TABLE[0][i]: + return _P_TABLE[1][i - 1] + + return _P_TABLE[1][i] + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def fit_ramp( + resultants_: cython.float[:], + t_bar_: cython.float[:], + tau_: cython.float[:], + n_reads_: cython.int[:], + read_noise: cython.float, + ramp: RampIndex, +) -> RampFit: """ Fit a single ramp using Casertano+22 algorithm. @@ -207,7 +221,7 @@ - read_var - poisson_var """ - cdef int n_resultants = ramp.end - ramp.start + 1 + n_resultants: cython.int = ramp.end - ramp.start + 1 # Special case where there is no or one resultant, there is no fit and # we bail out before any computations. @@ -218,58 +232,62 @@ return RampFit(NAN, NAN, NAN) # Compute the fit - cdef int i = 0, j = 0 + i: cython.int = 0 + j: cython.int = 0 # Setup data for fitting (work over subset of data) to make things cleaner # Recall that the RampIndex contains the index of the first and last # index of the ramp. Therefore, the Python slice needed to get all the # data within the ramp is: # ramp.start:ramp.end + 1 - cdef float[:] resultants = resultants_[ramp.start:ramp.end + 1] - cdef float[:] t_bar = t_bar_[ramp.start:ramp.end + 1] - cdef float[:] tau = tau_[ramp.start:ramp.end + 1] - cdef int[:] n_reads = n_reads_[ramp.start:ramp.end + 1] + resultants: cython.float[:] = resultants_[ramp.start : ramp.end + 1] + t_bar: cython.float[:] = t_bar_[ramp.start : ramp.end + 1] + tau: cython.float[:] = tau_[ramp.start : ramp.end + 1] + n_reads: cython.int[:] = n_reads_[ramp.start : ramp.end + 1] # Compute mid point time - cdef int end = n_resultants - 1 - cdef float t_bar_mid = (t_bar[0] + t_bar[end]) / 2 + end: cython.int = n_resultants - 1 + t_bar_mid: cython.float = (t_bar[0] + t_bar[end]) / 2 # Casertano+2022 Eq. 44 # Note we've departed from Casertano+22 slightly; # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., # a CR in the first resultant has boosted the whole ramp high but there # is no actual signal. - cdef float power = fmaxf(resultants[end] - resultants[0], 0) + power: cython.float = fmaxf(resultants[end] - resultants[0], 0) power = power / sqrt(read_noise**2 + power) power = _get_power(power) # It's easy to use up a lot of dynamic range on something like # (tbar - tbarmid) ** 10. Rescale these. - cdef float t_scale = (t_bar[end] - t_bar[0]) / 2 + t_scale: cython.float = (t_bar[end] - t_bar[0]) / 2 t_scale = 1 if t_scale == 0 else t_scale # Initialize the fit loop # it is faster to generate a c++ vector than a numpy array - cdef vector[float] weights = vector[float](n_resultants) - cdef vector[float] coeffs = vector[float](n_resultants) - cdef RampFit ramp_fit = RampFit(0, 0, 0) - cdef float f0 = 0, f1 = 0, f2 = 0 - cdef float coeff + weights: vector[cython.float] = vector[float](n_resultants) + coeffs: vector[cython.float] = vector[float](n_resultants) + ramp_fit: RampFit = RampFit(0, 0, 0) + f0: cython.float = 0 + f1: cython.float = 0 + f2: cython.float = 0 + coeff: cython.float # Issue when tbar[] == tbarmid causes exception otherwise - with cpow(True): + with cython.cpow(True): for i in range(n_resultants): # Casertano+22, Eq. 45 - weights[i] = ((((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * - fabs((t_bar[i] - t_bar_mid) / t_scale) ** power) + weights[i] = (((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * fabs( + (t_bar[i] - t_bar_mid) / t_scale + ) ** power # Casertano+22 Eq. 35 f0 += weights[i] f1 += weights[i] * t_bar[i] - f2 += weights[i] * t_bar[i]**2 + f2 += weights[i] * t_bar[i] ** 2 # Casertano+22 Eq. 36 - cdef float det = f2 * f0 - f1 ** 2 + det: cython.float = f2 * f0 - f1**2 if det == 0: return ramp_fit @@ -282,14 +300,14 @@ ramp_fit.slope += coeff * resultants[i] # Casertano+22 Eq. 39 - ramp_fit.read_var += (coeff ** 2 * read_noise ** 2 / n_reads[i]) + ramp_fit.read_var += coeff**2 * read_noise**2 / n_reads[i] # Casertano+22 Eq 40 # Note that this is an inversion of the indexing from the equation; # however, commutivity of addition results in the same answer. This # makes it so that we don't have to loop over all the resultants twice. - ramp_fit.poisson_var += coeff ** 2 * tau[i] + ramp_fit.poisson_var += coeff**2 * tau[i] for j in range(i): - ramp_fit.poisson_var += (2 * coeff * coeffs[j] * t_bar[j]) + ramp_fit.poisson_var += 2 * coeff * coeffs[j] * t_bar[j] return ramp_fit From 52e7e350a915a6c7ce7e5e9178b76a7faed5fbc6 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 12:20:38 -0500 Subject: [PATCH 04/31] Update jump detection code to use .py file instead of .pyx --- setup.py | 2 +- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 12 +- .../ols_cas22/{_jump.pyx => _jump.py} | 357 ++++++++++-------- 3 files changed, 202 insertions(+), 169 deletions(-) rename src/stcal/ramp_fitting/ols_cas22/{_jump.pyx => _jump.py} (70%) diff --git a/setup.py b/setup.py index 58a83184..215d40fb 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ ), Extension( "stcal.ramp_fitting.ols_cas22._jump", - ["src/stcal/ramp_fitting/ols_cas22/_jump.pyx"], + ["src/stcal/ramp_fitting/ols_cas22/_jump.py"], include_dirs=[np.get_include()], language="c++", ), diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 3fd38852..776d67a7 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -35,12 +35,12 @@ cdef struct JumpFits: RampQueue index -cpdef _fill_fixed_values(float[:, :] single_fixed, - float[:, :] double_fixed, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants) +cpdef void _fill_fixed_values(float[:, :] single_fixed, + float[:, :] double_fixed, + float[:] t_bar, + float[:] tau, + int[:] n_reads, + int n_resultants) cdef JumpFits fit_jumps(float[:] resultants, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx b/src/stcal/ramp_fitting/ols_cas22/_jump.py similarity index 70% rename from src/stcal/ramp_fitting/ols_cas22/_jump.pyx rename to src/stcal/ramp_fitting/ols_cas22/_jump.py index a2e0160b..1b060159 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.py @@ -3,7 +3,7 @@ """ This module contains all the functions needed to execute jump detection for the - Castentano+22 ramp fitting algorithm + Castentano+22 ramp fitting algorithm. The _ramp module contains the actual ramp fitting algorithm, this module contains a driver for the algorithm and detection of jumps/splitting ramps. @@ -53,24 +53,38 @@ meaning it automatically handles splitting ramps across dq flags in addition to splitting across detected jumps (if jump detection is turned on). """ - -from cython cimport boundscheck, cdivision, wraparound -from libc.math cimport NAN, fmaxf, isnan, log10, sqrt -from libcpp cimport bool - -from stcal.ramp_fitting.ols_cas22._jump cimport JUMP_DET, FixedOffsets, JumpFits, PixelOffsets, Thresh -from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampIndex, RampQueue, fit_ramp, init_ramps - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cpdef inline _fill_fixed_values(float[:, :] single_fixed, - float[:, :] double_fixed, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants): +import cython +from cython.cimports.libc.math import NAN, fmaxf, isnan, log10, sqrt +from cython.cimports.libcpp import bool as cpp_bool +from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( + JUMP_DET, + FixedOffsets, + JumpFits, + PixelOffsets, + Thresh, +) +from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import ( + RampFit, + RampIndex, + RampQueue, + fit_ramp, + init_ramps, +) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.ccall +def _fill_fixed_values( + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], + n_resultants: cython.int, +) -> cython.void: """ Pre-compute all the values needed for jump detection which only depend on the read pattern. @@ -104,15 +118,15 @@ """ # Cast the enum values into integers for indexing (otherwise compiler complains) # These will be optimized out - cdef int t_bar_diff = FixedOffsets.t_bar_diff - cdef int t_bar_diff_sqr = FixedOffsets.t_bar_diff_sqr - cdef int read_recip = FixedOffsets.read_recip - cdef int var_slope_val = FixedOffsets.var_slope_val + t_bar_diff: cython.int = FixedOffsets.t_bar_diff + t_bar_diff_sqr: cython.int = FixedOffsets.t_bar_diff_sqr + read_recip: cython.int = FixedOffsets.read_recip + var_slope_val: cython.int = FixedOffsets.var_slope_val # Coerce division to be using floats - cdef float num = 1 + num: cython.float = 1 - cdef int i + i: cython.int for i in range(n_resultants - 1): single_fixed[t_bar_diff, i] = t_bar[i + 1] - t_bar[i] single_fixed[t_bar_diff_sqr, i] = single_fixed[t_bar_diff, i] ** 2 @@ -126,16 +140,20 @@ double_fixed[var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cpdef inline _fill_pixel_values(float[:, :] single_pixel, - float[:, :] double_pixel, - float[:, :] single_fixed, - float[:, :] double_fixed, - float[:] resultants, - float read_noise, - int n_resultants): +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.ccall +def _fill_pixel_values( + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + resultants: cython.float[:], + read_noise: cython.float, + n_resultants: cython.int, +) -> cython.void: """ Pre-compute all the values needed for jump detection which only depend on the a specific pixel (independent of the given ramp for a pixel). @@ -163,15 +181,15 @@ read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, ] """ - cdef int t_bar_diff = FixedOffsets.t_bar_diff - cdef int read_recip = FixedOffsets.read_recip + t_bar_diff: cython.int = FixedOffsets.t_bar_diff + read_recip: cython.int = FixedOffsets.read_recip - cdef int local_slope = PixelOffsets.local_slope - cdef int var_read_noise = PixelOffsets.var_read_noise + local_slope: cython.int = PixelOffsets.local_slope + var_read_noise: cython.int = PixelOffsets.var_read_noise - cdef float read_noise_sqr = read_noise ** 2 + read_noise_sqr: cython.float = read_noise**2 - cdef int i + i: cython.int for i in range(n_resultants - 1): single_pixel[local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[t_bar_diff, i] single_pixel[var_read_noise, i] = read_noise_sqr * single_fixed[read_recip, i] @@ -181,9 +199,12 @@ double_pixel[var_read_noise, i] = read_noise_sqr * double_fixed[read_recip, i] -cdef inline float _threshold(Thresh thresh, float slope): +@cython.inline +@cython.cfunc +@cython.exceptval(check=False) +def _threshold(thresh: Thresh, slope: cython.float) -> cython.float: """ - Compute jump threshold + Compute jump threshold. Parameters ---------- @@ -202,12 +223,14 @@ return thresh.intercept - thresh.constant * log10(slope) -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cdef inline float _correction(float[:] t_bar, RampIndex ramp, float slope): +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def _correction(t_bar: cython.float[:], ramp: RampIndex, slope: cython.float) -> cython.float: """ - Compute the correction factor for the variance used by a statistic + Compute the correction factor for the variance used by a statistic. - slope / (t_bar[end] - t_bar[start]) @@ -220,24 +243,27 @@ slope : float The computed slope for the ramp """ - - cdef float diff = t_bar[ramp.end] - t_bar[ramp.start] - - return - slope / diff - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cdef inline float _statstic(float local_slope, - float var_read_noise, - float t_bar_diff_sqr, - float var_slope_coeff, - float slope, - float correct): + diff: cython.float = t_bar[ramp.end] - t_bar[ramp.start] + + return -slope / diff + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def _statistic( + local_slope: cython.float, + var_read_noise: cython.float, + t_bar_diff_sqr: cython.float, + var_slope_coeff: cython.float, + slope: cython.float, + correct: cython.float, +) -> cython.float: """ Compute a single fit statistic - delta / sqrt(var + correct) + delta / sqrt(var + correct). where: delta = local_slope - slope @@ -268,27 +294,33 @@ ------- Create a single instance of the stastic for the given parameters """ - - cdef float delta = local_slope - slope - cdef float var = (var_read_noise + slope * var_slope_coeff) / t_bar_diff_sqr + delta: cython.float = local_slope - slope + var: cython.float = (var_read_noise + slope * var_slope_coeff) / t_bar_diff_sqr return delta / sqrt(var + correct) -@boundscheck(False) -@wraparound(False) -cdef inline (int, float) _fit_statistic(float[:, :] single_pixel, - float[:, :] double_pixel, - float[:, :] single_fixed, - float[:, :] double_fixed, - float[:] t_bar, - float slope, - RampIndex ramp): +Stat = cython.struct(arg_max=cython.int, max_stat=cython.float) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.cfunc +def _fit_statistic( + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + t_bar: cython.float[:], + slope: cython.float, + ramp: RampIndex, +) -> Stat: """ Compute the maximum index and its value over all fit statistics for a given ramp. Each index's stat is the max of the single and double difference statistics: - all_stats = + all_stats = . Parameters ---------- @@ -309,19 +341,18 @@ """ # Cast the enum values into integers for indexing (otherwise compiler complains) # These will be optimized out - cdef int local_slope = PixelOffsets.local_slope - cdef int var_read_noise = PixelOffsets.var_read_noise - - cdef int t_bar_diff_sqr = FixedOffsets.t_bar_diff_sqr - cdef int var_slope_val = FixedOffsets.var_slope_val + local_slope: cython.int = PixelOffsets.local_slope + var_read_noise: cython.int = PixelOffsets.var_read_noise + t_bar_diff_sqr: cython.int = FixedOffsets.t_bar_diff_sqr + var_slope_val: cython.int = FixedOffsets.var_slope_val # Note that a ramp consisting of a single point is degenerate and has no # fit statistic so we bail out here if ramp.start == ramp.end: - return 0, NAN + return Stat(0, NAN) # Start computing fit statistics - cdef float correct = _correction(t_bar, ramp, slope) + correct: cython.float = _correction(t_bar, ramp, slope) # We are computing single and double differences of using the ramp's resultants. # Each of these computations requires two points meaning that there are @@ -330,59 +361,72 @@ # We use this point's single difference as our initial guess for the fit # statistic. Note that the fit statistic can technically be negative so # this makes it much easier to compute a "lazy" max. - cdef int index = ramp.end - 1 - cdef int argmax = ramp.end - ramp.start - 1 - cdef float max_stat = _statstic(single_pixel[local_slope, index], - single_pixel[var_read_noise, index], - single_fixed[t_bar_diff_sqr, index], - single_fixed[var_slope_val, index], - slope, - correct) + index: cython.int = ramp.end - 1 + stat: Stat = Stat( + ramp.end - ramp.start - 1, + _statistic( + single_pixel[local_slope, index], + single_pixel[var_read_noise, index], + single_fixed[t_bar_diff_sqr, index], + single_fixed[var_slope_val, index], + slope, + correct, + ), + ) # Compute the rest of the fit statistics - cdef float stat, stat1, stat2 - cdef int stat_index - for stat_index, index in enumerate(range(ramp.start, ramp.end - 1)): + max_stat: cython.float + single_stat: cython.float + double_stat: cython.float + arg_max: cython.int + for arg_max, index in enumerate(range(ramp.start, ramp.end - 1)): # Compute max of single and double difference statistics - stat1 = _statstic(single_pixel[local_slope, index], - single_pixel[var_read_noise, index], - single_fixed[t_bar_diff_sqr, index], - single_fixed[var_slope_val, index], - slope, - correct) - stat2 = _statstic(double_pixel[local_slope, index], - double_pixel[var_read_noise, index], - double_fixed[t_bar_diff_sqr, index], - double_fixed[var_slope_val, index], - slope, - correct) - stat = fmaxf(stat1, stat2) + single_stat = _statistic( + single_pixel[local_slope, index], + single_pixel[var_read_noise, index], + single_fixed[t_bar_diff_sqr, index], + single_fixed[var_slope_val, index], + slope, + correct, + ) + double_stat = _statistic( + double_pixel[local_slope, index], + double_pixel[var_read_noise, index], + double_fixed[t_bar_diff_sqr, index], + double_fixed[var_slope_val, index], + slope, + correct, + ) + max_stat = fmaxf(single_stat, double_stat) # If this is larger than the current max, update the max - if stat > max_stat: - max_stat = stat - argmax = stat_index - - return argmax, max_stat - - -@boundscheck(False) -@wraparound(False) -@cdivision(True) -cdef inline JumpFits fit_jumps(float[:] resultants, - int[:] dq, - float read_noise, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants, - float[:, :] single_pixel, - float[:, :] double_pixel, - float[:, :] single_fixed, - float[:, :] double_fixed, - Thresh thresh, - bool use_jump, - bool include_diagnostic): + if max_stat > stat.max_stat: + stat = Stat(arg_max, max_stat) + + return stat + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def fit_jumps( + resultants: cython.float[:], + dq: cython.int[:], + read_noise: cython.float, + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], + n_resultants: cython.int, + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + thresh: Thresh, + use_jump: cpp_bool, + include_diagnostic: cpp_bool, +) -> JumpFits: """ Compute all the ramps for a single pixel using the Casertano+22 algorithm with jump detection. @@ -425,30 +469,28 @@ RampFits struct of all the fits for a single pixel """ # Find initial set of ramps - cdef RampQueue ramps = init_ramps(dq, n_resultants) + ramps: RampQueue = init_ramps(dq, n_resultants) # Initialize algorithm - cdef JumpFits ramp_fits - cdef RampIndex ramp - cdef RampFit ramp_fit - + ramp_fits: JumpFits = JumpFits() ramp_fits.average.slope = 0 ramp_fits.average.read_var = 0 ramp_fits.average.poisson_var = 0 - cdef int argmax, jump0, jump1 - cdef float max_stat - cdef float weight, total_weight = 0 + # Declare variables for the loop + ramp: RampIndex + ramp_fit: RampFit + stat: Stat + jump0: cython.int + jump1: cython.int + weight: cython.float + total_weight: cython.float = 0 # Fill in the jump detection pre-compute values for a single pixel if use_jump: - _fill_pixel_values(single_pixel, - double_pixel, - single_fixed, - double_fixed, - resultants, - read_noise, - n_resultants) + _fill_pixel_values( + single_pixel, double_pixel, single_fixed, double_fixed, resultants, read_noise, n_resultants + ) # Run while the Queue is non-empty while not ramps.empty(): @@ -457,27 +499,18 @@ ramps.pop_back() # Compute fit using the Casertano+22 algorithm - ramp_fit = fit_ramp(resultants, - t_bar, - tau, - n_reads, - read_noise, - ramp) + ramp_fit = fit_ramp(resultants, t_bar, tau, n_reads, read_noise, ramp) # Run jump detection if enabled if use_jump: - argmax, max_stat = _fit_statistic(single_pixel, - double_pixel, - single_fixed, - double_fixed, - t_bar, - ramp_fit.slope, - ramp) + stat = _fit_statistic( + single_pixel, double_pixel, single_fixed, double_fixed, t_bar, ramp_fit.slope, ramp + ) # Note that when a "ramp" is a single point, _fit_statistic returns # a NaN for max_stat. Note that NaN > anything is always false so the # result drops through as desired. - if max_stat > _threshold(thresh, ramp_fit.slope): + if stat.max_stat > _threshold(thresh, ramp_fit.slope): # Compute jump point to create two new ramps # This jump point corresponds to the index of the largest # statistic: @@ -486,7 +519,7 @@ # ramp's range. Therefore, we need to add the start index # of the ramp to the result. # - jump0 = argmax + ramp.start + jump0 = stat.arg_max + ramp.start # Note that because the resultants are averages of reads, but # jumps occur in individual reads, it is possible that the From d9857afbb6164f98486c3f3a38a2ad2aae344fb0 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 12:33:01 -0500 Subject: [PATCH 05/31] Update setup.py to use .py file instead of .pyx for ols_cas22._fit --- setup.py | 2 +- src/stcal/ramp_fitting/ols_cas22/_fit.pxd | 11 ++ .../ols_cas22/{_fit.pyx => _fit.py} | 154 ++++++++---------- 3 files changed, 83 insertions(+), 84 deletions(-) create mode 100644 src/stcal/ramp_fitting/ols_cas22/_fit.pxd rename src/stcal/ramp_fitting/ols_cas22/{_fit.pyx => _fit.py} (67%) diff --git a/setup.py b/setup.py index 215d40fb..28414ff1 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ ), Extension( "stcal.ramp_fitting.ols_cas22._fit", - ["src/stcal/ramp_fitting/ols_cas22/_fit.pyx"], + ["src/stcal/ramp_fitting/ols_cas22/_fit.py"], include_dirs=[np.get_include()], language="c++", ), diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd new file mode 100644 index 00000000..9e2fd7f6 --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd @@ -0,0 +1,11 @@ +cpdef enum Parameter: + intercept + slope + n_param + + +cpdef enum Variance: + read_var + poisson_var + total_var + n_var diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx b/src/stcal/ramp_fitting/ols_cas22/_fit.py similarity index 67% rename from src/stcal/ramp_fitting/ols_cas22/_fit.pyx rename to src/stcal/ramp_fitting/ols_cas22/_fit.py index 9df3c1bd..6fa37dfc 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pyx +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -28,14 +28,14 @@ """ from __future__ import annotations -import numpy as np - -cimport numpy as cnp -from cython cimport boundscheck, wraparound -from libcpp cimport bool -from libcpp.list cimport list as cpp_list +from typing import TYPE_CHECKING, NamedTuple -from stcal.ramp_fitting.ols_cas22._jump cimport ( +import cython +import numpy as np +from cython.cimports import numpy as cnp +from cython.cimports.libcpp.list import list as cpp_list +from cython.cimports.stcal.ramp_fitting.ols_cas22._fit import Parameter, Variance +from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( JumpFits, Thresh, _fill_fixed_values, @@ -43,27 +43,15 @@ n_fixed_offsets, n_pixel_offsets, ) -from stcal.ramp_fitting.ols_cas22._ramp cimport _fill_metadata +from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import _fill_metadata -from typing import NamedTuple +if TYPE_CHECKING: + from cython.cimports.libcpp import bool as cpp_bool # Initialize numpy for cython use in this module cnp.import_array() -cpdef enum Parameter: - intercept - slope - n_param - - -cpdef enum Variance: - read_var - poisson_var - total_var - n_var - - class RampFitOutputs(NamedTuple): """ Simple tuple wrapper for outputs from the ramp fitting algorithm @@ -86,23 +74,26 @@ class RampFitOutputs(NamedTuple): the raw ramp fit outputs, these are all structs which will get mapped to python dictionaries. """ + parameters: np.ndarray variances: np.ndarray dq: np.ndarray fits: list | None = None -@boundscheck(False) -@wraparound(False) -def fit_ramps(float[:, :] resultants, - cnp.ndarray[int, ndim=2] dq, - float[:] read_noise, - float read_time, - list[list[int]] read_pattern, - bool use_jump=False, - float intercept=5.5, - float constant=1/3, - bool include_diagnostic=False): +@cython.boundscheck(False) +@cython.wraparound(False) +def fit_ramps( + resultants: cython.float[:, :], + dq: cython.int[:, :], + read_noise: cython.float[:], + read_time: cython.float, + read_pattern: list[list[int]], + use_jump: cpp_bool = False, + intercept: cython.float = 5.5, + constant: cython.float = 1 / 3, + include_diagnostic: cpp_bool = False, +) -> RampFitOutputs: """Fit ramps using the Casertano+22 algorithm. This implementation uses the Cas22 algorithm to fit ramps, where ramps are fit between bad resultants marked by dq flags for each pixel @@ -140,26 +131,28 @@ def fit_ramps(float[:, :] resultants, ------- A RampFitOutputs tuple """ - cdef int n_pixels, n_resultants - n_resultants = resultants.shape[0] - n_pixels = resultants.shape[1] + n_resultants: cython.int = resultants.shape[0] + n_pixels: cython.int = resultants.shape[1] # Raise error if input data is inconsistent if n_resultants != len(read_pattern): - raise RuntimeError(f'The read pattern length {len(read_pattern)} does not ' - f'match number of resultants {n_resultants}') + msg = ( + f"The read pattern length {len(read_pattern)} does " + f"not match number of resultants {n_resultants}" + ) + raise RuntimeError(msg) # Compute the main metadata from the read pattern and cast it to memory views - cdef float[:] t_bar = np.empty(n_resultants, dtype=np.float32) - cdef float[:] tau = np.empty(n_resultants, dtype=np.float32) - cdef int[:] n_reads = np.empty(n_resultants, dtype=np.int32) + t_bar: cython.float[:] = np.empty(n_resultants, dtype=np.float32) + tau: cython.float[:] = np.empty(n_resultants, dtype=np.float32) + n_reads: cython.int[:] = np.empty(n_resultants, dtype=np.int32) _fill_metadata(read_pattern, read_time, t_bar, tau, n_reads) # Setup pre-compute arrays for jump detection - cdef float[:, :] single_pixel - cdef float[:, :] double_pixel - cdef float[:, :] single_fixed - cdef float[:, :] double_fixed + single_pixel: cython.float[:, :] + double_pixel: cython.float[:, :] + single_fixed: cython.float[:, :] + double_fixed: cython.float[:, :] if use_jump: # Initialize arrays for the jump detection pre-computed values single_pixel = np.empty((n_pixel_offsets, n_resultants - 1), dtype=np.float32) @@ -169,12 +162,7 @@ def fit_ramps(float[:, :] resultants, double_fixed = np.empty((n_fixed_offsets, n_resultants - 2), dtype=np.float32) # Pre-compute the values from the read pattern - _fill_fixed_values(single_fixed, - double_fixed, - t_bar, - tau, - n_reads, - n_resultants) + _fill_fixed_values(single_fixed, double_fixed, t_bar, tau, n_reads, n_resultants) else: # "Initialize" the arrays when not using jump detection, they need to be # initialized because they do get passed around, but they don't need @@ -186,51 +174,49 @@ def fit_ramps(float[:, :] resultants, double_fixed = np.empty((0, 0), dtype=np.float32) # Create a threshold struct - cdef Thresh thresh = Thresh(intercept, constant) + thresh: Thresh = Thresh(intercept, constant) # Create variable to old the diagnostic data # Use list because this might grow very large which would require constant # reallocation. We don't need random access, and this gets cast to a python # list in the end. - cdef cpp_list[JumpFits] ramp_fits + ramp_fits: cpp_list[JumpFits] = cpp_list[JumpFits]() # Initialize the output arrays. Note that the fit intercept is currently always # zero, where as every variance is calculated and set. This means that the # parameters need to be filled with zeros, where as the variances can just # be allocated - cdef float[:, :] parameters = np.zeros((n_pixels, Parameter.n_param), dtype=np.float32) - cdef float[:, :] variances = np.empty((n_pixels, Variance.n_var), dtype=np.float32) + parameters: cython.float[:, :] = np.zeros((n_pixels, Parameter.n_param), dtype=np.float32) + variances: cython.float[:, :] = np.empty((n_pixels, Variance.n_var), dtype=np.float32) # Cast the enum values into integers for indexing (otherwise compiler complains) # These will be optimized out - cdef int slope = Parameter.slope - cdef int read_var = Variance.read_var - cdef int poisson_var = Variance.poisson_var - cdef int total_var = Variance.total_var - - # Pull memory view of dq for speed of access later - # changes to this array will backpropagate to the original numpy array - cdef int[:, :] dq_ = dq + slope: cython.int = Parameter.slope + read_var: cython.int = Variance.read_var + poisson_var: cython.int = Variance.poisson_var + total_var: cython.int = Variance.total_var # Run the jump fitting algorithm for each pixel - cdef JumpFits fit - cdef int index + fit: JumpFits + index: cython.int for index in range(n_pixels): # Fit all the ramps for the given pixel - fit = fit_jumps(resultants[:, index], - dq_[:, index], - read_noise[index], - t_bar, - tau, - n_reads, - n_resultants, - single_pixel, - double_pixel, - single_fixed, - double_fixed, - thresh, - use_jump, - include_diagnostic) + fit = fit_jumps( + resultants[:, index], + dq[:, index], + read_noise[index], + t_bar, + tau, + n_reads, + n_resultants, + single_pixel, + double_pixel, + single_fixed, + double_fixed, + thresh, + use_jump, + include_diagnostic, + ) # Extract the output fit's parameters parameters[index, slope] = fit.average.slope @@ -245,7 +231,9 @@ def fit_ramps(float[:, :] resultants, ramp_fits.push_back(fit) # Cast memory views into numpy arrays for ease of use in python. - return RampFitOutputs(np.array(parameters, dtype=np.float32), - np.array(variances, dtype=np.float32), - dq, - ramp_fits if include_diagnostic else None) + return RampFitOutputs( + np.array(parameters, dtype=np.float32), + np.array(variances, dtype=np.float32), + np.array(dq, dtype=np.uint32), + ramp_fits if include_diagnostic else None, + ) From b582f22e72bc448ebfcd8f667cba151edf06a178 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 12:40:18 -0500 Subject: [PATCH 06/31] Switch to all vectors --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 5 ++-- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 9 +++--- src/stcal/ramp_fitting/ols_cas22/_ramp.py | 34 +++++++++++++--------- tests/test_jump_cas22.py | 6 ++-- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 6fa37dfc..7c0fddb7 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -47,6 +47,7 @@ if TYPE_CHECKING: from cython.cimports.libcpp import bool as cpp_bool + from cython.cimports.libcpp.vector import vector # Initialize numpy for cython use in this module cnp.import_array() @@ -88,7 +89,7 @@ def fit_ramps( dq: cython.int[:, :], read_noise: cython.float[:], read_time: cython.float, - read_pattern: list[list[int]], + read_pattern: vector[vector[cython.int]], use_jump: cpp_bool = False, intercept: cython.float = 5.5, constant: cython.float = 1 / 3, @@ -146,7 +147,7 @@ def fit_ramps( t_bar: cython.float[:] = np.empty(n_resultants, dtype=np.float32) tau: cython.float[:] = np.empty(n_resultants, dtype=np.float32) n_reads: cython.int[:] = np.empty(n_resultants, dtype=np.int32) - _fill_metadata(read_pattern, read_time, t_bar, tau, n_reads) + _fill_metadata(t_bar, tau, n_reads, read_pattern, read_time, n_resultants) # Setup pre-compute arrays for jump detection single_pixel: cython.float[:, :] diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 5741abc6..1ce49556 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -3,11 +3,12 @@ from libcpp.vector cimport vector -cpdef void _fill_metadata(list[list[int]] read_pattern, - float read_time, - float[:] t_bar, +cpdef void _fill_metadata(float[:] t_bar, float[:] tau, - int[:] n_reads) + int[:] n_reads, + vector[vector[int]] read_pattern, + float read_time, + int n_resultants) cdef struct RampIndex: diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.py b/src/stcal/ramp_fitting/ols_cas22/_ramp.py index c3ad8f97..8a9b2a7c 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.py +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.py @@ -50,15 +50,10 @@ listed for a single pixel """ import cython -import numpy as np -from cython.cimports import numpy as cnp from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, sqrt from cython.cimports.libcpp.vector import vector from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import RampFit, RampIndex, RampQueue -# Initialize numpy for cython use in this module -cnp.import_array() - @cython.boundscheck(False) @cython.wraparound(False) @@ -66,21 +61,32 @@ @cython.inline @cython.ccall def _fill_metadata( - read_pattern: list[list[cython.int]], - read_time: cython.float, t_bar: cython.float[:], tau: cython.float[:], n_reads: cython.int[:], + read_pattern: vector[vector[cython.int]], + read_time: cython.float, + n_resultants: cython.int, ) -> cython.void: - index: cython.int n_read: cython.int - resultant: list[cython.int] - for index, resultant in enumerate(read_pattern): - n_read = len(resultant) - n_reads[index] = n_read - t_bar[index] = read_time * np.mean(resultant) - tau[index] = np.sum((2 * (n_read - np.arange(n_read)) - 1) * resultant) * read_time / n_read**2 + i: cython.int + j: cython.int + resultant: vector[cython.int] + for i in range(n_resultants): + resultant = read_pattern[i] + n_read = resultant.size() + + n_reads[i] = n_read + t_bar[i] = 0 + tau[i] = 0 + + for j in range(n_read): + t_bar[i] += read_time * resultant[j] + tau[i] += (2 * (n_read - j) - 1) * resultant[j] + + t_bar[i] /= n_read + tau[i] *= read_time / n_read**2 @cython.boundscheck(False) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 9b2f6067..8e4ae297 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -118,7 +118,7 @@ def test__fill_metadata(read_pattern): t_bar = np.empty(n_resultants, dtype=np.float32) tau = np.empty(n_resultants, dtype=np.float32) n_reads = np.empty(n_resultants, dtype=np.int32) - _fill_metadata(read_pattern, READ_TIME, t_bar, tau, n_reads) + _fill_metadata(t_bar, tau, n_reads, read_pattern, READ_TIME, n_resultants) assert t_bar.shape == (n_resultants,) assert tau.shape == (n_resultants,) @@ -126,7 +126,7 @@ def test__fill_metadata(read_pattern): # Check that the data is correct assert_allclose(t_bar, [7.6, 15.2, 21.279999, 41.040001, 60.799999, 88.159996]) - assert_allclose(tau, [5.7, 15.2, 19.928888, 36.023998, 59.448887, 80.593781]) + assert_allclose(tau, [5.7, 15.2, 19.928888, 36.024002, 59.448887, 80.59378]) assert np.all(n_reads == [4, 1, 3, 10, 3, 15]) # Check datatypes @@ -151,7 +151,7 @@ def ramp_data(read_pattern): t_bar = np.empty(n_resultants, dtype=np.float32) tau = np.empty(n_resultants, dtype=np.float32) n_reads = np.empty(n_resultants, dtype=np.int32) - _fill_metadata(read_pattern, READ_TIME, t_bar, tau, n_reads) + _fill_metadata(t_bar, tau, n_reads, read_pattern, READ_TIME, n_resultants) return t_bar, tau, n_reads, read_pattern From 32a8ee8b128e4ef8a4b09060784f4e6dff2b9061 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 12:52:13 -0500 Subject: [PATCH 07/31] Resolve cython compiler warnings --- src/stcal/ramp_fitting/ols_cas22/_fit.pxd | 3 +++ src/stcal/ramp_fitting/ols_cas22/_jump.py | 33 ++++++++++++----------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd index 9e2fd7f6..cceae5fc 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd @@ -1,3 +1,6 @@ +# cython: language_level=3str + + cpdef enum Parameter: intercept slope diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.py b/src/stcal/ramp_fitting/ols_cas22/_jump.py index 1b060159..faacd180 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.py +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.py @@ -56,6 +56,7 @@ import cython from cython.cimports.libc.math import NAN, fmaxf, isnan, log10, sqrt from cython.cimports.libcpp import bool as cpp_bool +from cython.cimports.libcpp.vector import vector from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( JUMP_DET, FixedOffsets, @@ -472,10 +473,10 @@ def fit_jumps( ramps: RampQueue = init_ramps(dq, n_resultants) # Initialize algorithm - ramp_fits: JumpFits = JumpFits() - ramp_fits.average.slope = 0 - ramp_fits.average.read_var = 0 - ramp_fits.average.poisson_var = 0 + average: RampFit = RampFit(0, 0, 0) + jumps: vector[cython.int] = vector[cython.int]() + fits: vector[RampFit] = vector[RampFit]() + index: RampQueue = RampQueue() # Declare variables for the loop ramp: RampIndex @@ -542,8 +543,8 @@ def fit_jumps( # Record jump diagnostics if include_diagnostic: - ramp_fits.jumps.push_back(jump0) - ramp_fits.jumps.push_back(jump1) + jumps.push_back(jump0) + jumps.push_back(jump1) # The two resultant indices need to be skipped, therefore # the two @@ -591,8 +592,8 @@ def fit_jumps( # Record the diagnositcs if include_diagnostic: - ramp_fits.fits.push_back(ramp_fit) - ramp_fits.index.push_back(ramp) + fits.push_back(ramp_fit) + index.push_back(ramp) # Start computing the averages using a lazy process # Note we do not do anything in the NaN case for degenerate ramps @@ -602,16 +603,16 @@ def fit_jumps( weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var total_weight += weight - ramp_fits.average.slope += weight * ramp_fit.slope - ramp_fits.average.read_var += weight**2 * ramp_fit.read_var - ramp_fits.average.poisson_var += weight**2 * ramp_fit.poisson_var + average.slope += weight * ramp_fit.slope + average.read_var += weight**2 * ramp_fit.read_var + average.poisson_var += weight**2 * ramp_fit.poisson_var # Finish computing averages using the lazy process - ramp_fits.average.slope /= total_weight if total_weight != 0 else 1 - ramp_fits.average.read_var /= total_weight**2 if total_weight != 0 else 1 - ramp_fits.average.poisson_var /= total_weight**2 if total_weight != 0 else 1 + average.slope /= total_weight if total_weight != 0 else 1 + average.read_var /= total_weight**2 if total_weight != 0 else 1 + average.poisson_var /= total_weight**2 if total_weight != 0 else 1 # Multiply poisson term by flux, (no negative fluxes) - ramp_fits.average.poisson_var *= max(ramp_fits.average.slope, 0) + average.poisson_var *= max(average.slope, 0) - return ramp_fits + return JumpFits(average, jumps, fits, index) From 9d6a520944fc11ddee3441dd2059c2a1f1cb4aed Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 13:21:35 -0500 Subject: [PATCH 08/31] Simplify the ramp initializer --- src/stcal/ramp_fitting/ols_cas22/_ramp.py | 32 ++++++++++------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.py b/src/stcal/ramp_fitting/ols_cas22/_ramp.py index 8a9b2a7c..5c0a3463 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.py +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.py @@ -121,27 +121,23 @@ def init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: ramp: RampIndex = RampIndex(-1, -1) index_resultant: cython.int for index_resultant in range(n_resultants): + # Checking for start of ramp if ramp.start == -1: - # Looking for the start of a ramp if dq[index_resultant] == 0: - # We have found the start of a ramp! + # This resultant is in the ramp + # => We have found the start of a ramp! ramp.start = index_resultant - else: - # This is not the start of the ramp yet - continue - else: # noqa: PLR5501 (makes more logical sense than to use elif) - # Looking for the end of a ramp - if dq[index_resultant] == 0: - # This pixel is in the ramp do nothing - continue - else: # noqa: RET507 (makes more logical sense than to remove else) - # This pixel is not in the ramp - # => index_resultant - 1 is the end of the ramp - ramp.end = index_resultant - 1 - - # Add completed ramp to stack and reset ramp - ramps.push_back(ramp) - ramp = RampIndex(-1, -1) + + # This resultant cannot be the start of a ramp + # => Checking for end of ramp + elif dq[index_resultant] != 0: + # This pixel is not in the ramp + # => index_resultant - 1 is the end of the ramp + ramp.end = index_resultant - 1 + + # Add completed ramp to the queue and reset ramp + ramps.push_back(ramp) + ramp = RampIndex(-1, -1) # Handle case where last resultant is in ramp (so no end has been set) if ramp.start != -1 and ramp.end == -1: From 9aa3d292cdefddf0421fdad82a741834103f5a93 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 13:37:58 -0500 Subject: [PATCH 09/31] Fix annoying inline declarations --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 22 +++--- src/stcal/ramp_fitting/ols_cas22/_jump.py | 95 ++++++++++------------- 2 files changed, 53 insertions(+), 64 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 7c0fddb7..ca1ff6c6 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -82,6 +82,13 @@ class RampFitOutputs(NamedTuple): fits: list | None = None +_slope = cython.declare(cython.int, Parameter.slope) + +_read_var = cython.declare(cython.int, Variance.read_var) +_poisson_var = cython.declare(cython.int, Variance.poisson_var) +_total_var = cython.declare(cython.int, Variance.total_var) + + @cython.boundscheck(False) @cython.wraparound(False) def fit_ramps( @@ -190,13 +197,6 @@ def fit_ramps( parameters: cython.float[:, :] = np.zeros((n_pixels, Parameter.n_param), dtype=np.float32) variances: cython.float[:, :] = np.empty((n_pixels, Variance.n_var), dtype=np.float32) - # Cast the enum values into integers for indexing (otherwise compiler complains) - # These will be optimized out - slope: cython.int = Parameter.slope - read_var: cython.int = Variance.read_var - poisson_var: cython.int = Variance.poisson_var - total_var: cython.int = Variance.total_var - # Run the jump fitting algorithm for each pixel fit: JumpFits index: cython.int @@ -220,12 +220,12 @@ def fit_ramps( ) # Extract the output fit's parameters - parameters[index, slope] = fit.average.slope + parameters[index, _slope] = fit.average.slope # Extract the output fit's variances - variances[index, read_var] = fit.average.read_var - variances[index, poisson_var] = fit.average.poisson_var - variances[index, total_var] = fit.average.read_var + fit.average.poisson_var + variances[index, _read_var] = fit.average.read_var + variances[index, _poisson_var] = fit.average.poisson_var + variances[index, _total_var] = fit.average.read_var + fit.average.poisson_var # Store diagnostic data if requested if include_diagnostic: diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.py b/src/stcal/ramp_fitting/ols_cas22/_jump.py index faacd180..ee48de38 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.py +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.py @@ -72,6 +72,11 @@ init_ramps, ) +_t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) +_t_bar_diff_sqr = cython.declare(cython.int, FixedOffsets.t_bar_diff_sqr) +_read_recip = cython.declare(cython.int, FixedOffsets.read_recip) +_var_slope_val = cython.declare(cython.int, FixedOffsets.var_slope_val) + @cython.boundscheck(False) @cython.wraparound(False) @@ -117,28 +122,25 @@ def _fill_fixed_values( <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, ] """ - # Cast the enum values into integers for indexing (otherwise compiler complains) - # These will be optimized out - t_bar_diff: cython.int = FixedOffsets.t_bar_diff - t_bar_diff_sqr: cython.int = FixedOffsets.t_bar_diff_sqr - read_recip: cython.int = FixedOffsets.read_recip - var_slope_val: cython.int = FixedOffsets.var_slope_val - # Coerce division to be using floats num: cython.float = 1 i: cython.int for i in range(n_resultants - 1): - single_fixed[t_bar_diff, i] = t_bar[i + 1] - t_bar[i] - single_fixed[t_bar_diff_sqr, i] = single_fixed[t_bar_diff, i] ** 2 - single_fixed[read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) - single_fixed[var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) + single_fixed[_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] + single_fixed[_t_bar_diff_sqr, i] = single_fixed[_t_bar_diff, i] ** 2 + single_fixed[_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) + single_fixed[_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) if i < n_resultants - 2: - double_fixed[t_bar_diff, i] = t_bar[i + 2] - t_bar[i] - double_fixed[t_bar_diff_sqr, i] = double_fixed[t_bar_diff, i] ** 2 - double_fixed[read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) - double_fixed[var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) + double_fixed[_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] + double_fixed[_t_bar_diff_sqr, i] = double_fixed[_t_bar_diff, i] ** 2 + double_fixed[_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) + double_fixed[_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) + + +_local_slope = cython.declare(cython.int, PixelOffsets.local_slope) +_var_read_noise = cython.declare(cython.int, PixelOffsets.var_read_noise) @cython.boundscheck(False) @@ -182,22 +184,16 @@ def _fill_pixel_values( read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, ] """ - t_bar_diff: cython.int = FixedOffsets.t_bar_diff - read_recip: cython.int = FixedOffsets.read_recip - - local_slope: cython.int = PixelOffsets.local_slope - var_read_noise: cython.int = PixelOffsets.var_read_noise - read_noise_sqr: cython.float = read_noise**2 i: cython.int for i in range(n_resultants - 1): - single_pixel[local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[t_bar_diff, i] - single_pixel[var_read_noise, i] = read_noise_sqr * single_fixed[read_recip, i] + single_pixel[_local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[_t_bar_diff, i] + single_pixel[_var_read_noise, i] = read_noise_sqr * single_fixed[_read_recip, i] if i < n_resultants - 2: - double_pixel[local_slope, i] = (resultants[i + 2] - resultants[i]) / double_fixed[t_bar_diff, i] - double_pixel[var_read_noise, i] = read_noise_sqr * double_fixed[read_recip, i] + double_pixel[_local_slope, i] = (resultants[i + 2] - resultants[i]) / double_fixed[_t_bar_diff, i] + double_pixel[_var_read_noise, i] = read_noise_sqr * double_fixed[_read_recip, i] @cython.inline @@ -258,7 +254,7 @@ def _statistic( local_slope: cython.float, var_read_noise: cython.float, t_bar_diff_sqr: cython.float, - var_slope_coeff: cython.float, + var_slope_val: cython.float, slope: cython.float, correct: cython.float, ) -> cython.float: @@ -267,8 +263,8 @@ def _statistic( delta / sqrt(var + correct). where: - delta = local_slope - slope - var = (var_read_noise + slope * var_slope_coeff) / t_bar_diff_sqr + delta = _local_slope - slope + var = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr pre-computed: local_slope = (resultant[i + j] - resultant[i]) / (t_bar[i + j] - t_bar[i]) @@ -280,12 +276,12 @@ def _statistic( ---------- local_slope : float The local slope the statistic is computed for - float : var_read_noise - The read noise variance for local_slope + var_read_noise: float + The read noise variance for _local_slope t_bar_diff_sqr : float - The square difference for the t_bar corresponding to local_slope - var_slope_coeff : float - The slope variance coefficient for local_slope + The square difference for the t_bar corresponding to _local_slope + var_slope_val : float + The slope variance coefficient for _local_slope slope : float The computed slope for the ramp correct : float @@ -296,7 +292,7 @@ def _statistic( Create a single instance of the stastic for the given parameters """ delta: cython.float = local_slope - slope - var: cython.float = (var_read_noise + slope * var_slope_coeff) / t_bar_diff_sqr + var: cython.float = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr return delta / sqrt(var + correct) @@ -340,13 +336,6 @@ def _fit_statistic( ------- argmax(all_stats), max(all_stats) """ - # Cast the enum values into integers for indexing (otherwise compiler complains) - # These will be optimized out - local_slope: cython.int = PixelOffsets.local_slope - var_read_noise: cython.int = PixelOffsets.var_read_noise - - t_bar_diff_sqr: cython.int = FixedOffsets.t_bar_diff_sqr - var_slope_val: cython.int = FixedOffsets.var_slope_val # Note that a ramp consisting of a single point is degenerate and has no # fit statistic so we bail out here if ramp.start == ramp.end: @@ -366,10 +355,10 @@ def _fit_statistic( stat: Stat = Stat( ramp.end - ramp.start - 1, _statistic( - single_pixel[local_slope, index], - single_pixel[var_read_noise, index], - single_fixed[t_bar_diff_sqr, index], - single_fixed[var_slope_val, index], + single_pixel[_local_slope, index], + single_pixel[_var_read_noise, index], + single_fixed[_t_bar_diff_sqr, index], + single_fixed[_var_slope_val, index], slope, correct, ), @@ -383,18 +372,18 @@ def _fit_statistic( for arg_max, index in enumerate(range(ramp.start, ramp.end - 1)): # Compute max of single and double difference statistics single_stat = _statistic( - single_pixel[local_slope, index], - single_pixel[var_read_noise, index], - single_fixed[t_bar_diff_sqr, index], - single_fixed[var_slope_val, index], + single_pixel[_local_slope, index], + single_pixel[_var_read_noise, index], + single_fixed[_t_bar_diff_sqr, index], + single_fixed[_var_slope_val, index], slope, correct, ) double_stat = _statistic( - double_pixel[local_slope, index], - double_pixel[var_read_noise, index], - double_fixed[t_bar_diff_sqr, index], - double_fixed[var_slope_val, index], + double_pixel[_local_slope, index], + double_pixel[_var_read_noise, index], + double_fixed[_t_bar_diff_sqr, index], + double_fixed[_var_slope_val, index], slope, correct, ) From 515afa50a6e985dac881614aeece2b2f716fb65f Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 15:01:12 -0500 Subject: [PATCH 10/31] Refactor enum definitions and extract output fit's parameters and variances in fit_ramps() --- src/stcal/ramp_fitting/ols_cas22/__init__.py | 4 +- src/stcal/ramp_fitting/ols_cas22/_fit.pxd | 13 ----- src/stcal/ramp_fitting/ols_cas22/_fit.py | 13 ++--- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 18 ++++++- src/stcal/ramp_fitting/ols_cas22/_jump.py | 31 +++++++---- tests/test_jump_cas22.py | 55 ++++++++------------ 6 files changed, 66 insertions(+), 68 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index 3d30b0ad..ab615755 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -1,4 +1,4 @@ -from ._fit import Parameter, RampFitOutputs, Variance, fit_ramps -from ._jump import JUMP_DET +from ._fit import RampFitOutputs, fit_ramps +from ._jump import JUMP_DET, Parameter, Variance __all__ = ["fit_ramps", "RampFitOutputs", "Parameter", "Variance", "Diff", "JUMP_DET"] diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd index cceae5fc..75f723ab 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd @@ -1,14 +1 @@ # cython: language_level=3str - - -cpdef enum Parameter: - intercept - slope - n_param - - -cpdef enum Variance: - read_var - poisson_var - total_var - n_var diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index ca1ff6c6..a3ed5a05 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -34,10 +34,11 @@ import numpy as np from cython.cimports import numpy as cnp from cython.cimports.libcpp.list import list as cpp_list -from cython.cimports.stcal.ramp_fitting.ols_cas22._fit import Parameter, Variance from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( JumpFits, + Parameter, Thresh, + Variance, _fill_fixed_values, fit_jumps, n_fixed_offsets, @@ -203,6 +204,8 @@ def fit_ramps( for index in range(n_pixels): # Fit all the ramps for the given pixel fit = fit_jumps( + parameters[index, :], + variances[index, :], resultants[:, index], dq[:, index], read_noise[index], @@ -219,14 +222,6 @@ def fit_ramps( include_diagnostic, ) - # Extract the output fit's parameters - parameters[index, _slope] = fit.average.slope - - # Extract the output fit's variances - variances[index, _read_var] = fit.average.read_var - variances[index, _poisson_var] = fit.average.poisson_var - variances[index, _total_var] = fit.average.read_var + fit.average.poisson_var - # Store diagnostic data if requested if include_diagnostic: ramp_fits.push_back(fit) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index 776d67a7..c5f3e0f4 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -20,6 +20,19 @@ cpdef enum PixelOffsets: n_pixel_offsets +cpdef enum Parameter: + intercept + slope + n_param + + +cpdef enum Variance: + read_var + poisson_var + total_var + n_var + + cpdef enum: JUMP_DET = 4 @@ -29,7 +42,6 @@ cdef struct Thresh: cdef struct JumpFits: - RampFit average vector[int] jumps vector[RampFit] fits RampQueue index @@ -43,7 +55,9 @@ cpdef void _fill_fixed_values(float[:, :] single_fixed, int n_resultants) -cdef JumpFits fit_jumps(float[:] resultants, +cdef JumpFits fit_jumps(float[:] parameters, + float[:] variances, + float[:] resultants, int[:] dq, float read_noise, float[:] t_bar, diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.py b/src/stcal/ramp_fitting/ols_cas22/_jump.py index ee48de38..40d3f177 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.py +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.py @@ -61,8 +61,10 @@ JUMP_DET, FixedOffsets, JumpFits, + Parameter, PixelOffsets, Thresh, + Variance, ) from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import ( RampFit, @@ -396,12 +398,20 @@ def _fit_statistic( return stat +_slope = cython.declare(cython.int, Parameter.slope) +_read_var = cython.declare(cython.int, Variance.read_var) +_poisson_var = cython.declare(cython.int, Variance.poisson_var) +_total_var = cython.declare(cython.int, Variance.total_var) + + @cython.boundscheck(False) @cython.wraparound(False) @cython.cdivision(True) @cython.inline @cython.cfunc def fit_jumps( + parameters: cython.float[:], + variances: cython.float[:], resultants: cython.float[:], dq: cython.int[:], read_noise: cython.float, @@ -462,7 +472,9 @@ def fit_jumps( ramps: RampQueue = init_ramps(dq, n_resultants) # Initialize algorithm - average: RampFit = RampFit(0, 0, 0) + parameters[:] = 0 + variances[:] = 0 + jumps: vector[cython.int] = vector[cython.int]() fits: vector[RampFit] = vector[RampFit]() index: RampQueue = RampQueue() @@ -592,16 +604,17 @@ def fit_jumps( weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var total_weight += weight - average.slope += weight * ramp_fit.slope - average.read_var += weight**2 * ramp_fit.read_var - average.poisson_var += weight**2 * ramp_fit.poisson_var + parameters[_slope] += weight * ramp_fit.slope + variances[_read_var] += weight**2 * ramp_fit.read_var + variances[_poisson_var] += weight**2 * ramp_fit.poisson_var # Finish computing averages using the lazy process - average.slope /= total_weight if total_weight != 0 else 1 - average.read_var /= total_weight**2 if total_weight != 0 else 1 - average.poisson_var /= total_weight**2 if total_weight != 0 else 1 + parameters[_slope] /= total_weight if total_weight != 0 else 1 + variances[_read_var] /= total_weight**2 if total_weight != 0 else 1 + variances[_poisson_var] /= total_weight**2 if total_weight != 0 else 1 # Multiply poisson term by flux, (no negative fluxes) - average.poisson_var *= max(average.slope, 0) + variances[_poisson_var] *= max(parameters[_slope], 0) + variances[_total_var] = variances[_read_var] + variances[_poisson_var] - return JumpFits(average, jumps, fits, index) + return JumpFits(jumps, fits, index) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 8e4ae297..c02d3f51 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,10 +2,12 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22 import JUMP_DET, Parameter, Variance, fit_ramps +from stcal.ramp_fitting.ols_cas22 import JUMP_DET, fit_ramps from stcal.ramp_fitting.ols_cas22._jump import ( FixedOffsets, + Parameter, PixelOffsets, + Variance, _fill_fixed_values, _fill_pixel_values, ) @@ -329,8 +331,12 @@ def test_fit_ramps(detector_data, use_jump, use_dq): ) assert len(output.fits) == N_PIXELS # sanity check that a fit is output for each pixel + slopes = output.parameters[:, Parameter.slope] + read_vars = output.variances[:, Variance.read_var] + poisson_vars = output.variances[:, Variance.poisson_var] + chi2 = 0 - for fit, use in zip(output.fits, okay): + for fit, slope, read_var, poisson_var, use in zip(output.fits, slopes, read_vars, poisson_vars, okay): if not use_dq and not use_jump: ##### The not use_jump makes this NOT test for false positives ##### # Check that the data generated does not generate any false positives @@ -343,14 +349,14 @@ def test_fit_ramps(detector_data, use_jump, use_dq): if use: # Add okay ramps to chi2 - total_var = fit["average"]["read_var"] + fit["average"]["poisson_var"] + total_var = read_var + poisson_var if total_var != 0: - chi2 += (fit["average"]["slope"] - FLUX) ** 2 / total_var + chi2 += (slope - FLUX) ** 2 / total_var else: # Check no slope fit for bad ramps - assert fit["average"]["slope"] == 0 - assert fit["average"]["read_var"] == 0 - assert fit["average"]["poisson_var"] == 0 + assert slope == 0 + assert read_var == 0 + assert poisson_var == 0 assert use_dq # sanity check that this branch is only encountered when use_dq = True @@ -358,29 +364,6 @@ def test_fit_ramps(detector_data, use_jump, use_dq): assert np.abs(chi2 - 1) < CHI2_TOL -@pytest.mark.parametrize("use_jump", [True, False]) -def test_fit_ramps_array_outputs(detector_data, use_jump): - """ - Test that the array outputs line up with the dictionary output - """ - resultants, read_noise, read_pattern = detector_data - dq = np.zeros(resultants.shape, dtype=np.int32) - - output = fit_ramps( - resultants, dq, read_noise, READ_TIME, read_pattern, use_jump=use_jump, include_diagnostic=True - ) - - for fit, par, var in zip(output.fits, output.parameters, output.variances): - assert par[Parameter.intercept] == 0 - assert par[Parameter.slope] == fit["average"]["slope"] - - assert var[Variance.read_var] == fit["average"]["read_var"] - assert var[Variance.poisson_var] == fit["average"]["poisson_var"] - assert var[Variance.total_var] == np.float32( - fit["average"]["read_var"] + fit["average"]["poisson_var"] - ) - - @pytest.fixture(scope="module") def jump_data(detector_data): """ @@ -445,12 +428,18 @@ def test_find_jumps(jump_data): ) assert len(output.fits) == len(jump_reads) # sanity check that a fit/jump is set for every pixel + slopes = output.parameters[:, Parameter.slope] + read_vars = output.variances[:, Variance.read_var] + poisson_vars = output.variances[:, Variance.poisson_var] + chi2 = 0 incorrect_too_few = 0 incorrect_too_many = 0 incorrect_does_not_capture = 0 incorrect_other = 0 - for fit, jump_index, resultant_index in zip(output.fits, jump_reads, jump_resultants): + for fit, slope, read_var, poisson_var, jump_index, resultant_index in zip( + output.fits, slopes, read_vars, poisson_vars, jump_reads, jump_resultants + ): # Check that the jumps are detected correctly if jump_index == 0: # There is no way to detect a jump if it is in the very first read @@ -503,8 +492,8 @@ def test_find_jumps(jump_data): # assert set(ramp_indices).union(fit['jumps']) == set(range(len(read_pattern))) # Compute the chi2 for the fit and add it to a running "total chi2" - total_var = fit["average"]["read_var"] + fit["average"]["poisson_var"] - chi2 += (fit["average"]["slope"] - FLUX) ** 2 / total_var + total_var = read_var + poisson_var + chi2 += (slope - FLUX) ** 2 / total_var # Check that the average chi2 is ~1. chi2 /= N_PIXELS - incorrect_too_few - incorrect_too_many - incorrect_does_not_capture - incorrect_other From 27437bc6ff5390db6a7ef7c2fb110b0e7c977360 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 15:25:08 -0500 Subject: [PATCH 11/31] Move parameter/variance allocation outside of cython --- src/stcal/ramp_fitting/ols_cas22/__init__.py | 4 +- src/stcal/ramp_fitting/ols_cas22/_fit.py | 65 ++----------- src/stcal/ramp_fitting/ols_cas22_fit.py | 46 ++++++++- tests/test_jump_cas22.py | 99 ++++++++++++++++---- 4 files changed, 130 insertions(+), 84 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index ab615755..843b7625 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -1,4 +1,4 @@ -from ._fit import RampFitOutputs, fit_ramps +from ._fit import fit_ramps from ._jump import JUMP_DET, Parameter, Variance -__all__ = ["fit_ramps", "RampFitOutputs", "Parameter", "Variance", "Diff", "JUMP_DET"] +__all__ = ["fit_ramps", "Parameter", "Variance", "Diff", "JUMP_DET"] diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index a3ed5a05..4ddd8474 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -26,19 +26,16 @@ for jumps (if use_jump is True) and bad pixels (via the dq array). This is the primary externally callable function. """ -from __future__ import annotations - -from typing import TYPE_CHECKING, NamedTuple import cython import numpy as np from cython.cimports import numpy as cnp +from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.list import list as cpp_list +from cython.cimports.libcpp.vector import vector from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( JumpFits, - Parameter, Thresh, - Variance, _fill_fixed_values, fit_jumps, n_fixed_offsets, @@ -46,50 +43,10 @@ ) from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import _fill_metadata -if TYPE_CHECKING: - from cython.cimports.libcpp import bool as cpp_bool - from cython.cimports.libcpp.vector import vector - # Initialize numpy for cython use in this module cnp.import_array() -class RampFitOutputs(NamedTuple): - """ - Simple tuple wrapper for outputs from the ramp fitting algorithm - This clarifies the meaning of the outputs via naming them something - descriptive. - - Attributes - ---------- - parameters: np.ndarray[n_pixel, 2] - the slope and intercept for each pixel's ramp fit. see Parameter enum - for indexing indicating slope/intercept in the second dimension. - variances: np.ndarray[n_pixel, 3] - the read, poisson, and total variances for each pixel's ramp fit. - see Variance enum for indexing indicating read/poisson/total in the - second dimension. - dq: np.ndarray[n_resultants, n_pixel] - the dq array, with additional flags set for jumps detected by the - jump detection algorithm. - fits: list of RampFits - the raw ramp fit outputs, these are all structs which will get mapped to - python dictionaries. - """ - - parameters: np.ndarray - variances: np.ndarray - dq: np.ndarray - fits: list | None = None - - -_slope = cython.declare(cython.int, Parameter.slope) - -_read_var = cython.declare(cython.int, Variance.read_var) -_poisson_var = cython.declare(cython.int, Variance.poisson_var) -_total_var = cython.declare(cython.int, Variance.total_var) - - @cython.boundscheck(False) @cython.wraparound(False) def fit_ramps( @@ -98,11 +55,13 @@ def fit_ramps( read_noise: cython.float[:], read_time: cython.float, read_pattern: vector[vector[cython.int]], + parameters: cython.float[:, :], + variances: cython.float[:, :], use_jump: cpp_bool = False, intercept: cython.float = 5.5, constant: cython.float = 1 / 3, include_diagnostic: cpp_bool = False, -) -> RampFitOutputs: +) -> cpp_list[JumpFits]: """Fit ramps using the Casertano+22 algorithm. This implementation uses the Cas22 algorithm to fit ramps, where ramps are fit between bad resultants marked by dq flags for each pixel @@ -191,13 +150,6 @@ def fit_ramps( # list in the end. ramp_fits: cpp_list[JumpFits] = cpp_list[JumpFits]() - # Initialize the output arrays. Note that the fit intercept is currently always - # zero, where as every variance is calculated and set. This means that the - # parameters need to be filled with zeros, where as the variances can just - # be allocated - parameters: cython.float[:, :] = np.zeros((n_pixels, Parameter.n_param), dtype=np.float32) - variances: cython.float[:, :] = np.empty((n_pixels, Variance.n_var), dtype=np.float32) - # Run the jump fitting algorithm for each pixel fit: JumpFits index: cython.int @@ -227,9 +179,4 @@ def fit_ramps( ramp_fits.push_back(fit) # Cast memory views into numpy arrays for ease of use in python. - return RampFitOutputs( - np.array(parameters, dtype=np.float32), - np.array(variances, dtype=np.float32), - np.array(dq, dtype=np.uint32), - ramp_fits if include_diagnostic else None, - ) + return ramp_fits diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22_fit.py index 9203686e..d9b50f2e 100644 --- a/src/stcal/ramp_fitting/ols_cas22_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22_fit.py @@ -29,12 +29,42 @@ So the routines in these packages construct these different matrices, store them, and interpolate between them for different different fluxes and ratios. """ +from typing import NamedTuple + import numpy as np from astropy import units as u from . import ols_cas22 +class RampFitOutputs(NamedTuple): + """ + Simple tuple wrapper for outputs from the ramp fitting algorithm + This clarifies the meaning of the outputs via naming them something + descriptive. + + Attributes + ---------- + parameters: np.ndarray[n_pixel, 2] + the slope and intercept for each pixel's ramp fit. see Parameter enum + for indexing indicating slope/intercept in the second dimension. + variances: np.ndarray[n_pixel, 3] + the read, poisson, and total variances for each pixel's ramp fit. + see Variance enum for indexing indicating read/poisson/total in the + second dimension. + dq: np.ndarray[n_resultants, n_pixel] + the dq array, with additional flags set for jumps detected by the + jump detection algorithm. + fits: list of RampFits + the raw ramp fit outputs, these are all structs which will get mapped to + python dictionaries. + """ + + parameters: np.ndarray + variances: np.ndarray + dq: np.ndarray + + def fit_ramps_casertano( resultants, dq, @@ -118,19 +148,25 @@ def fit_ramps_casertano( dq = dq.reshape((*orig_shape, 1)) read_noise = read_noise.reshape(orig_shape[1:] + (1,)) - output = ols_cas22.fit_ramps( + n_pixels = np.prod(resultants.shape[1:]) + parameters = np.empty((n_pixels, ols_cas22.Parameter.n_param), dtype=np.float32) + variances = np.empty((n_pixels, ols_cas22.Variance.n_var), dtype=np.float32) + + ols_cas22.fit_ramps( resultants.reshape(resultants.shape[0], -1), dq.reshape(resultants.shape[0], -1), read_noise.reshape(-1), read_time, read_pattern, + parameters, + variances, use_jump, **kwargs, ) - parameters = output.parameters.reshape(orig_shape[1:] + (2,)) - variances = output.variances.reshape(orig_shape[1:] + (3,)) - dq = output.dq.reshape(orig_shape) + parameters = parameters.reshape(orig_shape[1:] + (2,)) + variances = variances.reshape(orig_shape[1:] + (3,)) + dq = dq.reshape(orig_shape) if resultants.shape != orig_shape: parameters = parameters[0] @@ -140,4 +176,4 @@ def fit_ramps_casertano( parameters = parameters * resultants_unit # return ols_cas22.RampFitOutputs(output.fits, parameters, variances, dq) - return ols_cas22.RampFitOutputs(parameters, variances, dq) + return RampFitOutputs(parameters, variances, dq) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index c02d3f51..db561788 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -326,17 +326,31 @@ def test_fit_ramps(detector_data, use_jump, use_dq): if not use_dq: assert okay.all() + # Initialize the output arrays + parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) + variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) output = fit_ramps( - resultants, dq, read_noise, READ_TIME, read_pattern, use_jump=use_jump, include_diagnostic=True + resultants, + dq, + read_noise, + READ_TIME, + read_pattern, + parameters, + variances, + use_jump=use_jump, + include_diagnostic=True, ) - assert len(output.fits) == N_PIXELS # sanity check that a fit is output for each pixel + assert len(output) == N_PIXELS # sanity check that a fit is output for each pixel - slopes = output.parameters[:, Parameter.slope] - read_vars = output.variances[:, Variance.read_var] - poisson_vars = output.variances[:, Variance.poisson_var] + # Check that the intercept is always zero + assert np.all(parameters[:, Parameter.intercept] == 0) + + slopes = parameters[:, Parameter.slope] + read_vars = variances[:, Variance.read_var] + poisson_vars = variances[:, Variance.poisson_var] chi2 = 0 - for fit, slope, read_var, poisson_var, use in zip(output.fits, slopes, read_vars, poisson_vars, okay): + for fit, slope, read_var, poisson_var, use in zip(output, slopes, read_vars, poisson_vars, okay): if not use_dq and not use_jump: ##### The not use_jump makes this NOT test for false positives ##### # Check that the data generated does not generate any false positives @@ -423,14 +437,28 @@ def test_find_jumps(jump_data): resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) + # Initialize the output arrays + parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) + variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) output = fit_ramps( - resultants, dq, read_noise, READ_TIME, read_pattern, use_jump=True, include_diagnostic=True + resultants, + dq, + read_noise, + READ_TIME, + read_pattern, + parameters, + variances, + use_jump=True, + include_diagnostic=True, ) - assert len(output.fits) == len(jump_reads) # sanity check that a fit/jump is set for every pixel + assert len(output) == len(jump_reads) # sanity check that a fit/jump is set for every pixel + + # Check that the intercept is always zero + assert np.all(parameters[:, Parameter.intercept] == 0) - slopes = output.parameters[:, Parameter.slope] - read_vars = output.variances[:, Variance.read_var] - poisson_vars = output.variances[:, Variance.poisson_var] + slopes = parameters[:, Parameter.slope] + read_vars = variances[:, Variance.read_var] + poisson_vars = variances[:, Variance.poisson_var] chi2 = 0 incorrect_too_few = 0 @@ -438,7 +466,7 @@ def test_find_jumps(jump_data): incorrect_does_not_capture = 0 incorrect_other = 0 for fit, slope, read_var, poisson_var, jump_index, resultant_index in zip( - output.fits, slopes, read_vars, poisson_vars, jump_reads, jump_resultants + output, slopes, read_vars, poisson_vars, jump_reads, jump_resultants ): # Check that the jumps are detected correctly if jump_index == 0: @@ -505,14 +533,38 @@ def test_override_default_threshold(jump_data): resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) - standard = fit_ramps(resultants, dq, read_noise, READ_TIME, read_pattern, use_jump=True) - override = fit_ramps( - resultants, dq, read_noise, READ_TIME, read_pattern, use_jump=True, intercept=0, constant=0 + # Initialize the output arrays + standard_parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) + standard_variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) + fit_ramps( + resultants, + dq, + read_noise, + READ_TIME, + read_pattern, + standard_parameters, + standard_variances, + use_jump=True, + ) + + override_parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) + override_variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) + fit_ramps( + resultants, + dq, + read_noise, + READ_TIME, + read_pattern, + override_parameters, + override_variances, + use_jump=True, + intercept=0, + constant=0, ) # All this is intended to do is show that with all other things being equal passing non-default # threshold parameters changes the results. - assert (standard.parameters != override.parameters).any() + assert (standard_parameters != override_parameters).any() def test_jump_dq_set(jump_data): @@ -522,11 +574,22 @@ def test_jump_dq_set(jump_data): resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) + # Initialize the output arrays + parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) + variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) output = fit_ramps( - resultants, dq, read_noise, READ_TIME, read_pattern, use_jump=True, include_diagnostic=True + resultants, + dq, + read_noise, + READ_TIME, + read_pattern, + parameters, + variances, + use_jump=True, + include_diagnostic=True, ) - for fit, pixel_dq in zip(output.fits, output.dq.transpose()): + for fit, pixel_dq in zip(output, dq.transpose()): # Check that all jumps found get marked assert (pixel_dq[fit["jumps"]] == JUMP_DET).all() From 01ba76e12eda425a5e0b1ac8b9d5c14184fba1f9 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 15:45:22 -0500 Subject: [PATCH 12/31] Pre-allocate output and working memory arrays for improved performance in ols_cas22_fit.py and _fit.py --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 38 ++--------- src/stcal/ramp_fitting/ols_cas22_fit.py | 26 ++++++++ tests/test_jump_cas22.py | 85 ++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 31 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 4ddd8474..5ff5d73b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -28,8 +28,6 @@ """ import cython -import numpy as np -from cython.cimports import numpy as cnp from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.list import list as cpp_list from cython.cimports.libcpp.vector import vector @@ -38,14 +36,9 @@ Thresh, _fill_fixed_values, fit_jumps, - n_fixed_offsets, - n_pixel_offsets, ) from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import _fill_metadata -# Initialize numpy for cython use in this module -cnp.import_array() - @cython.boundscheck(False) @cython.wraparound(False) @@ -57,6 +50,13 @@ def fit_ramps( read_pattern: vector[vector[cython.int]], parameters: cython.float[:, :], variances: cython.float[:, :], + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], use_jump: cpp_bool = False, intercept: cython.float = 5.5, constant: cython.float = 1 / 3, @@ -111,35 +111,11 @@ def fit_ramps( raise RuntimeError(msg) # Compute the main metadata from the read pattern and cast it to memory views - t_bar: cython.float[:] = np.empty(n_resultants, dtype=np.float32) - tau: cython.float[:] = np.empty(n_resultants, dtype=np.float32) - n_reads: cython.int[:] = np.empty(n_resultants, dtype=np.int32) _fill_metadata(t_bar, tau, n_reads, read_pattern, read_time, n_resultants) - # Setup pre-compute arrays for jump detection - single_pixel: cython.float[:, :] - double_pixel: cython.float[:, :] - single_fixed: cython.float[:, :] - double_fixed: cython.float[:, :] if use_jump: - # Initialize arrays for the jump detection pre-computed values - single_pixel = np.empty((n_pixel_offsets, n_resultants - 1), dtype=np.float32) - double_pixel = np.empty((n_pixel_offsets, n_resultants - 2), dtype=np.float32) - - single_fixed = np.empty((n_fixed_offsets, n_resultants - 1), dtype=np.float32) - double_fixed = np.empty((n_fixed_offsets, n_resultants - 2), dtype=np.float32) - # Pre-compute the values from the read pattern _fill_fixed_values(single_fixed, double_fixed, t_bar, tau, n_reads, n_resultants) - else: - # "Initialize" the arrays when not using jump detection, they need to be - # initialized because they do get passed around, but they don't need - # to actually have any entries - single_pixel = np.empty((0, 0), dtype=np.float32) - double_pixel = np.empty((0, 0), dtype=np.float32) - - single_fixed = np.empty((0, 0), dtype=np.float32) - double_fixed = np.empty((0, 0), dtype=np.float32) # Create a threshold struct thresh: Thresh = Thresh(intercept, constant) diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22_fit.py index d9b50f2e..db8bf65d 100644 --- a/src/stcal/ramp_fitting/ols_cas22_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22_fit.py @@ -148,10 +148,29 @@ def fit_ramps_casertano( dq = dq.reshape((*orig_shape, 1)) read_noise = read_noise.reshape(orig_shape[1:] + (1,)) + # Pre-allocate the output arrays n_pixels = np.prod(resultants.shape[1:]) parameters = np.empty((n_pixels, ols_cas22.Parameter.n_param), dtype=np.float32) variances = np.empty((n_pixels, ols_cas22.Variance.n_var), dtype=np.float32) + # Pre-allocate the working memory arrays + # This prevents bouncing to and from cython for this allocation, which + # is slower than just doing it all in python to start. + n_resultants = resultants.shape[0] + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + if use_jump: + single_pixel = np.empty((ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + else: + single_pixel = np.empty((0, 0), dtype=np.float32) + double_pixel = np.empty((0, 0), dtype=np.float32) + single_fixed = np.empty((0, 0), dtype=np.float32) + double_fixed = np.empty((0, 0), dtype=np.float32) + ols_cas22.fit_ramps( resultants.reshape(resultants.shape[0], -1), dq.reshape(resultants.shape[0], -1), @@ -160,6 +179,13 @@ def fit_ramps_casertano( read_pattern, parameters, variances, + t_bar, + tau, + n_reads, + single_pixel, + double_pixel, + single_fixed, + double_fixed, use_jump, **kwargs, ) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index db561788..fc57512b 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -329,6 +329,23 @@ def test_fit_ramps(detector_data, use_jump, use_dq): # Initialize the output arrays parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) + + # Initialize scratch storage + n_resultants = resultants.shape[0] + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + if use_jump: + single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + else: + single_pixel = np.empty((0, 0), dtype=np.float32) + double_pixel = np.empty((0, 0), dtype=np.float32) + single_fixed = np.empty((0, 0), dtype=np.float32) + double_fixed = np.empty((0, 0), dtype=np.float32) + output = fit_ramps( resultants, dq, @@ -337,6 +354,13 @@ def test_fit_ramps(detector_data, use_jump, use_dq): read_pattern, parameters, variances, + t_bar, + tau, + n_reads, + single_pixel, + double_pixel, + single_fixed, + double_fixed, use_jump=use_jump, include_diagnostic=True, ) @@ -440,6 +464,17 @@ def test_find_jumps(jump_data): # Initialize the output arrays parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) + + # Initialize scratch storage + n_resultants = resultants.shape[0] + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + output = fit_ramps( resultants, dq, @@ -448,6 +483,13 @@ def test_find_jumps(jump_data): read_pattern, parameters, variances, + t_bar, + tau, + n_reads, + single_pixel, + double_pixel, + single_fixed, + double_fixed, use_jump=True, include_diagnostic=True, ) @@ -533,6 +575,16 @@ def test_override_default_threshold(jump_data): resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) + # Initialize scratch storage + n_resultants = resultants.shape[0] + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + # Initialize the output arrays standard_parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) standard_variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) @@ -544,6 +596,13 @@ def test_override_default_threshold(jump_data): read_pattern, standard_parameters, standard_variances, + t_bar, + tau, + n_reads, + single_pixel, + double_pixel, + single_fixed, + double_fixed, use_jump=True, ) @@ -557,6 +616,13 @@ def test_override_default_threshold(jump_data): read_pattern, override_parameters, override_variances, + t_bar, + tau, + n_reads, + single_pixel, + double_pixel, + single_fixed, + double_fixed, use_jump=True, intercept=0, constant=0, @@ -565,6 +631,7 @@ def test_override_default_threshold(jump_data): # All this is intended to do is show that with all other things being equal passing non-default # threshold parameters changes the results. assert (standard_parameters != override_parameters).any() + assert (standard_variances != override_variances).any() def test_jump_dq_set(jump_data): @@ -577,6 +644,17 @@ def test_jump_dq_set(jump_data): # Initialize the output arrays parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) + + # Initialize scratch storage + n_resultants = resultants.shape[0] + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + output = fit_ramps( resultants, dq, @@ -585,6 +663,13 @@ def test_jump_dq_set(jump_data): read_pattern, parameters, variances, + t_bar, + tau, + n_reads, + single_pixel, + double_pixel, + single_fixed, + double_fixed, use_jump=True, include_diagnostic=True, ) From 37a310b385488db428a61358b9b65c714fca3a0a Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 15:48:10 -0500 Subject: [PATCH 13/31] Move consistency checking outside of cython --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 8 -------- src/stcal/ramp_fitting/ols_cas22_fit.py | 10 +++++++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 5ff5d73b..9ebed4ae 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -102,14 +102,6 @@ def fit_ramps( n_resultants: cython.int = resultants.shape[0] n_pixels: cython.int = resultants.shape[1] - # Raise error if input data is inconsistent - if n_resultants != len(read_pattern): - msg = ( - f"The read pattern length {len(read_pattern)} does " - f"not match number of resultants {n_resultants}" - ) - raise RuntimeError(msg) - # Compute the main metadata from the read pattern and cast it to memory views _fill_metadata(t_bar, tau, n_reads, read_pattern, read_time, n_resultants) diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22_fit.py index db8bf65d..123e5e25 100644 --- a/src/stcal/ramp_fitting/ols_cas22_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22_fit.py @@ -141,6 +141,15 @@ def fit_ramps_casertano( read_noise = read_noise * np.ones(resultants.shape[1:]) read_noise = np.array(read_noise).astype(np.float32) + # Raise error if input data is inconsistent + n_resultants = resultants.shape[0] + if n_resultants != len(read_pattern): + msg = ( + f"The read pattern length {len(read_pattern)} does " + f"not match number of resultants {n_resultants}" + ) + raise RuntimeError(msg) + orig_shape = resultants.shape if len(resultants.shape) == 1: # single ramp. @@ -156,7 +165,6 @@ def fit_ramps_casertano( # Pre-allocate the working memory arrays # This prevents bouncing to and from cython for this allocation, which # is slower than just doing it all in python to start. - n_resultants = resultants.shape[0] t_bar = np.empty(n_resultants, dtype=np.float32) tau = np.empty(n_resultants, dtype=np.float32) n_reads = np.empty(n_resultants, dtype=np.int32) From df94f43cef5b24ebbfb58c27b00cd7577bbd4b5c Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 17:28:19 -0500 Subject: [PATCH 14/31] Remove kwarg from cython function --- src/stcal/ramp_fitting/ols_cas22/__init__.py | 12 +++++++- src/stcal/ramp_fitting/ols_cas22/_fit.py | 10 +++--- src/stcal/ramp_fitting/ols_cas22_fit.py | 15 ++++----- tests/test_jump_cas22.py | 32 +++++++++++++------- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index 843b7625..9eb3725f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -1,4 +1,14 @@ +from enum import Enum + +import numpy as np + from ._fit import fit_ramps from ._jump import JUMP_DET, Parameter, Variance -__all__ = ["fit_ramps", "Parameter", "Variance", "Diff", "JUMP_DET"] + +class DefaultThreshold(Enum): + INTERCEPT = np.float32(5.5) + CONSTANT = np.float32(1 / 3) + + +__all__ = ["fit_ramps", "Parameter", "Variance", "Diff", "JUMP_DET", "DefaultThreshold"] diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 9ebed4ae..2dd5fe1b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -26,7 +26,6 @@ for jumps (if use_jump is True) and bad pixels (via the dq array). This is the primary externally callable function. """ - import cython from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.list import list as cpp_list @@ -42,6 +41,7 @@ @cython.boundscheck(False) @cython.wraparound(False) +@cython.ccall def fit_ramps( resultants: cython.float[:, :], dq: cython.int[:, :], @@ -57,10 +57,10 @@ def fit_ramps( double_pixel: cython.float[:, :], single_fixed: cython.float[:, :], double_fixed: cython.float[:, :], - use_jump: cpp_bool = False, - intercept: cython.float = 5.5, - constant: cython.float = 1 / 3, - include_diagnostic: cpp_bool = False, + use_jump: cpp_bool, + intercept: cython.float, + constant: cython.float, + include_diagnostic: cpp_bool, ) -> cpp_list[JumpFits]: """Fit ramps using the Casertano+22 algorithm. This implementation uses the Cas22 algorithm to fit ramps, where diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22_fit.py index 123e5e25..c44ac614 100644 --- a/src/stcal/ramp_fitting/ols_cas22_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22_fit.py @@ -73,8 +73,9 @@ def fit_ramps_casertano( read_pattern, use_jump=False, *, - threshold_intercept=None, - threshold_constant=None, + threshold_intercept=ols_cas22.DefaultThreshold.INTERCEPT.value, + threshold_constant=ols_cas22.DefaultThreshold.CONSTANT.value, + include_diagnostic=False, ): """Fit ramps following Casertano+2022, including averaging partial ramps. @@ -124,12 +125,6 @@ def fit_ramps_casertano( """ # Trickery to avoid having to specify the defaults for the threshold # parameters outside the cython code. - kwargs = {} - if threshold_intercept is not None: - kwargs["intercept"] = threshold_intercept - if threshold_constant is not None: - kwargs["constant"] = threshold_constant - resultants_unit = getattr(resultants, "unit", None) if resultants_unit is not None: resultants = resultants.to(u.electron).value @@ -195,7 +190,9 @@ def fit_ramps_casertano( single_fixed, double_fixed, use_jump, - **kwargs, + threshold_intercept, + threshold_constant, + include_diagnostic, ) parameters = parameters.reshape(orig_shape[1:] + (2,)) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index fc57512b..bfdd4423 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -2,7 +2,7 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22 import JUMP_DET, fit_ramps +from stcal.ramp_fitting.ols_cas22 import JUMP_DET, DefaultThreshold, fit_ramps from stcal.ramp_fitting.ols_cas22._jump import ( FixedOffsets, Parameter, @@ -361,8 +361,10 @@ def test_fit_ramps(detector_data, use_jump, use_dq): double_pixel, single_fixed, double_fixed, - use_jump=use_jump, - include_diagnostic=True, + use_jump, + DefaultThreshold.INTERCEPT.value, + DefaultThreshold.CONSTANT.value, + True, ) assert len(output) == N_PIXELS # sanity check that a fit is output for each pixel @@ -490,8 +492,10 @@ def test_find_jumps(jump_data): double_pixel, single_fixed, double_fixed, - use_jump=True, - include_diagnostic=True, + True, + DefaultThreshold.INTERCEPT.value, + DefaultThreshold.CONSTANT.value, + True, ) assert len(output) == len(jump_reads) # sanity check that a fit/jump is set for every pixel @@ -603,7 +607,10 @@ def test_override_default_threshold(jump_data): double_pixel, single_fixed, double_fixed, - use_jump=True, + True, + DefaultThreshold.INTERCEPT.value, + DefaultThreshold.CONSTANT.value, + False, ) override_parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) @@ -623,9 +630,10 @@ def test_override_default_threshold(jump_data): double_pixel, single_fixed, double_fixed, - use_jump=True, - intercept=0, - constant=0, + True, + 0, + 0, + False, ) # All this is intended to do is show that with all other things being equal passing non-default @@ -670,8 +678,10 @@ def test_jump_dq_set(jump_data): double_pixel, single_fixed, double_fixed, - use_jump=True, - include_diagnostic=True, + True, + DefaultThreshold.INTERCEPT.value, + DefaultThreshold.CONSTANT.value, + True, ) for fit, pixel_dq in zip(output, dq.transpose()): From dc75dd847d6cf750a48498e268aa637fdd08262a Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 17:50:11 -0500 Subject: [PATCH 15/31] Move code out of ramp and into other modules --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 35 ++- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 16 +- src/stcal/ramp_fitting/ols_cas22/_jump.py | 238 ++++++++++++++++++- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 36 --- src/stcal/ramp_fitting/ols_cas22/_ramp.py | 264 --------------------- tests/test_jump_cas22.py | 7 +- 6 files changed, 282 insertions(+), 314 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 2dd5fe1b..ecdd40e2 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -36,7 +36,6 @@ _fill_fixed_values, fit_jumps, ) -from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import _fill_metadata @cython.boundscheck(False) @@ -148,3 +147,37 @@ def fit_ramps( # Cast memory views into numpy arrays for ease of use in python. return ramp_fits + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.ccall +def _fill_metadata( + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], + read_pattern: vector[vector[cython.int]], + read_time: cython.float, + n_resultants: cython.int, +) -> cython.void: + n_read: cython.int + + i: cython.int + j: cython.int + resultant: vector[cython.int] + for i in range(n_resultants): + resultant = read_pattern[i] + n_read = resultant.size() + + n_reads[i] = n_read + t_bar[i] = 0 + tau[i] = 0 + + for j in range(n_read): + t_bar[i] += read_time * resultant[j] + tau[i] += (2 * (n_read - j) - 1) * resultant[j] + + t_bar[i] /= n_read + tau[i] *= read_time / n_read**2 diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index c5f3e0f4..c2c381b2 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -3,8 +3,6 @@ from libcpp cimport bool from libcpp.vector cimport vector -from stcal.ramp_fitting.ols_cas22._ramp cimport RampFit, RampQueue - cpdef enum FixedOffsets: t_bar_diff @@ -41,6 +39,20 @@ cdef struct Thresh: float constant +cdef struct RampIndex: + int start + int end + + +ctypedef vector[RampIndex] RampQueue + + +cdef struct RampFit: + float slope + float read_var + float poisson_var + + cdef struct JumpFits: vector[int] jumps vector[RampFit] fits diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.py b/src/stcal/ramp_fitting/ols_cas22/_jump.py index 40d3f177..9f2881d4 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.py +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.py @@ -54,7 +54,7 @@ to splitting across detected jumps (if jump detection is turned on). """ import cython -from cython.cimports.libc.math import NAN, fmaxf, isnan, log10, sqrt +from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, isnan, log10, sqrt from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.vector import vector from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( @@ -63,15 +63,11 @@ JumpFits, Parameter, PixelOffsets, - Thresh, - Variance, -) -from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import ( RampFit, RampIndex, RampQueue, - fit_ramp, - init_ramps, + Thresh, + Variance, ) _t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) @@ -469,7 +465,7 @@ def fit_jumps( RampFits struct of all the fits for a single pixel """ # Find initial set of ramps - ramps: RampQueue = init_ramps(dq, n_resultants) + ramps: RampQueue = _init_ramps(dq, n_resultants) # Initialize algorithm parameters[:] = 0 @@ -618,3 +614,229 @@ def fit_jumps( variances[_total_var] = variances[_read_var] + variances[_poisson_var] return JumpFits(jumps, fits, index) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def fit_ramp( + resultants_: cython.float[:], + t_bar_: cython.float[:], + tau_: cython.float[:], + n_reads_: cython.int[:], + read_noise: cython.float, + ramp: RampIndex, +) -> RampFit: + """ + Fit a single ramp using Casertano+22 algorithm. + + Parameters + ---------- + resultants_ : float[:] + All of the resultants for the pixel + t_bar_ : float[:] + All the t_bar values + tau_ : float[:] + All the tau values + n_reads_ : int[:] + All the n_reads values + read_noise : float + The read noise for the pixel + ramp : RampIndex + Struct for start and end of ramp to fit + + Returns + ------- + RampFit + struct containing + - slope + - read_var + - poisson_var + """ + n_resultants: cython.int = ramp.end - ramp.start + 1 + + # Special case where there is no or one resultant, there is no fit and + # we bail out before any computations. + # Note that in this case, we cannot compute the slope or the variances + # because these computations require at least two resultants. Therefore, + # this case is degernate and we return NaNs for the values. + if n_resultants <= 1: + return RampFit(NAN, NAN, NAN) + + # Compute the fit + i: cython.int = 0 + j: cython.int = 0 + + # Setup data for fitting (work over subset of data) to make things cleaner + # Recall that the RampIndex contains the index of the first and last + # index of the ramp. Therefore, the Python slice needed to get all the + # data within the ramp is: + # ramp.start:ramp.end + 1 + resultants: cython.float[:] = resultants_[ramp.start : ramp.end + 1] + t_bar: cython.float[:] = t_bar_[ramp.start : ramp.end + 1] + tau: cython.float[:] = tau_[ramp.start : ramp.end + 1] + n_reads: cython.int[:] = n_reads_[ramp.start : ramp.end + 1] + + # Compute mid point time + end: cython.int = n_resultants - 1 + t_bar_mid: cython.float = (t_bar[0] + t_bar[end]) / 2 + + # Casertano+2022 Eq. 44 + # Note we've departed from Casertano+22 slightly; + # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., + # a CR in the first resultant has boosted the whole ramp high but there + # is no actual signal. + power: cython.float = fmaxf(resultants[end] - resultants[0], 0) + power = power / sqrt(read_noise**2 + power) + power = _get_power(power) + + # It's easy to use up a lot of dynamic range on something like + # (tbar - tbarmid) ** 10. Rescale these. + t_scale: cython.float = (t_bar[end] - t_bar[0]) / 2 + t_scale = 1 if t_scale == 0 else t_scale + + # Initialize the fit loop + # it is faster to generate a c++ vector than a numpy array + weights: vector[cython.float] = vector[float](n_resultants) + coeffs: vector[cython.float] = vector[float](n_resultants) + ramp_fit: RampFit = RampFit(0, 0, 0) + f0: cython.float = 0 + f1: cython.float = 0 + f2: cython.float = 0 + coeff: cython.float + + # Issue when tbar[] == tbarmid causes exception otherwise + with cython.cpow(True): + for i in range(n_resultants): + # Casertano+22, Eq. 45 + weights[i] = (((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * fabs( + (t_bar[i] - t_bar_mid) / t_scale + ) ** power + + # Casertano+22 Eq. 35 + f0 += weights[i] + f1 += weights[i] * t_bar[i] + f2 += weights[i] * t_bar[i] ** 2 + + # Casertano+22 Eq. 36 + det: cython.float = f2 * f0 - f1**2 + if det == 0: + return ramp_fit + + for i in range(n_resultants): + # Casertano+22 Eq. 37 + coeff = (f0 * t_bar[i] - f1) * weights[i] / det + coeffs[i] = coeff + + # Casertano+22 Eq. 38 + ramp_fit.slope += coeff * resultants[i] + + # Casertano+22 Eq. 39 + ramp_fit.read_var += coeff**2 * read_noise**2 / n_reads[i] + + # Casertano+22 Eq 40 + # Note that this is an inversion of the indexing from the equation; + # however, commutivity of addition results in the same answer. This + # makes it so that we don't have to loop over all the resultants twice. + ramp_fit.poisson_var += coeff**2 * tau[i] + for j in range(i): + ramp_fit.poisson_var += 2 * coeff * coeffs[j] * t_bar[j] + + return ramp_fit + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.ccall +def _init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: + """ + Create the initial ramp "queue" for each pixel + if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp + otherwise, the resultant is not in a ramp. + + Parameters + ---------- + dq : int[n_resultants] + DQ array + n_resultants : int + Number of resultants + + Returns + ------- + RampQueue + vector of RampIndex objects + - vector with entry for each ramp found (last entry is last ramp found) + - RampIndex with start and end indices of the ramp in the resultants + """ + ramps: RampQueue = RampQueue() + + # Note: if start/end are -1, then no value has been assigned + # ramp.start == -1 means we have not started a ramp + # dq[index_resultant, index_pixel] == 0 means resultant is in ramp + ramp: RampIndex = RampIndex(-1, -1) + index_resultant: cython.int + for index_resultant in range(n_resultants): + # Checking for start of ramp + if ramp.start == -1: + if dq[index_resultant] == 0: + # This resultant is in the ramp + # => We have found the start of a ramp! + ramp.start = index_resultant + + # This resultant cannot be the start of a ramp + # => Checking for end of ramp + elif dq[index_resultant] != 0: + # This pixel is not in the ramp + # => index_resultant - 1 is the end of the ramp + ramp.end = index_resultant - 1 + + # Add completed ramp to the queue and reset ramp + ramps.push_back(ramp) + ramp = RampIndex(-1, -1) + + # Handle case where last resultant is in ramp (so no end has been set) + if ramp.start != -1 and ramp.end == -1: + # Last resultant is end of the ramp => set then add to stack + ramp.end = n_resultants - 1 + ramps.push_back(ramp) + + return ramps + + +# Casertano+2022, Table 2 +_P_TABLE = cython.declare( + cython.float[6][2], + [ + [-INFINITY, 5, 10, 20, 50, 100], + [0, 0.4, 1, 3, 6, 10], + ], +) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.cfunc +@cython.exceptval(check=False) +def _get_power(signal: cython.float) -> cython.float: + """ + Return the power from Casertano+22, Table 2. + + Parameters + ---------- + signal: float + signal from the resultants + + Returns + ------- + signal power from Table 2 + """ + i: cython.int + for i in range(6): + if signal < _P_TABLE[0][i]: + return _P_TABLE[1][i - 1] + + return _P_TABLE[1][i] diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd index 1ce49556..75f723ab 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd @@ -1,37 +1 @@ # cython: language_level=3str - -from libcpp.vector cimport vector - - -cpdef void _fill_metadata(float[:] t_bar, - float[:] tau, - int[:] n_reads, - vector[vector[int]] read_pattern, - float read_time, - int n_resultants) - - -cdef struct RampIndex: - int start - int end - - -ctypedef vector[RampIndex] RampQueue - - -cpdef RampQueue init_ramps(int[:] dq, - int n_resultants) - - -cdef struct RampFit: - float slope - float read_var - float poisson_var - - -cdef RampFit fit_ramp(float[:] resultants_, - float[:] t_bar_, - float[:] tau_, - int[:] n_reads, - float read_noise, - RampIndex ramp) diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.py b/src/stcal/ramp_fitting/ols_cas22/_ramp.py index 5c0a3463..072c6f0b 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.py +++ b/src/stcal/ramp_fitting/ols_cas22/_ramp.py @@ -49,267 +49,3 @@ Implementation of running the Casertano+22 algorithm on a (sub)set of resultants listed for a single pixel """ -import cython -from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, sqrt -from cython.cimports.libcpp.vector import vector -from cython.cimports.stcal.ramp_fitting.ols_cas22._ramp import RampFit, RampIndex, RampQueue - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.ccall -def _fill_metadata( - t_bar: cython.float[:], - tau: cython.float[:], - n_reads: cython.int[:], - read_pattern: vector[vector[cython.int]], - read_time: cython.float, - n_resultants: cython.int, -) -> cython.void: - n_read: cython.int - - i: cython.int - j: cython.int - resultant: vector[cython.int] - for i in range(n_resultants): - resultant = read_pattern[i] - n_read = resultant.size() - - n_reads[i] = n_read - t_bar[i] = 0 - tau[i] = 0 - - for j in range(n_read): - t_bar[i] += read_time * resultant[j] - tau[i] += (2 * (n_read - j) - 1) * resultant[j] - - t_bar[i] /= n_read - tau[i] *= read_time / n_read**2 - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.inline -@cython.ccall -def init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: - """ - Create the initial ramp "queue" for each pixel - if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp - otherwise, the resultant is not in a ramp. - - Parameters - ---------- - dq : int[n_resultants] - DQ array - n_resultants : int - Number of resultants - - Returns - ------- - RampQueue - vector of RampIndex objects - - vector with entry for each ramp found (last entry is last ramp found) - - RampIndex with start and end indices of the ramp in the resultants - """ - ramps: RampQueue = RampQueue() - - # Note: if start/end are -1, then no value has been assigned - # ramp.start == -1 means we have not started a ramp - # dq[index_resultant, index_pixel] == 0 means resultant is in ramp - ramp: RampIndex = RampIndex(-1, -1) - index_resultant: cython.int - for index_resultant in range(n_resultants): - # Checking for start of ramp - if ramp.start == -1: - if dq[index_resultant] == 0: - # This resultant is in the ramp - # => We have found the start of a ramp! - ramp.start = index_resultant - - # This resultant cannot be the start of a ramp - # => Checking for end of ramp - elif dq[index_resultant] != 0: - # This pixel is not in the ramp - # => index_resultant - 1 is the end of the ramp - ramp.end = index_resultant - 1 - - # Add completed ramp to the queue and reset ramp - ramps.push_back(ramp) - ramp = RampIndex(-1, -1) - - # Handle case where last resultant is in ramp (so no end has been set) - if ramp.start != -1 and ramp.end == -1: - # Last resultant is end of the ramp => set then add to stack - ramp.end = n_resultants - 1 - ramps.push_back(ramp) - - return ramps - - -# Casertano+2022, Table 2 -_P_TABLE = cython.declare( - cython.float[6][2], - [ - [-INFINITY, 5, 10, 20, 50, 100], - [0, 0.4, 1, 3, 6, 10], - ], -) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.inline -@cython.cfunc -@cython.exceptval(check=False) -def _get_power(signal: cython.float) -> cython.float: - """ - Return the power from Casertano+22, Table 2. - - Parameters - ---------- - signal: float - signal from the resultants - - Returns - ------- - signal power from Table 2 - """ - i: cython.int - for i in range(6): - if signal < _P_TABLE[0][i]: - return _P_TABLE[1][i - 1] - - return _P_TABLE[1][i] - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.cfunc -def fit_ramp( - resultants_: cython.float[:], - t_bar_: cython.float[:], - tau_: cython.float[:], - n_reads_: cython.int[:], - read_noise: cython.float, - ramp: RampIndex, -) -> RampFit: - """ - Fit a single ramp using Casertano+22 algorithm. - - Parameters - ---------- - resultants_ : float[:] - All of the resultants for the pixel - t_bar_ : float[:] - All the t_bar values - tau_ : float[:] - All the tau values - n_reads_ : int[:] - All the n_reads values - read_noise : float - The read noise for the pixel - ramp : RampIndex - Struct for start and end of ramp to fit - - Returns - ------- - RampFit - struct containing - - slope - - read_var - - poisson_var - """ - n_resultants: cython.int = ramp.end - ramp.start + 1 - - # Special case where there is no or one resultant, there is no fit and - # we bail out before any computations. - # Note that in this case, we cannot compute the slope or the variances - # because these computations require at least two resultants. Therefore, - # this case is degernate and we return NaNs for the values. - if n_resultants <= 1: - return RampFit(NAN, NAN, NAN) - - # Compute the fit - i: cython.int = 0 - j: cython.int = 0 - - # Setup data for fitting (work over subset of data) to make things cleaner - # Recall that the RampIndex contains the index of the first and last - # index of the ramp. Therefore, the Python slice needed to get all the - # data within the ramp is: - # ramp.start:ramp.end + 1 - resultants: cython.float[:] = resultants_[ramp.start : ramp.end + 1] - t_bar: cython.float[:] = t_bar_[ramp.start : ramp.end + 1] - tau: cython.float[:] = tau_[ramp.start : ramp.end + 1] - n_reads: cython.int[:] = n_reads_[ramp.start : ramp.end + 1] - - # Compute mid point time - end: cython.int = n_resultants - 1 - t_bar_mid: cython.float = (t_bar[0] + t_bar[end]) / 2 - - # Casertano+2022 Eq. 44 - # Note we've departed from Casertano+22 slightly; - # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., - # a CR in the first resultant has boosted the whole ramp high but there - # is no actual signal. - power: cython.float = fmaxf(resultants[end] - resultants[0], 0) - power = power / sqrt(read_noise**2 + power) - power = _get_power(power) - - # It's easy to use up a lot of dynamic range on something like - # (tbar - tbarmid) ** 10. Rescale these. - t_scale: cython.float = (t_bar[end] - t_bar[0]) / 2 - t_scale = 1 if t_scale == 0 else t_scale - - # Initialize the fit loop - # it is faster to generate a c++ vector than a numpy array - weights: vector[cython.float] = vector[float](n_resultants) - coeffs: vector[cython.float] = vector[float](n_resultants) - ramp_fit: RampFit = RampFit(0, 0, 0) - f0: cython.float = 0 - f1: cython.float = 0 - f2: cython.float = 0 - coeff: cython.float - - # Issue when tbar[] == tbarmid causes exception otherwise - with cython.cpow(True): - for i in range(n_resultants): - # Casertano+22, Eq. 45 - weights[i] = (((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * fabs( - (t_bar[i] - t_bar_mid) / t_scale - ) ** power - - # Casertano+22 Eq. 35 - f0 += weights[i] - f1 += weights[i] * t_bar[i] - f2 += weights[i] * t_bar[i] ** 2 - - # Casertano+22 Eq. 36 - det: cython.float = f2 * f0 - f1**2 - if det == 0: - return ramp_fit - - for i in range(n_resultants): - # Casertano+22 Eq. 37 - coeff = (f0 * t_bar[i] - f1) * weights[i] / det - coeffs[i] = coeff - - # Casertano+22 Eq. 38 - ramp_fit.slope += coeff * resultants[i] - - # Casertano+22 Eq. 39 - ramp_fit.read_var += coeff**2 * read_noise**2 / n_reads[i] - - # Casertano+22 Eq 40 - # Note that this is an inversion of the indexing from the equation; - # however, commutivity of addition results in the same answer. This - # makes it so that we don't have to loop over all the resultants twice. - ramp_fit.poisson_var += coeff**2 * tau[i] - for j in range(i): - ramp_fit.poisson_var += 2 * coeff * coeffs[j] * t_bar[j] - - return ramp_fit diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index bfdd4423..6bb8275b 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -3,6 +3,7 @@ from numpy.testing import assert_allclose from stcal.ramp_fitting.ols_cas22 import JUMP_DET, DefaultThreshold, fit_ramps +from stcal.ramp_fitting.ols_cas22._fit import _fill_metadata from stcal.ramp_fitting.ols_cas22._jump import ( FixedOffsets, Parameter, @@ -10,8 +11,8 @@ Variance, _fill_fixed_values, _fill_pixel_values, + _init_ramps, ) -from stcal.ramp_fitting.ols_cas22._ramp import _fill_metadata, init_ramps # Purposefully set a fixed seed so that the tests in this module are deterministic RNG = np.random.default_rng(619) @@ -38,7 +39,7 @@ GOOD_PROB = 0.7 -def test_init_ramps(): +def test__init_ramps(): """ Test turning dq flags into initial ramp splits Note that because `init_ramps` itself returns a stack, which does not have @@ -58,7 +59,7 @@ def test_init_ramps(): ) n_resultants, n_pixels = dq.shape - ramps = [init_ramps(dq[:, index], n_resultants) for index in range(n_pixels)] + ramps = [_init_ramps(dq[:, index], n_resultants) for index in range(n_pixels)] assert len(ramps) == dq.shape[1] == 16 From 11891ba5c32f5f6344386eace9405c97c43b8327 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 17:56:48 -0500 Subject: [PATCH 16/31] Reorder jump so that most important methods are first --- src/stcal/ramp_fitting/ols_cas22/_jump.py | 1014 ++++++++++----------- 1 file changed, 507 insertions(+), 507 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.py b/src/stcal/ramp_fitting/ols_cas22/_jump.py index 9f2881d4..d7897756 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.py +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.py @@ -70,34 +70,51 @@ Variance, ) -_t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) -_t_bar_diff_sqr = cython.declare(cython.int, FixedOffsets.t_bar_diff_sqr) -_read_recip = cython.declare(cython.int, FixedOffsets.read_recip) -_var_slope_val = cython.declare(cython.int, FixedOffsets.var_slope_val) +_slope = cython.declare(cython.int, Parameter.slope) +_read_var = cython.declare(cython.int, Variance.read_var) +_poisson_var = cython.declare(cython.int, Variance.poisson_var) +_total_var = cython.declare(cython.int, Variance.total_var) @cython.boundscheck(False) @cython.wraparound(False) @cython.cdivision(True) @cython.inline -@cython.ccall -def _fill_fixed_values( - single_fixed: cython.float[:, :], - double_fixed: cython.float[:, :], +@cython.cfunc +def fit_jumps( + parameters: cython.float[:], + variances: cython.float[:], + resultants: cython.float[:], + dq: cython.int[:], + read_noise: cython.float, t_bar: cython.float[:], tau: cython.float[:], n_reads: cython.int[:], n_resultants: cython.int, -) -> cython.void: + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + thresh: Thresh, + use_jump: cpp_bool, + include_diagnostic: cpp_bool, +) -> JumpFits: """ - Pre-compute all the values needed for jump detection which only depend on - the read pattern. + Compute all the ramps for a single pixel using the Casertano+22 algorithm + with jump detection. Parameters ---------- - fixed : float[:, :] - A pre-allocated memoryview to store the pre-computed values in, its faster - to allocate outside this function. + resultants : float[:] + The resultants for the pixel + dq : int[:] + The dq flags for the pixel. This is modified in place, so the external + dq flag array will be modified as a side-effect. + read_noise : float + The read noise for the pixel. + ramps : RampQueue + RampQueue for initial ramps to fit for the pixel + multiple ramps are possible due to dq flags t_bar : float[:] The average time for each resultant tau : float[:] @@ -105,117 +122,174 @@ def _fill_fixed_values( n_reads : int[:] The number of reads for each resultant n_resultants : int - The number of resultants for the read pattern + The number of resultants for the pixel + fixed : float[:, :] + The jump detection pre-computed values for a given read_pattern + pixel : float[:, :] + A pre-allocated array for the jump detection fixed values for the + given pixel. This will be modified in place, it is passed in to avoid + re-allocating it for each pixel. + thresh : Thresh + The threshold parameter struct for jump detection + use_jump : bool + Turn on or off jump detection. + include_diagnostic : bool + Turn on or off recording all the diaganostic information on the fit Returns ------- - [ - , - , - ** 2, - ** 2, - <(1/n_reads[i+1] + 1/n_reads[i])>, - <(1/n_reads[i+2] + 1/n_reads[i])>, - <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, - <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, - ] + RampFits struct of all the fits for a single pixel """ - # Coerce division to be using floats - num: cython.float = 1 + # Find initial set of ramps + ramps: RampQueue = _init_ramps(dq, n_resultants) - i: cython.int - for i in range(n_resultants - 1): - single_fixed[_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] - single_fixed[_t_bar_diff_sqr, i] = single_fixed[_t_bar_diff, i] ** 2 - single_fixed[_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) - single_fixed[_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) + # Initialize algorithm + parameters[:] = 0 + variances[:] = 0 - if i < n_resultants - 2: - double_fixed[_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] - double_fixed[_t_bar_diff_sqr, i] = double_fixed[_t_bar_diff, i] ** 2 - double_fixed[_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) - double_fixed[_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) + jumps: vector[cython.int] = vector[cython.int]() + fits: vector[RampFit] = vector[RampFit]() + index: RampQueue = RampQueue() + # Declare variables for the loop + ramp: RampIndex + ramp_fit: RampFit + stat: Stat + jump0: cython.int + jump1: cython.int + weight: cython.float + total_weight: cython.float = 0 -_local_slope = cython.declare(cython.int, PixelOffsets.local_slope) -_var_read_noise = cython.declare(cython.int, PixelOffsets.var_read_noise) + # Fill in the jump detection pre-compute values for a single pixel + if use_jump: + _fill_pixel_values( + single_pixel, double_pixel, single_fixed, double_fixed, resultants, read_noise, n_resultants + ) + # Run while the Queue is non-empty + while not ramps.empty(): + # Remove top ramp of the stack to use + ramp = ramps.back() + ramps.pop_back() -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.ccall -def _fill_pixel_values( - single_pixel: cython.float[:, :], - double_pixel: cython.float[:, :], - single_fixed: cython.float[:, :], - double_fixed: cython.float[:, :], - resultants: cython.float[:], - read_noise: cython.float, - n_resultants: cython.int, -) -> cython.void: - """ - Pre-compute all the values needed for jump detection which only depend on - the a specific pixel (independent of the given ramp for a pixel). + # Compute fit using the Casertano+22 algorithm + ramp_fit = _fit_ramp(resultants, t_bar, tau, n_reads, read_noise, ramp) - Parameters - ---------- - pixel : float[:, :] - A pre-allocated memoryview to store the pre-computed values in, its faster - to allocate outside this function. - resultants : float[:] - The resultants for the pixel in question. - fixed : float[:, :] - The pre-computed fixed values for the read_pattern - read_noise : float - The read noise for the pixel - n_resultants : int - The number of resultants for the read_pattern + # Run jump detection if enabled + if use_jump: + stat = _fit_statistic( + single_pixel, double_pixel, single_fixed, double_fixed, t_bar, ramp_fit.slope, ramp + ) - Returns - ------- - [ - <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, - <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, - read_noise**2 * <(1/n_reads[i+1] + 1/n_reads[i])>, - read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, - ] - """ - read_noise_sqr: cython.float = read_noise**2 + # Note that when a "ramp" is a single point, _fit_statistic returns + # a NaN for max_stat. Note that NaN > anything is always false so the + # result drops through as desired. + if stat.max_stat > _threshold(thresh, ramp_fit.slope): + # Compute jump point to create two new ramps + # This jump point corresponds to the index of the largest + # statistic: + # argmax = argmax(stats) + # These statistics are indexed relative to the + # ramp's range. Therefore, we need to add the start index + # of the ramp to the result. + # + jump0 = stat.arg_max + ramp.start - i: cython.int - for i in range(n_resultants - 1): - single_pixel[_local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[_t_bar_diff, i] - single_pixel[_var_read_noise, i] = read_noise_sqr * single_fixed[_read_recip, i] + # Note that because the resultants are averages of reads, but + # jumps occur in individual reads, it is possible that the + # jump is averaged down by the resultant with the actual jump + # causing the computed jump to be off by one index. + # In the idealized case this is when the jump occurs near + # the start of the resultant with the jump. In this case, + # the statistic for the resultant will be maximized at + # index - 1 rather than index. This means that we have to + # remove argmax(stats) + 1 as it is also a possible jump. + # This case is difficult to distinguish from the case where + # argmax(stats) does correspond to the jump resultant. + # Therefore, we just remove both possible resultants from + # consideration. + jump1 = jump0 + 1 - if i < n_resultants - 2: - double_pixel[_local_slope, i] = (resultants[i + 2] - resultants[i]) / double_fixed[_t_bar_diff, i] - double_pixel[_var_read_noise, i] = read_noise_sqr * double_fixed[_read_recip, i] + # Update the dq flags + dq[jump0] = JUMP_DET + dq[jump1] = JUMP_DET + # Record jump diagnostics + if include_diagnostic: + jumps.push_back(jump0) + jumps.push_back(jump1) -@cython.inline -@cython.cfunc -@cython.exceptval(check=False) -def _threshold(thresh: Thresh, slope: cython.float) -> cython.float: - """ - Compute jump threshold. + # The two resultant indices need to be skipped, therefore + # the two + # possible new ramps are: + # RampIndex(ramp.start, jump0 - 1) + # RampIndex(jump1 + 1, ramp.end) + # This is because the RampIndex contains the index of the + # first and last resulants in the sub-ramp it describes. + # Note: The algorithm works via working over the sub-ramps + # backward in time. Therefore, since we are using a stack, + # we need to add the ramps in the time order they were + # observed in. This results in the last observation ramp + # being the top of the stack; meaning that, + # it will be the next ramp handled. - Parameters - ---------- - thresh : Thresh - threshold parameters struct - slope : float - slope of the ramp in question + if jump0 > ramp.start: + # Note that when jump0 == ramp.start, we have detected a + # jump in the first resultant of the ramp. This means + # there is no sub-ramp before jump0. + # Also, note that this will produce bad results as + # the ramp indexing will go out of bounds. So it is + # important that we exclude it. + # Note that jump0 < ramp.start is not possible because + # the argmax is always >= 0 + ramps.push_back(RampIndex(ramp.start, jump0 - 1)) - Returns - ------- - intercept - constant * log10(slope) - """ - slope = slope if slope > 1 else 1 - slope = slope if slope < 1e4 else 1e4 + if jump1 < ramp.end: + # Note that if jump1 == ramp.end, we have detected a + # jump in the last resultant of the ramp. This means + # there is no sub-ramp after jump1. + # Also, note that this will produce bad results as + # the ramp indexing will go out of bounds. So it is + # important that we exclude it. + # Note that jump1 > ramp.end is technically possible + # however in those potential cases it will draw on + # resultants which are not considered part of the ramp + # under consideration. Therefore, we have to exclude all + # of those values. + ramps.push_back(RampIndex(jump1 + 1, ramp.end)) - return thresh.intercept - thresh.constant * log10(slope) + # Skip recording the ramp as it has a detected jump + continue + + # Start recording the the fit (no jump detected) + + # Record the diagnositcs + if include_diagnostic: + fits.push_back(ramp_fit) + index.push_back(ramp) + + # Start computing the averages using a lazy process + # Note we do not do anything in the NaN case for degenerate ramps + if not isnan(ramp_fit.slope): + # protect weight against the extremely unlikely case of a zero + # variance + weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var + total_weight += weight + + parameters[_slope] += weight * ramp_fit.slope + variances[_read_var] += weight**2 * ramp_fit.read_var + variances[_poisson_var] += weight**2 * ramp_fit.poisson_var + + # Finish computing averages using the lazy process + parameters[_slope] /= total_weight if total_weight != 0 else 1 + variances[_read_var] /= total_weight**2 if total_weight != 0 else 1 + variances[_poisson_var] /= total_weight**2 if total_weight != 0 else 1 + + # Multiply poisson term by flux, (no negative fluxes) + variances[_poisson_var] *= max(parameters[_slope], 0) + variances[_total_var] = variances[_read_var] + variances[_poisson_var] + + return JumpFits(jumps, fits, index) @cython.boundscheck(False) @@ -223,76 +297,213 @@ def _threshold(thresh: Thresh, slope: cython.float) -> cython.float: @cython.cdivision(True) @cython.inline @cython.cfunc -def _correction(t_bar: cython.float[:], ramp: RampIndex, slope: cython.float) -> cython.float: +def _fit_ramp( + resultants_: cython.float[:], + t_bar_: cython.float[:], + tau_: cython.float[:], + n_reads_: cython.int[:], + read_noise: cython.float, + ramp: RampIndex, +) -> RampFit: """ - Compute the correction factor for the variance used by a statistic. - - - slope / (t_bar[end] - t_bar[start]) + Fit a single ramp using Casertano+22 algorithm. Parameters ---------- - t_bar : float[:] - The computed t_bar values for the ramp + resultants_ : float[:] + All of the resultants for the pixel + t_bar_ : float[:] + All the t_bar values + tau_ : float[:] + All the tau values + n_reads_ : int[:] + All the n_reads values + read_noise : float + The read noise for the pixel ramp : RampIndex - Struct for start and end indices resultants for the ramp - slope : float - The computed slope for the ramp + Struct for start and end of ramp to fit + + Returns + ------- + RampFit + struct containing + - slope + - read_var + - poisson_var """ - diff: cython.float = t_bar[ramp.end] - t_bar[ramp.start] + n_resultants: cython.int = ramp.end - ramp.start + 1 - return -slope / diff + # Special case where there is no or one resultant, there is no fit and + # we bail out before any computations. + # Note that in this case, we cannot compute the slope or the variances + # because these computations require at least two resultants. Therefore, + # this case is degernate and we return NaNs for the values. + if n_resultants <= 1: + return RampFit(NAN, NAN, NAN) + + # Compute the fit + i: cython.int = 0 + j: cython.int = 0 + + # Setup data for fitting (work over subset of data) to make things cleaner + # Recall that the RampIndex contains the index of the first and last + # index of the ramp. Therefore, the Python slice needed to get all the + # data within the ramp is: + # ramp.start:ramp.end + 1 + resultants: cython.float[:] = resultants_[ramp.start : ramp.end + 1] + t_bar: cython.float[:] = t_bar_[ramp.start : ramp.end + 1] + tau: cython.float[:] = tau_[ramp.start : ramp.end + 1] + n_reads: cython.int[:] = n_reads_[ramp.start : ramp.end + 1] + + # Compute mid point time + end: cython.int = n_resultants - 1 + t_bar_mid: cython.float = (t_bar[0] + t_bar[end]) / 2 + + # Casertano+2022 Eq. 44 + # Note we've departed from Casertano+22 slightly; + # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., + # a CR in the first resultant has boosted the whole ramp high but there + # is no actual signal. + power: cython.float = fmaxf(resultants[end] - resultants[0], 0) + power = power / sqrt(read_noise**2 + power) + power = _get_power(power) + + # It's easy to use up a lot of dynamic range on something like + # (tbar - tbarmid) ** 10. Rescale these. + t_scale: cython.float = (t_bar[end] - t_bar[0]) / 2 + t_scale = 1 if t_scale == 0 else t_scale + + # Initialize the fit loop + # it is faster to generate a c++ vector than a numpy array + weights: vector[cython.float] = vector[float](n_resultants) + coeffs: vector[cython.float] = vector[float](n_resultants) + ramp_fit: RampFit = RampFit(0, 0, 0) + f0: cython.float = 0 + f1: cython.float = 0 + f2: cython.float = 0 + coeff: cython.float + + # Issue when tbar[] == tbarmid causes exception otherwise + with cython.cpow(True): + for i in range(n_resultants): + # Casertano+22, Eq. 45 + weights[i] = (((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * fabs( + (t_bar[i] - t_bar_mid) / t_scale + ) ** power + + # Casertano+22 Eq. 35 + f0 += weights[i] + f1 += weights[i] * t_bar[i] + f2 += weights[i] * t_bar[i] ** 2 + + # Casertano+22 Eq. 36 + det: cython.float = f2 * f0 - f1**2 + if det == 0: + return ramp_fit + + for i in range(n_resultants): + # Casertano+22 Eq. 37 + coeff = (f0 * t_bar[i] - f1) * weights[i] / det + coeffs[i] = coeff + + # Casertano+22 Eq. 38 + ramp_fit.slope += coeff * resultants[i] + + # Casertano+22 Eq. 39 + ramp_fit.read_var += coeff**2 * read_noise**2 / n_reads[i] + + # Casertano+22 Eq 40 + # Note that this is an inversion of the indexing from the equation; + # however, commutivity of addition results in the same answer. This + # makes it so that we don't have to loop over all the resultants twice. + ramp_fit.poisson_var += coeff**2 * tau[i] + for j in range(i): + ramp_fit.poisson_var += 2 * coeff * coeffs[j] * t_bar[j] + + return ramp_fit @cython.boundscheck(False) @cython.wraparound(False) -@cython.cdivision(True) @cython.inline -@cython.cfunc -def _statistic( - local_slope: cython.float, - var_read_noise: cython.float, - t_bar_diff_sqr: cython.float, - var_slope_val: cython.float, - slope: cython.float, - correct: cython.float, -) -> cython.float: +@cython.ccall +def _init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: """ - Compute a single fit statistic - delta / sqrt(var + correct). + Create the initial ramp "queue" for each pixel + if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp + otherwise, the resultant is not in a ramp. - where: - delta = _local_slope - slope - var = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr + Parameters + ---------- + dq : int[n_resultants] + DQ array + n_resultants : int + Number of resultants + + Returns + ------- + RampQueue + vector of RampIndex objects + - vector with entry for each ramp found (last entry is last ramp found) + - RampIndex with start and end indices of the ramp in the resultants + """ + ramps: RampQueue = RampQueue() + + # Note: if start/end are -1, then no value has been assigned + # ramp.start == -1 means we have not started a ramp + # dq[index_resultant, index_pixel] == 0 means resultant is in ramp + ramp: RampIndex = RampIndex(-1, -1) + index_resultant: cython.int + for index_resultant in range(n_resultants): + # Checking for start of ramp + if ramp.start == -1: + if dq[index_resultant] == 0: + # This resultant is in the ramp + # => We have found the start of a ramp! + ramp.start = index_resultant + + # This resultant cannot be the start of a ramp + # => Checking for end of ramp + elif dq[index_resultant] != 0: + # This pixel is not in the ramp + # => index_resultant - 1 is the end of the ramp + ramp.end = index_resultant - 1 + + # Add completed ramp to the queue and reset ramp + ramps.push_back(ramp) + ramp = RampIndex(-1, -1) + + # Handle case where last resultant is in ramp (so no end has been set) + if ramp.start != -1 and ramp.end == -1: + # Last resultant is end of the ramp => set then add to stack + ramp.end = n_resultants - 1 + ramps.push_back(ramp) + + return ramps - pre-computed: - local_slope = (resultant[i + j] - resultant[i]) / (t_bar[i + j] - t_bar[i]) - var_read_noise = read_noise ** 2 * (1/n_reads[i + j] + 1/n_reads[i]) - var_slope_coeff = tau[i + j] + tau[i] - 2 * min(t_bar[i + j], t_bar[i]) - t_bar_diff_sqr = (t_bar[i + j] - t_bar[i]) ** 2 + +@cython.inline +@cython.cfunc +@cython.exceptval(check=False) +def _threshold(thresh: Thresh, slope: cython.float) -> cython.float: + """ + Compute jump threshold. Parameters ---------- - local_slope : float - The local slope the statistic is computed for - var_read_noise: float - The read noise variance for _local_slope - t_bar_diff_sqr : float - The square difference for the t_bar corresponding to _local_slope - var_slope_val : float - The slope variance coefficient for _local_slope + thresh : Thresh + threshold parameters struct slope : float - The computed slope for the ramp - correct : float - The correction factor needed + slope of the ramp in question Returns ------- - Create a single instance of the stastic for the given parameters + intercept - constant * log10(slope) """ - delta: cython.float = local_slope - slope - var: cython.float = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr + slope = slope if slope > 1 else 1 + slope = slope if slope < 1e4 else 1e4 - return delta / sqrt(var + correct) + return thresh.intercept - thresh.constant * log10(slope) Stat = cython.struct(arg_max=cython.int, max_stat=cython.float) @@ -394,226 +605,56 @@ def _fit_statistic( return stat -_slope = cython.declare(cython.int, Parameter.slope) -_read_var = cython.declare(cython.int, Variance.read_var) -_poisson_var = cython.declare(cython.int, Variance.poisson_var) -_total_var = cython.declare(cython.int, Variance.total_var) - - @cython.boundscheck(False) @cython.wraparound(False) @cython.cdivision(True) @cython.inline @cython.cfunc -def fit_jumps( - parameters: cython.float[:], - variances: cython.float[:], - resultants: cython.float[:], - dq: cython.int[:], - read_noise: cython.float, - t_bar: cython.float[:], - tau: cython.float[:], - n_reads: cython.int[:], - n_resultants: cython.int, - single_pixel: cython.float[:, :], - double_pixel: cython.float[:, :], - single_fixed: cython.float[:, :], - double_fixed: cython.float[:, :], - thresh: Thresh, - use_jump: cpp_bool, - include_diagnostic: cpp_bool, -) -> JumpFits: +def _statistic( + local_slope: cython.float, + var_read_noise: cython.float, + t_bar_diff_sqr: cython.float, + var_slope_val: cython.float, + slope: cython.float, + correct: cython.float, +) -> cython.float: """ - Compute all the ramps for a single pixel using the Casertano+22 algorithm - with jump detection. + Compute a single fit statistic + delta / sqrt(var + correct). + + where: + delta = _local_slope - slope + var = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr + + pre-computed: + local_slope = (resultant[i + j] - resultant[i]) / (t_bar[i + j] - t_bar[i]) + var_read_noise = read_noise ** 2 * (1/n_reads[i + j] + 1/n_reads[i]) + var_slope_coeff = tau[i + j] + tau[i] - 2 * min(t_bar[i + j], t_bar[i]) + t_bar_diff_sqr = (t_bar[i + j] - t_bar[i]) ** 2 Parameters ---------- - resultants : float[:] - The resultants for the pixel - dq : int[:] - The dq flags for the pixel. This is modified in place, so the external - dq flag array will be modified as a side-effect. - read_noise : float - The read noise for the pixel. - ramps : RampQueue - RampQueue for initial ramps to fit for the pixel - multiple ramps are possible due to dq flags - t_bar : float[:] - The average time for each resultant - tau : float[:] - The time variance for each resultant - n_reads : int[:] - The number of reads for each resultant - n_resultants : int - The number of resultants for the pixel - fixed : float[:, :] - The jump detection pre-computed values for a given read_pattern - pixel : float[:, :] - A pre-allocated array for the jump detection fixed values for the - given pixel. This will be modified in place, it is passed in to avoid - re-allocating it for each pixel. - thresh : Thresh - The threshold parameter struct for jump detection - use_jump : bool - Turn on or off jump detection. - include_diagnostic : bool - Turn on or off recording all the diaganostic information on the fit + local_slope : float + The local slope the statistic is computed for + var_read_noise: float + The read noise variance for _local_slope + t_bar_diff_sqr : float + The square difference for the t_bar corresponding to _local_slope + var_slope_val : float + The slope variance coefficient for _local_slope + slope : float + The computed slope for the ramp + correct : float + The correction factor needed Returns ------- - RampFits struct of all the fits for a single pixel + Create a single instance of the stastic for the given parameters """ - # Find initial set of ramps - ramps: RampQueue = _init_ramps(dq, n_resultants) - - # Initialize algorithm - parameters[:] = 0 - variances[:] = 0 - - jumps: vector[cython.int] = vector[cython.int]() - fits: vector[RampFit] = vector[RampFit]() - index: RampQueue = RampQueue() - - # Declare variables for the loop - ramp: RampIndex - ramp_fit: RampFit - stat: Stat - jump0: cython.int - jump1: cython.int - weight: cython.float - total_weight: cython.float = 0 - - # Fill in the jump detection pre-compute values for a single pixel - if use_jump: - _fill_pixel_values( - single_pixel, double_pixel, single_fixed, double_fixed, resultants, read_noise, n_resultants - ) - - # Run while the Queue is non-empty - while not ramps.empty(): - # Remove top ramp of the stack to use - ramp = ramps.back() - ramps.pop_back() - - # Compute fit using the Casertano+22 algorithm - ramp_fit = fit_ramp(resultants, t_bar, tau, n_reads, read_noise, ramp) - - # Run jump detection if enabled - if use_jump: - stat = _fit_statistic( - single_pixel, double_pixel, single_fixed, double_fixed, t_bar, ramp_fit.slope, ramp - ) - - # Note that when a "ramp" is a single point, _fit_statistic returns - # a NaN for max_stat. Note that NaN > anything is always false so the - # result drops through as desired. - if stat.max_stat > _threshold(thresh, ramp_fit.slope): - # Compute jump point to create two new ramps - # This jump point corresponds to the index of the largest - # statistic: - # argmax = argmax(stats) - # These statistics are indexed relative to the - # ramp's range. Therefore, we need to add the start index - # of the ramp to the result. - # - jump0 = stat.arg_max + ramp.start - - # Note that because the resultants are averages of reads, but - # jumps occur in individual reads, it is possible that the - # jump is averaged down by the resultant with the actual jump - # causing the computed jump to be off by one index. - # In the idealized case this is when the jump occurs near - # the start of the resultant with the jump. In this case, - # the statistic for the resultant will be maximized at - # index - 1 rather than index. This means that we have to - # remove argmax(stats) + 1 as it is also a possible jump. - # This case is difficult to distinguish from the case where - # argmax(stats) does correspond to the jump resultant. - # Therefore, we just remove both possible resultants from - # consideration. - jump1 = jump0 + 1 - - # Update the dq flags - dq[jump0] = JUMP_DET - dq[jump1] = JUMP_DET - - # Record jump diagnostics - if include_diagnostic: - jumps.push_back(jump0) - jumps.push_back(jump1) - - # The two resultant indices need to be skipped, therefore - # the two - # possible new ramps are: - # RampIndex(ramp.start, jump0 - 1) - # RampIndex(jump1 + 1, ramp.end) - # This is because the RampIndex contains the index of the - # first and last resulants in the sub-ramp it describes. - # Note: The algorithm works via working over the sub-ramps - # backward in time. Therefore, since we are using a stack, - # we need to add the ramps in the time order they were - # observed in. This results in the last observation ramp - # being the top of the stack; meaning that, - # it will be the next ramp handled. - - if jump0 > ramp.start: - # Note that when jump0 == ramp.start, we have detected a - # jump in the first resultant of the ramp. This means - # there is no sub-ramp before jump0. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump0 < ramp.start is not possible because - # the argmax is always >= 0 - ramps.push_back(RampIndex(ramp.start, jump0 - 1)) - - if jump1 < ramp.end: - # Note that if jump1 == ramp.end, we have detected a - # jump in the last resultant of the ramp. This means - # there is no sub-ramp after jump1. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump1 > ramp.end is technically possible - # however in those potential cases it will draw on - # resultants which are not considered part of the ramp - # under consideration. Therefore, we have to exclude all - # of those values. - ramps.push_back(RampIndex(jump1 + 1, ramp.end)) - - # Skip recording the ramp as it has a detected jump - continue - - # Start recording the the fit (no jump detected) - - # Record the diagnositcs - if include_diagnostic: - fits.push_back(ramp_fit) - index.push_back(ramp) - - # Start computing the averages using a lazy process - # Note we do not do anything in the NaN case for degenerate ramps - if not isnan(ramp_fit.slope): - # protect weight against the extremely unlikely case of a zero - # variance - weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var - total_weight += weight - - parameters[_slope] += weight * ramp_fit.slope - variances[_read_var] += weight**2 * ramp_fit.read_var - variances[_poisson_var] += weight**2 * ramp_fit.poisson_var - - # Finish computing averages using the lazy process - parameters[_slope] /= total_weight if total_weight != 0 else 1 - variances[_read_var] /= total_weight**2 if total_weight != 0 else 1 - variances[_poisson_var] /= total_weight**2 if total_weight != 0 else 1 - - # Multiply poisson term by flux, (no negative fluxes) - variances[_poisson_var] *= max(parameters[_slope], 0) - variances[_total_var] = variances[_read_var] + variances[_poisson_var] + delta: cython.float = local_slope - slope + var: cython.float = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr - return JumpFits(jumps, fits, index) + return delta / sqrt(var + correct) @cython.boundscheck(False) @@ -621,189 +662,148 @@ def fit_jumps( @cython.cdivision(True) @cython.inline @cython.cfunc -def fit_ramp( - resultants_: cython.float[:], - t_bar_: cython.float[:], - tau_: cython.float[:], - n_reads_: cython.int[:], - read_noise: cython.float, - ramp: RampIndex, -) -> RampFit: +def _correction(t_bar: cython.float[:], ramp: RampIndex, slope: cython.float) -> cython.float: """ - Fit a single ramp using Casertano+22 algorithm. + Compute the correction factor for the variance used by a statistic. + + - slope / (t_bar[end] - t_bar[start]) Parameters ---------- - resultants_ : float[:] - All of the resultants for the pixel - t_bar_ : float[:] - All the t_bar values - tau_ : float[:] - All the tau values - n_reads_ : int[:] - All the n_reads values - read_noise : float - The read noise for the pixel + t_bar : float[:] + The computed t_bar values for the ramp ramp : RampIndex - Struct for start and end of ramp to fit - - Returns - ------- - RampFit - struct containing - - slope - - read_var - - poisson_var + Struct for start and end indices resultants for the ramp + slope : float + The computed slope for the ramp """ - n_resultants: cython.int = ramp.end - ramp.start + 1 - - # Special case where there is no or one resultant, there is no fit and - # we bail out before any computations. - # Note that in this case, we cannot compute the slope or the variances - # because these computations require at least two resultants. Therefore, - # this case is degernate and we return NaNs for the values. - if n_resultants <= 1: - return RampFit(NAN, NAN, NAN) - - # Compute the fit - i: cython.int = 0 - j: cython.int = 0 - - # Setup data for fitting (work over subset of data) to make things cleaner - # Recall that the RampIndex contains the index of the first and last - # index of the ramp. Therefore, the Python slice needed to get all the - # data within the ramp is: - # ramp.start:ramp.end + 1 - resultants: cython.float[:] = resultants_[ramp.start : ramp.end + 1] - t_bar: cython.float[:] = t_bar_[ramp.start : ramp.end + 1] - tau: cython.float[:] = tau_[ramp.start : ramp.end + 1] - n_reads: cython.int[:] = n_reads_[ramp.start : ramp.end + 1] - - # Compute mid point time - end: cython.int = n_resultants - 1 - t_bar_mid: cython.float = (t_bar[0] + t_bar[end]) / 2 + diff: cython.float = t_bar[ramp.end] - t_bar[ramp.start] - # Casertano+2022 Eq. 44 - # Note we've departed from Casertano+22 slightly; - # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., - # a CR in the first resultant has boosted the whole ramp high but there - # is no actual signal. - power: cython.float = fmaxf(resultants[end] - resultants[0], 0) - power = power / sqrt(read_noise**2 + power) - power = _get_power(power) + return -slope / diff - # It's easy to use up a lot of dynamic range on something like - # (tbar - tbarmid) ** 10. Rescale these. - t_scale: cython.float = (t_bar[end] - t_bar[0]) / 2 - t_scale = 1 if t_scale == 0 else t_scale - # Initialize the fit loop - # it is faster to generate a c++ vector than a numpy array - weights: vector[cython.float] = vector[float](n_resultants) - coeffs: vector[cython.float] = vector[float](n_resultants) - ramp_fit: RampFit = RampFit(0, 0, 0) - f0: cython.float = 0 - f1: cython.float = 0 - f2: cython.float = 0 - coeff: cython.float +_t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) +_t_bar_diff_sqr = cython.declare(cython.int, FixedOffsets.t_bar_diff_sqr) +_read_recip = cython.declare(cython.int, FixedOffsets.read_recip) +_var_slope_val = cython.declare(cython.int, FixedOffsets.var_slope_val) - # Issue when tbar[] == tbarmid causes exception otherwise - with cython.cpow(True): - for i in range(n_resultants): - # Casertano+22, Eq. 45 - weights[i] = (((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * fabs( - (t_bar[i] - t_bar_mid) / t_scale - ) ** power - # Casertano+22 Eq. 35 - f0 += weights[i] - f1 += weights[i] * t_bar[i] - f2 += weights[i] * t_bar[i] ** 2 +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.ccall +def _fill_fixed_values( + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], + n_resultants: cython.int, +) -> cython.void: + """ + Pre-compute all the values needed for jump detection which only depend on + the read pattern. - # Casertano+22 Eq. 36 - det: cython.float = f2 * f0 - f1**2 - if det == 0: - return ramp_fit + Parameters + ---------- + fixed : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. + t_bar : float[:] + The average time for each resultant + tau : float[:] + The time variance for each resultant + n_reads : int[:] + The number of reads for each resultant + n_resultants : int + The number of resultants for the read pattern - for i in range(n_resultants): - # Casertano+22 Eq. 37 - coeff = (f0 * t_bar[i] - f1) * weights[i] / det - coeffs[i] = coeff + Returns + ------- + [ + , + , + ** 2, + ** 2, + <(1/n_reads[i+1] + 1/n_reads[i])>, + <(1/n_reads[i+2] + 1/n_reads[i])>, + <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, + <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, + ] + """ + # Coerce division to be using floats + num: cython.float = 1 - # Casertano+22 Eq. 38 - ramp_fit.slope += coeff * resultants[i] + i: cython.int + for i in range(n_resultants - 1): + single_fixed[_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] + single_fixed[_t_bar_diff_sqr, i] = single_fixed[_t_bar_diff, i] ** 2 + single_fixed[_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) + single_fixed[_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) - # Casertano+22 Eq. 39 - ramp_fit.read_var += coeff**2 * read_noise**2 / n_reads[i] + if i < n_resultants - 2: + double_fixed[_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] + double_fixed[_t_bar_diff_sqr, i] = double_fixed[_t_bar_diff, i] ** 2 + double_fixed[_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) + double_fixed[_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) - # Casertano+22 Eq 40 - # Note that this is an inversion of the indexing from the equation; - # however, commutivity of addition results in the same answer. This - # makes it so that we don't have to loop over all the resultants twice. - ramp_fit.poisson_var += coeff**2 * tau[i] - for j in range(i): - ramp_fit.poisson_var += 2 * coeff * coeffs[j] * t_bar[j] - return ramp_fit +_local_slope = cython.declare(cython.int, PixelOffsets.local_slope) +_var_read_noise = cython.declare(cython.int, PixelOffsets.var_read_noise) @cython.boundscheck(False) @cython.wraparound(False) +@cython.cdivision(True) @cython.inline @cython.ccall -def _init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: +def _fill_pixel_values( + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + resultants: cython.float[:], + read_noise: cython.float, + n_resultants: cython.int, +) -> cython.void: """ - Create the initial ramp "queue" for each pixel - if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp - otherwise, the resultant is not in a ramp. + Pre-compute all the values needed for jump detection which only depend on + the a specific pixel (independent of the given ramp for a pixel). Parameters ---------- - dq : int[n_resultants] - DQ array + pixel : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. + resultants : float[:] + The resultants for the pixel in question. + fixed : float[:, :] + The pre-computed fixed values for the read_pattern + read_noise : float + The read noise for the pixel n_resultants : int - Number of resultants + The number of resultants for the read_pattern Returns ------- - RampQueue - vector of RampIndex objects - - vector with entry for each ramp found (last entry is last ramp found) - - RampIndex with start and end indices of the ramp in the resultants + [ + <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, + <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, + read_noise**2 * <(1/n_reads[i+1] + 1/n_reads[i])>, + read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, + ] """ - ramps: RampQueue = RampQueue() - - # Note: if start/end are -1, then no value has been assigned - # ramp.start == -1 means we have not started a ramp - # dq[index_resultant, index_pixel] == 0 means resultant is in ramp - ramp: RampIndex = RampIndex(-1, -1) - index_resultant: cython.int - for index_resultant in range(n_resultants): - # Checking for start of ramp - if ramp.start == -1: - if dq[index_resultant] == 0: - # This resultant is in the ramp - # => We have found the start of a ramp! - ramp.start = index_resultant - - # This resultant cannot be the start of a ramp - # => Checking for end of ramp - elif dq[index_resultant] != 0: - # This pixel is not in the ramp - # => index_resultant - 1 is the end of the ramp - ramp.end = index_resultant - 1 - - # Add completed ramp to the queue and reset ramp - ramps.push_back(ramp) - ramp = RampIndex(-1, -1) + read_noise_sqr: cython.float = read_noise**2 - # Handle case where last resultant is in ramp (so no end has been set) - if ramp.start != -1 and ramp.end == -1: - # Last resultant is end of the ramp => set then add to stack - ramp.end = n_resultants - 1 - ramps.push_back(ramp) + i: cython.int + for i in range(n_resultants - 1): + single_pixel[_local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[_t_bar_diff, i] + single_pixel[_var_read_noise, i] = read_noise_sqr * single_fixed[_read_recip, i] - return ramps + if i < n_resultants - 2: + double_pixel[_local_slope, i] = (resultants[i + 2] - resultants[i]) / double_fixed[_t_bar_diff, i] + double_pixel[_var_read_noise, i] = read_noise_sqr * double_fixed[_read_recip, i] # Casertano+2022, Table 2 From 5cb9689b8df2c1bb9969f952e9a3425d7fe016eb Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 18:16:22 -0500 Subject: [PATCH 17/31] Merge all cas22 modules together. This enables cython and the C compiler to auto inline stuff for maximum speed. --- src/stcal/ramp_fitting/ols_cas22/__init__.py | 3 +- src/stcal/ramp_fitting/ols_cas22/_fit.pxd | 31 + src/stcal/ramp_fitting/ols_cas22/_fit.py | 806 ++++++++++++++++++- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 84 -- src/stcal/ramp_fitting/ols_cas22/_jump.py | 787 ------------------ tests/test_jump_cas22.py | 4 +- 6 files changed, 834 insertions(+), 881 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index 9eb3725f..6d7795a2 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -2,8 +2,7 @@ import numpy as np -from ._fit import fit_ramps -from ._jump import JUMP_DET, Parameter, Variance +from ._fit import JUMP_DET, Parameter, Variance, fit_ramps class DefaultThreshold(Enum): diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd index 75f723ab..4df8af8c 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.pxd @@ -1 +1,32 @@ # cython: language_level=3str + + +cpdef enum Parameter: + intercept + slope + n_param + + +cpdef enum Variance: + read_var + poisson_var + total_var + n_var + + +cpdef enum: + JUMP_DET = 4 + + +cpdef enum FixedOffsets: + t_bar_diff + t_bar_diff_sqr + read_recip + var_slope_val + n_fixed_offsets + + +cpdef enum PixelOffsets: + local_slope + var_read_noise + n_pixel_offsets diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index ecdd40e2..1d067bdf 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -27,14 +27,36 @@ is the primary externally callable function. """ import cython +from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, isnan, log10, sqrt from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.list import list as cpp_list from cython.cimports.libcpp.vector import vector -from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( - JumpFits, - Thresh, - _fill_fixed_values, - fit_jumps, +from cython.cimports.stcal.ramp_fitting.ols_cas22._fit import ( + JUMP_DET, + FixedOffsets, + Parameter, + PixelOffsets, + Variance, +) + +RampIndex = cython.struct( + start=cython.int, + end=cython.int, +) +RampQueue = cython.typedef(vector[RampIndex]) +RampFit = cython.struct( + slope=cython.float, + read_var=cython.float, + poisson_var=cython.float, +) +JumpFits = cython.struct( + jumps=vector[cython.int], + fits=vector[RampFit], + index=RampQueue, +) +Thresh = cython.struct( + intercept=cython.float, + constant=cython.float, ) @@ -122,7 +144,7 @@ def fit_ramps( index: cython.int for index in range(n_pixels): # Fit all the ramps for the given pixel - fit = fit_jumps( + fit = _fit_jumps( parameters[index, :], variances[index, :], resultants[:, index], @@ -149,6 +171,654 @@ def fit_ramps( return ramp_fits +_slope = cython.declare(cython.int, Parameter.slope) +_read_var = cython.declare(cython.int, Variance.read_var) +_poisson_var = cython.declare(cython.int, Variance.poisson_var) +_total_var = cython.declare(cython.int, Variance.total_var) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def _fit_jumps( + parameters: cython.float[:], + variances: cython.float[:], + resultants: cython.float[:], + dq: cython.int[:], + read_noise: cython.float, + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], + n_resultants: cython.int, + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + thresh: Thresh, + use_jump: cpp_bool, + include_diagnostic: cpp_bool, +) -> JumpFits: + """ + Compute all the ramps for a single pixel using the Casertano+22 algorithm + with jump detection. + + Parameters + ---------- + resultants : float[:] + The resultants for the pixel + dq : int[:] + The dq flags for the pixel. This is modified in place, so the external + dq flag array will be modified as a side-effect. + read_noise : float + The read noise for the pixel. + ramps : RampQueue + RampQueue for initial ramps to fit for the pixel + multiple ramps are possible due to dq flags + t_bar : float[:] + The average time for each resultant + tau : float[:] + The time variance for each resultant + n_reads : int[:] + The number of reads for each resultant + n_resultants : int + The number of resultants for the pixel + fixed : float[:, :] + The jump detection pre-computed values for a given read_pattern + pixel : float[:, :] + A pre-allocated array for the jump detection fixed values for the + given pixel. This will be modified in place, it is passed in to avoid + re-allocating it for each pixel. + thresh : Thresh + The threshold parameter struct for jump detection + use_jump : bool + Turn on or off jump detection. + include_diagnostic : bool + Turn on or off recording all the diaganostic information on the fit + + Returns + ------- + RampFits struct of all the fits for a single pixel + """ + # Find initial set of ramps + ramps: RampQueue = _init_ramps(dq, n_resultants) + + # Initialize algorithm + parameters[:] = 0 + variances[:] = 0 + + jumps: vector[cython.int] = vector[cython.int]() + fits: vector[RampFit] = vector[RampFit]() + index: RampQueue = RampQueue() + + # Declare variables for the loop + ramp: RampIndex + ramp_fit: RampFit + stat: Stat + jump0: cython.int + jump1: cython.int + weight: cython.float + total_weight: cython.float = 0 + + # Fill in the jump detection pre-compute values for a single pixel + if use_jump: + _fill_pixel_values( + single_pixel, double_pixel, single_fixed, double_fixed, resultants, read_noise, n_resultants + ) + + # Run while the Queue is non-empty + while not ramps.empty(): + # Remove top ramp of the stack to use + ramp = ramps.back() + ramps.pop_back() + + # Compute fit using the Casertano+22 algorithm + ramp_fit = _fit_ramp(resultants, t_bar, tau, n_reads, read_noise, ramp) + + # Run jump detection if enabled + if use_jump: + stat = _fit_statistic( + single_pixel, double_pixel, single_fixed, double_fixed, t_bar, ramp_fit.slope, ramp + ) + + # Note that when a "ramp" is a single point, _fit_statistic returns + # a NaN for max_stat. Note that NaN > anything is always false so the + # result drops through as desired. + if stat.max_stat > _threshold(thresh, ramp_fit.slope): + # Compute jump point to create two new ramps + # This jump point corresponds to the index of the largest + # statistic: + # argmax = argmax(stats) + # These statistics are indexed relative to the + # ramp's range. Therefore, we need to add the start index + # of the ramp to the result. + # + jump0 = stat.arg_max + ramp.start + + # Note that because the resultants are averages of reads, but + # jumps occur in individual reads, it is possible that the + # jump is averaged down by the resultant with the actual jump + # causing the computed jump to be off by one index. + # In the idealized case this is when the jump occurs near + # the start of the resultant with the jump. In this case, + # the statistic for the resultant will be maximized at + # index - 1 rather than index. This means that we have to + # remove argmax(stats) + 1 as it is also a possible jump. + # This case is difficult to distinguish from the case where + # argmax(stats) does correspond to the jump resultant. + # Therefore, we just remove both possible resultants from + # consideration. + jump1 = jump0 + 1 + + # Update the dq flags + dq[jump0] = JUMP_DET + dq[jump1] = JUMP_DET + + # Record jump diagnostics + if include_diagnostic: + jumps.push_back(jump0) + jumps.push_back(jump1) + + # The two resultant indices need to be skipped, therefore + # the two + # possible new ramps are: + # RampIndex(ramp.start, jump0 - 1) + # RampIndex(jump1 + 1, ramp.end) + # This is because the RampIndex contains the index of the + # first and last resulants in the sub-ramp it describes. + # Note: The algorithm works via working over the sub-ramps + # backward in time. Therefore, since we are using a stack, + # we need to add the ramps in the time order they were + # observed in. This results in the last observation ramp + # being the top of the stack; meaning that, + # it will be the next ramp handled. + + if jump0 > ramp.start: + # Note that when jump0 == ramp.start, we have detected a + # jump in the first resultant of the ramp. This means + # there is no sub-ramp before jump0. + # Also, note that this will produce bad results as + # the ramp indexing will go out of bounds. So it is + # important that we exclude it. + # Note that jump0 < ramp.start is not possible because + # the argmax is always >= 0 + ramps.push_back(RampIndex(ramp.start, jump0 - 1)) + + if jump1 < ramp.end: + # Note that if jump1 == ramp.end, we have detected a + # jump in the last resultant of the ramp. This means + # there is no sub-ramp after jump1. + # Also, note that this will produce bad results as + # the ramp indexing will go out of bounds. So it is + # important that we exclude it. + # Note that jump1 > ramp.end is technically possible + # however in those potential cases it will draw on + # resultants which are not considered part of the ramp + # under consideration. Therefore, we have to exclude all + # of those values. + ramps.push_back(RampIndex(jump1 + 1, ramp.end)) + + # Skip recording the ramp as it has a detected jump + continue + + # Start recording the the fit (no jump detected) + + # Record the diagnositcs + if include_diagnostic: + fits.push_back(ramp_fit) + index.push_back(ramp) + + # Start computing the averages using a lazy process + # Note we do not do anything in the NaN case for degenerate ramps + if not isnan(ramp_fit.slope): + # protect weight against the extremely unlikely case of a zero + # variance + weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var + total_weight += weight + + parameters[_slope] += weight * ramp_fit.slope + variances[_read_var] += weight**2 * ramp_fit.read_var + variances[_poisson_var] += weight**2 * ramp_fit.poisson_var + + # Finish computing averages using the lazy process + parameters[_slope] /= total_weight if total_weight != 0 else 1 + variances[_read_var] /= total_weight**2 if total_weight != 0 else 1 + variances[_poisson_var] /= total_weight**2 if total_weight != 0 else 1 + + # Multiply poisson term by flux, (no negative fluxes) + variances[_poisson_var] *= max(parameters[_slope], 0) + variances[_total_var] = variances[_read_var] + variances[_poisson_var] + + return JumpFits(jumps, fits, index) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def _fit_ramp( + resultants_: cython.float[:], + t_bar_: cython.float[:], + tau_: cython.float[:], + n_reads_: cython.int[:], + read_noise: cython.float, + ramp: RampIndex, +) -> RampFit: + """ + Fit a single ramp using Casertano+22 algorithm. + + Parameters + ---------- + resultants_ : float[:] + All of the resultants for the pixel + t_bar_ : float[:] + All the t_bar values + tau_ : float[:] + All the tau values + n_reads_ : int[:] + All the n_reads values + read_noise : float + The read noise for the pixel + ramp : RampIndex + Struct for start and end of ramp to fit + + Returns + ------- + RampFit + struct containing + - slope + - read_var + - poisson_var + """ + n_resultants: cython.int = ramp.end - ramp.start + 1 + + # Special case where there is no or one resultant, there is no fit and + # we bail out before any computations. + # Note that in this case, we cannot compute the slope or the variances + # because these computations require at least two resultants. Therefore, + # this case is degernate and we return NaNs for the values. + if n_resultants <= 1: + return RampFit(NAN, NAN, NAN) + + # Compute the fit + i: cython.int = 0 + j: cython.int = 0 + + # Setup data for fitting (work over subset of data) to make things cleaner + # Recall that the RampIndex contains the index of the first and last + # index of the ramp. Therefore, the Python slice needed to get all the + # data within the ramp is: + # ramp.start:ramp.end + 1 + resultants: cython.float[:] = resultants_[ramp.start : ramp.end + 1] + t_bar: cython.float[:] = t_bar_[ramp.start : ramp.end + 1] + tau: cython.float[:] = tau_[ramp.start : ramp.end + 1] + n_reads: cython.int[:] = n_reads_[ramp.start : ramp.end + 1] + + # Compute mid point time + end: cython.int = n_resultants - 1 + t_bar_mid: cython.float = (t_bar[0] + t_bar[end]) / 2 + + # Casertano+2022 Eq. 44 + # Note we've departed from Casertano+22 slightly; + # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., + # a CR in the first resultant has boosted the whole ramp high but there + # is no actual signal. + power: cython.float = fmaxf(resultants[end] - resultants[0], 0) + power = power / sqrt(read_noise**2 + power) + power = _get_power(power) + + # It's easy to use up a lot of dynamic range on something like + # (tbar - tbarmid) ** 10. Rescale these. + t_scale: cython.float = (t_bar[end] - t_bar[0]) / 2 + t_scale = 1 if t_scale == 0 else t_scale + + # Initialize the fit loop + # it is faster to generate a c++ vector than a numpy array + weights: vector[cython.float] = vector[float](n_resultants) + coeffs: vector[cython.float] = vector[float](n_resultants) + ramp_fit: RampFit = RampFit(0, 0, 0) + f0: cython.float = 0 + f1: cython.float = 0 + f2: cython.float = 0 + coeff: cython.float + + # Issue when tbar[] == tbarmid causes exception otherwise + with cython.cpow(True): + for i in range(n_resultants): + # Casertano+22, Eq. 45 + weights[i] = (((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * fabs( + (t_bar[i] - t_bar_mid) / t_scale + ) ** power + + # Casertano+22 Eq. 35 + f0 += weights[i] + f1 += weights[i] * t_bar[i] + f2 += weights[i] * t_bar[i] ** 2 + + # Casertano+22 Eq. 36 + det: cython.float = f2 * f0 - f1**2 + if det == 0: + return ramp_fit + + for i in range(n_resultants): + # Casertano+22 Eq. 37 + coeff = (f0 * t_bar[i] - f1) * weights[i] / det + coeffs[i] = coeff + + # Casertano+22 Eq. 38 + ramp_fit.slope += coeff * resultants[i] + + # Casertano+22 Eq. 39 + ramp_fit.read_var += coeff**2 * read_noise**2 / n_reads[i] + + # Casertano+22 Eq 40 + # Note that this is an inversion of the indexing from the equation; + # however, commutivity of addition results in the same answer. This + # makes it so that we don't have to loop over all the resultants twice. + ramp_fit.poisson_var += coeff**2 * tau[i] + for j in range(i): + ramp_fit.poisson_var += 2 * coeff * coeffs[j] * t_bar[j] + + return ramp_fit + + +# Casertano+2022, Table 2 +_P_TABLE = cython.declare( + cython.float[6][2], + [ + [-INFINITY, 5, 10, 20, 50, 100], + [0, 0.4, 1, 3, 6, 10], + ], +) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.cfunc +@cython.exceptval(check=False) +def _get_power(signal: cython.float) -> cython.float: + """ + Return the power from Casertano+22, Table 2. + + Parameters + ---------- + signal: float + signal from the resultants + + Returns + ------- + signal power from Table 2 + """ + i: cython.int + for i in range(6): + if signal < _P_TABLE[0][i]: + return _P_TABLE[1][i - 1] + + return _P_TABLE[1][i] + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.ccall +def _init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: + """ + Create the initial ramp "queue" for each pixel + if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp + otherwise, the resultant is not in a ramp. + + Parameters + ---------- + dq : int[n_resultants] + DQ array + n_resultants : int + Number of resultants + + Returns + ------- + RampQueue + vector of RampIndex objects + - vector with entry for each ramp found (last entry is last ramp found) + - RampIndex with start and end indices of the ramp in the resultants + """ + ramps: RampQueue = RampQueue() + + # Note: if start/end are -1, then no value has been assigned + # ramp.start == -1 means we have not started a ramp + # dq[index_resultant, index_pixel] == 0 means resultant is in ramp + ramp: RampIndex = RampIndex(-1, -1) + index_resultant: cython.int + for index_resultant in range(n_resultants): + # Checking for start of ramp + if ramp.start == -1: + if dq[index_resultant] == 0: + # This resultant is in the ramp + # => We have found the start of a ramp! + ramp.start = index_resultant + + # This resultant cannot be the start of a ramp + # => Checking for end of ramp + elif dq[index_resultant] != 0: + # This pixel is not in the ramp + # => index_resultant - 1 is the end of the ramp + ramp.end = index_resultant - 1 + + # Add completed ramp to the queue and reset ramp + ramps.push_back(ramp) + ramp = RampIndex(-1, -1) + + # Handle case where last resultant is in ramp (so no end has been set) + if ramp.start != -1 and ramp.end == -1: + # Last resultant is end of the ramp => set then add to stack + ramp.end = n_resultants - 1 + ramps.push_back(ramp) + + return ramps + + +@cython.inline +@cython.cfunc +@cython.exceptval(check=False) +def _threshold(thresh: Thresh, slope: cython.float) -> cython.float: + """ + Compute jump threshold. + + Parameters + ---------- + thresh : Thresh + threshold parameters struct + slope : float + slope of the ramp in question + + Returns + ------- + intercept - constant * log10(slope) + """ + slope = slope if slope > 1 else 1 + slope = slope if slope < 1e4 else 1e4 + + return thresh.intercept - thresh.constant * log10(slope) + + +Stat = cython.struct(arg_max=cython.int, max_stat=cython.float) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.cfunc +def _fit_statistic( + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + t_bar: cython.float[:], + slope: cython.float, + ramp: RampIndex, +) -> Stat: + """ + Compute the maximum index and its value over all fit statistics for a given + ramp. Each index's stat is the max of the single and double difference + statistics: + all_stats = . + + Parameters + ---------- + pixel : float[:, :] + The pre-computed fixed values for a given pixel + fixed : float[:, :] + The pre-computed fixed values for a given read_pattern + t_bar : float[:, :] + The average time for each resultant + slope : float + The computed slope for the ramp + ramp : RampIndex + Struct for start and end of ramp to fit + + Returns + ------- + argmax(all_stats), max(all_stats) + """ + # Note that a ramp consisting of a single point is degenerate and has no + # fit statistic so we bail out here + if ramp.start == ramp.end: + return Stat(0, NAN) + + # Start computing fit statistics + correct: cython.float = _correction(t_bar, ramp, slope) + + # We are computing single and double differences of using the ramp's resultants. + # Each of these computations requires two points meaning that there are + # start - end - 1 possible differences. However, we cannot compute a double + # difference for the last point as there is no point after it. Therefore, + # We use this point's single difference as our initial guess for the fit + # statistic. Note that the fit statistic can technically be negative so + # this makes it much easier to compute a "lazy" max. + index: cython.int = ramp.end - 1 + stat: Stat = Stat( + ramp.end - ramp.start - 1, + _statistic( + single_pixel[_local_slope, index], + single_pixel[_var_read_noise, index], + single_fixed[_t_bar_diff_sqr, index], + single_fixed[_var_slope_val, index], + slope, + correct, + ), + ) + + # Compute the rest of the fit statistics + max_stat: cython.float + single_stat: cython.float + double_stat: cython.float + arg_max: cython.int + for arg_max, index in enumerate(range(ramp.start, ramp.end - 1)): + # Compute max of single and double difference statistics + single_stat = _statistic( + single_pixel[_local_slope, index], + single_pixel[_var_read_noise, index], + single_fixed[_t_bar_diff_sqr, index], + single_fixed[_var_slope_val, index], + slope, + correct, + ) + double_stat = _statistic( + double_pixel[_local_slope, index], + double_pixel[_var_read_noise, index], + double_fixed[_t_bar_diff_sqr, index], + double_fixed[_var_slope_val, index], + slope, + correct, + ) + max_stat = fmaxf(single_stat, double_stat) + + # If this is larger than the current max, update the max + if max_stat > stat.max_stat: + stat = Stat(arg_max, max_stat) + + return stat + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def _statistic( + local_slope: cython.float, + var_read_noise: cython.float, + t_bar_diff_sqr: cython.float, + var_slope_val: cython.float, + slope: cython.float, + correct: cython.float, +) -> cython.float: + """ + Compute a single fit statistic + delta / sqrt(var + correct). + + where: + delta = _local_slope - slope + var = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr + + pre-computed: + local_slope = (resultant[i + j] - resultant[i]) / (t_bar[i + j] - t_bar[i]) + var_read_noise = read_noise ** 2 * (1/n_reads[i + j] + 1/n_reads[i]) + var_slope_coeff = tau[i + j] + tau[i] - 2 * min(t_bar[i + j], t_bar[i]) + t_bar_diff_sqr = (t_bar[i + j] - t_bar[i]) ** 2 + + Parameters + ---------- + local_slope : float + The local slope the statistic is computed for + var_read_noise: float + The read noise variance for _local_slope + t_bar_diff_sqr : float + The square difference for the t_bar corresponding to _local_slope + var_slope_val : float + The slope variance coefficient for _local_slope + slope : float + The computed slope for the ramp + correct : float + The correction factor needed + + Returns + ------- + Create a single instance of the stastic for the given parameters + """ + delta: cython.float = local_slope - slope + var: cython.float = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr + + return delta / sqrt(var + correct) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.cfunc +def _correction(t_bar: cython.float[:], ramp: RampIndex, slope: cython.float) -> cython.float: + """ + Compute the correction factor for the variance used by a statistic. + + - slope / (t_bar[end] - t_bar[start]) + + Parameters + ---------- + t_bar : float[:] + The computed t_bar values for the ramp + ramp : RampIndex + Struct for start and end indices resultants for the ramp + slope : float + The computed slope for the ramp + """ + diff: cython.float = t_bar[ramp.end] - t_bar[ramp.start] + + return -slope / diff + + @cython.boundscheck(False) @cython.wraparound(False) @cython.cdivision(True) @@ -181,3 +851,127 @@ def _fill_metadata( t_bar[i] /= n_read tau[i] *= read_time / n_read**2 + + +_t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) +_t_bar_diff_sqr = cython.declare(cython.int, FixedOffsets.t_bar_diff_sqr) +_read_recip = cython.declare(cython.int, FixedOffsets.read_recip) +_var_slope_val = cython.declare(cython.int, FixedOffsets.var_slope_val) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.ccall +def _fill_fixed_values( + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + t_bar: cython.float[:], + tau: cython.float[:], + n_reads: cython.int[:], + n_resultants: cython.int, +) -> cython.void: + """ + Pre-compute all the values needed for jump detection which only depend on + the read pattern. + + Parameters + ---------- + fixed : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. + t_bar : float[:] + The average time for each resultant + tau : float[:] + The time variance for each resultant + n_reads : int[:] + The number of reads for each resultant + n_resultants : int + The number of resultants for the read pattern + + Returns + ------- + [ + , + , + ** 2, + ** 2, + <(1/n_reads[i+1] + 1/n_reads[i])>, + <(1/n_reads[i+2] + 1/n_reads[i])>, + <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, + <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, + ] + """ + # Coerce division to be using floats + num: cython.float = 1 + + i: cython.int + for i in range(n_resultants - 1): + single_fixed[_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] + single_fixed[_t_bar_diff_sqr, i] = single_fixed[_t_bar_diff, i] ** 2 + single_fixed[_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) + single_fixed[_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) + + if i < n_resultants - 2: + double_fixed[_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] + double_fixed[_t_bar_diff_sqr, i] = double_fixed[_t_bar_diff, i] ** 2 + double_fixed[_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) + double_fixed[_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) + + +_local_slope = cython.declare(cython.int, PixelOffsets.local_slope) +_var_read_noise = cython.declare(cython.int, PixelOffsets.var_read_noise) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.cdivision(True) +@cython.inline +@cython.ccall +def _fill_pixel_values( + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + resultants: cython.float[:], + read_noise: cython.float, + n_resultants: cython.int, +) -> cython.void: + """ + Pre-compute all the values needed for jump detection which only depend on + the a specific pixel (independent of the given ramp for a pixel). + + Parameters + ---------- + pixel : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. + resultants : float[:] + The resultants for the pixel in question. + fixed : float[:, :] + The pre-computed fixed values for the read_pattern + read_noise : float + The read noise for the pixel + n_resultants : int + The number of resultants for the read_pattern + + Returns + ------- + [ + <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, + <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, + read_noise**2 * <(1/n_reads[i+1] + 1/n_reads[i])>, + read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, + ] + """ + read_noise_sqr: cython.float = read_noise**2 + + i: cython.int + for i in range(n_resultants - 1): + single_pixel[_local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[_t_bar_diff, i] + single_pixel[_var_read_noise, i] = read_noise_sqr * single_fixed[_read_recip, i] + + if i < n_resultants - 2: + double_pixel[_local_slope, i] = (resultants[i + 2] - resultants[i]) / double_fixed[_t_bar_diff, i] + double_pixel[_var_read_noise, i] = read_noise_sqr * double_fixed[_read_recip, i] diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd index c2c381b2..75f723ab 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd @@ -1,85 +1 @@ # cython: language_level=3str - -from libcpp cimport bool -from libcpp.vector cimport vector - - -cpdef enum FixedOffsets: - t_bar_diff - t_bar_diff_sqr - read_recip - var_slope_val - n_fixed_offsets - - -cpdef enum PixelOffsets: - local_slope - var_read_noise - n_pixel_offsets - - -cpdef enum Parameter: - intercept - slope - n_param - - -cpdef enum Variance: - read_var - poisson_var - total_var - n_var - - -cpdef enum: - JUMP_DET = 4 - -cdef struct Thresh: - float intercept - float constant - - -cdef struct RampIndex: - int start - int end - - -ctypedef vector[RampIndex] RampQueue - - -cdef struct RampFit: - float slope - float read_var - float poisson_var - - -cdef struct JumpFits: - vector[int] jumps - vector[RampFit] fits - RampQueue index - - -cpdef void _fill_fixed_values(float[:, :] single_fixed, - float[:, :] double_fixed, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants) - - -cdef JumpFits fit_jumps(float[:] parameters, - float[:] variances, - float[:] resultants, - int[:] dq, - float read_noise, - float[:] t_bar, - float[:] tau, - int[:] n_reads, - int n_resultants, - float[:, :] single_pixel, - float[:, :] double_pixel, - float[:, :] single_fixed, - float[:, :] double_fixed, - Thresh thresh, - bool use_jump, - bool include_diagnostic) diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.py b/src/stcal/ramp_fitting/ols_cas22/_jump.py index d7897756..d82eabf5 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.py +++ b/src/stcal/ramp_fitting/ols_cas22/_jump.py @@ -53,790 +53,3 @@ meaning it automatically handles splitting ramps across dq flags in addition to splitting across detected jumps (if jump detection is turned on). """ -import cython -from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, isnan, log10, sqrt -from cython.cimports.libcpp import bool as cpp_bool -from cython.cimports.libcpp.vector import vector -from cython.cimports.stcal.ramp_fitting.ols_cas22._jump import ( - JUMP_DET, - FixedOffsets, - JumpFits, - Parameter, - PixelOffsets, - RampFit, - RampIndex, - RampQueue, - Thresh, - Variance, -) - -_slope = cython.declare(cython.int, Parameter.slope) -_read_var = cython.declare(cython.int, Variance.read_var) -_poisson_var = cython.declare(cython.int, Variance.poisson_var) -_total_var = cython.declare(cython.int, Variance.total_var) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.cfunc -def fit_jumps( - parameters: cython.float[:], - variances: cython.float[:], - resultants: cython.float[:], - dq: cython.int[:], - read_noise: cython.float, - t_bar: cython.float[:], - tau: cython.float[:], - n_reads: cython.int[:], - n_resultants: cython.int, - single_pixel: cython.float[:, :], - double_pixel: cython.float[:, :], - single_fixed: cython.float[:, :], - double_fixed: cython.float[:, :], - thresh: Thresh, - use_jump: cpp_bool, - include_diagnostic: cpp_bool, -) -> JumpFits: - """ - Compute all the ramps for a single pixel using the Casertano+22 algorithm - with jump detection. - - Parameters - ---------- - resultants : float[:] - The resultants for the pixel - dq : int[:] - The dq flags for the pixel. This is modified in place, so the external - dq flag array will be modified as a side-effect. - read_noise : float - The read noise for the pixel. - ramps : RampQueue - RampQueue for initial ramps to fit for the pixel - multiple ramps are possible due to dq flags - t_bar : float[:] - The average time for each resultant - tau : float[:] - The time variance for each resultant - n_reads : int[:] - The number of reads for each resultant - n_resultants : int - The number of resultants for the pixel - fixed : float[:, :] - The jump detection pre-computed values for a given read_pattern - pixel : float[:, :] - A pre-allocated array for the jump detection fixed values for the - given pixel. This will be modified in place, it is passed in to avoid - re-allocating it for each pixel. - thresh : Thresh - The threshold parameter struct for jump detection - use_jump : bool - Turn on or off jump detection. - include_diagnostic : bool - Turn on or off recording all the diaganostic information on the fit - - Returns - ------- - RampFits struct of all the fits for a single pixel - """ - # Find initial set of ramps - ramps: RampQueue = _init_ramps(dq, n_resultants) - - # Initialize algorithm - parameters[:] = 0 - variances[:] = 0 - - jumps: vector[cython.int] = vector[cython.int]() - fits: vector[RampFit] = vector[RampFit]() - index: RampQueue = RampQueue() - - # Declare variables for the loop - ramp: RampIndex - ramp_fit: RampFit - stat: Stat - jump0: cython.int - jump1: cython.int - weight: cython.float - total_weight: cython.float = 0 - - # Fill in the jump detection pre-compute values for a single pixel - if use_jump: - _fill_pixel_values( - single_pixel, double_pixel, single_fixed, double_fixed, resultants, read_noise, n_resultants - ) - - # Run while the Queue is non-empty - while not ramps.empty(): - # Remove top ramp of the stack to use - ramp = ramps.back() - ramps.pop_back() - - # Compute fit using the Casertano+22 algorithm - ramp_fit = _fit_ramp(resultants, t_bar, tau, n_reads, read_noise, ramp) - - # Run jump detection if enabled - if use_jump: - stat = _fit_statistic( - single_pixel, double_pixel, single_fixed, double_fixed, t_bar, ramp_fit.slope, ramp - ) - - # Note that when a "ramp" is a single point, _fit_statistic returns - # a NaN for max_stat. Note that NaN > anything is always false so the - # result drops through as desired. - if stat.max_stat > _threshold(thresh, ramp_fit.slope): - # Compute jump point to create two new ramps - # This jump point corresponds to the index of the largest - # statistic: - # argmax = argmax(stats) - # These statistics are indexed relative to the - # ramp's range. Therefore, we need to add the start index - # of the ramp to the result. - # - jump0 = stat.arg_max + ramp.start - - # Note that because the resultants are averages of reads, but - # jumps occur in individual reads, it is possible that the - # jump is averaged down by the resultant with the actual jump - # causing the computed jump to be off by one index. - # In the idealized case this is when the jump occurs near - # the start of the resultant with the jump. In this case, - # the statistic for the resultant will be maximized at - # index - 1 rather than index. This means that we have to - # remove argmax(stats) + 1 as it is also a possible jump. - # This case is difficult to distinguish from the case where - # argmax(stats) does correspond to the jump resultant. - # Therefore, we just remove both possible resultants from - # consideration. - jump1 = jump0 + 1 - - # Update the dq flags - dq[jump0] = JUMP_DET - dq[jump1] = JUMP_DET - - # Record jump diagnostics - if include_diagnostic: - jumps.push_back(jump0) - jumps.push_back(jump1) - - # The two resultant indices need to be skipped, therefore - # the two - # possible new ramps are: - # RampIndex(ramp.start, jump0 - 1) - # RampIndex(jump1 + 1, ramp.end) - # This is because the RampIndex contains the index of the - # first and last resulants in the sub-ramp it describes. - # Note: The algorithm works via working over the sub-ramps - # backward in time. Therefore, since we are using a stack, - # we need to add the ramps in the time order they were - # observed in. This results in the last observation ramp - # being the top of the stack; meaning that, - # it will be the next ramp handled. - - if jump0 > ramp.start: - # Note that when jump0 == ramp.start, we have detected a - # jump in the first resultant of the ramp. This means - # there is no sub-ramp before jump0. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump0 < ramp.start is not possible because - # the argmax is always >= 0 - ramps.push_back(RampIndex(ramp.start, jump0 - 1)) - - if jump1 < ramp.end: - # Note that if jump1 == ramp.end, we have detected a - # jump in the last resultant of the ramp. This means - # there is no sub-ramp after jump1. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump1 > ramp.end is technically possible - # however in those potential cases it will draw on - # resultants which are not considered part of the ramp - # under consideration. Therefore, we have to exclude all - # of those values. - ramps.push_back(RampIndex(jump1 + 1, ramp.end)) - - # Skip recording the ramp as it has a detected jump - continue - - # Start recording the the fit (no jump detected) - - # Record the diagnositcs - if include_diagnostic: - fits.push_back(ramp_fit) - index.push_back(ramp) - - # Start computing the averages using a lazy process - # Note we do not do anything in the NaN case for degenerate ramps - if not isnan(ramp_fit.slope): - # protect weight against the extremely unlikely case of a zero - # variance - weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var - total_weight += weight - - parameters[_slope] += weight * ramp_fit.slope - variances[_read_var] += weight**2 * ramp_fit.read_var - variances[_poisson_var] += weight**2 * ramp_fit.poisson_var - - # Finish computing averages using the lazy process - parameters[_slope] /= total_weight if total_weight != 0 else 1 - variances[_read_var] /= total_weight**2 if total_weight != 0 else 1 - variances[_poisson_var] /= total_weight**2 if total_weight != 0 else 1 - - # Multiply poisson term by flux, (no negative fluxes) - variances[_poisson_var] *= max(parameters[_slope], 0) - variances[_total_var] = variances[_read_var] + variances[_poisson_var] - - return JumpFits(jumps, fits, index) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.cfunc -def _fit_ramp( - resultants_: cython.float[:], - t_bar_: cython.float[:], - tau_: cython.float[:], - n_reads_: cython.int[:], - read_noise: cython.float, - ramp: RampIndex, -) -> RampFit: - """ - Fit a single ramp using Casertano+22 algorithm. - - Parameters - ---------- - resultants_ : float[:] - All of the resultants for the pixel - t_bar_ : float[:] - All the t_bar values - tau_ : float[:] - All the tau values - n_reads_ : int[:] - All the n_reads values - read_noise : float - The read noise for the pixel - ramp : RampIndex - Struct for start and end of ramp to fit - - Returns - ------- - RampFit - struct containing - - slope - - read_var - - poisson_var - """ - n_resultants: cython.int = ramp.end - ramp.start + 1 - - # Special case where there is no or one resultant, there is no fit and - # we bail out before any computations. - # Note that in this case, we cannot compute the slope or the variances - # because these computations require at least two resultants. Therefore, - # this case is degernate and we return NaNs for the values. - if n_resultants <= 1: - return RampFit(NAN, NAN, NAN) - - # Compute the fit - i: cython.int = 0 - j: cython.int = 0 - - # Setup data for fitting (work over subset of data) to make things cleaner - # Recall that the RampIndex contains the index of the first and last - # index of the ramp. Therefore, the Python slice needed to get all the - # data within the ramp is: - # ramp.start:ramp.end + 1 - resultants: cython.float[:] = resultants_[ramp.start : ramp.end + 1] - t_bar: cython.float[:] = t_bar_[ramp.start : ramp.end + 1] - tau: cython.float[:] = tau_[ramp.start : ramp.end + 1] - n_reads: cython.int[:] = n_reads_[ramp.start : ramp.end + 1] - - # Compute mid point time - end: cython.int = n_resultants - 1 - t_bar_mid: cython.float = (t_bar[0] + t_bar[end]) / 2 - - # Casertano+2022 Eq. 44 - # Note we've departed from Casertano+22 slightly; - # there s is just resultants[ramp.end]. But that doesn't seem good if, e.g., - # a CR in the first resultant has boosted the whole ramp high but there - # is no actual signal. - power: cython.float = fmaxf(resultants[end] - resultants[0], 0) - power = power / sqrt(read_noise**2 + power) - power = _get_power(power) - - # It's easy to use up a lot of dynamic range on something like - # (tbar - tbarmid) ** 10. Rescale these. - t_scale: cython.float = (t_bar[end] - t_bar[0]) / 2 - t_scale = 1 if t_scale == 0 else t_scale - - # Initialize the fit loop - # it is faster to generate a c++ vector than a numpy array - weights: vector[cython.float] = vector[float](n_resultants) - coeffs: vector[cython.float] = vector[float](n_resultants) - ramp_fit: RampFit = RampFit(0, 0, 0) - f0: cython.float = 0 - f1: cython.float = 0 - f2: cython.float = 0 - coeff: cython.float - - # Issue when tbar[] == tbarmid causes exception otherwise - with cython.cpow(True): - for i in range(n_resultants): - # Casertano+22, Eq. 45 - weights[i] = (((1 + power) * n_reads[i]) / (1 + power * n_reads[i])) * fabs( - (t_bar[i] - t_bar_mid) / t_scale - ) ** power - - # Casertano+22 Eq. 35 - f0 += weights[i] - f1 += weights[i] * t_bar[i] - f2 += weights[i] * t_bar[i] ** 2 - - # Casertano+22 Eq. 36 - det: cython.float = f2 * f0 - f1**2 - if det == 0: - return ramp_fit - - for i in range(n_resultants): - # Casertano+22 Eq. 37 - coeff = (f0 * t_bar[i] - f1) * weights[i] / det - coeffs[i] = coeff - - # Casertano+22 Eq. 38 - ramp_fit.slope += coeff * resultants[i] - - # Casertano+22 Eq. 39 - ramp_fit.read_var += coeff**2 * read_noise**2 / n_reads[i] - - # Casertano+22 Eq 40 - # Note that this is an inversion of the indexing from the equation; - # however, commutivity of addition results in the same answer. This - # makes it so that we don't have to loop over all the resultants twice. - ramp_fit.poisson_var += coeff**2 * tau[i] - for j in range(i): - ramp_fit.poisson_var += 2 * coeff * coeffs[j] * t_bar[j] - - return ramp_fit - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.inline -@cython.ccall -def _init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: - """ - Create the initial ramp "queue" for each pixel - if dq[index_resultant, index_pixel] == 0, then the resultant is in a ramp - otherwise, the resultant is not in a ramp. - - Parameters - ---------- - dq : int[n_resultants] - DQ array - n_resultants : int - Number of resultants - - Returns - ------- - RampQueue - vector of RampIndex objects - - vector with entry for each ramp found (last entry is last ramp found) - - RampIndex with start and end indices of the ramp in the resultants - """ - ramps: RampQueue = RampQueue() - - # Note: if start/end are -1, then no value has been assigned - # ramp.start == -1 means we have not started a ramp - # dq[index_resultant, index_pixel] == 0 means resultant is in ramp - ramp: RampIndex = RampIndex(-1, -1) - index_resultant: cython.int - for index_resultant in range(n_resultants): - # Checking for start of ramp - if ramp.start == -1: - if dq[index_resultant] == 0: - # This resultant is in the ramp - # => We have found the start of a ramp! - ramp.start = index_resultant - - # This resultant cannot be the start of a ramp - # => Checking for end of ramp - elif dq[index_resultant] != 0: - # This pixel is not in the ramp - # => index_resultant - 1 is the end of the ramp - ramp.end = index_resultant - 1 - - # Add completed ramp to the queue and reset ramp - ramps.push_back(ramp) - ramp = RampIndex(-1, -1) - - # Handle case where last resultant is in ramp (so no end has been set) - if ramp.start != -1 and ramp.end == -1: - # Last resultant is end of the ramp => set then add to stack - ramp.end = n_resultants - 1 - ramps.push_back(ramp) - - return ramps - - -@cython.inline -@cython.cfunc -@cython.exceptval(check=False) -def _threshold(thresh: Thresh, slope: cython.float) -> cython.float: - """ - Compute jump threshold. - - Parameters - ---------- - thresh : Thresh - threshold parameters struct - slope : float - slope of the ramp in question - - Returns - ------- - intercept - constant * log10(slope) - """ - slope = slope if slope > 1 else 1 - slope = slope if slope < 1e4 else 1e4 - - return thresh.intercept - thresh.constant * log10(slope) - - -Stat = cython.struct(arg_max=cython.int, max_stat=cython.float) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.inline -@cython.cfunc -def _fit_statistic( - single_pixel: cython.float[:, :], - double_pixel: cython.float[:, :], - single_fixed: cython.float[:, :], - double_fixed: cython.float[:, :], - t_bar: cython.float[:], - slope: cython.float, - ramp: RampIndex, -) -> Stat: - """ - Compute the maximum index and its value over all fit statistics for a given - ramp. Each index's stat is the max of the single and double difference - statistics: - all_stats = . - - Parameters - ---------- - pixel : float[:, :] - The pre-computed fixed values for a given pixel - fixed : float[:, :] - The pre-computed fixed values for a given read_pattern - t_bar : float[:, :] - The average time for each resultant - slope : float - The computed slope for the ramp - ramp : RampIndex - Struct for start and end of ramp to fit - - Returns - ------- - argmax(all_stats), max(all_stats) - """ - # Note that a ramp consisting of a single point is degenerate and has no - # fit statistic so we bail out here - if ramp.start == ramp.end: - return Stat(0, NAN) - - # Start computing fit statistics - correct: cython.float = _correction(t_bar, ramp, slope) - - # We are computing single and double differences of using the ramp's resultants. - # Each of these computations requires two points meaning that there are - # start - end - 1 possible differences. However, we cannot compute a double - # difference for the last point as there is no point after it. Therefore, - # We use this point's single difference as our initial guess for the fit - # statistic. Note that the fit statistic can technically be negative so - # this makes it much easier to compute a "lazy" max. - index: cython.int = ramp.end - 1 - stat: Stat = Stat( - ramp.end - ramp.start - 1, - _statistic( - single_pixel[_local_slope, index], - single_pixel[_var_read_noise, index], - single_fixed[_t_bar_diff_sqr, index], - single_fixed[_var_slope_val, index], - slope, - correct, - ), - ) - - # Compute the rest of the fit statistics - max_stat: cython.float - single_stat: cython.float - double_stat: cython.float - arg_max: cython.int - for arg_max, index in enumerate(range(ramp.start, ramp.end - 1)): - # Compute max of single and double difference statistics - single_stat = _statistic( - single_pixel[_local_slope, index], - single_pixel[_var_read_noise, index], - single_fixed[_t_bar_diff_sqr, index], - single_fixed[_var_slope_val, index], - slope, - correct, - ) - double_stat = _statistic( - double_pixel[_local_slope, index], - double_pixel[_var_read_noise, index], - double_fixed[_t_bar_diff_sqr, index], - double_fixed[_var_slope_val, index], - slope, - correct, - ) - max_stat = fmaxf(single_stat, double_stat) - - # If this is larger than the current max, update the max - if max_stat > stat.max_stat: - stat = Stat(arg_max, max_stat) - - return stat - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.cfunc -def _statistic( - local_slope: cython.float, - var_read_noise: cython.float, - t_bar_diff_sqr: cython.float, - var_slope_val: cython.float, - slope: cython.float, - correct: cython.float, -) -> cython.float: - """ - Compute a single fit statistic - delta / sqrt(var + correct). - - where: - delta = _local_slope - slope - var = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr - - pre-computed: - local_slope = (resultant[i + j] - resultant[i]) / (t_bar[i + j] - t_bar[i]) - var_read_noise = read_noise ** 2 * (1/n_reads[i + j] + 1/n_reads[i]) - var_slope_coeff = tau[i + j] + tau[i] - 2 * min(t_bar[i + j], t_bar[i]) - t_bar_diff_sqr = (t_bar[i + j] - t_bar[i]) ** 2 - - Parameters - ---------- - local_slope : float - The local slope the statistic is computed for - var_read_noise: float - The read noise variance for _local_slope - t_bar_diff_sqr : float - The square difference for the t_bar corresponding to _local_slope - var_slope_val : float - The slope variance coefficient for _local_slope - slope : float - The computed slope for the ramp - correct : float - The correction factor needed - - Returns - ------- - Create a single instance of the stastic for the given parameters - """ - delta: cython.float = local_slope - slope - var: cython.float = (var_read_noise + slope * var_slope_val) / t_bar_diff_sqr - - return delta / sqrt(var + correct) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.cfunc -def _correction(t_bar: cython.float[:], ramp: RampIndex, slope: cython.float) -> cython.float: - """ - Compute the correction factor for the variance used by a statistic. - - - slope / (t_bar[end] - t_bar[start]) - - Parameters - ---------- - t_bar : float[:] - The computed t_bar values for the ramp - ramp : RampIndex - Struct for start and end indices resultants for the ramp - slope : float - The computed slope for the ramp - """ - diff: cython.float = t_bar[ramp.end] - t_bar[ramp.start] - - return -slope / diff - - -_t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) -_t_bar_diff_sqr = cython.declare(cython.int, FixedOffsets.t_bar_diff_sqr) -_read_recip = cython.declare(cython.int, FixedOffsets.read_recip) -_var_slope_val = cython.declare(cython.int, FixedOffsets.var_slope_val) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.ccall -def _fill_fixed_values( - single_fixed: cython.float[:, :], - double_fixed: cython.float[:, :], - t_bar: cython.float[:], - tau: cython.float[:], - n_reads: cython.int[:], - n_resultants: cython.int, -) -> cython.void: - """ - Pre-compute all the values needed for jump detection which only depend on - the read pattern. - - Parameters - ---------- - fixed : float[:, :] - A pre-allocated memoryview to store the pre-computed values in, its faster - to allocate outside this function. - t_bar : float[:] - The average time for each resultant - tau : float[:] - The time variance for each resultant - n_reads : int[:] - The number of reads for each resultant - n_resultants : int - The number of resultants for the read pattern - - Returns - ------- - [ - , - , - ** 2, - ** 2, - <(1/n_reads[i+1] + 1/n_reads[i])>, - <(1/n_reads[i+2] + 1/n_reads[i])>, - <(tau[i] + tau[i+1] - 2 * min(t_bar[i], t_bar[i+1]))>, - <(tau[i] + tau[i+2] - 2 * min(t_bar[i], t_bar[i+2]))>, - ] - """ - # Coerce division to be using floats - num: cython.float = 1 - - i: cython.int - for i in range(n_resultants - 1): - single_fixed[_t_bar_diff, i] = t_bar[i + 1] - t_bar[i] - single_fixed[_t_bar_diff_sqr, i] = single_fixed[_t_bar_diff, i] ** 2 - single_fixed[_read_recip, i] = (num / n_reads[i + 1]) + (num / n_reads[i]) - single_fixed[_var_slope_val, i] = tau[i + 1] + tau[i] - 2 * min(t_bar[i + 1], t_bar[i]) - - if i < n_resultants - 2: - double_fixed[_t_bar_diff, i] = t_bar[i + 2] - t_bar[i] - double_fixed[_t_bar_diff_sqr, i] = double_fixed[_t_bar_diff, i] ** 2 - double_fixed[_read_recip, i] = (num / n_reads[i + 2]) + (num / n_reads[i]) - double_fixed[_var_slope_val, i] = tau[i + 2] + tau[i] - 2 * min(t_bar[i + 2], t_bar[i]) - - -_local_slope = cython.declare(cython.int, PixelOffsets.local_slope) -_var_read_noise = cython.declare(cython.int, PixelOffsets.var_read_noise) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.ccall -def _fill_pixel_values( - single_pixel: cython.float[:, :], - double_pixel: cython.float[:, :], - single_fixed: cython.float[:, :], - double_fixed: cython.float[:, :], - resultants: cython.float[:], - read_noise: cython.float, - n_resultants: cython.int, -) -> cython.void: - """ - Pre-compute all the values needed for jump detection which only depend on - the a specific pixel (independent of the given ramp for a pixel). - - Parameters - ---------- - pixel : float[:, :] - A pre-allocated memoryview to store the pre-computed values in, its faster - to allocate outside this function. - resultants : float[:] - The resultants for the pixel in question. - fixed : float[:, :] - The pre-computed fixed values for the read_pattern - read_noise : float - The read noise for the pixel - n_resultants : int - The number of resultants for the read_pattern - - Returns - ------- - [ - <(resultants[i+1] - resultants[i])> / <(t_bar[i+1] - t_bar[i])>, - <(resultants[i+2] - resultants[i])> / <(t_bar[i+2] - t_bar[i])>, - read_noise**2 * <(1/n_reads[i+1] + 1/n_reads[i])>, - read_noise**2 * <(1/n_reads[i+2] + 1/n_reads[i])>, - ] - """ - read_noise_sqr: cython.float = read_noise**2 - - i: cython.int - for i in range(n_resultants - 1): - single_pixel[_local_slope, i] = (resultants[i + 1] - resultants[i]) / single_fixed[_t_bar_diff, i] - single_pixel[_var_read_noise, i] = read_noise_sqr * single_fixed[_read_recip, i] - - if i < n_resultants - 2: - double_pixel[_local_slope, i] = (resultants[i + 2] - resultants[i]) / double_fixed[_t_bar_diff, i] - double_pixel[_var_read_noise, i] = read_noise_sqr * double_fixed[_read_recip, i] - - -# Casertano+2022, Table 2 -_P_TABLE = cython.declare( - cython.float[6][2], - [ - [-INFINITY, 5, 10, 20, 50, 100], - [0, 0.4, 1, 3, 6, 10], - ], -) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.inline -@cython.cfunc -@cython.exceptval(check=False) -def _get_power(signal: cython.float) -> cython.float: - """ - Return the power from Casertano+22, Table 2. - - Parameters - ---------- - signal: float - signal from the resultants - - Returns - ------- - signal power from Table 2 - """ - i: cython.int - for i in range(6): - if signal < _P_TABLE[0][i]: - return _P_TABLE[1][i - 1] - - return _P_TABLE[1][i] diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 6bb8275b..4ffce84d 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -3,13 +3,13 @@ from numpy.testing import assert_allclose from stcal.ramp_fitting.ols_cas22 import JUMP_DET, DefaultThreshold, fit_ramps -from stcal.ramp_fitting.ols_cas22._fit import _fill_metadata -from stcal.ramp_fitting.ols_cas22._jump import ( +from stcal.ramp_fitting.ols_cas22._fit import ( FixedOffsets, Parameter, PixelOffsets, Variance, _fill_fixed_values, + _fill_metadata, _fill_pixel_values, _init_ramps, ) From 02f854483e36a372de69df96c1b5806abcb3aec0 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 18:29:22 -0500 Subject: [PATCH 18/31] Reduce calls back to python --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 1d067bdf..e9f0f1e0 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -144,7 +144,7 @@ def fit_ramps( index: cython.int for index in range(n_pixels): # Fit all the ramps for the given pixel - fit = _fit_jumps( + fit = _fit_pixel( parameters[index, :], variances[index, :], resultants[:, index], @@ -182,7 +182,7 @@ def fit_ramps( @cython.cdivision(True) @cython.inline @cython.cfunc -def _fit_jumps( +def _fit_pixel( parameters: cython.float[:], variances: cython.float[:], resultants: cython.float[:], @@ -747,6 +747,7 @@ def _fit_statistic( @cython.cdivision(True) @cython.inline @cython.cfunc +@cython.exceptval(check=False) def _statistic( local_slope: cython.float, var_read_noise: cython.float, @@ -799,6 +800,7 @@ def _statistic( @cython.cdivision(True) @cython.inline @cython.cfunc +@cython.exceptval(check=False) def _correction(t_bar: cython.float[:], ramp: RampIndex, slope: cython.float) -> cython.float: """ Compute the correction factor for the variance used by a statistic. From fbf662f7bc8c1d85a4958f95fb1681a96858d91e Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 15 Nov 2023 18:46:53 -0500 Subject: [PATCH 19/31] Clean up tests slightly --- tests/test_jump_cas22.py | 161 +++++++++++++++------------------------ 1 file changed, 61 insertions(+), 100 deletions(-) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 4ffce84d..21b11000 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -1,3 +1,5 @@ +from typing import NamedTuple + import numpy as np import pytest from numpy.testing import assert_allclose @@ -301,9 +303,46 @@ def detector_data(read_pattern): return resultants, read_noise, read_pattern +class AllocatedData(NamedTuple): + parameters: np.ndarray + variances: np.ndarray + t_bar: np.ndarray + tau: np.ndarray + n_reads: np.ndarray + single_pixel: np.ndarray + double_pixel: np.ndarray + single_fixed: np.ndarray + double_fixed: np.ndarray + + +@pytest.fixture() +def allocated_data(read_pattern): + """ + Allocate the working data for the tests + """ + n_resultants = len(read_pattern) + + # Initialize the output arrays + parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) + variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) + + # Initialize scratch storage + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + + return AllocatedData( + parameters, variances, t_bar, tau, n_reads, single_pixel, double_pixel, single_fixed, double_fixed + ) + + @pytest.mark.parametrize("use_jump", [True, False]) @pytest.mark.parametrize("use_dq", [True, False]) -def test_fit_ramps(detector_data, use_jump, use_dq): +def test_fit_ramps(detector_data, allocated_data, use_jump, use_dq): """ Test fitting ramps Since no jumps are simulated in the data, jump detection shouldn't pick @@ -327,20 +366,12 @@ def test_fit_ramps(detector_data, use_jump, use_dq): if not use_dq: assert okay.all() - # Initialize the output arrays - parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) - variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) - - # Initialize scratch storage - n_resultants = resultants.shape[0] - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) + # Mirror what python supporting code does if use_jump: - single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) - single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + single_pixel = allocated_data.single_pixel + double_pixel = allocated_data.double_pixel + single_fixed = allocated_data.single_fixed + double_fixed = allocated_data.double_fixed else: single_pixel = np.empty((0, 0), dtype=np.float32) double_pixel = np.empty((0, 0), dtype=np.float32) @@ -353,11 +384,7 @@ def test_fit_ramps(detector_data, use_jump, use_dq): read_noise, READ_TIME, read_pattern, - parameters, - variances, - t_bar, - tau, - n_reads, + *allocated_data[:-4], single_pixel, double_pixel, single_fixed, @@ -370,11 +397,11 @@ def test_fit_ramps(detector_data, use_jump, use_dq): assert len(output) == N_PIXELS # sanity check that a fit is output for each pixel # Check that the intercept is always zero - assert np.all(parameters[:, Parameter.intercept] == 0) + assert np.all(allocated_data.parameters[:, Parameter.intercept] == 0) - slopes = parameters[:, Parameter.slope] - read_vars = variances[:, Variance.read_var] - poisson_vars = variances[:, Variance.poisson_var] + slopes = allocated_data.parameters[:, Parameter.slope] + read_vars = allocated_data.variances[:, Variance.read_var] + poisson_vars = allocated_data.variances[:, Variance.poisson_var] chi2 = 0 for fit, slope, read_var, poisson_var, use in zip(output, slopes, read_vars, poisson_vars, okay): @@ -455,7 +482,7 @@ def jump_data(detector_data): return resultants, read_noise, read_pattern, jump_reads, jump_resultants -def test_find_jumps(jump_data): +def test_find_jumps(jump_data, allocated_data): """ Full unit tests to demonstrate that we can detect jumps in any read (except the first one) and that we correctly remove these reads from the fit to recover @@ -464,35 +491,13 @@ def test_find_jumps(jump_data): resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) - # Initialize the output arrays - parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) - variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) - - # Initialize scratch storage - n_resultants = resultants.shape[0] - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) - single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) - single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) - output = fit_ramps( resultants, dq, read_noise, READ_TIME, read_pattern, - parameters, - variances, - t_bar, - tau, - n_reads, - single_pixel, - double_pixel, - single_fixed, - double_fixed, + *allocated_data, True, DefaultThreshold.INTERCEPT.value, DefaultThreshold.CONSTANT.value, @@ -501,11 +506,11 @@ def test_find_jumps(jump_data): assert len(output) == len(jump_reads) # sanity check that a fit/jump is set for every pixel # Check that the intercept is always zero - assert np.all(parameters[:, Parameter.intercept] == 0) + assert np.all(allocated_data.parameters[:, Parameter.intercept] == 0) - slopes = parameters[:, Parameter.slope] - read_vars = variances[:, Variance.read_var] - poisson_vars = variances[:, Variance.poisson_var] + slopes = allocated_data.parameters[:, Parameter.slope] + read_vars = allocated_data.variances[:, Variance.read_var] + poisson_vars = allocated_data.variances[:, Variance.poisson_var] chi2 = 0 incorrect_too_few = 0 @@ -575,21 +580,11 @@ def test_find_jumps(jump_data): assert np.abs(chi2 - 1) < CHI2_TOL -def test_override_default_threshold(jump_data): +def test_override_default_threshold(jump_data, allocated_data): """This tests that we can override the default jump detection threshold constants""" resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) - # Initialize scratch storage - n_resultants = resultants.shape[0] - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) - single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) - single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) - # Initialize the output arrays standard_parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) standard_variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) @@ -601,13 +596,7 @@ def test_override_default_threshold(jump_data): read_pattern, standard_parameters, standard_variances, - t_bar, - tau, - n_reads, - single_pixel, - double_pixel, - single_fixed, - double_fixed, + *allocated_data[2:], True, DefaultThreshold.INTERCEPT.value, DefaultThreshold.CONSTANT.value, @@ -624,13 +613,7 @@ def test_override_default_threshold(jump_data): read_pattern, override_parameters, override_variances, - t_bar, - tau, - n_reads, - single_pixel, - double_pixel, - single_fixed, - double_fixed, + *allocated_data[2:], True, 0, 0, @@ -643,42 +626,20 @@ def test_override_default_threshold(jump_data): assert (standard_variances != override_variances).any() -def test_jump_dq_set(jump_data): +def test_jump_dq_set(jump_data, allocated_data): # Check the DQ flag value to start assert 2**2 == JUMP_DET resultants, read_noise, read_pattern, jump_reads, jump_resultants = jump_data dq = np.zeros(resultants.shape, dtype=np.int32) - # Initialize the output arrays - parameters = np.empty((N_PIXELS, Parameter.n_param), dtype=np.float32) - variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) - - # Initialize scratch storage - n_resultants = resultants.shape[0] - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) - single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) - single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) - output = fit_ramps( resultants, dq, read_noise, READ_TIME, read_pattern, - parameters, - variances, - t_bar, - tau, - n_reads, - single_pixel, - double_pixel, - single_fixed, - double_fixed, + *allocated_data, True, DefaultThreshold.INTERCEPT.value, DefaultThreshold.CONSTANT.value, From 6293bc23d58a5c4a92edf40bda3ec3f0b0959184 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 13:38:07 -0500 Subject: [PATCH 20/31] bugfix for ramp tests. --- src/stcal/ramp_fitting/ols_cas22/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index 6d7795a2..fd497b4c 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -2,7 +2,7 @@ import numpy as np -from ._fit import JUMP_DET, Parameter, Variance, fit_ramps +from ._fit import JUMP_DET, Parameter, Variance, fit_ramps, PixelOffsets, FixedOffsets class DefaultThreshold(Enum): @@ -10,4 +10,4 @@ class DefaultThreshold(Enum): CONSTANT = np.float32(1 / 3) -__all__ = ["fit_ramps", "Parameter", "Variance", "Diff", "JUMP_DET", "DefaultThreshold"] +__all__ = ["fit_ramps", "Parameter", "Variance", "PixelOffsets", "FixedOffsets", "JUMP_DET", "DefaultThreshold"] From 2281b7f4de15cbcaffec7aecfbcdaef48d115251 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 14:38:56 -0500 Subject: [PATCH 21/31] Partial revert of converting _fill_metadata to vectors This makes it so that we get no numerical differences, it does not effect average computation time --- src/stcal/ramp_fitting/ols_cas22/__init__.py | 12 ++++++++++-- src/stcal/ramp_fitting/ols_cas22/_fit.py | 16 ++++++---------- tests/test_jump_cas22.py | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/ols_cas22/__init__.py index fd497b4c..7472ebf9 100644 --- a/src/stcal/ramp_fitting/ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/ols_cas22/__init__.py @@ -2,7 +2,7 @@ import numpy as np -from ._fit import JUMP_DET, Parameter, Variance, fit_ramps, PixelOffsets, FixedOffsets +from ._fit import JUMP_DET, FixedOffsets, Parameter, PixelOffsets, Variance, fit_ramps class DefaultThreshold(Enum): @@ -10,4 +10,12 @@ class DefaultThreshold(Enum): CONSTANT = np.float32(1 / 3) -__all__ = ["fit_ramps", "Parameter", "Variance", "PixelOffsets", "FixedOffsets", "JUMP_DET", "DefaultThreshold"] +__all__ = [ + "fit_ramps", + "Parameter", + "Variance", + "PixelOffsets", + "FixedOffsets", + "JUMP_DET", + "DefaultThreshold", +] diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index e9f0f1e0..f56fc620 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -27,6 +27,8 @@ is the primary externally callable function. """ import cython +import numpy as np +from cython.cimports import numpy as cnp from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, isnan, log10, sqrt from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.list import list as cpp_list @@ -39,6 +41,8 @@ Variance, ) +cnp.import_array() + RampIndex = cython.struct( start=cython.int, end=cython.int, @@ -837,22 +841,14 @@ def _fill_metadata( n_read: cython.int i: cython.int - j: cython.int resultant: vector[cython.int] for i in range(n_resultants): resultant = read_pattern[i] n_read = resultant.size() n_reads[i] = n_read - t_bar[i] = 0 - tau[i] = 0 - - for j in range(n_read): - t_bar[i] += read_time * resultant[j] - tau[i] += (2 * (n_read - j) - 1) * resultant[j] - - t_bar[i] /= n_read - tau[i] *= read_time / n_read**2 + t_bar[i] = read_time * np.mean(resultant) + tau[i] = np.sum((2 * (n_read - np.arange(n_read)) - 1) * resultant) * read_time / n_read**2 _t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 21b11000..37b8134d 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -131,7 +131,7 @@ def test__fill_metadata(read_pattern): # Check that the data is correct assert_allclose(t_bar, [7.6, 15.2, 21.279999, 41.040001, 60.799999, 88.159996]) - assert_allclose(tau, [5.7, 15.2, 19.928888, 36.024002, 59.448887, 80.59378]) + assert_allclose(tau, [5.7, 15.2, 19.928888, 36.023998, 59.448887, 80.593781]) assert np.all(n_reads == [4, 1, 3, 10, 3, 15]) # Check datatypes From d62faa9332f7faacfdd1bed35b1b1cb93b7a32a2 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 15:20:37 -0500 Subject: [PATCH 22/31] Move metadata creation out of cython completely This prevents a bounce out of cython and does not affect compute times --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 35 ---------------------- src/stcal/ramp_fitting/ols_cas22_fit.py | 23 +++++++++++---- tests/test_jump_cas22.py | 37 +++++++----------------- 3 files changed, 29 insertions(+), 66 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index f56fc620..deae4d14 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -27,8 +27,6 @@ is the primary externally callable function. """ import cython -import numpy as np -from cython.cimports import numpy as cnp from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, isnan, log10, sqrt from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.list import list as cpp_list @@ -41,8 +39,6 @@ Variance, ) -cnp.import_array() - RampIndex = cython.struct( start=cython.int, end=cython.int, @@ -71,8 +67,6 @@ def fit_ramps( resultants: cython.float[:, :], dq: cython.int[:, :], read_noise: cython.float[:], - read_time: cython.float, - read_pattern: vector[vector[cython.int]], parameters: cython.float[:, :], variances: cython.float[:, :], t_bar: cython.float[:], @@ -127,9 +121,6 @@ def fit_ramps( n_resultants: cython.int = resultants.shape[0] n_pixels: cython.int = resultants.shape[1] - # Compute the main metadata from the read pattern and cast it to memory views - _fill_metadata(t_bar, tau, n_reads, read_pattern, read_time, n_resultants) - if use_jump: # Pre-compute the values from the read pattern _fill_fixed_values(single_fixed, double_fixed, t_bar, tau, n_reads, n_resultants) @@ -825,32 +816,6 @@ def _correction(t_bar: cython.float[:], ramp: RampIndex, slope: cython.float) -> return -slope / diff -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -@cython.inline -@cython.ccall -def _fill_metadata( - t_bar: cython.float[:], - tau: cython.float[:], - n_reads: cython.int[:], - read_pattern: vector[vector[cython.int]], - read_time: cython.float, - n_resultants: cython.int, -) -> cython.void: - n_read: cython.int - - i: cython.int - resultant: vector[cython.int] - for i in range(n_resultants): - resultant = read_pattern[i] - n_read = resultant.size() - - n_reads[i] = n_read - t_bar[i] = read_time * np.mean(resultant) - tau[i] = np.sum((2 * (n_read - np.arange(n_read)) - 1) * resultant) * read_time / n_read**2 - - _t_bar_diff = cython.declare(cython.int, FixedOffsets.t_bar_diff) _t_bar_diff_sqr = cython.declare(cython.int, FixedOffsets.t_bar_diff_sqr) _read_recip = cython.declare(cython.int, FixedOffsets.read_recip) diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22_fit.py index c44ac614..dc0d75bd 100644 --- a/src/stcal/ramp_fitting/ols_cas22_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22_fit.py @@ -160,9 +160,7 @@ def fit_ramps_casertano( # Pre-allocate the working memory arrays # This prevents bouncing to and from cython for this allocation, which # is slower than just doing it all in python to start. - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) + t_bar, tau, n_reads = _create_metadata(read_pattern, read_time) if use_jump: single_pixel = np.empty((ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) double_pixel = np.empty((ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) @@ -178,8 +176,6 @@ def fit_ramps_casertano( resultants.reshape(resultants.shape[0], -1), dq.reshape(resultants.shape[0], -1), read_noise.reshape(-1), - read_time, - read_pattern, parameters, variances, t_bar, @@ -208,3 +204,20 @@ def fit_ramps_casertano( # return ols_cas22.RampFitOutputs(output.fits, parameters, variances, dq) return RampFitOutputs(parameters, variances, dq) + + +def _create_metadata(read_pattern, read_time): + n_resultants = len(read_pattern) + + t_bar = np.empty(n_resultants, dtype=np.float32) + tau = np.empty(n_resultants, dtype=np.float32) + n_reads = np.empty(n_resultants, dtype=np.int32) + + for i, resultant in enumerate(read_pattern): + n_read = len(resultant) + + n_reads[i] = n_read + t_bar[i] = read_time * np.mean(resultant) + tau[i] = np.sum((2 * (n_read - np.arange(n_read)) - 1) * resultant) * read_time / n_read**2 + + return t_bar, tau, n_reads diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 37b8134d..7ddd2892 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -11,10 +11,10 @@ PixelOffsets, Variance, _fill_fixed_values, - _fill_metadata, _fill_pixel_values, _init_ramps, ) +from stcal.ramp_fitting.ols_cas22_fit import _create_metadata # Purposefully set a fixed seed so that the tests in this module are deterministic RNG = np.random.default_rng(619) @@ -116,14 +116,11 @@ def read_pattern(): ] -def test__fill_metadata(read_pattern): +def test__create_metadata(read_pattern): """Test turning read_pattern into the time data""" n_resultants = len(read_pattern) - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) - _fill_metadata(t_bar, tau, n_reads, read_pattern, READ_TIME, n_resultants) + t_bar, tau, n_reads = _create_metadata(read_pattern, READ_TIME) assert t_bar.shape == (n_resultants,) assert tau.shape == (n_resultants,) @@ -152,13 +149,8 @@ def ramp_data(read_pattern): metadata : dict The metadata computed from the read pattern """ - n_resultants = len(read_pattern) - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) - _fill_metadata(t_bar, tau, n_reads, read_pattern, READ_TIME, n_resultants) - return t_bar, tau, n_reads, read_pattern + return *_create_metadata(read_pattern, READ_TIME), read_pattern def test_fill_fixed_values(ramp_data): @@ -327,16 +319,19 @@ def allocated_data(read_pattern): variances = np.empty((N_PIXELS, Variance.n_var), dtype=np.float32) # Initialize scratch storage - t_bar = np.empty(n_resultants, dtype=np.float32) - tau = np.empty(n_resultants, dtype=np.float32) - n_reads = np.empty(n_resultants, dtype=np.int32) single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) return AllocatedData( - parameters, variances, t_bar, tau, n_reads, single_pixel, double_pixel, single_fixed, double_fixed + parameters, + variances, + *_create_metadata(read_pattern, READ_TIME), + single_pixel, + double_pixel, + single_fixed, + double_fixed, ) @@ -382,8 +377,6 @@ def test_fit_ramps(detector_data, allocated_data, use_jump, use_dq): resultants, dq, read_noise, - READ_TIME, - read_pattern, *allocated_data[:-4], single_pixel, double_pixel, @@ -495,8 +488,6 @@ def test_find_jumps(jump_data, allocated_data): resultants, dq, read_noise, - READ_TIME, - read_pattern, *allocated_data, True, DefaultThreshold.INTERCEPT.value, @@ -592,8 +583,6 @@ def test_override_default_threshold(jump_data, allocated_data): resultants, dq, read_noise, - READ_TIME, - read_pattern, standard_parameters, standard_variances, *allocated_data[2:], @@ -609,8 +598,6 @@ def test_override_default_threshold(jump_data, allocated_data): resultants, dq, read_noise, - READ_TIME, - read_pattern, override_parameters, override_variances, *allocated_data[2:], @@ -637,8 +624,6 @@ def test_jump_dq_set(jump_data, allocated_data): resultants, dq, read_noise, - READ_TIME, - read_pattern, *allocated_data, True, DefaultThreshold.INTERCEPT.value, From 59d4ed93497c5236c3d1e5f4bd2a3eab9b5ccba8 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 15:41:31 -0500 Subject: [PATCH 23/31] Update docs --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 104 ++++++++++++++++++----- 1 file changed, 83 insertions(+), 21 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index deae4d14..365ac76f 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -100,10 +100,31 @@ def fit_ramps( be working on memory views of this array) read_noise : float[n_pixel] the read noise in electrons for each pixel (same note as the resultants) - read_time : float - Time to perform a readout. For Roman data, this is FRAME_TIME. - read_pattern : list[list[int]] - the read pattern for the image + parameters : float[n_pixel, 2] + The output array for the fit parameters. The first dimension is the + intercept, the second dimension is the slope. + variances : float[n_pixel, 3] + The output array for the fit variances. The first dimension is the + The first dimension is the read noise variance, the second dimension + is the poissson variance, and the third dimension is the total variance. + t_bar : float[n_resultants] + The average times for each resultant computed from the read pattern + tau : float[n_resultants] + The variance in the time for each resultant computed from the read pattern + n_reads : int[n_resultants] + The number of reads for each resultant computed from the read pattern + single_pixel : float[2, n_resultants - 1] + Pre-allocated array for the jump detection fixed values for a given pixel. + These will hold single difference values. + double_pixel : float[2, n_resultants - 2] + Pre-allocated array for the jump detection fixed values for a given pixel. + These will hold double difference values. + single_fixed : float[4, n_resultants - 1] + Pre-allocated array for the jump detection fixed values for all pixels. + These will hold single difference values. + double_fixed : float[4, n_resultants - 2] + Pre-allocated array for the jump detection fixed values for all pixels. + These will hold double difference values. use_jump : bool If True, use the jump detection algorithm to identify CRs. If False, use the DQ array to identify CRs. @@ -114,9 +135,18 @@ def fit_ramps( include_diagnostic : bool If True, include the raw ramp fits in the output. Default=False + Notes + ----- + The single_pixel, double_pixel, single_fixed, and double_fixed arrays + are passed in so that python can use numpy to pre-allocate the arrays + in python code. Surprisingly this is more efficient than using numpy + to allocate these arrays in cython code. This is because numpy requires + a back and forth jump between python and cython calls which induces a + lot of overhead. + Returns ------- - A RampFitOutputs tuple + list of JumpFits (if include_diagnostic is True) """ n_resultants: cython.int = resultants.shape[0] n_pixels: cython.int = resultants.shape[1] @@ -201,16 +231,21 @@ def _fit_pixel( Parameters ---------- + parameters : float[:] + 2 element array for the output parameters (slice of the total parameters array). + This will be modified in place, so the array it is a slice of will be modified + as a side effect. + variance : float[:] + 3 element array for the output variances (slice of the total variances array) + This will be modified in place, so the array it is a slice of will be modified + as a side effect. resultants : float[:] The resultants for the pixel dq : int[:] - The dq flags for the pixel. This is modified in place, so the external - dq flag array will be modified as a side-effect. + The dq flags for the pixel. This is a slice of the dq array. This is modified + in place, so the external dq flag array will be modified as a side-effect. read_noise : float The read noise for the pixel. - ramps : RampQueue - RampQueue for initial ramps to fit for the pixel - multiple ramps are possible due to dq flags t_bar : float[:] The average time for each resultant tau : float[:] @@ -219,12 +254,21 @@ def _fit_pixel( The number of reads for each resultant n_resultants : int The number of resultants for the pixel - fixed : float[:, :] - The jump detection pre-computed values for a given read_pattern - pixel : float[:, :] + single_pixel : float[:, :] A pre-allocated array for the jump detection fixed values for the given pixel. This will be modified in place, it is passed in to avoid re-allocating it for each pixel. + These will hold single difference values. + double_pixel : float[:, :] + A pre-allocated array for the jump detection fixed values for the + given pixel. This will be modified in place, it is passed in to avoid + re-allocating it for each pixel. These will hold double difference values. + single-fixed : float[:, :] + The jump detection pre-computed values for a given read_pattern. + These will hold single difference values. + double-fixed : float[:, :] + The jump detection pre-computed values for a given read_pattern. + These will hold double difference values. thresh : Thresh The threshold parameter struct for jump detection use_jump : bool @@ -662,10 +706,18 @@ def _fit_statistic( Parameters ---------- - pixel : float[:, :] + single_pixel : float[:, :] The pre-computed fixed values for a given pixel - fixed : float[:, :] + These will hold single difference values. + double_pixel : float[:, :] + The pre-computed fixed values for a given pixel + These will hold double difference values. + single_fixed : float[:, :] + The pre-computed fixed values for a given read_pattern + These will hold single difference values. + double_fixed : float[:, :] The pre-computed fixed values for a given read_pattern + These will hold double difference values. t_bar : float[:, :] The average time for each resultant slope : float @@ -841,9 +893,12 @@ def _fill_fixed_values( Parameters ---------- - fixed : float[:, :] + single_fixed : float[:, :] A pre-allocated memoryview to store the pre-computed values in, its faster - to allocate outside this function. + to allocate outside this function. These will hold single difference values. + double_fixed : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. These will hold double difference values. t_bar : float[:] The average time for each resultant tau : float[:] @@ -907,13 +962,20 @@ def _fill_pixel_values( Parameters ---------- - pixel : float[:, :] + single_pixel : float[:, :] + A pre-allocated memoryview to store the pre-computed values in, its faster + to allocate outside this function. These will hold single difference values. + double_pixel : float[:, :] A pre-allocated memoryview to store the pre-computed values in, its faster - to allocate outside this function. + to allocate outside this function. These will hold double difference values. + single_fixed : float[:, :] + The pre-computed fixed values for the read_pattern + These will hold single difference values. + double_fixed : float[:, :] + The pre-computed fixed values for the read_pattern + These will hold double difference values. resultants : float[:] The resultants for the pixel in question. - fixed : float[:, :] - The pre-computed fixed values for the read_pattern read_noise : float The read noise for the pixel n_resultants : int From 4aef566e32a52b7e58f734ff57ba0b9bfbed6682 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 17 Nov 2023 11:06:13 -0500 Subject: [PATCH 24/31] Refactor jump detection into its own function This significantly enhances readability of the single pixel fitting function. --- src/stcal/ramp_fitting/ols_cas22/_fit.py | 329 ++++++++++++++--------- 1 file changed, 202 insertions(+), 127 deletions(-) diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/ols_cas22/_fit.py index 365ac76f..46904158 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22/_fit.py @@ -1,30 +1,42 @@ # cython: language_level=3str """ -External interface module for the Casertano+22 ramp fitting algorithm with jump detection. - This module is intended to contain everything needed by external code. +Cython implementation of the Casertano+22 algorithm for fitting ramps with jump detection. + + Note that this is written in annotated Python using Cython annotations, meaning + that Cython 3 can interpret this "python" file as if it were a Cython file. This + enables one to use Python tooling to write Cython code, and prevents the need to + context switch between the Python and Cython programming languages. + + Note that everything is crammed into a single file because it enables Cython to + directly optimize the C code it generates. This is because Cython can only optimize + across a single file (i.e. inlining functions only works if the function is in the + same file as the function it is being inlined into). This helps aid the C compiler + in optimizing C code when it compiles. Enums ----- -Parameter : - Enumerate the index for the output parameters array. - -Variance : - Enumerate the index for the output variances array. - -Classes -------- -RampFitOutputs : NamedTuple - Simple tuple wrapper for outputs from the ramp fitting algorithm - This clarifies the meaning of the outputs via naming them something - descriptive. - -(Public) Functions ------------------- -fit_ramps : function - Fit ramps using the Castenario+22 algorithm to a set of pixels accounting - for jumps (if use_jump is True) and bad pixels (via the dq array). This - is the primary externally callable function. +Parameter: + This enum is used to index into the parameter output array. + slope: 0 + intercept: 1 + n_param: 2 (number of parameters output) +Variance: + This enum is used to index into the variance output array. + read_var: 0 + poisson_var: 1 + total_var: 2 + n_var: 3 (number of variances output) + +Functions +--------- +fit_ramps: + This is the main driver program for the Casertano+22 algorithm. It fits ramps + with jump detection (if enabled) to a series of pixels, returning the Cas22 + fit parameters and variances for each pixel. This function is not intended to + be called outside of stcal itself as it requires a lot of pre-allocation of + memory views to be passed in. Use the `stcal.ramp_fitting.ols_cas22.fit_ramps` + function instead. """ import cython from cython.cimports.libc.math import INFINITY, NAN, fabs, fmaxf, isnan, log10, sqrt @@ -292,11 +304,9 @@ def _fit_pixel( index: RampQueue = RampQueue() # Declare variables for the loop + jump: Jump ramp: RampIndex ramp_fit: RampFit - stat: Stat - jump0: cython.int - jump1: cython.int weight: cython.float total_weight: cython.float = 0 @@ -315,110 +325,74 @@ def _fit_pixel( # Compute fit using the Casertano+22 algorithm ramp_fit = _fit_ramp(resultants, t_bar, tau, n_reads, read_noise, ramp) - # Run jump detection if enabled - if use_jump: - stat = _fit_statistic( - single_pixel, double_pixel, single_fixed, double_fixed, t_bar, ramp_fit.slope, ramp - ) - - # Note that when a "ramp" is a single point, _fit_statistic returns - # a NaN for max_stat. Note that NaN > anything is always false so the - # result drops through as desired. - if stat.max_stat > _threshold(thresh, ramp_fit.slope): - # Compute jump point to create two new ramps - # This jump point corresponds to the index of the largest - # statistic: - # argmax = argmax(stats) - # These statistics are indexed relative to the - # ramp's range. Therefore, we need to add the start index - # of the ramp to the result. - # - jump0 = stat.arg_max + ramp.start - - # Note that because the resultants are averages of reads, but - # jumps occur in individual reads, it is possible that the - # jump is averaged down by the resultant with the actual jump - # causing the computed jump to be off by one index. - # In the idealized case this is when the jump occurs near - # the start of the resultant with the jump. In this case, - # the statistic for the resultant will be maximized at - # index - 1 rather than index. This means that we have to - # remove argmax(stats) + 1 as it is also a possible jump. - # This case is difficult to distinguish from the case where - # argmax(stats) does correspond to the jump resultant. - # Therefore, we just remove both possible resultants from - # consideration. - jump1 = jump0 + 1 - - # Update the dq flags - dq[jump0] = JUMP_DET - dq[jump1] = JUMP_DET - - # Record jump diagnostics - if include_diagnostic: - jumps.push_back(jump0) - jumps.push_back(jump1) - - # The two resultant indices need to be skipped, therefore - # the two - # possible new ramps are: - # RampIndex(ramp.start, jump0 - 1) - # RampIndex(jump1 + 1, ramp.end) - # This is because the RampIndex contains the index of the - # first and last resulants in the sub-ramp it describes. - # Note: The algorithm works via working over the sub-ramps - # backward in time. Therefore, since we are using a stack, - # we need to add the ramps in the time order they were - # observed in. This results in the last observation ramp - # being the top of the stack; meaning that, - # it will be the next ramp handled. - - if jump0 > ramp.start: - # Note that when jump0 == ramp.start, we have detected a - # jump in the first resultant of the ramp. This means - # there is no sub-ramp before jump0. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump0 < ramp.start is not possible because - # the argmax is always >= 0 - ramps.push_back(RampIndex(ramp.start, jump0 - 1)) - - if jump1 < ramp.end: - # Note that if jump1 == ramp.end, we have detected a - # jump in the last resultant of the ramp. This means - # there is no sub-ramp after jump1. - # Also, note that this will produce bad results as - # the ramp indexing will go out of bounds. So it is - # important that we exclude it. - # Note that jump1 > ramp.end is technically possible - # however in those potential cases it will draw on - # resultants which are not considered part of the ramp - # under consideration. Therefore, we have to exclude all - # of those values. - ramps.push_back(RampIndex(jump1 + 1, ramp.end)) - - # Skip recording the ramp as it has a detected jump - continue - - # Start recording the the fit (no jump detected) - - # Record the diagnositcs - if include_diagnostic: - fits.push_back(ramp_fit) - index.push_back(ramp) - - # Start computing the averages using a lazy process - # Note we do not do anything in the NaN case for degenerate ramps - if not isnan(ramp_fit.slope): - # protect weight against the extremely unlikely case of a zero - # variance - weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var - total_weight += weight + # Run jump detection + jump = _jump_detection( + single_pixel, + double_pixel, + single_fixed, + double_fixed, + t_bar, + ramp_fit.slope, + ramp, + thresh, + use_jump, + ) - parameters[_slope] += weight * ramp_fit.slope - variances[_read_var] += weight**2 * ramp_fit.read_var - variances[_poisson_var] += weight**2 * ramp_fit.poisson_var + if jump.detected: + # A jump was detected! + # => Split the ramp and record the jumps + # Note that we have to do this splitting and recording here because + # vectors cannot be modified in place, so we have to copy the vectors + # if they were updated in a separate function, which is expensive. + + # Update the dq flags + dq[jump.jump0] = JUMP_DET + dq[jump.jump1] = JUMP_DET + + # Record jump diagnostics + if include_diagnostic: + jumps.push_back(jump.jump0) + jumps.push_back(jump.jump1) + + # The two resultant indices need to be skipped, therefore + # the two possible new ramps are: + # RampIndex(ramp.start, jump0 - 1) + # RampIndex(jump1 + 1, ramp.end) + # This is because the RampIndex contains the index of the + # first and last resultants in the sub-ramp it describes. + if jump.jump0 > ramp.start: + # Note that when jump0 == ramp.start, we have detected a + # jump in the first resultant of the ramp. + + # Add ramp from start to right before jump0 + ramps.push_back(RampIndex(ramp.start, jump.jump0 - 1)) + + if jump.jump1 < ramp.end: + # Note that if jump1 == ramp.end, we have detected a + # jump in the last resultant of the ramp. + + # Add ramp from right after jump1 to end + ramps.push_back(RampIndex(jump.jump1 + 1, ramp.end)) + else: + # No jump was detected! + # => Record the fit. + + # Record the diagnostics + if include_diagnostic: + fits.push_back(ramp_fit) + index.push_back(ramp) + + # Start computing the averages using a lazy process + # Note we do not do anything in the NaN case for degenerate ramps + if not isnan(ramp_fit.slope): + # protect weight against the extremely unlikely case of a zero + # variance + weight = 0 if ramp_fit.read_var == 0 else 1 / ramp_fit.read_var + total_weight += weight + + parameters[_slope] += weight * ramp_fit.slope + variances[_read_var] += weight**2 * ramp_fit.read_var + variances[_poisson_var] += weight**2 * ramp_fit.poisson_var # Finish computing averages using the lazy process parameters[_slope] /= total_weight if total_weight != 0 else 1 @@ -658,6 +632,107 @@ def _init_ramps(dq: cython.int[:], n_resultants: cython.int) -> RampQueue: return ramps +# Note everything below this comment is to support jump detection. + + +Jump = cython.struct( + detected=cpp_bool, + jump0=cython.int, + jump1=cython.int, +) + + +@cython.boundscheck(False) +@cython.wraparound(False) +@cython.inline +@cython.cfunc +def _jump_detection( + single_pixel: cython.float[:, :], + double_pixel: cython.float[:, :], + single_fixed: cython.float[:, :], + double_fixed: cython.float[:, :], + t_bar: cython.float[:], + slope: cython.float, + ramp: RampIndex, + thresh: Thresh, + use_jump: cpp_bool, +) -> Jump: + """ + Run jump detection on a single ramp fit. + + Parameters + ---------- + single_pixel : float[:, :] + The pre-computed fixed values for a given pixel + These will hold single difference values. + double_pixel : float[:, :] + The pre-computed fixed values for a given pixel + These will hold double difference values. + single_fixed : float[:, :] + The pre-computed fixed values for a given read_pattern + These will hold single difference values. + double_fixed : float[:, :] + The pre-computed fixed values for a given read_pattern + These will hold double difference values. + t_bar : float[:, :] + The average time for each resultant + slope : float + The computed slope for the ramp + ramp : RampIndex + Struct for start and end of ramp to fit + thresh : Thresh + Threshold parameters struct + use_jump : bool + Turn on or off jump detection. + + Returns + ------- + Jump: struct + - detected: bool + True if a jump was detected + - jump0: int + Index of the first resultant of the jump + - jump1: int + Index of the second resultant of the jump + """ + jump: cython.int + + # Run jump detection if enabled + if use_jump: + stat: Stat = _fit_statistic( + single_pixel, double_pixel, single_fixed, double_fixed, t_bar, slope, ramp + ) + # Note that when a "ramp" is a single point, _fit_statistic returns + # a NaN for max_stat. Note that NaN > anything is always false so the + # result drops through as desired. + if stat.max_stat > _threshold(thresh, slope): + # Compute jump point to create two new ramps + # This jump point corresponds to the index of the largest + # statistic: + # argmax = argmax(stats) + # These statistics are indexed relative to the ramp's range. + # Therefore, we need to add the start index of the ramp to the + # result. + jump = stat.arg_max + ramp.start + + # Note that because the resultants are averages of reads, but + # jumps occur in individual reads, it is possible that the + # jump is averaged down by the resultant with the actual jump + # causing the computed jump to be off by one index. + # In the idealized case this is when the jump occurs near + # the start of the resultant with the jump. In this case, + # the statistic for the resultant will be maximized at + # index - 1 rather than index. This means that we have to + # remove argmax(stats) + 1 as it is also a possible jump. + # This case is difficult to distinguish from the case where + # argmax(stats) does correspond to the jump resultant. + # Therefore, we just remove both possible resultants from + # consideration. + return Jump(True, jump, jump + 1) + + return Jump(False, -1, -1) + + @cython.inline @cython.cfunc @cython.exceptval(check=False) From 618fa98c1333e2c3e97e1227bb898d9962bb81b5 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 17 Nov 2023 11:49:09 -0500 Subject: [PATCH 25/31] Remove dead files --- setup.py | 12 ----- src/stcal/ramp_fitting/ols_cas22/_jump.pxd | 1 - src/stcal/ramp_fitting/ols_cas22/_jump.py | 55 ---------------------- src/stcal/ramp_fitting/ols_cas22/_ramp.pxd | 1 - src/stcal/ramp_fitting/ols_cas22/_ramp.py | 51 -------------------- 5 files changed, 120 deletions(-) delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_jump.pxd delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_jump.py delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_ramp.pxd delete mode 100644 src/stcal/ramp_fitting/ols_cas22/_ramp.py diff --git a/setup.py b/setup.py index 28414ff1..231d2c5d 100644 --- a/setup.py +++ b/setup.py @@ -7,18 +7,6 @@ Options.annotate = False extensions = [ - Extension( - "stcal.ramp_fitting.ols_cas22._ramp", - ["src/stcal/ramp_fitting/ols_cas22/_ramp.py"], - include_dirs=[np.get_include()], - language="c++", - ), - Extension( - "stcal.ramp_fitting.ols_cas22._jump", - ["src/stcal/ramp_fitting/ols_cas22/_jump.py"], - include_dirs=[np.get_include()], - language="c++", - ), Extension( "stcal.ramp_fitting.ols_cas22._fit", ["src/stcal/ramp_fitting/ols_cas22/_fit.py"], diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd b/src/stcal/ramp_fitting/ols_cas22/_jump.pxd deleted file mode 100644 index 75f723ab..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.pxd +++ /dev/null @@ -1 +0,0 @@ -# cython: language_level=3str diff --git a/src/stcal/ramp_fitting/ols_cas22/_jump.py b/src/stcal/ramp_fitting/ols_cas22/_jump.py deleted file mode 100644 index d82eabf5..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_jump.py +++ /dev/null @@ -1,55 +0,0 @@ -# cython: language_level=3str - - -""" -This module contains all the functions needed to execute jump detection for the - Castentano+22 ramp fitting algorithm. - - The _ramp module contains the actual ramp fitting algorithm, this module - contains a driver for the algorithm and detection of jumps/splitting ramps. - -Structs -------- -Thresh : struct - intercept - constant * log10(slope) - - intercept : float - The intercept of the jump threshold - - constant : float - The constant of the jump threshold - -JumpFits : struct - All the data on a given pixel's ramp fit with (or without) jump detection - - average : RampFit - The average of all the ramps fit for the pixel - - jumps : vector[int] - The indices of the resultants which were detected as jumps - - fits : vector[RampFit] - All of the fits for each ramp fit for the pixel - - index : RampQueue - The RampIndex representations corresponding to each fit in fits - -Enums ------ -FixedOffsets : enum - Enumerate the different pieces of information computed for jump detection - which only depend on the read pattern. - -PixelOffsets : enum - Enumerate the different pieces of information computed for jump detection - which only depend on the given pixel (independent of specific ramp). - -JUMP_DET : value - A the fixed value for the jump detection dq flag. - -(Public) Functions ------------------- -fill_fixed_values : function - Pre-compute all the values needed for jump detection for a given read_pattern, - this is independent of the pixel involved. - -fit_jumps : function - Compute all the ramps for a single pixel using the Casertano+22 algorithm - with jump detection. This is a driver for the ramp fit algorithm in general - meaning it automatically handles splitting ramps across dq flags in addition - to splitting across detected jumps (if jump detection is turned on). -""" diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd b/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd deleted file mode 100644 index 75f723ab..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.pxd +++ /dev/null @@ -1 +0,0 @@ -# cython: language_level=3str diff --git a/src/stcal/ramp_fitting/ols_cas22/_ramp.py b/src/stcal/ramp_fitting/ols_cas22/_ramp.py deleted file mode 100644 index 072c6f0b..00000000 --- a/src/stcal/ramp_fitting/ols_cas22/_ramp.py +++ /dev/null @@ -1,51 +0,0 @@ -# cython: language_level=3str - -""" -This module contains all the functions needed to execute the Casertano+22 ramp - fitting algorithm on its own without jump detection. - - The _jump module contains a driver function which calls the `fit_ramp` function - from this module iteratively. This evvetively handles dq flags and detected - jumps simultaneously. - -Structs -------- -RampIndex : struct - - start : int - Index of the first resultant in the ramp - - end : int - Index of the last resultant in the ramp (so indexing of ramp requires end + 1) - -RampFit : struct - - slope : float - The slope fit to the ramp - - read_var : float - The read noise variance for the fit - - poisson_var : float - The poisson variance for the fit - -RampQueue : vector[RampIndex] - Vector of RampIndex objects (convenience typedef) - -Classes -------- -ReadPattern : - Container class for all the metadata derived from the read pattern, this - is just a temporary object to allow us to return multiple memory views from - a single function. - -(Public) Functions ------------------- -init_ramps : function - Create the initial ramp "queue" for each pixel in order to handle any initial - "dq" flags passed in from outside. - -from_read_pattern : function - Derive the input data from the the read pattern - This is faster than using __init__ or __cinit__ to construct the object with - these calls. - -fit_ramps : function - Implementation of running the Casertano+22 algorithm on a (sub)set of resultants - listed for a single pixel -""" From 587fed753effdb02af799260da664f9e59b04363 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 17 Nov 2023 11:57:24 -0500 Subject: [PATCH 26/31] Hide the cython module by making it private. It is complicated to use and the helper function is there to reduce that complexity. --- .gitignore | 6 +++--- pyproject.toml | 2 +- setup.py | 4 ++-- .../{ols_cas22 => _ols_cas22}/__init__.py | 0 .../{ols_cas22 => _ols_cas22}/_fit.pxd | 0 .../{ols_cas22 => _ols_cas22}/_fit.py | 2 +- src/stcal/ramp_fitting/ols_cas22_fit.py | 20 +++++++++---------- tests/test_jump_cas22.py | 4 ++-- 8 files changed, 19 insertions(+), 19 deletions(-) rename src/stcal/ramp_fitting/{ols_cas22 => _ols_cas22}/__init__.py (100%) rename src/stcal/ramp_fitting/{ols_cas22 => _ols_cas22}/_fit.pxd (100%) rename src/stcal/ramp_fitting/{ols_cas22 => _ols_cas22}/_fit.py (99%) diff --git a/.gitignore b/.gitignore index d35d7af2..136e50d0 100644 --- a/.gitignore +++ b/.gitignore @@ -139,9 +139,9 @@ dmypy.json # Cython debug symbols cython_debug/ -src/stcal/ramp_fitting/ols_cas22/*.c -src/stcal/ramp_fitting/ols_cas22/*.cpp -src/stcal/ramp_fitting/ols_cas22/*.html +src/stcal/ramp_fitting/_ols_cas22/*.c +src/stcal/ramp_fitting/_ols_cas22/*.cpp +src/stcal/ramp_fitting/_ols_cas22/*.html # setuptools-scm generated module src/stcal/_version.py diff --git a/pyproject.toml b/pyproject.toml index 62dfdc56..c1e946a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ testpaths = [ "docs", ] norecursedirs = [ - 'src/stcal/ramp_fitting/ols_cas22', + 'src/stcal/ramp_fitting/_ols_cas22', 'benchmarks', '.asv', '.eggs', diff --git a/setup.py b/setup.py index 231d2c5d..923c8719 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ extensions = [ Extension( - "stcal.ramp_fitting.ols_cas22._fit", - ["src/stcal/ramp_fitting/ols_cas22/_fit.py"], + "stcal.ramp_fitting._ols_cas22._fit", + ["src/stcal/ramp_fitting/_ols_cas22/_fit.py"], include_dirs=[np.get_include()], language="c++", ), diff --git a/src/stcal/ramp_fitting/ols_cas22/__init__.py b/src/stcal/ramp_fitting/_ols_cas22/__init__.py similarity index 100% rename from src/stcal/ramp_fitting/ols_cas22/__init__.py rename to src/stcal/ramp_fitting/_ols_cas22/__init__.py diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.pxd b/src/stcal/ramp_fitting/_ols_cas22/_fit.pxd similarity index 100% rename from src/stcal/ramp_fitting/ols_cas22/_fit.pxd rename to src/stcal/ramp_fitting/_ols_cas22/_fit.pxd diff --git a/src/stcal/ramp_fitting/ols_cas22/_fit.py b/src/stcal/ramp_fitting/_ols_cas22/_fit.py similarity index 99% rename from src/stcal/ramp_fitting/ols_cas22/_fit.py rename to src/stcal/ramp_fitting/_ols_cas22/_fit.py index 46904158..b8c7f5ac 100644 --- a/src/stcal/ramp_fitting/ols_cas22/_fit.py +++ b/src/stcal/ramp_fitting/_ols_cas22/_fit.py @@ -43,7 +43,7 @@ from cython.cimports.libcpp import bool as cpp_bool from cython.cimports.libcpp.list import list as cpp_list from cython.cimports.libcpp.vector import vector -from cython.cimports.stcal.ramp_fitting.ols_cas22._fit import ( +from cython.cimports.stcal.ramp_fitting._ols_cas22._fit import ( JUMP_DET, FixedOffsets, Parameter, diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22_fit.py index dc0d75bd..56823a75 100644 --- a/src/stcal/ramp_fitting/ols_cas22_fit.py +++ b/src/stcal/ramp_fitting/ols_cas22_fit.py @@ -34,7 +34,7 @@ import numpy as np from astropy import units as u -from . import ols_cas22 +from . import _ols_cas22 class RampFitOutputs(NamedTuple): @@ -73,8 +73,8 @@ def fit_ramps_casertano( read_pattern, use_jump=False, *, - threshold_intercept=ols_cas22.DefaultThreshold.INTERCEPT.value, - threshold_constant=ols_cas22.DefaultThreshold.CONSTANT.value, + threshold_intercept=_ols_cas22.DefaultThreshold.INTERCEPT.value, + threshold_constant=_ols_cas22.DefaultThreshold.CONSTANT.value, include_diagnostic=False, ): """Fit ramps following Casertano+2022, including averaging partial ramps. @@ -154,25 +154,25 @@ def fit_ramps_casertano( # Pre-allocate the output arrays n_pixels = np.prod(resultants.shape[1:]) - parameters = np.empty((n_pixels, ols_cas22.Parameter.n_param), dtype=np.float32) - variances = np.empty((n_pixels, ols_cas22.Variance.n_var), dtype=np.float32) + parameters = np.empty((n_pixels, _ols_cas22.Parameter.n_param), dtype=np.float32) + variances = np.empty((n_pixels, _ols_cas22.Variance.n_var), dtype=np.float32) # Pre-allocate the working memory arrays # This prevents bouncing to and from cython for this allocation, which # is slower than just doing it all in python to start. t_bar, tau, n_reads = _create_metadata(read_pattern, read_time) if use_jump: - single_pixel = np.empty((ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - double_pixel = np.empty((ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) - single_fixed = np.empty((ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - double_fixed = np.empty((ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + single_pixel = np.empty((_ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((_ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((_ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((_ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) else: single_pixel = np.empty((0, 0), dtype=np.float32) double_pixel = np.empty((0, 0), dtype=np.float32) single_fixed = np.empty((0, 0), dtype=np.float32) double_fixed = np.empty((0, 0), dtype=np.float32) - ols_cas22.fit_ramps( + _ols_cas22.fit_ramps( resultants.reshape(resultants.shape[0], -1), dq.reshape(resultants.shape[0], -1), read_noise.reshape(-1), diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 7ddd2892..981fccbb 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -4,8 +4,8 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting.ols_cas22 import JUMP_DET, DefaultThreshold, fit_ramps -from stcal.ramp_fitting.ols_cas22._fit import ( +from stcal.ramp_fitting._ols_cas22 import JUMP_DET, DefaultThreshold, fit_ramps +from stcal.ramp_fitting._ols_cas22._fit import ( FixedOffsets, Parameter, PixelOffsets, From 2fc6832ac0b0771b747937384e95cd7e642b249b Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 17 Nov 2023 11:59:06 -0500 Subject: [PATCH 27/31] Simplify the name of the wrapping module --- src/stcal/ramp_fitting/{ols_cas22_fit.py => ols_cas22.py} | 0 tests/test_jump_cas22.py | 2 +- tests/test_ramp_fitting_cas22.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/stcal/ramp_fitting/{ols_cas22_fit.py => ols_cas22.py} (100%) diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22.py similarity index 100% rename from src/stcal/ramp_fitting/ols_cas22_fit.py rename to src/stcal/ramp_fitting/ols_cas22.py diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 981fccbb..74c8e991 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -14,7 +14,7 @@ _fill_pixel_values, _init_ramps, ) -from stcal.ramp_fitting.ols_cas22_fit import _create_metadata +from stcal.ramp_fitting.ols_cas22 import _create_metadata # Purposefully set a fixed seed so that the tests in this module are deterministic RNG = np.random.default_rng(619) diff --git a/tests/test_ramp_fitting_cas22.py b/tests/test_ramp_fitting_cas22.py index e6266fb9..18289981 100644 --- a/tests/test_ramp_fitting_cas22.py +++ b/tests/test_ramp_fitting_cas22.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from stcal.ramp_fitting import ols_cas22_fit as ramp +from stcal.ramp_fitting import ols_cas22 as ramp # Purposefully set a fixed seed so that the tests in this module are deterministic RNG = np.random.default_rng(42) From 47693ed347cce0b4341d2f973f97f405d7bdd5c5 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 17 Nov 2023 12:08:26 -0500 Subject: [PATCH 28/31] Clean up imports --- src/stcal/ramp_fitting/_ols_cas22/__init__.py | 19 ++++-------- src/stcal/ramp_fitting/ols_cas22.py | 31 ++++++++++++------- tests/test_jump_cas22.py | 9 ++---- tests/test_ramp_fitting_cas22.py | 2 +- 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/stcal/ramp_fitting/_ols_cas22/__init__.py b/src/stcal/ramp_fitting/_ols_cas22/__init__.py index 7472ebf9..764b3783 100644 --- a/src/stcal/ramp_fitting/_ols_cas22/__init__.py +++ b/src/stcal/ramp_fitting/_ols_cas22/__init__.py @@ -1,14 +1,9 @@ -from enum import Enum - -import numpy as np - -from ._fit import JUMP_DET, FixedOffsets, Parameter, PixelOffsets, Variance, fit_ramps - - -class DefaultThreshold(Enum): - INTERCEPT = np.float32(5.5) - CONSTANT = np.float32(1 / 3) - +""" +This subpackage exists to hold the Cython implementation of the OLS cas22 algorithm + This subpackage is private, and should not be imported directly by users. Instead, + import from stcal.ramp_fitting.ols_cas22. +""" +from ._fit import FixedOffsets, Parameter, PixelOffsets, Variance, fit_ramps __all__ = [ "fit_ramps", @@ -16,6 +11,4 @@ class DefaultThreshold(Enum): "Variance", "PixelOffsets", "FixedOffsets", - "JUMP_DET", - "DefaultThreshold", ] diff --git a/src/stcal/ramp_fitting/ols_cas22.py b/src/stcal/ramp_fitting/ols_cas22.py index 56823a75..6e652f93 100644 --- a/src/stcal/ramp_fitting/ols_cas22.py +++ b/src/stcal/ramp_fitting/ols_cas22.py @@ -29,12 +29,21 @@ So the routines in these packages construct these different matrices, store them, and interpolate between them for different different fluxes and ratios. """ +from enum import Enum from typing import NamedTuple import numpy as np from astropy import units as u -from . import _ols_cas22 +from ._ols_cas22 import FixedOffsets, Parameter, PixelOffsets, Variance +from ._ols_cas22 import fit_ramps as _fit_ramps + +__all__ = ["fit_ramps", "Parameter", "Variance", "DefaultThreshold", "RampFitOutputs"] + + +class DefaultThreshold(Enum): + INTERCEPT = np.float32(5.5) + CONSTANT = np.float32(1 / 3) class RampFitOutputs(NamedTuple): @@ -65,7 +74,7 @@ class RampFitOutputs(NamedTuple): dq: np.ndarray -def fit_ramps_casertano( +def fit_ramps( resultants, dq, read_noise, @@ -73,8 +82,8 @@ def fit_ramps_casertano( read_pattern, use_jump=False, *, - threshold_intercept=_ols_cas22.DefaultThreshold.INTERCEPT.value, - threshold_constant=_ols_cas22.DefaultThreshold.CONSTANT.value, + threshold_intercept=DefaultThreshold.INTERCEPT.value, + threshold_constant=DefaultThreshold.CONSTANT.value, include_diagnostic=False, ): """Fit ramps following Casertano+2022, including averaging partial ramps. @@ -154,25 +163,25 @@ def fit_ramps_casertano( # Pre-allocate the output arrays n_pixels = np.prod(resultants.shape[1:]) - parameters = np.empty((n_pixels, _ols_cas22.Parameter.n_param), dtype=np.float32) - variances = np.empty((n_pixels, _ols_cas22.Variance.n_var), dtype=np.float32) + parameters = np.empty((n_pixels, Parameter.n_param), dtype=np.float32) + variances = np.empty((n_pixels, Variance.n_var), dtype=np.float32) # Pre-allocate the working memory arrays # This prevents bouncing to and from cython for this allocation, which # is slower than just doing it all in python to start. t_bar, tau, n_reads = _create_metadata(read_pattern, read_time) if use_jump: - single_pixel = np.empty((_ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) - double_pixel = np.empty((_ols_cas22.PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) - single_fixed = np.empty((_ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) - double_fixed = np.empty((_ols_cas22.FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) + single_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 1), dtype=np.float32) + double_pixel = np.empty((PixelOffsets.n_pixel_offsets, n_resultants - 2), dtype=np.float32) + single_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 1), dtype=np.float32) + double_fixed = np.empty((FixedOffsets.n_fixed_offsets, n_resultants - 2), dtype=np.float32) else: single_pixel = np.empty((0, 0), dtype=np.float32) double_pixel = np.empty((0, 0), dtype=np.float32) single_fixed = np.empty((0, 0), dtype=np.float32) double_fixed = np.empty((0, 0), dtype=np.float32) - _ols_cas22.fit_ramps( + _fit_ramps( resultants.reshape(resultants.shape[0], -1), dq.reshape(resultants.shape[0], -1), read_noise.reshape(-1), diff --git a/tests/test_jump_cas22.py b/tests/test_jump_cas22.py index 74c8e991..b2e6e695 100644 --- a/tests/test_jump_cas22.py +++ b/tests/test_jump_cas22.py @@ -4,17 +4,14 @@ import pytest from numpy.testing import assert_allclose -from stcal.ramp_fitting._ols_cas22 import JUMP_DET, DefaultThreshold, fit_ramps +from stcal.ramp_fitting._ols_cas22 import FixedOffsets, Parameter, PixelOffsets, Variance, fit_ramps from stcal.ramp_fitting._ols_cas22._fit import ( - FixedOffsets, - Parameter, - PixelOffsets, - Variance, + JUMP_DET, _fill_fixed_values, _fill_pixel_values, _init_ramps, ) -from stcal.ramp_fitting.ols_cas22 import _create_metadata +from stcal.ramp_fitting.ols_cas22 import DefaultThreshold, _create_metadata # Purposefully set a fixed seed so that the tests in this module are deterministic RNG = np.random.default_rng(619) diff --git a/tests/test_ramp_fitting_cas22.py b/tests/test_ramp_fitting_cas22.py index 18289981..123a69c1 100644 --- a/tests/test_ramp_fitting_cas22.py +++ b/tests/test_ramp_fitting_cas22.py @@ -40,7 +40,7 @@ def test_simulated_ramps(use_unit, use_dq): bad = RNG.uniform(size=resultants.shape) > 0.7 dq |= bad - output = ramp.fit_ramps_casertano( + output = ramp.fit_ramps( resultants, dq, read_noise, ROMAN_READ_TIME, read_pattern, threshold_constant=0, threshold_intercept=0 ) # set the threshold parameters # to demo the interface. This From e09983a2b16ccbd244fa2eb6ff226a517801d9e2 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 17 Nov 2023 12:14:11 -0500 Subject: [PATCH 29/31] Fix interface to match previous one This is done so that romancal does not break when this gets merged --- src/stcal/ramp_fitting/ols_cas22_fit.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/stcal/ramp_fitting/ols_cas22_fit.py diff --git a/src/stcal/ramp_fitting/ols_cas22_fit.py b/src/stcal/ramp_fitting/ols_cas22_fit.py new file mode 100644 index 00000000..05e03e8f --- /dev/null +++ b/src/stcal/ramp_fitting/ols_cas22_fit.py @@ -0,0 +1,13 @@ +"""This module exists to keep the interfaces the same as before a refactoring.""" +import warnings + +from .ols_cas22 import fit_ramps as fit_ramps_casertano + +__all__ = ["fit_ramps_casertano"] + +warnings.warn( + "The module stcal.ramp_fitting.ols_cas22_fit is deprecated. " + "Please use stcal.ramp_fitting.ols_cas22 instead.", + DeprecationWarning, + stacklevel=2, +) From 9b296248c70210f8eca7c6c8b674641c7fbdd3ad Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 17 Nov 2023 13:08:58 -0500 Subject: [PATCH 30/31] Bump cython version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c1e946a9..84e2e30e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ test = [ requires = [ 'setuptools >=61', 'setuptools_scm[toml] >=3.4', - 'Cython >=0.29.21', + 'Cython >=3.0.0', 'numpy >=1.18', ] build-backend = 'setuptools.build_meta' From 34e59dce70e0049b2b2602f5267265324e1edc9b Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Wed, 31 Jan 2024 10:49:02 -0500 Subject: [PATCH 31/31] Update changes --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 094c6404..7f85b560 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,10 @@ jump ---- - Fix the code to at least always flag the group with the shower and the requested groups after the primary shower. [#237] + + ramp + ---- +- Convert Cython code to using annotated .py files. [#232] 1.5.2 (2023-12-13) ==================