Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Set DQ of ref pixels to DO_NOT_USE after refpix correction #7017

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+72 −45
Open

Set DQ of ref pixels to DO_NOT_USE after refpix correction #7017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,11 @@ ramp_fitting
removing NaNs from the rateints product and setting appropriate DQ
flags. [#6949]

refpix
------

- Set DQ of ref pixels to DO_NOT_USE after `refpix` correction [#7017]

Comment on lines +580 to +584
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
refpix
------
- Set DQ of ref pixels to DO_NOT_USE after `refpix` correction [#7017]

now that #8671 is merged (switching change log handling to towncrier) this change log entry should be a file in changes/ instead:

echo "Set DQ of ref pixels to ``DO_NOT_USE`` after ``refpix`` correction" > changes/7017.refpix.rst

(new PRs will include instructions on how to do this)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(ignore this if this PR is no longer active or needed)

resample
--------

Expand Down
2 changes: 2 additions & 0 deletions docs/jwst/refpix/description.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ NIR Detector Data
subtracted from the full group on a row-by-row basis. Note that the ``odd_even_rows``
parameter is ignored for NIR data when the side reference pixels are processed.
#. Transform the data back to the JWST focal plane, or DMS, frame.
#. Flag reference pixel locations in DQ array as DO_NOT_USE now that the correction is
complete, so that they are not used in ramp fitting or any other downstream steps.

MIR Detector Data
+++++++++++++++++
Expand Down
46 changes: 23 additions & 23 deletions jwst/refpix/reference_pixels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,13 @@
from ..datamodels import dqflags
from ..lib import reffile_utils


log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)

#
DO_NOT_USE = dqflags.pixel["DO_NOT_USE"]
REFERENCE_PIXEL = dqflags.pixel["REFERENCE_PIXEL"]

# NIR Reference section dictionaries are zero indexed and specify the values
# to be used in the following slice:
# (rowstart: rowstop, colstart:colstop)
Expand Down Expand Up @@ -216,15 +219,14 @@ def sigma_clip(self, data, dq, low=3.0, high=3.0):

"""

#
# Only calculate the clipped mean for pixels that don't have the DO_NOT_USE
# DQ bit set
goodpixels = np.where(np.bitwise_and(dq, dqflags.pixel['DO_NOT_USE']) == 0)
#
# DQ bit set. Create a boolean mask.
goodpixels = np.bitwise_and(dq, DO_NOT_USE) != DO_NOT_USE

# If there are no good pixels, return None
if len(goodpixels[0]) == 0:
if not goodpixels.any():
return None
#

# scipy routine fails if the pixels all have exactly the same value
if np.std(data[goodpixels], dtype=np.float64) != 0.0:
clipped_ref, lowlim, uplim = stats.sigmaclip(data[goodpixels],
Expand Down Expand Up @@ -261,11 +263,10 @@ def get_pixeldq(self):
fullcols = 2048
self.full_shape = (fullrows, fullcols)
pixeldq = np.zeros(self.full_shape, dtype=self.input_model.pixeldq.dtype)
refpixdq_dontuse = dqflags.pixel['DO_NOT_USE'] | dqflags.pixel['REFERENCE_PIXEL']
pixeldq[0:4, :] = refpixdq_dontuse
pixeldq[fullrows - 4:fullrows, :] = refpixdq_dontuse
pixeldq[4:fullrows - 4, 0:4] = refpixdq_dontuse
pixeldq[4:fullrows - 4, fullcols - 4:fullcols] = refpixdq_dontuse
pixeldq[0:4, :] = DO_NOT_USE | REFERENCE_PIXEL
pixeldq[fullrows - 4:fullrows, :] = DO_NOT_USE | REFERENCE_PIXEL
pixeldq[4:fullrows - 4, 0:4] = DO_NOT_USE | REFERENCE_PIXEL
pixeldq[4:fullrows - 4, fullcols - 4:fullcols] = DO_NOT_USE | REFERENCE_PIXEL
pixeldq[self.rowstart:self.rowstop, self.colstart:self.colstop] = self.input_model.pixeldq.copy()
else:
pixeldq = self.input_model.pixeldq.copy()
Expand Down Expand Up @@ -394,21 +395,19 @@ def log_parameters(self):
log.info('refpix processing skipped for this mode')

def count_good_side_refpixels(self):
donotuse = dqflags.pixel['DO_NOT_USE']
ngood = 0
for amplifier in 'AD':
rowstart, rowstop, colstart, colstop = NIR_reference_sections[amplifier]['side']
good = np.where(np.bitwise_and(self.pixeldq[rowstart:rowstop, colstart:colstop], donotuse) != donotuse)
good = np.where(np.bitwise_and(self.pixeldq[rowstart:rowstop, colstart:colstop], DO_NOT_USE) != DO_NOT_USE)
ngood += len(good[0])
return ngood

def count_good_top_bottom_refpixels(self):
donotuse = dqflags.pixel['DO_NOT_USE']
ngood = 0
for edge in ['top', 'bottom']:
for amplifier in 'ABCD':
rowstart, rowstop, colstart, colstop = NIR_reference_sections[amplifier][edge]
good = np.where(np.bitwise_and(self.pixeldq[rowstart:rowstop, colstart:colstop], donotuse) != donotuse)
good = np.where(np.bitwise_and(self.pixeldq[rowstart:rowstop, colstart:colstop], DO_NOT_USE) != DO_NOT_USE)
ngood += len(good[0])
return ngood

Expand Down Expand Up @@ -825,7 +824,7 @@ def median_filter(self, data, dq, smoothing_length):
rowstart = i
rowstop = rowstart + smoothing_length
goodpixels = np.where(np.bitwise_and(augmented_dq[rowstart:rowstop],
dqflags.pixel['DO_NOT_USE']) == 0)
DO_NOT_USE) == 0)
if len(goodpixels[0]) == 0:
result[i] = np.nan
else:
Expand Down Expand Up @@ -1012,17 +1011,14 @@ def do_subarray_corrections(self):
"""Do corrections for subarray. Reference pixel value calculated
separately for odd and even columns if odd_even_columns is True,
otherwise a single number calculated from all reference pixels"""
#

# First transform to detector coordinates
#
refdq = dqflags.pixel['REFERENCE_PIXEL']
donotuse = dqflags.pixel['DO_NOT_USE']
#
# This transforms the pixeldq array from DMS to detector coordinates,
# only needs to be done once
self.DMS_to_detector_dq()
# Determined refpix indices to use on each group
refpixindices = np.where((self.pixeldq & refdq == refdq) & (self.pixeldq & donotuse != donotuse))
refpixindices = np.where((self.pixeldq & REFERENCE_PIXEL == REFERENCE_PIXEL) & (self.pixeldq & DO_NOT_USE != DO_NOT_USE))
nrefpixels = len(refpixindices[0])
if nrefpixels == 0:
self.bad_reference_pixels = True
Expand Down Expand Up @@ -1931,6 +1927,10 @@ def correct_model(input_model, odd_even_columns,
input_dataset.log_parameters()
reference_pixel_correction(input_dataset)

# Now that they are corrected, flag reference pixels as DO_NOT_USE
mask = input_model.pixeldq & REFERENCE_PIXEL == REFERENCE_PIXEL
input_model.pixeldq[mask] |= DO_NOT_USE

return REFPIX_OK


Expand Down Expand Up @@ -2044,7 +2044,7 @@ def setup_dataset_for_zeroframe(input_dataset, saved_values):
gdtype = input_dataset.input_model.groupdq.dtype
gdq = np.zeros(dims, dtype=gdtype)
wh_zero = saved_values[-1]
gdq[wh_zero] = dqflags.pixel['DO_NOT_USE']
gdq[wh_zero] = DO_NOT_USE
gdq = gdq.reshape(new_dims)

# Setup dataset with ZEROFRAME data
Expand Down
64 changes: 42 additions & 22 deletions jwst/refpix/tests/test_refpix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
from jwst.refpix.reference_pixels import Dataset, NIRDataset, correct_model, create_dataset


REFERENCE_PIXEL = dqflags.pixel["REFERENCE_PIXEL"]
DO_NOT_USE = dqflags.pixel["DO_NOT_USE"]


def test_refpix_subarray():
'''Check that the correction is skipped for MIR subarray data '''

Expand Down Expand Up @@ -59,8 +63,8 @@ def test_each_amp():
im.data[:, 1:, :, 1031] = 4.0

# set reference pixels to 'REFERENCE_PIXEL'
im.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, 1028:] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, :4] = REFERENCE_PIXEL
im.pixeldq[:, 1028:] = REFERENCE_PIXEL

# run the step
out = RefPixStep.call(im)
Expand Down Expand Up @@ -103,8 +107,8 @@ def test_firstframe_sub():
im.data[:, :, :, 1031] = 4.0

# set reference pixels to 'REFERENCE_PIXEL'
im.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, 1028:] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, :4] = REFERENCE_PIXEL
im.pixeldq[:, 1028:] = REFERENCE_PIXEL

# run the step
outim = RefPixStep.call(im)
Expand Down Expand Up @@ -148,8 +152,8 @@ def test_odd_even():
im.data[:, 1:, 0:ysize - 1:2, 1031] = 8.0

# set reference pixels to 'REFERENCE_PIXEL'
im.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, 1028:] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, :4] = REFERENCE_PIXEL
im.pixeldq[:, 1028:] = REFERENCE_PIXEL

# run the step
out = RefPixStep.call(im)
Expand All @@ -165,6 +169,11 @@ def test_odd_even():
assert out.data[0, 5, 101, 6] == 47.0
assert out.data[0, 5, 101, 7] == 46.0

# Make sure reference pixel DQs are set to DO_NOT_USE
mask1 = out.pixeldq & REFERENCE_PIXEL == REFERENCE_PIXEL
mask2 = out.pixeldq & DO_NOT_USE == DO_NOT_USE
np.testing.assert_array_equal(mask1, mask1 & mask2)


def test_no_odd_even():
'''Check that odd/even rows are not applied if flag is set to False'''
Expand Down Expand Up @@ -200,8 +209,8 @@ def test_no_odd_even():
im.data[:, 1:, 0:ysize - 1:2, 1031] = 8.0

# set reference pixels to 'REFERENCE_PIXEL'
im.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, 1028:] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, :4] = REFERENCE_PIXEL
im.pixeldq[:, 1028:] = REFERENCE_PIXEL

# run the step
out = RefPixStep.call(im, odd_even_rows=False)
Expand Down Expand Up @@ -240,8 +249,8 @@ def test_side_averaging():
im.data[:, 1:, :, 1028:] = 2.0

# set reference pixels to 'REFERENCE_PIXEL'
im.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, 1028:] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, :4] = REFERENCE_PIXEL
im.pixeldq[:, 1028:] = REFERENCE_PIXEL

# run the step
out = RefPixStep.call(im)
Expand Down Expand Up @@ -271,8 +280,8 @@ def test_above_sigma():
im.data[0, 3, 50, 3] = 35.0

# set reference pixels to 'REFERENCE_PIXEL'
im.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, 1028:] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, :4] = REFERENCE_PIXEL
im.pixeldq[:, 1028:] = REFERENCE_PIXEL

# run the step
out = RefPixStep.call(im)
Expand Down Expand Up @@ -305,9 +314,9 @@ def test_nan_refpix():
im.data[0, 3, 50, 3] = np.nan

# set reference pixels to 'REFERENCE_PIXEL'
im.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[:, 1028:] = dqflags.pixel['REFERENCE_PIXEL']
im.pixeldq[50, 3] = dqflags.pixel['DO_NOT_USE']
im.pixeldq[:, :4] = REFERENCE_PIXEL
im.pixeldq[:, 1028:] = REFERENCE_PIXEL
im.pixeldq[50, 3] = DO_NOT_USE

# run the step
out = RefPixStep.call(im)
Expand Down Expand Up @@ -341,8 +350,8 @@ def test_do_corrections_subarray_no_oddEven(setup_subarray_cube):
input_model.data[0, 0, :, :] = dataval
input_model.data[0, 0, :4, :] = bottom_rpix
input_model.data[0, 0, :, :4] = left_rpix
input_model.pixeldq[:4, :] = dqflags.pixel['REFERENCE_PIXEL']
input_model.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
input_model.pixeldq[:4, :] = REFERENCE_PIXEL
input_model.pixeldq[:, :4] = REFERENCE_PIXEL

init_dataset = create_dataset(input_model,
odd_even_columns,
Expand Down Expand Up @@ -383,8 +392,8 @@ def test_do_corrections_subarray(setup_subarray_cube):
input_model.data[0, 0, :, :] = dataval
input_model.data[0, 0, :4, :] = bottom_rpix
input_model.data[0, 0, :, :4] = left_rpix
input_model.pixeldq[:4, :] = dqflags.pixel['REFERENCE_PIXEL']
input_model.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
input_model.pixeldq[:4, :] = REFERENCE_PIXEL
input_model.pixeldq[:, :4] = REFERENCE_PIXEL

init_dataset = create_dataset(input_model,
odd_even_columns,
Expand Down Expand Up @@ -453,9 +462,9 @@ def test_do_corrections_subarray_4amp(setup_subarray_cube):
input_model.data[0, 0, :, 1:4:2] = left_rpix + bottom_rpix_a_even
input_model.data[0, 0, :, -4::2] = right_rpix + bottom_rpix_d_odd
input_model.data[0, 0, :, -3::2] = right_rpix + bottom_rpix_d_even
input_model.pixeldq[:4, :] = dqflags.pixel['REFERENCE_PIXEL']
input_model.pixeldq[:, :4] = dqflags.pixel['REFERENCE_PIXEL']
input_model.pixeldq[:, -4:] = dqflags.pixel['REFERENCE_PIXEL']
input_model.pixeldq[:4, :] = REFERENCE_PIXEL
input_model.pixeldq[:, :4] = REFERENCE_PIXEL
input_model.pixeldq[:, -4:] = REFERENCE_PIXEL

init_dataset = create_dataset(input_model,
odd_even_columns,
Expand Down Expand Up @@ -752,6 +761,12 @@ def test_correct_model(setup_cube, instr, det):
input_model.data[0, 0, :, :] = rpix
input_model.data[0, 0, 4:-4, 4:-4] = dataval

# Populate DQ array with REFERENCE_PIXEL where appropriate
input_model.pixeldq[:4,:] = REFERENCE_PIXEL
input_model.pixeldq[-4:,:] = REFERENCE_PIXEL
input_model.pixeldq[:,:4] = REFERENCE_PIXEL
input_model.pixeldq[:,-4:] = REFERENCE_PIXEL

correct_model(input_model,
odd_even_columns,
use_side_ref_pixels,
Expand All @@ -762,6 +777,11 @@ def test_correct_model(setup_cube, instr, det):
np.testing.assert_almost_equal(np.mean(input_model.data[0, 0, :4, 4:-4]), 0, decimal=0)
np.testing.assert_almost_equal(np.mean(input_model.data[0, 0, 4:-4, 4:-4]), dataval - rpix, decimal=0)

# Make sure reference pixel DQs are set to DO_NOT_USE
mask1 = input_model.pixeldq & REFERENCE_PIXEL == REFERENCE_PIXEL
mask2 = input_model.pixeldq & DO_NOT_USE == DO_NOT_USE
np.testing.assert_array_equal(mask1, mask1 & mask2)


def test_zero_frame(setup_cube):
"""
Expand Down