From 435553cba9f3183a376c933804606cf48d21eeca Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 15 Mar 2024 13:35:07 -0300 Subject: [PATCH 1/7] port change from #35 --- ioos_qc/qartod.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index c256dfc..0eaf2e5 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -372,8 +372,9 @@ def check(self, tinp, inp, zinp): with np.errstate(invalid='ignore'): z_idx = (~zinp.mask) & (zinp >= m.zspan.minv) & (zinp <= m.zspan.maxv) else: - # Only test the values with masked Z, ie values with no Z - z_idx = zinp.mask + # If there is no z data in the config, don't try to filter by depth! + # Set z_idx to all True to prevent filtering + z_idx = np.ones(inp.size, dtype=bool) # Combine the T and Z indexes values_idx = (t_idx & z_idx) From f2ec6b20f4b7074de7b73d519ad10b5b2513e7bd Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 15 Mar 2024 13:47:25 -0300 Subject: [PATCH 2/7] my guess is that this should be 3, fail --- tests/test_qartod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_qartod.py b/tests/test_qartod.py index 51c2bcc..26e2e4e 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -857,7 +857,7 @@ def test_climatology_test_depths(self): 101 ) ] - expected_result = [1, 1, 1, 3, 3, 2] + expected_result = [1, 1, 1, 3, 3, 3] self._run_test(test_inputs, expected_result) From 518c9f7f01181a79da193828203c6061f819b90f Mon Sep 17 00:00:00 2001 From: Iwen Su Date: Mon, 1 Apr 2024 05:51:50 -0700 Subject: [PATCH 3/7] Add QartodClimtologyMissingTest() to Test to capture test scenario provided here https://github.com/ioos/ioos_qc/issues/65 --- tests/test_qartod.py | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_qartod.py b/tests/test_qartod.py index 26e2e4e..cfa65f4 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -720,6 +720,62 @@ def test_climatology_test_all_unknown(self): self._run_test(test_inputs, expected_result) + +class QartodClimatologyMissingTest(unittest.TestCase): + def setUp(self): + self.cc = qartod.ClimatologyConfig() + # different time range, no depth + self.cc.add( + tspan=(np.datetime64('2021-07'), np.datetime64('2021-09')), + vspan=(3.4, 5) + ) + + def _run_test(self, test_inputs, expected_result): + times, values, depths = zip(*test_inputs) + inputs = [ + values, + np.asarray(values, dtype=np.float64), + dask_arr(np.asarray(values, dtype=np.float64)) + ] + + for i in inputs: + results = qartod.climatology_test( + config=self.cc, + tinp=times, + inp=i, + zinp=depths + ) + npt.assert_array_equal( + results, + np.ma.array(expected_result) + ) + + def test_climatology_missing_values(self): + test_inputs = [ + # Not missing value or depth, value out of bounds + ( + np.datetime64('2021-07-16'), + 0, + 0 + ), + # Missing value and depth + ( + np.datetime64('2021-07-16'), + np.nan, + np.nan + ), + # Not missing value and depth, value within bounds + ( + np.datetime64('2021-07-16'), + 4.16743, + 0.08931513 + ) + ] + expected_result = [3, 9, 1] + self._run_test(test_inputs, expected_result) + + + class QartodClimatologyTest(unittest.TestCase): def setUp(self): @@ -859,6 +915,7 @@ def test_climatology_test_depths(self): ] expected_result = [1, 1, 1, 3, 3, 3] self._run_test(test_inputs, expected_result) + class QartodSpikeTest(unittest.TestCase): From 71efc11716f4b93e6e6958f64729fbb9a1585c75 Mon Sep 17 00:00:00 2001 From: Iwen Su Date: Mon, 1 Apr 2024 05:52:57 -0700 Subject: [PATCH 4/7] Set MISSING flag at the end of check() to prevent override --- ioos_qc/qartod.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 0eaf2e5..04f8b58 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -335,9 +335,6 @@ def check(self, tinp, inp, zinp): flag_arr = np.ma.empty(inp.size, dtype='uint8') flag_arr.fill(QartodFlags.UNKNOWN) - # If the value is masked set the flag to MISSING - flag_arr[inp.mask] = QartodFlags.MISSING - # Iterate over each member and apply its spans on the input data. # Member spans are applied in order and any data points that fall into # more than one member are flagged by each one. @@ -388,11 +385,14 @@ def check(self, tinp, inp, zinp): fail_idx = np.zeros(inp.size, dtype=bool) suspect_idx = (inp < m.vspan.minv) | (inp > m.vspan.maxv) - + with np.errstate(invalid='ignore'): flag_arr[(values_idx & fail_idx)] = QartodFlags.FAIL flag_arr[(values_idx & ~fail_idx & suspect_idx)] = QartodFlags.SUSPECT flag_arr[(values_idx & ~fail_idx & ~suspect_idx)] = QartodFlags.GOOD + + # If the value is masked set the flag to MISSING + flag_arr[inp.mask] = QartodFlags.MISSING return flag_arr @@ -423,11 +423,11 @@ def climatology_test(config : Union[ClimatologyConfig, Sequence[Dict[str, Tuple] config: A ClimatologyConfig object or a list of dicts containing tuples that can be used to create a ClimatologyConfig object. See ClimatologyConfig docs for more info. + inp: Input data as a numeric numpy array or a list of numbers. tinp: Time data as a sequence of datetime objects compatible with pandas DatetimeIndex. This includes numpy datetime64, python datetime objects and pandas Timestamp object. ie. pd.DatetimeIndex([datetime.utcnow(), np.datetime64(), pd.Timestamp.now()] If anything else is passed in the format is assumed to be seconds since the unix epoch. - vinp: Input data as a numeric numpy array or a list of numbers. zinp: Z (depth) data, in meters positive down, as a numeric numpy array or a list of numbers. Returns: From 485d63535eb1f281f4b0473a364dea723c068ab8 Mon Sep 17 00:00:00 2001 From: Iwen Su Date: Mon, 1 Apr 2024 15:39:26 -0700 Subject: [PATCH 5/7] Remove extra line between class definitions --- tests/test_qartod.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_qartod.py b/tests/test_qartod.py index cfa65f4..92b5a74 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -720,7 +720,6 @@ def test_climatology_test_all_unknown(self): self._run_test(test_inputs, expected_result) - class QartodClimatologyMissingTest(unittest.TestCase): def setUp(self): self.cc = qartod.ClimatologyConfig() @@ -775,7 +774,6 @@ def test_climatology_missing_values(self): self._run_test(test_inputs, expected_result) - class QartodClimatologyTest(unittest.TestCase): def setUp(self): @@ -917,7 +915,6 @@ def test_climatology_test_depths(self): self._run_test(test_inputs, expected_result) - class QartodSpikeTest(unittest.TestCase): def setUp(self): From 8b6d5f03ebe3da18dc4f434a3c0ee53adca3ae85 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 2 Apr 2024 15:37:59 +0200 Subject: [PATCH 6/7] configure pre-commit --- .pre-commit-config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8430946..989a95a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,3 +24,14 @@ repos: )$ args: - --ignore-words-list=pres,ba,ot + +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autofix_prs: false + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: monthly + skip: [] + submodules: false From 8a870386bfce3361ad6c1135905e77e53ef672ac Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 2 Apr 2024 15:38:24 +0200 Subject: [PATCH 7/7] run pre-commit --- ioos_qc/qartod.py | 4 ++-- tests/test_qartod.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 04f8b58..5242bc2 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -385,12 +385,12 @@ def check(self, tinp, inp, zinp): fail_idx = np.zeros(inp.size, dtype=bool) suspect_idx = (inp < m.vspan.minv) | (inp > m.vspan.maxv) - + with np.errstate(invalid='ignore'): flag_arr[(values_idx & fail_idx)] = QartodFlags.FAIL flag_arr[(values_idx & ~fail_idx & suspect_idx)] = QartodFlags.SUSPECT flag_arr[(values_idx & ~fail_idx & ~suspect_idx)] = QartodFlags.GOOD - + # If the value is masked set the flag to MISSING flag_arr[inp.mask] = QartodFlags.MISSING diff --git a/tests/test_qartod.py b/tests/test_qartod.py index 92b5a74..d3f25b9 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -913,7 +913,7 @@ def test_climatology_test_depths(self): ] expected_result = [1, 1, 1, 3, 3, 3] self._run_test(test_inputs, expected_result) - + class QartodSpikeTest(unittest.TestCase):