adding a test with a more diverse set of buffers errors, add initial doc

docs/api/noise_detection.md

@@ -0,0 +1,51 @@
+# denoising / pre-processing
+
+## Filter broken buffers (block of images) in a frame 
+
+explain how to filter, different methods we have now
+how to use them
+testing preo-processing algorithms on 200x200 pixel frames
+
+### Types of broken buffer (blocks of images)
+#### 'Check pattern' broken buffers:
+Black and white pixels
+
+- **Less than one row broken ('check pattern' <200 px)**:
+
+
+![Image 1](../images/preprocess_broken_buffers/onerow1.png)
+![Image 2](../images/preprocess_broken_buffers/onerow2.png)
+
+
+- **More than one row but clearly less than a block broken ('check pattern' 2-19 rows of pixels)**:
+
+
+![Image 3](../images/preprocess_broken_buffers/2_19_rows1.png)
+![Image 4](../images/preprocess_broken_buffers/2_19_rows2.png)
+
+- **One block broken ('check-pattern' >19 rows of pixels)**:
+
+![Image 5](../images/preprocess_broken_buffers/oneblock1.png)
+![Image 6](../images/preprocess_broken_buffers/oneblock2.png)
+
+
+- **Several blocks brocken ('check pattern' distinctive blocks)**:
+
+![Image 7](../images/preprocess_broken_buffers/severalblocks1.png)
+![Image 8](../images/preprocess_broken_buffers/severalblocks2.png)
+
+
+#### Black-out broken buffers:
+Entirely black pixels
+
+- **One block broken ('black-out')**:
+
+![Image 8](../images/preprocess_broken_buffers/black_oneblock1.png)
+![Image 9](../images/preprocess_broken_buffers/black_oneblock2.png)
+
+- **Majority of frame broken ('black-out')**:
+
+![Image 9](../images/preprocess_broken_buffers/major_black_frame1.png)
+
+##
+add parts for spaitial mask filtering of horizontal stripes

docs/index.md

@@ -33,6 +33,7 @@ api/utils
 api/stream_daq
 api/bit_operation
 api/vendor/index
+api/noise_detection
 ```
 
 ```{toctree}

tests/data/wireless_corrupted_extended.yaml

@@ -0,0 +1,97 @@
+frames: # frame indexing in cv2 starts from 0
+  check_pattern:
+    less_than_one_row:
+      - 71
+      - 117
+      - 413
+      - 682
+      - 752
+    several_rows:
+      - 1
+      - 4
+      - 7
+      - 8
+      - 11
+      - 15
+      - 32
+      - 88
+      - 89
+      - 96
+      - 101
+      - 124
+      - 144
+      - 214
+      - 270
+      - 284
+      - 310
+      - 335
+      - 367
+      - 400
+      - 427
+      - 467
+      - 502
+      - 570
+      - 580
+      - 583
+      - 595
+      - 596
+      - 637
+      - 641
+      - 643
+      - 665
+      - 676
+      - 708
+      - 719
+      - 723
+      - 726
+      - 729
+      - 731
+      - 733
+      - 742
+      - 744
+      - 749
+      - 751
+      - 767
+      - 769
+      - 783
+      - 796
+      - 797
+      - 798
+      - 800
+      - 801
+      - 803
+      - 804
+      - 805
+      - 809
+      - 826
+      - 832
+      - 872
+    one_block:
+      - 0
+      - 41
+      - 46
+      - 67
+      - 98
+      - 195
+      - 208
+      - 685
+      - 718
+      - 737
+      - 747
+      - 750
+      - 755
+      - 770
+      - 799
+      - 802
+    several_blocks:
+      - 142
+      - 159
+      - 608
+      - 794
+      - 869
+  blacked_out:
+    one_block:
+      - 18
+      - 718
+    majority_of_frame:
+      - 19

tests/test_process/test_extended_frame_helper.py

@@ -0,0 +1,114 @@
+import cv2
+import yaml
+import numpy as np
+import pytest
+
+from mio.models.process import DenoiseConfig, NoisePatchConfig
+from mio.process.frame_helper import NoiseDetectionHelper
+
+from ..conftest import DATA_DIR
+
+@pytest.mark.parametrize(
+    "noise_detection_method",
+    [
+        "gradient",
+        "mean_error",
+    ],
+)
+
+def test_noise_detection_contrast(noise_detection_method):
+    """
+    Test broken buffer noise with detailed categorization of real wirelessly imaging noise.
+    Two main categories should be correctly detected: check-pattern and blacked-out labeled in YAML file.
+    Subcategories are dependent on the number of pixels (number of pixel rows) that are broken.
+    """
+    if noise_detection_method == "gradient":
+        global_config: DenoiseConfig = DenoiseConfig.from_id("denoise_example")
+    elif noise_detection_method == "mean_error":
+        global_config: DenoiseConfig = DenoiseConfig.from_id("denoise_example_mean_error")
+    else:
+        raise ValueError("Invalid noise detection method")
+
+    config: NoisePatchConfig = global_config.noise_patch
+    config.method = noise_detection_method
+
+    with open(DATA_DIR / "wireless_corrupted_extended.yaml") as yfile:
+        expected = yaml.safe_load(yfile)
+
+    video = cv2.VideoCapture(str(DATA_DIR / "wireless_corrupted_extended.avi"))
+
+    detector = NoiseDetectionHelper(
+        height=int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)),
+        width=int(video.get(cv2.CAP_PROP_FRAME_WIDTH)),
+    )
+
+    missed_frames = {category: {sub: [] for sub in subs} for category, subs in expected["frames"].items()}
+    false_positives = []
+
+    previous_frame = None
+
+    while video.isOpened():
+        ret, frame = video.read()
+        if not ret:
+            break
+
+        frame_number = int(video.get(cv2.CAP_PROP_POS_FRAMES)) - 1
+        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+
+        if noise_detection_method == "gradient":
+            is_noisy, _ = detector.detect_frame_with_noisy_buffer(
+                current_frame=frame, previous_frame=None, config=config
+            )
+        elif noise_detection_method == "mean_error":
+            if previous_frame is None:
+                previous_frame = frame
+            is_noisy, _ = detector.detect_frame_with_noisy_buffer(
+                current_frame=frame, previous_frame=previous_frame, config=config
+            )
+            if not is_noisy:
+                previous_frame = frame
+
+        # Track missed frames
+        detected_in_category = False
+        for category, subcategories in expected["frames"].items():
+            for subcategory, frame_list in subcategories.items():
+                if frame_number in frame_list:
+                    detected_in_category = True
+                    if not is_noisy:  # Mark as missed if not detected
+                        missed_frames[category][subcategory].append(frame_number)
+
+        # Track false positives
+        if is_noisy and not detected_in_category:
+            false_positives.append(frame_number)
+
+    # Prepare missed message, grouped by method
+    missed_message = f"=== Missed Frames ({noise_detection_method}) ==="
+    for category, subcategories in missed_frames.items():
+        for subcategory, frames in subcategories.items():
+            if frames:  # Only include non-empty missed frames
+                missed_message += f"\n  Category: {category} -> Subcategory: {subcategory}: {frames}"
+
+    # Prepare false positive message, grouped by method
+    false_positive_message = (
+        f"\n=== False Positives ({noise_detection_method}) ===\n  Frames: {false_positives}"
+        if false_positives
+        else f"\n=== False Positives ({noise_detection_method}) ===\n  None"
+    )
+
+    print(f"Missed Message: {missed_message}")
+    print(f"False Positive Message: {false_positive_message}")
+
+
+    # Combine detailed message
+    detailed_message = (
+    f"=== Detailed Noise Detection Report ({noise_detection_method}) ===\n"
+    f"{missed_message}\n{false_positive_message}"
+)
+
+    # Debug: Print the detailed message for verification
+    print(detailed_message)
+
+    # Raise assertion error with detailed message
+    assert not any(
+        missed_frames[cat][sub] for cat in missed_frames for sub in missed_frames[cat]
+    ) and not false_positives, detailed_message

tests/test_process/test_frame_helper.py

@@ -18,8 +18,9 @@
 )
 def test_noise_detection_contrast(noise_detection_method):
     """
-    Contrast method of noise detection should correctly label frames corrupted
-    by speckled noise
+    Test broken buffer noise with detailed categorization of real wirelessly imaging noise.
+    Two main categories should be correctly detected: check-pattern and blacked-out labeled in YAML file.
+    Subcategories are dependent on the number of pixels (number of pixel rows) that are broken.
     """
     if noise_detection_method == "gradient":
         global_config: DenoiseConfig = DenoiseConfig.from_id("denoise_example")
@@ -34,35 +35,56 @@ def test_noise_detection_contrast(noise_detection_method):
     with open(DATA_DIR / "wireless_corrupted.yaml") as yfile:
         expected = yaml.safe_load(yfile)
 
-    video = cv2.VideoCapture(str(DATA_DIR / "wireless_corrupted.avi"))
+    video = cv2.VideoCapture(str(DATA_DIR / "wireless_corrupted_extended.avi"))
 
     detector = NoiseDetectionHelper(
         height=int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)),
         width=int(video.get(cv2.CAP_PROP_FRAME_WIDTH)),
     )
 
-    frames = []
-    masks = []
+    missed_frames = {category: {sub: [] for sub in subs} for category, subs in expected["frames"].items()}
+    false_positives = []
+
     previous_frame = None
+
     while video.isOpened():
         ret, frame = video.read()
         if not ret:
             break
 
+        frame_number = int(video.get(cv2.CAP_PROP_POS_FRAMES)) - 1
         frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+
         if noise_detection_method == "gradient":
-            is_noisy, mask = detector.detect_frame_with_noisy_buffer(
+            is_noisy, _ = detector.detect_frame_with_noisy_buffer(
                 current_frame=frame, previous_frame=None, config=config
             )
-        if noise_detection_method == "mean_error":
+        elif noise_detection_method == "mean_error":
             if previous_frame is None:
                 previous_frame = frame
-            is_noisy, mask = detector.detect_frame_with_noisy_buffer(
+            is_noisy, _ = detector.detect_frame_with_noisy_buffer(
                 current_frame=frame, previous_frame=previous_frame, config=config
             )
             if not is_noisy:
                 previous_frame = frame
-        frames.append(is_noisy)
-        masks.append(mask)
 
-    assert np.array_equal(np.where(frames)[0], expected["frames"])
+        # Track missed frames
+        detected_in_category = False
+        for category, subcategories in expected["frames"].items():
+            for subcategory, frame_list in subcategories.items():
+                if frame_number in frame_list:
+                    detected_in_category = True
+                    if not is_noisy:  # Mark as missed if not detected
+                        missed_frames[category][subcategory].append(frame_number)
+
+        # Track false positives
+        if is_noisy and not detected_in_category:
+            false_positives.append(frame_number)
+
+    # Print the report for this specific method
+    print(f"\n=== Detailed Noise Detection Report for Method: {noise_detection_method} ===")
+
+    print("\nMissed Frames:")
+    for category, subcategories in missed_frames.items():
+        print(f"\nCategory: {category}")
+        for subcategory, frames in subcategories.items():