fix NEON waterfill

Author: Gin

SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_BinaryMatrix.cpp

@@ -31,10 +31,9 @@ BinaryMatrixType get_BinaryMatrixType(){
         return BinaryMatrixType::i64x8_x64_SSE42;
     }
 #elif PA_ARCH_arm64
-    // TODO: enable this once binary matrix is ready!
-    // if (CPU_CAPABILITY_CURRENT.OK_M1){
-    //     return BinaryMatrixType::arm64x8_x64_NEON;
-    // }
+    if (CPU_CAPABILITY_CURRENT.OK_M1){
+        return BinaryMatrixType::arm64x8_x64_NEON;
+    }
 #endif
 
 //    return BinaryMatrixType::i64x8_Default;

SerialPrograms/Source/Kernels/Waterfill/Kernels_Waterfill_Core_64x8_arm64_NEON.cpp

@@ -6,7 +6,7 @@
 
 #ifdef PA_AutoDispatch_arm64_20_M1
 
-#define USE_CPP_TEMPLATE_IMPL
+// #define USE_CPP_TEMPLATE_IMPL
 
 #include "Kernels_Waterfill_Routines.h"
 #include "Kernels_Waterfill_Core_64x8_arm64_NEON.h"
@@ -24,15 +24,15 @@ namespace Waterfill{
 using Waterfill_64x8_Default = Waterfill_64xH_Default<BinaryTile_64x8_arm64_NEON>;
 
 std::vector<WaterfillObject> find_objects_inplace_64x8_arm64_NEON(PackedBinaryMatrix_IB& matrix, size_t min_area){
-    return find_objects_inplace<BinaryTile_64x8_arm64_NEON, Waterfill_64x8_arm64_NEON>(
+    return find_objects_inplace<BinaryTile_64x8_arm64_NEON, Waterfill_64x8_Default>(
         static_cast<PackedBinaryMatrix_64x8_arm64_NEON&>(matrix).get(),
         min_area
     );
 }
 std::unique_ptr<WaterfillSession> make_WaterfillSession_64x8_arm64_NEON(PackedBinaryMatrix_IB* matrix){
     return matrix == nullptr
-        ? std::make_unique<WaterfillSession_t<BinaryTile_64x8_arm64_NEON, Waterfill_64x8_arm64_NEON>>()
-        : std::make_unique<WaterfillSession_t<BinaryTile_64x8_arm64_NEON, Waterfill_64x8_arm64_NEON>>(
+        ? std::make_unique<WaterfillSession_t<BinaryTile_64x8_arm64_NEON, Waterfill_64x8_Default>>()
+        : std::make_unique<WaterfillSession_t<BinaryTile_64x8_arm64_NEON, Waterfill_64x8_Default>>(
             static_cast<PackedBinaryMatrix_64x8_arm64_NEON*>(matrix)->get()
         );
 }

SerialPrograms/Source/Kernels/Waterfill/Kernels_Waterfill_Core_64x8_arm64_NEON.h

@@ -55,14 +55,14 @@ PA_FORCE_INLINE uint64x2_t bit_reverse(uint64x2_t x){
 
 
 
-struct Waterfill_64x8_x64_SSE42_ProcessedMask{
+struct Waterfill_64x8_arm64_NEON_ProcessedMask{
     uint64x2_t m0, m1, m2, m3; //  Copy of logical OR of the mask `m` and recorded tile `x` bits 
     uint64x2_t b0, b1, b2, b3; //  Bit-reversed copy of m0, m1, m2, m3.
     uint64x2_t t0, t1, t2, t3; //  Transposed masks.
     uint64x2_t f1, f2, f3;     //  Forward-carry mask.
     uint64x2_t r0, r1, r2;     //  Reverse-carry mask.
 
-    PA_FORCE_INLINE Waterfill_64x8_x64_SSE42_ProcessedMask(
+    PA_FORCE_INLINE Waterfill_64x8_arm64_NEON_ProcessedMask(
         const BinaryTile_64x8_arm64_NEON& m,
         uint64x2_t x0, uint64x2_t x1, uint64x2_t x2, uint64x2_t x3
     ){
@@ -104,7 +104,7 @@ struct Waterfill_64x8_x64_SSE42_ProcessedMask{
 
 
 PA_FORCE_INLINE bool keep_going(
-    const Waterfill_64x8_x64_SSE42_ProcessedMask& mask,
+    const Waterfill_64x8_arm64_NEON_ProcessedMask& mask,
     uint64x2_t& m0, uint64x2_t& m1, uint64x2_t& m2, uint64x2_t& m3,
     uint64x2_t& x0, uint64x2_t& x1, uint64x2_t& x2, uint64x2_t& x3
 ){
@@ -145,7 +145,7 @@ PA_FORCE_INLINE bool keep_going(
 
 
 PA_FORCE_INLINE void expand_forward(
-    const Waterfill_64x8_x64_SSE42_ProcessedMask& mask,
+    const Waterfill_64x8_arm64_NEON_ProcessedMask& mask,
     uint64x2_t& x0, uint64x2_t& x1, uint64x2_t& x2, uint64x2_t& x3
 ){
     uint64x2_t s0 = vaddq_u64(x0, mask.m0);
@@ -170,7 +170,7 @@ PA_FORCE_INLINE void expand_reverse(uint64x2_t m, uint64x2_t b, uint64x2_t& x){
     x = vorrq_u64(x, s);
 }
 PA_FORCE_INLINE void expand_reverse(
-    const Waterfill_64x8_x64_SSE42_ProcessedMask& mask,
+    const Waterfill_64x8_arm64_NEON_ProcessedMask& mask,
     uint64x2_t& x0, uint64x2_t& x1, uint64x2_t& x2, uint64x2_t& x3
 ){
     expand_reverse(mask.m0, mask.b0, x0);
@@ -179,7 +179,7 @@ PA_FORCE_INLINE void expand_reverse(
     expand_reverse(mask.m3, mask.b3, x3);
 }
 PA_FORCE_INLINE void expand_vertical(
-    const Waterfill_64x8_x64_SSE42_ProcessedMask& mask,
+    const Waterfill_64x8_arm64_NEON_ProcessedMask& mask,
     uint64x2_t& x0, uint64x2_t& x1, uint64x2_t& x2, uint64x2_t& x3
 ){
     //  Carry across adjacent rows.
@@ -435,7 +435,7 @@ static PA_FORCE_INLINE void waterfill_expand(BinaryTile_64x8_arm64_NEON& m, Bina
     uint64x2_t x2 = x.vec.val[2];
     uint64x2_t x3 = x.vec.val[3];
 
-    Waterfill_64x8_x64_SSE42_ProcessedMask mask(m, x0, x1, x2, x3);
+    Waterfill_64x8_arm64_NEON_ProcessedMask mask(m, x0, x1, x2, x3);
     expand_forward(mask, x0, x1, x2, x3);
 
     uint64x2_t m0, m1, m2, m3;

SerialPrograms/Source/Tests/Kernels_Tests.cpp

@@ -17,11 +17,18 @@
 #ifdef PA_AutoDispatch_arm64_20_M1
     #include "Kernels/BinaryMatrix/Kernels_BinaryMatrixTile_64x8_arm64_NEON.h"
     #include "Kernels/PartialWordAccess/Kernels_PartialWordAccess_arm64_NEON.h"
+    #include "Kernels/Waterfill/Kernels_Waterfill_Core_64x8_arm64_NEON.h"
 #endif
+#include "Kernels/BinaryMatrix/Kernels_BinaryMatrix_Arch_64xH_Default.h"
+#include "Kernels/BinaryMatrix/Kernels_BinaryMatrixTile_64x4_Default.h"
 #include "Kernels/BinaryMatrix/Kernels_BinaryMatrixTile_64xH_Default.h"
 #include "Kernels/BinaryImageFilters/Kernels_BinaryImage_BasicFilters.h"
 #include "Kernels/ImageFilters/Kernels_ImageFilter_Basic.h"
 #include "Kernels/ImageScaleBrightness/Kernels_ImageScaleBrightness.h"
+#include "Kernels/Waterfill/Kernels_Waterfill.h"
+#include "Kernels/Waterfill/Kernels_Waterfill_Session.h"
+#include "Kernels/Waterfill/Kernels_Waterfill_Core_64xH_Default.h"
+#include "Kernels/Waterfill/Kernels_Waterfill_Routines.h"
 #include "Kernels_Tests.h"
 #include "TestUtils.h"
 
@@ -593,20 +600,74 @@ int test_kernels_CompressRGB32ToBinaryEuclidean(const ImageViewRGB32& image){
 
 
 int test_kernels_Waterfill(const ImageViewRGB32& image){
+    const size_t width = image.width();
+    const size_t height = image.height();
+    cout << "Testing test_kernels_Waterfill(), image size " << width << " x " << height << endl;
 
-    ImagePixelBox box(0, 0, image.width(), image.height());
-    ImageViewRGB32 sub_image = extract_box_reference(image, box);
-
-    PackedBinaryMatrix matrix(sub_image.width(), sub_image.height());
+    PackedBinaryMatrix matrix(width, height);
     uint32_t mins = combine_rgb(0, 0, 0);
     // uint32_t maxs = combine_rgb(255, 255, 255);
     uint32_t maxs = combine_rgb(63, 63, 63);
     Kernels::compress_rgb32_to_binary_range(
-        sub_image.data(), sub_image.bytes_per_row(),
+        image.data(), image.bytes_per_row(),
         matrix, mins, maxs
     );
 
-    cout << matrix.dump() << flush;
+    PackedBinaryMatrix source_matrix = matrix.copy();
+
+    PackedBinaryMatrix gt_matrix = matrix.copy();
+    Kernels::PackedBinaryMatrix_IB& gt_matrix_ib = gt_matrix;
+
+    size_t min_area = 10;
+    std::vector<Kernels::Waterfill::WaterfillObject> gt_objects;
+    bool gt_computed = false;
+
+#ifdef PA_AutoDispatch_arm64_20_M1
+    if (CPU_CAPABILITY_CURRENT.OK_M1){
+        using Waterfill_64x8_Default = Kernels::Waterfill::Waterfill_64xH_Default<Kernels::BinaryTile_64x8_arm64_NEON>;
+        gt_objects = Kernels::Waterfill::find_objects_inplace<Kernels::BinaryTile_64x8_arm64_NEON, Waterfill_64x8_Default>(
+            static_cast<Kernels::PackedBinaryMatrix_64x8_arm64_NEON&>(gt_matrix_ib).get(),
+            min_area
+        );
+        gt_computed = true;
+    }
+#endif
+    if (gt_computed == false){
+        using Waterfill_64x4_Default = Kernels::Waterfill::Waterfill_64xH_Default<Kernels::BinaryTile_64x4_Default>;
+        gt_objects = Kernels::Waterfill::find_objects_inplace<Kernels::BinaryTile_64x4_Default, Waterfill_64x4_Default>(
+            static_cast<Kernels::PackedBinaryMatrix_64x4_Default&>(gt_matrix_ib).get(),
+            min_area
+        );
+    }
+    cout << "num objects: " << gt_objects.size() << endl;
+
+    auto time_start = current_time();
+    std::vector<Kernels::Waterfill::WaterfillObject> objects = Kernels::Waterfill::find_objects_inplace(matrix, min_area);
+    auto time_end = current_time();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(time_end - time_start).count();
+    auto ms = ns / 1000000.;
+    cout << "One waterfill time: " << ms << " ms" << endl;
+
+    for(size_t i = 0; i < objects.size(); ++i){
+        TEST_RESULT_COMPONENT_EQUAL(objects[i].area, gt_objects[i].area, "object " + std::to_string(i) + " area");
+        TEST_RESULT_COMPONENT_EQUAL(objects[i].min_x, gt_objects[i].min_x, "object " + std::to_string(i) + " min_x");
+        TEST_RESULT_COMPONENT_EQUAL(objects[i].min_y, gt_objects[i].min_y, "object " + std::to_string(i) + " min_y");
+        TEST_RESULT_COMPONENT_EQUAL(objects[i].max_x, gt_objects[i].max_x, "object " + std::to_string(i) + " max_x");
+        TEST_RESULT_COMPONENT_EQUAL(objects[i].max_y, gt_objects[i].max_y, "object " + std::to_string(i) + " max_y");
+    }
+
+    // We try to wait for three seconds:
+    const size_t num_iters = size_t(3000 / ms);
+    time_start = current_time();
+    for(size_t i = 0; i < num_iters; i++){
+        matrix = source_matrix.copy();
+        objects = Kernels::Waterfill::find_objects_inplace(matrix, min_area);
+    }
+    time_end = current_time();
+    ms = (double)std::chrono::duration_cast<Milliseconds>(time_end - time_start).count();
+    cout << "Running " << num_iters << " iters, avg filter time: " << ms / num_iters << " ms" << endl;
+
+
 
     return 0;
 }