Made has_equal_in a new format callable

[SadiinsoSnowfall]

No description provided.

[SadiinsoSnowfall]

Waiting for the SVE2 CI

[SadiinsoSnowfall]

Closes #2065

[SadiinsoSnowfall]

Codegen for has_equal_in on SVE using g++ (codegen improved on clang too but the compilers fails to perform basic optimisations on this target)

; exactly 128 bits of data
test(eve::arm_sve256_v0::wide<unsigned char, eve::fixed<16l>>, eve::arm_sve256_v0::wide<unsigned char, eve::fixed<16l>>):
        ptrue   p0.b, vl32
        match   p0.b, p0/z, z0.b, z1.b
        ret
        
; less than 128 bits of data (32 bits)
test(eve::arm_sve256_v0::wide<unsigned char, eve::fixed<4l>>, eve::arm_sve256_v0::wide<unsigned char, eve::fixed<4l>>):
        mov     z1.s, s1
        ptrue   p0.b, vl4
        match   p0.b, p0/z, z0.b, z1.b
        ret
       
; more than 128 bits of data (256, in SVE2-256 mode)
test(eve::arm_sve256_v0::wide<unsigned char, eve::fixed<32l>>, eve::arm_sve256_v0::wide<unsigned char, eve::fixed<32l>>):
        ptrue   p0.b, vl32
        movprfx z26, z1
        ext     z26.b, z26.b, z1.b, #16
        match   p3.b, p0/z, z0.b, z26.b
        match   p2.b, p0/z, z0.b, z1.b
        movprfx z27, z0
        ext     z27.b, z27.b, z0.b, #16
        orr     p2.b, p0/z, p2.b, p3.b
        match   p1.b, p0/z, z27.b, z26.b
        match   p3.b, p0/z, z27.b, z1.b
        orr     p3.b, p0/z, p3.b, p1.b
        index   z29.b, #0, #1
        mov     z30.b, p3/z, #-1
        cmpls   p1.b, p0/z, z29.b, #15
        mov     z31.b, p2/z, #-1
        mov     z28.b, p1/z, #-1
        and     z28.b, z28.b, #0x10
        cmplo   p3.b, p0/z, z29.b, z28.b
        splice  z31.b, p3, z31.b, z30.b
        cmpne   p0.b, p0/z, z31.b, #0
        ret

The code for the first two cases is optimal (the special "broadcasting move" is used to round up the second operand to 128 bits while keeping the same values, and the mask used by the match instruction is used to disable the inactive lanes in the first operand.

The code for the third case is good enough, we have two pairs of movprfx+ext to split both operands into chunks of 128 bits, then two pairs of match+match+orr to perform the operation on all combinations of the 128-bits-wide blocks. Around half of the code is used to perform the final concatenation of the result back to 256 bits. This kind of operation isn't directly supported on SVE/2 and so results in a few instructions ending with a splice.

[DenisYaroshevskiy]

I don't like 256 codegen. I think it's a bit much. You can do 16bytes with 16 bytes. Then swap two sides and do that again.

[SadiinsoSnowfall]

Updated codegen with new algorithm :
SVE-512 - 256 bits (2 match lanes)

test(eve::arm_sve512_v0::wide<unsigned char, eve::fixed<32l> >, eve::arm_sve512_v0::wide<unsigned char, eve::fixed<32l> >):
       ptrue   p3.b, vl64
       movprfx z31, z1
       ext     z31.b, z31.b, z1.b, #32
       ext     z31.b, z31.b, z1.b, #32
       movprfx z30, z31
       ext     z30.b, z30.b, z31.b, #16
       match   p0.b, p3/z, z0.b, z31.b
       match   p2.b, p3/z, z0.b, z30.b
       orr     p0.b, p3/z, p0.b, p2.b
       ret

SVE-512 - 512 bits (4 match lanes)

test(eve::arm_sve512_v0::wide<unsigned char, eve::fixed<64l> >, eve::arm_sve512_v0::wide<unsigned char, eve::fixed<64l> >):
        ptrue   p3.b, vl64
        movprfx z31, z1
        ext     z31.b, z31.b, z1.b, #0
        ext     z31.b, z31.b, z1.b, #0
        movprfx z30, z31
        ext     z30.b, z30.b, z31.b, #16
        match   p1.b, p3/z, z0.b, z30.b
        movprfx z29, z30
        ext     z29.b, z29.b, z30.b, #16
        match   p2.b, p3/z, z0.b, z31.b
        movprfx z28, z29
        ext     z28.b, z28.b, z29.b, #16
        orr     p2.b, p3/z, p2.b, p1.b
        match   p0.b, p3/z, z0.b, z28.b
        match   p1.b, p3/z, z0.b, z29.b
        orr     p2.b, p3/z, p2.b, p1.b
        orr     p0.b, p3/z, p2.b, p0.b
        ret

SVE-256 256 bits (4 match lanes)

test(eve::arm_sve256_v0::wide<unsigned char, eve::fixed<32l> >, eve::arm_sve256_v0::wide<unsigned char, eve::fixed<32l> >):
        ptrue   p3.b, vl32
        movprfx z31, z1
        ext     z31.b, z31.b, z1.b, #0
        ext     z31.b, z31.b, z1.b, #0
        movprfx z30, z31
        ext     z30.b, z30.b, z31.b, #16
        match   p0.b, p3/z, z0.b, z31.b
        match   p2.b, p3/z, z0.b, z30.b
        orr     p0.b, p3/z, p0.b, p2.b
        ret

Tested using :

#include <eve/eve.hpp>

using namespace eve;

using T = wide<unsigned char, fixed<64>>; // change size & type here

auto test(T a, T b) {
    return has_equal_in(a, b);
}

[DenisYaroshevskiy]

std::result_of_t

[DenisYaroshevskiy]

shuffle_l<3>(fw_t{match_against}, [](auto i, auto) { return i % N::value; });