From 8227c918a8f99ac081e630fb2a1bfa14be6e82c6 Mon Sep 17 00:00:00 2001 From: Tex Riddell Date: Thu, 3 Oct 2024 20:46:41 -0700 Subject: [PATCH 1/6] Fix scalar overload name constructed by ReplaceWithVeclib.cpp ReplaceWithVeclib.cpp would construct overload name using all the arguments in the intrinsic, but overloads should only be constructed from arguments for which isVectorIntrinsicWithOverloadTypeAtArg returns true, including the return (-1). Fixes #111093 --- llvm/lib/CodeGen/ReplaceWithVeclib.cpp | 16 +++++++++++++++- .../AArch64/replace-with-veclib-armpl.ll | 17 +++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index 9fbb7b461364b1..551210db85713a 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -108,8 +108,22 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // all vector operands match the previously found EC. SmallVector ScalarArgTypes; Intrinsic::ID IID = II->getIntrinsicID(); + + // OloadTys collects types used in scalar intrinsic overload name. + SmallVector OloadTys; + if (VTy && isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) + OloadTys.push_back(VTy->getElementType()); + for (auto Arg : enumerate(II->args())) { auto *ArgTy = Arg.value()->getType(); + // Gather type if it is used in the overload name. + if (isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index())) { + if (!isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index()) && isa(ArgTy)) + OloadTys.push_back(cast(ArgTy)->getElementType()); + else + OloadTys.push_back(ArgTy); + } + if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) { ScalarArgTypes.push_back(ArgTy); } else if (auto *VectorArgTy = dyn_cast(ArgTy)) { @@ -129,7 +143,7 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // using scalar argument types. std::string ScalarName = Intrinsic::isOverloaded(IID) - ? Intrinsic::getName(IID, ScalarArgTypes, II->getModule()) + ? Intrinsic::getName(IID, OloadTys, II->getModule()) : Intrinsic::getName(IID).str(); // Try to find the mapping for the scalar version of this intrinsic and the diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll index f7e95008b71237..7b173bda561553 100644 --- a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll +++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll @@ -15,7 +15,7 @@ declare @llvm.cos.nxv2f64() declare @llvm.cos.nxv4f32() ;. -; CHECK: @llvm.compiler.used = appending global [60 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vacosq_f64, ptr @armpl_vacosq_f32, ptr @armpl_svacos_f64_x, ptr @armpl_svacos_f32_x, ptr @armpl_vasinq_f64, ptr @armpl_vasinq_f32, ptr @armpl_svasin_f64_x, ptr @armpl_svasin_f32_x, ptr @armpl_vatanq_f64, ptr @armpl_vatanq_f32, ptr @armpl_svatan_f64_x, ptr @armpl_svatan_f32_x, ptr @armpl_vcoshq_f64, ptr @armpl_vcoshq_f32, ptr @armpl_svcosh_f64_x, ptr @armpl_svcosh_f32_x, ptr @armpl_vsinhq_f64, ptr @armpl_vsinhq_f32, ptr @armpl_svsinh_f64_x, ptr @armpl_svsinh_f32_x, ptr @armpl_vtanhq_f64, ptr @armpl_vtanhq_f32, ptr @armpl_svtanh_f64_x, ptr @armpl_svtanh_f32_x], section "llvm.metadata" +; CHECK: @llvm.compiler.used = appending global [64 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vpowq_f64, ptr @armpl_vpowq_f32, ptr @armpl_svpow_f64_x, ptr @armpl_svpow_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vacosq_f64, ptr @armpl_vacosq_f32, ptr @armpl_svacos_f64_x, ptr @armpl_svacos_f32_x, ptr @armpl_vasinq_f64, ptr @armpl_vasinq_f32, ptr @armpl_svasin_f64_x, ptr @armpl_svasin_f32_x, ptr @armpl_vatanq_f64, ptr @armpl_vatanq_f32, ptr @armpl_svatan_f64_x, ptr @armpl_svatan_f32_x, ptr @armpl_vcoshq_f64, ptr @armpl_vcoshq_f32, ptr @armpl_svcosh_f64_x, ptr @armpl_svcosh_f32_x, ptr @armpl_vsinhq_f64, ptr @armpl_vsinhq_f32, ptr @armpl_svsinh_f64_x, ptr @armpl_svsinh_f32_x, ptr @armpl_vtanhq_f64, ptr @armpl_vtanhq_f32, ptr @armpl_svtanh_f64_x, ptr @armpl_svtanh_f32_x], section "llvm.metadata" ;. define <2 x double> @llvm_cos_f64(<2 x double> %in) { @@ -333,17 +333,10 @@ declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>) declare @llvm.pow.nxv2f64(, ) declare @llvm.pow.nxv4f32(, ) -; -; There is a bug in the replace-with-veclib pass, and for intrinsics which take -; more than one arguments, but has just one overloaded type, it incorrectly -; reconstructs the scalar name, for pow specifically it is searching for: -; llvm.pow.f64.f64 and llvm.pow.f32.f32 -; - define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %power) { ; CHECK-LABEL: define <2 x double> @llvm_pow_f64 ; CHECK-SAME: (<2 x double> [[IN:%.*]], <2 x double> [[POWER:%.*]]) { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x double> @llvm.pow.v2f64(<2 x double> [[IN]], <2 x double> [[POWER]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x double> @armpl_vpowq_f64(<2 x double> [[IN]], <2 x double> [[POWER]]) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %in, <2 x double> %power) @@ -353,7 +346,7 @@ define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %power) { define <4 x float> @llvm_pow_f32(<4 x float> %in, <4 x float> %power) { ; CHECK-LABEL: define <4 x float> @llvm_pow_f32 ; CHECK-SAME: (<4 x float> [[IN:%.*]], <4 x float> [[POWER:%.*]]) { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.pow.v4f32(<4 x float> [[IN]], <4 x float> [[POWER]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @armpl_vpowq_f32(<4 x float> [[IN]], <4 x float> [[POWER]]) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %in, <4 x float> %power) @@ -363,7 +356,7 @@ define <4 x float> @llvm_pow_f32(<4 x float> %in, <4 x float> %power) { define @llvm_pow_vscale_f64( %in, %power) #0 { ; CHECK-LABEL: define @llvm_pow_vscale_f64 ; CHECK-SAME: ( [[IN:%.*]], [[POWER:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast @llvm.pow.nxv2f64( [[IN]], [[POWER]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast @armpl_svpow_f64_x( [[IN]], [[POWER]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: ret [[TMP1]] ; %1 = call fast @llvm.pow.nxv2f64( %in, %power) @@ -373,7 +366,7 @@ define @llvm_pow_vscale_f64( %in, @llvm_pow_vscale_f32( %in, %power) #0 { ; CHECK-LABEL: define @llvm_pow_vscale_f32 ; CHECK-SAME: ( [[IN:%.*]], [[POWER:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast @llvm.pow.nxv4f32( [[IN]], [[POWER]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast @armpl_svpow_f32_x( [[IN]], [[POWER]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: ret [[TMP1]] ; %1 = call fast @llvm.pow.nxv4f32( %in, %power) From f0557de28123377baff92fc12d712244451b3708 Mon Sep 17 00:00:00 2001 From: Tex Riddell Date: Mon, 7 Oct 2024 11:48:16 -0700 Subject: [PATCH 2/6] Simplify logic a bit, handle scalar return overload type. --- llvm/lib/CodeGen/ReplaceWithVeclib.cpp | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index 551210db85713a..740712d17fe68a 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -104,6 +104,7 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // a void type. auto *VTy = dyn_cast(II->getType()); ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(0)); + Type *ScalarRetTy = II->getType()->getScalarType(); // Compute the argument types of the corresponding scalar call and check that // all vector operands match the previously found EC. SmallVector ScalarArgTypes; @@ -111,30 +112,23 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // OloadTys collects types used in scalar intrinsic overload name. SmallVector OloadTys; - if (VTy && isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) - OloadTys.push_back(VTy->getElementType()); + if (isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) + OloadTys.push_back(ScalarRetTy); for (auto Arg : enumerate(II->args())) { auto *ArgTy = Arg.value()->getType(); - // Gather type if it is used in the overload name. - if (isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index())) { - if (!isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index()) && isa(ArgTy)) - OloadTys.push_back(cast(ArgTy)->getElementType()); - else - OloadTys.push_back(ArgTy); - } - - if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) { - ScalarArgTypes.push_back(ArgTy); - } else if (auto *VectorArgTy = dyn_cast(ArgTy)) { - ScalarArgTypes.push_back(VectorArgTy->getElementType()); + auto *ScalarArgTy = ArgTy->getScalarType(); + ScalarArgTypes.push_back(ScalarArgTy); + if (isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index())) + OloadTys.push_back(ScalarArgTy); + if (auto *VectorArgTy = dyn_cast(ArgTy)) { // When return type is void, set EC to the first vector argument, and // disallow vector arguments with different ECs. if (EC.isZero()) EC = VectorArgTy->getElementCount(); else if (EC != VectorArgTy->getElementCount()) return false; - } else + } else if (!isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) // Exit when it is supposed to be a vector argument but it isn't. return false; } @@ -160,7 +154,6 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // Replace the call to the intrinsic with a call to the vector library // function. - Type *ScalarRetTy = II->getType()->getScalarType(); FunctionType *ScalarFTy = FunctionType::get(ScalarRetTy, ScalarArgTypes, /*isVarArg*/ false); const std::string MangledName = VD->getVectorFunctionABIVariantString(); From 2c7028519c6b141838b0b013d9e4ba74d9cca1cd Mon Sep 17 00:00:00 2001 From: Tex Riddell Date: Mon, 7 Oct 2024 14:30:22 -0700 Subject: [PATCH 3/6] Skip not_intrinsic; skip void return for OloadTys; update tests --- llvm/lib/CodeGen/ReplaceWithVeclib.cpp | 7 +++++-- .../CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll | 6 +++--- llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll | 6 +++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index 740712d17fe68a..3e359123d51d71 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -100,6 +100,9 @@ static void replaceWithTLIFunction(IntrinsicInst *II, VFInfo &Info, static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, IntrinsicInst *II) { assert(II != nullptr && "Intrinsic cannot be null"); + Intrinsic::ID IID = II->getIntrinsicID(); + if (IID == Intrinsic::not_intrinsic) + return false; // At the moment VFABI assumes the return type is always widened unless it is // a void type. auto *VTy = dyn_cast(II->getType()); @@ -108,11 +111,11 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, // Compute the argument types of the corresponding scalar call and check that // all vector operands match the previously found EC. SmallVector ScalarArgTypes; - Intrinsic::ID IID = II->getIntrinsicID(); // OloadTys collects types used in scalar intrinsic overload name. SmallVector OloadTys; - if (isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) + if (!ScalarRetTy->isVoidTy() && + isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) OloadTys.push_back(ScalarRetTy); for (auto Arg : enumerate(II->args())) { diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll index f3f27344ad80e3..155587026b464b 100644 --- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll +++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll @@ -4,7 +4,7 @@ target triple = "aarch64-unknown-linux-gnu" ;. -; CHECK: @llvm.compiler.used = appending global [30 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxv_acos, ptr @_ZGVsMxv_acosf, ptr @_ZGVsMxv_asin, ptr @_ZGVsMxv_asinf, ptr @_ZGVsMxv_atan, ptr @_ZGVsMxv_atanf, ptr @_ZGVsMxv_cosh, ptr @_ZGVsMxv_coshf, ptr @_ZGVsMxv_sinh, ptr @_ZGVsMxv_sinhf, ptr @_ZGVsMxv_tanh, ptr @_ZGVsMxv_tanhf], section "llvm.metadata" +; CHECK: @llvm.compiler.used = appending global [32 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxvv_pow, ptr @_ZGVsMxvv_powf, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxv_acos, ptr @_ZGVsMxv_acosf, ptr @_ZGVsMxv_asin, ptr @_ZGVsMxv_asinf, ptr @_ZGVsMxv_atan, ptr @_ZGVsMxv_atanf, ptr @_ZGVsMxv_cosh, ptr @_ZGVsMxv_coshf, ptr @_ZGVsMxv_sinh, ptr @_ZGVsMxv_sinhf, ptr @_ZGVsMxv_tanh, ptr @_ZGVsMxv_tanhf], section "llvm.metadata" ;. define @llvm_ceil_vscale_f64( %in) { ; CHECK-LABEL: @llvm_ceil_vscale_f64( @@ -278,7 +278,7 @@ define @llvm_nearbyint_vscale_f32( %in) define @llvm_pow_vscale_f64( %in, %pow) { ; CHECK-LABEL: @llvm_pow_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast @llvm.pow.nxv2f64( [[IN:%.*]], [[POW:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast @_ZGVsMxvv_pow( [[IN:%.*]], [[POW:%.*]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: ret [[TMP1]] ; %1 = call fast @llvm.pow.nxv2f64( %in, %pow) @@ -287,7 +287,7 @@ define @llvm_pow_vscale_f64( %in, @llvm_pow_vscale_f32( %in, %pow) { ; CHECK-LABEL: @llvm_pow_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast @llvm.pow.nxv4f32( [[IN:%.*]], [[POW:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast @_ZGVsMxvv_powf( [[IN:%.*]], [[POW:%.*]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: ret [[TMP1]] ; %1 = call fast @llvm.pow.nxv4f32( %in, %pow) diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll index 59c2f94e167633..b89bf3d6f2ca97 100644 --- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll +++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll @@ -4,7 +4,7 @@ target triple = "aarch64-unknown-linux-gnu" ;. -; CHECK: @llvm.compiler.used = appending global [30 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2v_acos, ptr @_ZGVnN4v_acosf, ptr @_ZGVnN2v_asin, ptr @_ZGVnN4v_asinf, ptr @_ZGVnN2v_atan, ptr @_ZGVnN4v_atanf, ptr @_ZGVnN2v_cosh, ptr @_ZGVnN4v_coshf, ptr @_ZGVnN2v_sinh, ptr @_ZGVnN4v_sinhf, ptr @_ZGVnN2v_tanh, ptr @_ZGVnN4v_tanhf], section "llvm.metadata" +; CHECK: @llvm.compiler.used = appending global [32 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2vv_pow, ptr @_ZGVnN4vv_powf, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2v_acos, ptr @_ZGVnN4v_acosf, ptr @_ZGVnN2v_asin, ptr @_ZGVnN4v_asinf, ptr @_ZGVnN2v_atan, ptr @_ZGVnN4v_atanf, ptr @_ZGVnN2v_cosh, ptr @_ZGVnN4v_coshf, ptr @_ZGVnN2v_sinh, ptr @_ZGVnN4v_sinhf, ptr @_ZGVnN2v_tanh, ptr @_ZGVnN4v_tanhf], section "llvm.metadata" ;. define <2 x double> @llvm_ceil_f64(<2 x double> %in) { ; CHECK-LABEL: @llvm_ceil_f64( @@ -278,7 +278,7 @@ define <4 x float> @llvm_nearbyint_f32(<4 x float> %in) { define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %pow) { ; CHECK-LABEL: @llvm_pow_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x double> @llvm.pow.v2f64(<2 x double> [[IN:%.*]], <2 x double> [[POW:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2vv_pow(<2 x double> [[IN:%.*]], <2 x double> [[POW:%.*]]) ; CHECK-NEXT: ret <2 x double> [[TMP1]] ; %1 = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %in, <2 x double> %pow) @@ -287,7 +287,7 @@ define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %pow) { define <4 x float> @llvm_pow_f32(<4 x float> %in, <4 x float> %pow) { ; CHECK-LABEL: @llvm_pow_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.pow.v4f32(<4 x float> [[IN:%.*]], <4 x float> [[POW:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4vv_powf(<4 x float> [[IN:%.*]], <4 x float> [[POW:%.*]]) ; CHECK-NEXT: ret <4 x float> [[TMP1]] ; %1 = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %in, <4 x float> %pow) From 5e92c3d8b39178dc0d8286465268305d1ab79a9d Mon Sep 17 00:00:00 2001 From: Tex Riddell Date: Tue, 8 Oct 2024 18:37:02 -0700 Subject: [PATCH 4/6] Rework logic to preserve "scalar" operand/return type, even if vector If isVectorIntrinsicWithScalarOpAtArg() returns true for an argument or return, we should not attempt to modify the type or use the element count, even if it is a vector. This allows the op to be identified as something that doesn't vectorize/scalarize with the width of the intrinsic, but remains the same, even if it is a vector argument. Initial ElementCount will only be the return element count if it's a vector and isVectorIntrinsicWithScalarOpAtArg returns false. This also returns the control flow in the loop to be closer to the original. Remove inconsistent comment about VFABI return type assumption, since we have the accessors that tell us how to handle it for the operation. Fix formatting. --- llvm/lib/CodeGen/ReplaceWithVeclib.cpp | 34 ++++++++++++++++---------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index 3e359123d51d71..f14f165a98715d 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -103,35 +103,43 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, Intrinsic::ID IID = II->getIntrinsicID(); if (IID == Intrinsic::not_intrinsic) return false; - // At the moment VFABI assumes the return type is always widened unless it is - // a void type. - auto *VTy = dyn_cast(II->getType()); - ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(0)); - Type *ScalarRetTy = II->getType()->getScalarType(); + + // RetIsScalar: Return type is not widened. + bool RetIsScalar = isVectorIntrinsicWithScalarOpAtArg(IID, -1); + Type *RetTy = II->getType(); + Type *ScalarRetTy = RetTy->getScalarType(); + // Compute the argument types of the corresponding scalar call and check that // all vector operands match the previously found EC. SmallVector ScalarArgTypes; + auto *VTy = dyn_cast(RetTy); + ElementCount EC(!RetIsScalar && VTy ? VTy->getElementCount() + : ElementCount::getFixed(0)); // OloadTys collects types used in scalar intrinsic overload name. SmallVector OloadTys; - if (!ScalarRetTy->isVoidTy() && - isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) - OloadTys.push_back(ScalarRetTy); + if (!RetTy->isVoidTy() && isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) + OloadTys.push_back(RetIsScalar ? RetTy : ScalarRetTy); for (auto Arg : enumerate(II->args())) { auto *ArgTy = Arg.value()->getType(); - auto *ScalarArgTy = ArgTy->getScalarType(); - ScalarArgTypes.push_back(ScalarArgTy); - if (isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index())) + bool IsOloadTy = isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index()); + if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) { + ScalarArgTypes.push_back(ArgTy); + if (IsOloadTy) + OloadTys.push_back(ArgTy); + } else if (auto *VectorArgTy = dyn_cast(ArgTy)) { + auto *ScalarArgTy = VectorArgTy->getElementType(); + ScalarArgTypes.push_back(ScalarArgTy); + if (IsOloadTy) OloadTys.push_back(ScalarArgTy); - if (auto *VectorArgTy = dyn_cast(ArgTy)) { // When return type is void, set EC to the first vector argument, and // disallow vector arguments with different ECs. if (EC.isZero()) EC = VectorArgTy->getElementCount(); else if (EC != VectorArgTy->getElementCount()) return false; - } else if (!isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) + } else // Exit when it is supposed to be a vector argument but it isn't. return false; } From 9adc0469e5443740553025434a2bb840d4dea5bd Mon Sep 17 00:00:00 2001 From: Tex Riddell Date: Wed, 9 Oct 2024 09:57:14 -0700 Subject: [PATCH 5/6] fix formatting --- llvm/lib/CodeGen/ReplaceWithVeclib.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index f14f165a98715d..e835a8651cac05 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -114,7 +114,7 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, SmallVector ScalarArgTypes; auto *VTy = dyn_cast(RetTy); ElementCount EC(!RetIsScalar && VTy ? VTy->getElementCount() - : ElementCount::getFixed(0)); + : ElementCount::getFixed(0)); // OloadTys collects types used in scalar intrinsic overload name. SmallVector OloadTys; From 6a07812555f39b5e2c0885b756cc0364c63c7d62 Mon Sep 17 00:00:00 2001 From: Tex Riddell Date: Wed, 9 Oct 2024 15:22:52 -0700 Subject: [PATCH 6/6] Revert change to use isVectorIntrinsicWithScalarOpAtArg for return type --- llvm/lib/CodeGen/ReplaceWithVeclib.cpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index e835a8651cac05..2e63e289910934 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -103,24 +103,21 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, Intrinsic::ID IID = II->getIntrinsicID(); if (IID == Intrinsic::not_intrinsic) return false; - - // RetIsScalar: Return type is not widened. - bool RetIsScalar = isVectorIntrinsicWithScalarOpAtArg(IID, -1); Type *RetTy = II->getType(); Type *ScalarRetTy = RetTy->getScalarType(); - - // Compute the argument types of the corresponding scalar call and check that - // all vector operands match the previously found EC. - SmallVector ScalarArgTypes; + // At the moment VFABI assumes the return type is always widened unless it is + // a void type. auto *VTy = dyn_cast(RetTy); - ElementCount EC(!RetIsScalar && VTy ? VTy->getElementCount() - : ElementCount::getFixed(0)); + ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(0)); // OloadTys collects types used in scalar intrinsic overload name. SmallVector OloadTys; if (!RetTy->isVoidTy() && isVectorIntrinsicWithOverloadTypeAtArg(IID, -1)) - OloadTys.push_back(RetIsScalar ? RetTy : ScalarRetTy); + OloadTys.push_back(ScalarRetTy); + // Compute the argument types of the corresponding scalar call and check that + // all vector operands match the previously found EC. + SmallVector ScalarArgTypes; for (auto Arg : enumerate(II->args())) { auto *ArgTy = Arg.value()->getType(); bool IsOloadTy = isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index());