[SVE][AArch64] Adding pattern matching for some SVE instructions.

[mgudim]

Summary

This is a follow-up with https://reviews.llvm.org/D68098.

We added straight-forward patten matching for two SVE intrinsics: frecps and frsqrts. We also added patterns for fsub and fmul - these SDNodes directly correspond to machine instructions.

[mgudim]

llvm/include/llvm/IR/IntrinsicsAArch64.td

782

I will fix this before committing or in the next revision.

[amehsan]

This looks straighforward. So LGTM. Since this is @mgudim's first patch, I will commit it for him. But I will wait a bit more (before committing) just in case @huntergr has any comments.

[huntergr]

Hi, thanks for waiting a bit. LGTM.

[amehsan]

In D68476#1717715, @huntergr wrote:

Hi, thanks for waiting a bit. LGTM.

Thanks @huntergr and Sorry for my delay. I will merge this very soon and get to the other patches. Thanks again.

[huntergr]

llvm/include/llvm/IR/IntrinsicsAArch64.td

816

One minor change I'd like to request is to do with the naming convention we have for these...

int_aarch64_sve_frecps_x and int_aarch64_sve_frsqrts_x would be the preferred names. Adding f to the start will match the instruction name, and the _x suffix is to indicate an unpredicated intrinsic.

[mgudim]

Renamed the inrinsics according to the convention.

[huntergr]

In D68476#1725466, @mgudim wrote:

Renamed the inrinsics according to the convention.

Thanks :)

[amehsan]

Committed in https://reviews.llvm.org/rG1e9de0215f04

llvm/include/llvm/IR/IntrinsicsAArch64.td

: Intrinsic<[LLVMVectorOfBitcastsToInt<0>],
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  llvm_anyvector_ty],
                 [IntrNoMem]>;
 
   class AdvSIMD_SVE_PUNPKHI_Intrinsic
     : Intrinsic<[LLVMHalfElementsVectorType<0>],
                 [llvm_anyvector_ty],
                [IntrNoMem]>;
-
   // This class of intrinsics are not intended to be useful within LLVM IR but
   // are instead here to support some of the more regid parts of the ACLE.
   class Builtin_SVCVT<string name, LLVMType OUT, LLVMType IN>
   : GCCBuiltin<"__builtin_sve_" # name>,
     Intrinsic<[OUT], [OUT, llvm_nxv16i1_ty, IN], [IntrNoMem]>;
 }
 
 //===----------------------------------------------------------------------===//

 
 //
 // Floating-point comparisons
 //
 
 def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv8f16_ty>;
 
 //
+// Floating-point arithmetic
+//
+def int_aarch64_sve_recps: AdvSIMD_2VectorArg_Intrinsic;
+def int_aarch64_sve_rsqrts: AdvSIMD_2VectorArg_Intrinsic;
+
+//
 // Predicate operations
 //
 
 def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
 def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
 }

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm FMIN_ZPmZ   : sve_fp_2op_p_zds<0b0111, "fmin">;
   defm FABD_ZPmZ   : sve_fp_2op_p_zds<0b1000, "fabd">;
   defm FSCALE_ZPmZ : sve_fp_2op_p_zds<0b1001, "fscale">;
   defm FMULX_ZPmZ  : sve_fp_2op_p_zds<0b1010, "fmulx">;
   defm FDIVR_ZPmZ  : sve_fp_2op_p_zds<0b1100, "fdivr">;
   defm FDIV_ZPmZ   : sve_fp_2op_p_zds<0b1101, "fdiv">;
 
   defm FADD_ZZZ    : sve_fp_3op_u_zd<0b000, "fadd", fadd>;
-  defm FSUB_ZZZ    : sve_fp_3op_u_zd<0b001, "fsub", null_frag>;
-  defm FMUL_ZZZ    : sve_fp_3op_u_zd<0b010, "fmul", null_frag>;
+  defm FSUB_ZZZ    : sve_fp_3op_u_zd<0b001, "fsub", fsub>;
+  defm FMUL_ZZZ    : sve_fp_3op_u_zd<0b010, "fmul", fmul>;
   defm FTSMUL_ZZZ  : sve_fp_3op_u_zd<0b011, "ftsmul", null_frag>;
-  defm FRECPS_ZZZ  : sve_fp_3op_u_zd<0b110, "frecps", null_frag>;
-  defm FRSQRTS_ZZZ : sve_fp_3op_u_zd<0b111, "frsqrts", null_frag>;
+  defm FRECPS_ZZZ  : sve_fp_3op_u_zd<0b110, "frecps", int_aarch64_sve_recps>;
+  defm FRSQRTS_ZZZ : sve_fp_3op_u_zd<0b111, "frsqrts", int_aarch64_sve_rsqrts>;
 
   defm FTSSEL_ZZZ : sve_int_bin_cons_misc_0_b<"ftssel">;
 
   defm FCADD_ZPmZ : sve_fp_fcadd<"fcadd">;
   defm FCMLA_ZPmZZ : sve_fp_fcmla<"fcmla">;
 
   defm FMLA_ZPmZZ  : sve_fp_3op_p_zds_a<0b00, "fmla">;
   defm FMLS_ZPmZZ  : sve_fp_3op_p_zds_a<0b01, "fmls">;

llvm/test/CodeGen/AArch64/sve-fp.ll

 
 define <vscale x 2 x double> @fadd_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
 ; CHECK-LABEL: fadd_d:
 ; CHECK: fadd z0.d, z0.d, z1.d
 ; CHECK-NEXT: ret
   %res = fadd <vscale x 2 x double> %a, %b
   ret <vscale x 2 x double> %res
 }
+
+define <vscale x 8 x half> @fsub_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fsub_h:
+; CHECK: fsub z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = fsub <vscale x 8 x half> %a, %b
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fsub_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fsub_s:
+; CHECK: fsub z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = fsub <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fsub_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fsub_d:
+; CHECK: fsub z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = fsub <vscale x 2 x double> %a, %b
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x half> @fmul_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fmul_h:
+; CHECK: fmul z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = fmul <vscale x 8 x half> %a, %b
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @fmul_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fmul_s:
+; CHECK: fmul z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = fmul <vscale x 4 x float> %a, %b
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fmul_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fmul_d:
+; CHECK: fmul z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = fmul <vscale x 2 x double> %a, %b
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x half> @frecps_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frecps_h:
+; CHECK: frecps z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.recps.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @frecps_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frecps_s:
+; CHECK: frecps z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.recps.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frecps_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frecps_d:
+; CHECK: frecps z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.recps.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x half> @frsqrts_h(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frsqrts_h:
+; CHECK: frsqrts z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.rsqrts.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x float> @frsqrts_s(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frsqrts_s:
+; CHECK: frsqrts z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.rsqrts.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frsqrts_d(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frsqrts_d:
+; CHECK: frsqrts z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.rsqrts.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.recps.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float>  @llvm.aarch64.sve.recps.nxv4f32(<vscale x 4 x float> , <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.recps.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.rsqrts.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.rsqrts.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.rsqrts.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)