[AArch64] Add missing intrinsics for scalar fp rounding

[SjoerdMeijer]

llvm/include/llvm/IR/IntrinsicsAArch64.td

48

Better is that the 2nd llvm_anyfloat_ty is LLVMMatchType<0>.
Same for the other patterns below.

[jaykang10]

llvm/include/llvm/IR/IntrinsicsAArch64.td

48

Yep, I will update it.

[jaykang10]

Use LLVMMatchType

[jaykang10]

Any other comments please?

[SjoerdMeijer]

Thanks, looks very reasonable to me.

clang/include/clang/Basic/BuiltinsAArch64.def

 BUILTIN(__builtin_sponentry, "v*", "c")
 
 // Transactional Memory Extension
 BUILTIN(__builtin_arm_tstart, "WUi", "nj")
 BUILTIN(__builtin_arm_tcommit, "v", "n")
 BUILTIN(__builtin_arm_tcancel, "vWUIi", "n")
 BUILTIN(__builtin_arm_ttest, "WUi", "nc")
 
+// Armv8.5-A FP rounding intrinsics
+BUILTIN(__builtin_arm_frint32zf, "ff", "")
+BUILTIN(__builtin_arm_frint32z, "dd", "")
+BUILTIN(__builtin_arm_frint64zf, "ff", "")
+BUILTIN(__builtin_arm_frint64z, "dd", "")
+BUILTIN(__builtin_arm_frint32xf, "ff", "")
+BUILTIN(__builtin_arm_frint32x, "dd", "")
+BUILTIN(__builtin_arm_frint64xf, "ff", "")
+BUILTIN(__builtin_arm_frint64x, "dd", "")
+
 // Armv8.7-A load/store 64-byte intrinsics
 BUILTIN(__builtin_arm_ld64b, "vvC*WUi*", "n")
 BUILTIN(__builtin_arm_st64b, "vv*WUiC*", "n")
 BUILTIN(__builtin_arm_st64bv, "WUiv*WUiC*", "n")
 BUILTIN(__builtin_arm_st64bv0, "WUiv*WUiC*", "n")
 
 TARGET_HEADER_BUILTIN(_BitScanForward, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_BitScanReverse, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")

clang/lib/CodeGen/CGBuiltin.cpp

return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
                               "cls");
   }
   if (BuiltinID == AArch64::BI__builtin_arm_cls64) {
     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
                               "cls");
   }
 
+  if (BuiltinID == AArch64::BI__builtin_arm_frint32zf ||
+      BuiltinID == AArch64::BI__builtin_arm_frint32z) {
+    llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
+    llvm::Type *Ty = Arg->getType();
+    return Builder.CreateCall(
+        CGM.getIntrinsic(Intrinsic::aarch64_frint32z, {Ty, Ty}), Arg,
+        "frint32z");
+  }
+
+  if (BuiltinID == AArch64::BI__builtin_arm_frint64zf ||
+      BuiltinID == AArch64::BI__builtin_arm_frint64z) {
+    llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
+    llvm::Type *Ty = Arg->getType();
+    return Builder.CreateCall(
+        CGM.getIntrinsic(Intrinsic::aarch64_frint64z, {Ty, Ty}), Arg,
+        "frint64z");
+  }
+
+  if (BuiltinID == AArch64::BI__builtin_arm_frint32xf ||
+      BuiltinID == AArch64::BI__builtin_arm_frint32x) {
+    llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
+    llvm::Type *Ty = Arg->getType();
+    return Builder.CreateCall(
+        CGM.getIntrinsic(Intrinsic::aarch64_frint32x, {Ty, Ty}), Arg,
+        "frint32x");
+  }
+
+  if (BuiltinID == AArch64::BI__builtin_arm_frint64xf ||
+      BuiltinID == AArch64::BI__builtin_arm_frint64x) {
+    llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
+    llvm::Type *Ty = Arg->getType();
+    return Builder.CreateCall(
+        CGM.getIntrinsic(Intrinsic::aarch64_frint64x, {Ty, Ty}), Arg,
+        "frint64x");
+  }
+
   if (BuiltinID == AArch64::BI__builtin_arm_jcvt) {
     assert((getContext().getTypeSize(E->getType()) == 32) &&
            "__jcvt of unusual size!");
     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
     return Builder.CreateCall(
         CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
   }
 

clang/lib/Headers/arm_acle.h

 /*===---- arm_acle.h - ARM Non-Neon intrinsics -----------------------------===
  *
  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  * See https://llvm.org/LICENSE.txt for license information.
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===-----------------------------------------------------------------------===
  */
 
 #ifndef __ARM_ACLE_H
 #define __ARM_ACLE_H
 
 #ifndef __ARM_ACLE
 #error "ACLE intrinsics support not enabled."
 #endif
 
 #include <stdint.h>
 
 #if defined(__cplusplus)
 extern "C" {
 #endif
 
 /* 8 SYNCHRONIZATION, BARRIER AND HINT INTRINSICS */
 /* 8.3 Memory barriers */
 #if !__has_builtin(__dmb)
 #define __dmb(i) __builtin_arm_dmb(i)
 #endif
 #if !__has_builtin(__dsb)
 #define __dsb(i) __builtin_arm_dsb(i)
 #endif
 #if !__has_builtin(__isb)
 #define __isb(i) __builtin_arm_isb(i)
 #endif
 
 /* 8.4 Hints */
 
 #if !__has_builtin(__wfi)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __wfi(void) {
   __builtin_arm_wfi();
 }
 #endif
 
 #if !__has_builtin(__wfe)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __wfe(void) {
   __builtin_arm_wfe();
 }
 #endif
 
 #if !__has_builtin(__sev)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __sev(void) {
   __builtin_arm_sev();
 }
 #endif
 
 #if !__has_builtin(__sevl)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __sevl(void) {
   __builtin_arm_sevl();
 }
 #endif
 
 #if !__has_builtin(__yield)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __yield(void) {
   __builtin_arm_yield();
 }
 #endif
 
 #if __ARM_32BIT_STATE
 #define __dbg(t) __builtin_arm_dbg(t)
 #endif
 
 /* 8.5 Swap */
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __swp(uint32_t __x, volatile uint32_t *__p) {
   uint32_t v;
   do
     v = __builtin_arm_ldrex(__p);
   while (__builtin_arm_strex(__x, __p));
   return v;
 }
 
 /* 8.6 Memory prefetch intrinsics */
 /* 8.6.1 Data prefetch */
 #define __pld(addr) __pldx(0, 0, 0, addr)
 
 #if __ARM_32BIT_STATE

 #else
 #define __plix(cache_level, retention_policy, addr) \
   __builtin_arm_prefetch(addr, 0, cache_level, retention_policy, 0)
 #endif
 
 /* 8.7 NOP */
 #if !defined(_MSC_VER) || !defined(__aarch64__)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __nop(void) {
   __builtin_arm_nop();
 }
 #endif
 
 /* 9 DATA-PROCESSING INTRINSICS */
 /* 9.2 Miscellaneous data-processing intrinsics */
 /* ROR */
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __ror(uint32_t __x, uint32_t __y) {

 static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
 __clzll(uint64_t __t) {
   return __builtin_clzll(__t);
 }
 
 /* CLS */
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __cls(uint32_t __t) {
   return __builtin_arm_cls(__t);
 }
 
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __clsl(unsigned long __t) {
 #if __SIZEOF_LONG__ == 4
   return __builtin_arm_cls(__t);
 #else
   return __builtin_arm_cls64(__t);
 #endif
 }
 
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __clsll(uint64_t __t) {
   return __builtin_arm_cls64(__t);
 }
 
 /* REV */
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __rev(uint32_t __t) {
   return __builtin_bswap32(__t);
 }
 

 static __inline__ int16_t __attribute__((__always_inline__, __nodebug__))
 __revsh(int16_t __t) {
   return __builtin_bswap16(__t);
 }
 
 /* RBIT */
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __rbit(uint32_t __t) {
   return __builtin_arm_rbit(__t);
 }
 
 static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
 __rbitll(uint64_t __t) {
 #if __ARM_32BIT_STATE
   return (((uint64_t)__builtin_arm_rbit(__t)) << 32) |
          __builtin_arm_rbit(__t >> 32);
 #else
   return __builtin_arm_rbit64(__t);
 #endif
 }
 
 static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
 __rbitl(unsigned long __t) {
 #if __SIZEOF_LONG__ == 4
   return __rbit(__t);
 #else

 /* Armv8.3-A Javascript conversion intrinsic */
 #if __ARM_64BIT_STATE && defined(__ARM_FEATURE_JCVT)
 static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
 __jcvt(double __a) {
   return __builtin_arm_jcvt(__a);
 }
 #endif
 
+/* Armv8.5-A FP rounding intrinsics */
+#if __ARM_64BIT_STATE && defined(__ARM_FEATURE_FRINT)
+static __inline__ float __attribute__((__always_inline__, __nodebug__))
+__frint32zf(float __a) {
+  return __builtin_arm_frint32zf(__a);
+}
+
+static __inline__ double __attribute__((__always_inline__, __nodebug__))
+__frint32z(double __a) {
+  return __builtin_arm_frint32z(__a);
+}
+
+static __inline__ float __attribute__((__always_inline__, __nodebug__))
+__frint64zf(float __a) {
+  return __builtin_arm_frint64zf(__a);
+}
+
+static __inline__ double __attribute__((__always_inline__, __nodebug__))
+__frint64z(double __a) {
+  return __builtin_arm_frint64z(__a);
+}
+
+static __inline__ float __attribute__((__always_inline__, __nodebug__))
+__frint32xf(float __a) {
+  return __builtin_arm_frint32xf(__a);
+}
+
+static __inline__ double __attribute__((__always_inline__, __nodebug__))
+__frint32x(double __a) {
+  return __builtin_arm_frint32x(__a);
+}
+
+static __inline__ float __attribute__((__always_inline__, __nodebug__))
+__frint64xf(float __a) {
+  return __builtin_arm_frint64xf(__a);
+}
+
+static __inline__ double __attribute__((__always_inline__, __nodebug__))
+__frint64x(double __a) {
+  return __builtin_arm_frint64x(__a);
+}
+#endif
+
 /* Armv8.7-A load/store 64-byte intrinsics */
 #if __ARM_64BIT_STATE && defined(__ARM_FEATURE_LS64)
 typedef struct {
     uint64_t val[8];
 } data512_t;
 
 static __inline__ data512_t __attribute__((__always_inline__, __nodebug__))
 __arm_ld64b(const void *__addr) {

clang/test/CodeGen/aarch64-v8.5a-scalar-frint3264-intrinsic.c

+// RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +v8.5a\
+// RUN: -flax-vector-conversions=none -S -disable-O0-optnone -emit-llvm -o - %s \
+// RUN: | opt -S -mem2reg \
+// RUN: | FileCheck %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_acle.h>
+
+// CHECK-LABEL: test_frint32zf
+// CHECK:  [[RND:%.*]] =  call float @llvm.aarch64.frint32z.f32.f32(float %a)
+// CHECK:  ret float [[RND]]
+float test_frint32zf(float a) {
+  return __frint32zf(a);
+}
+
+// CHECK-LABEL: test_frint32z
+// CHECK:  [[RND:%.*]] =  call double @llvm.aarch64.frint32z.f64.f64(double %a)
+// CHECK:  ret double [[RND]]
+double test_frint32z(double a) {
+  return __frint32z(a);
+}
+
+// CHECK-LABEL: test_frint64zf
+// CHECK:  [[RND:%.*]] =  call float @llvm.aarch64.frint64z.f32.f32(float %a)
+// CHECK:  ret float [[RND]]
+float test_frint64zf(float a) {
+  return __frint64zf(a);
+}
+
+// CHECK-LABEL: test_frint64z
+// CHECK:  [[RND:%.*]] =  call double @llvm.aarch64.frint64z.f64.f64(double %a)
+// CHECK:  ret double [[RND]]
+double test_frint64z(double a) {
+  return __frint64z(a);
+}
+
+// CHECK-LABEL: test_frint32xf
+// CHECK:  [[RND:%.*]] =  call float @llvm.aarch64.frint32x.f32.f32(float %a)
+// CHECK:  ret float [[RND]]
+float test_frint32xf(float a) {
+  return __frint32xf(a);
+}
+
+// CHECK-LABEL: test_frint32x
+// CHECK:  [[RND:%.*]] =  call double @llvm.aarch64.frint32x.f64.f64(double %a)
+// CHECK:  ret double [[RND]]
+double test_frint32x(double a) {
+  return __frint32x(a);
+}
+
+// CHECK-LABEL: test_frint64xf
+// CHECK:  [[RND:%.*]] =  call float @llvm.aarch64.frint64x.f32.f32(float %a)
+// CHECK:  ret float [[RND]]
+float test_frint64xf(float a) {
+  return __frint64xf(a);
+}
+
+// CHECK-LABEL: test_frint64x
+// CHECK:  [[RND:%.*]] =  call double @llvm.aarch64.frint64x.f64.f64(double %a)
+// CHECK:  ret double [[RND]]
+double test_frint64x(double a) {
+  return __frint64x(a);
+}

llvm/include/llvm/IR/IntrinsicsAArch64.td

 def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
                                 LLVMMatchType<0>], [IntrNoMem]>;
 
 def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
 
 def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
 
+def int_aarch64_frint32z
+    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ llvm_anyfloat_ty ],
+                            [ IntrNoMem ]>;
+def int_aarch64_frint64z
+    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ llvm_anyfloat_ty ],
+                            [ IntrNoMem ]>;
+def int_aarch64_frint32x
+    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ llvm_anyfloat_ty ],
+                            [ IntrNoMem ]>;
+def int_aarch64_frint64x
+    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ llvm_anyfloat_ty ],
+                            [ IntrNoMem ]>;
+
 //===----------------------------------------------------------------------===//
 // HINT
 
 def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;
 
 //===----------------------------------------------------------------------===//
 // Data Barrier Instructions
 

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 defm FRINTX : SingleOperandFPData<0b1110, "frintx", frint>;
 defm FRINTZ : SingleOperandFPData<0b1011, "frintz", ftrunc>;
 
 let SchedRW = [WriteFDiv] in {
 defm FSQRT  : SingleOperandFPData<0b0011, "fsqrt", fsqrt>;
 }
 
 let Predicates = [HasFRInt3264] in {
-  defm FRINT32Z : FRIntNNT<0b00, "frint32z">;
-  defm FRINT64Z : FRIntNNT<0b10, "frint64z">;
-  defm FRINT32X : FRIntNNT<0b01, "frint32x">;
-  defm FRINT64X : FRIntNNT<0b11, "frint64x">;
+  defm FRINT32Z : FRIntNNT<0b00, "frint32z", int_aarch64_frint32z>;
+  defm FRINT64Z : FRIntNNT<0b10, "frint64z", int_aarch64_frint64z>;
+  defm FRINT32X : FRIntNNT<0b01, "frint32x", int_aarch64_frint32x>;
+  defm FRINT64X : FRIntNNT<0b11, "frint64x", int_aarch64_frint64x>;
 } // HasFRInt3264
 
 let Predicates = [HasFullFP16] in {
   def : Pat<(i32 (lrint f16:$Rn)),
             (FCVTZSUWHr (!cast<Instruction>(FRINTXHr) f16:$Rn))>;
   def : Pat<(i64 (lrint f16:$Rn)),
             (FCVTZSUXHr (!cast<Instruction>(FRINTXHr) f16:$Rn))>;
   def : Pat<(i64 (llrint f16:$Rn)),

llvm/test/CodeGen/AArch64/v8.5a-scalar-frint3264-intrinsic.ll

+; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.5a  | FileCheck %s
+
+declare float @llvm.aarch64.frint32z.f32.f32(float)
+declare double @llvm.aarch64.frint32z.f64.f64(double)
+declare float @llvm.aarch64.frint64z.f32.f32(float)
+declare double @llvm.aarch64.frint64z.f64.f64(double)
+
+define dso_local float @t_frint32z(float %a) {
+; CHECK-LABEL: t_frint32z:
+; CHECK:         frint32z s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call float @llvm.aarch64.frint32z.f32.f32(float %a)
+  ret float %val
+}
+
+define dso_local double @t_frint32zf(double %a) {
+; CHECK-LABEL: t_frint32zf:
+; CHECK:         frint32z d0, d0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call double @llvm.aarch64.frint32z.f64.f64(double %a)
+  ret double %val
+}
+
+define dso_local float @t_frint64z(float %a) {
+; CHECK-LABEL: t_frint64z:
+; CHECK:         frint64z s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call float @llvm.aarch64.frint64z.f32.f32(float %a)
+  ret float %val
+}
+
+define dso_local double @t_frint64zf(double %a) {
+; CHECK-LABEL: t_frint64zf:
+; CHECK:         frint64z d0, d0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call double @llvm.aarch64.frint64z.f64.f64(double %a)
+  ret double %val
+}
+
+declare float @llvm.aarch64.frint32x.f32.f32(float)
+declare double @llvm.aarch64.frint32x.f64.f64(double)
+declare float @llvm.aarch64.frint64x.f32.f32(float)
+declare double @llvm.aarch64.frint64x.f64.f64(double)
+
+define dso_local float @t_frint32x(float %a) {
+; CHECK-LABEL: t_frint32x:
+; CHECK:         frint32x s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call float @llvm.aarch64.frint32x.f32.f32(float %a)
+  ret float %val
+}
+
+define dso_local double @t_frint32xf(double %a) {
+; CHECK-LABEL: t_frint32xf:
+; CHECK:         frint32x d0, d0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call double @llvm.aarch64.frint32x.f64.f64(double %a)
+  ret double %val
+}
+
+define dso_local float @t_frint64x(float %a) {
+; CHECK-LABEL: t_frint64x:
+; CHECK:         frint64x s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call float @llvm.aarch64.frint64x.f32.f32(float %a)
+  ret float %val
+}
+
+define dso_local double @t_frint64xf(double %a) {
+; CHECK-LABEL: t_frint64xf:
+; CHECK:         frint64x d0, d0
+; CHECK-NEXT:    ret
+entry:
+  %val = tail call double @llvm.aarch64.frint64x.f64.f64(double %a)
+  ret double %val
+}