[ARM] Pass half or i16 types for NEON intrinsics

[SjoerdMeijer]

Summary

For generating NEON intrinsics, this determines the NEON data type, and
whether it should be a half type or an i16 type. I.e., we always pass a half
type for AArch64, this hasn't changed, but now also for ARM but only when
FullFP16 is enabled, and i16 otherwise.

This is intended to be non-functional change, but together with the backend
work in https://reviews.llvm.org/D44538 which adds support for f16 vectors,
this enables adding the AArch32 FP16 (vector) intrinsics.

[t.p.northover]

include/clang/Basic/TargetInfo.h

366

_Float16 doesn't seem to be supported anywhere in Clang (__fp16 is).

But we should probably clarify exactly what kind of support we mean here. This variable doesn't affect:

• Whether __fp16 can be used at all in source.
• Its ABI.

I'm actually slightly worried that when we document what it does affect it'll end up being an ARM implementation-detail.

lib/Basic/Targets/ARM.cpp

381–382

This is now a bool rather than a bitfield, so false is definitely the best initializer (probably was before, too, but whatever).

lib/CodeGen/CGBuiltin.cpp

3455

This check is equivalent to setting HasFloat16 in AArch64TargetInfo at the moment. Are you intending to change that (say by adding more uses of HasFloat16)?

[SjoerdMeijer]

Thanks for the review. Please see a first comment inline.

include/clang/Basic/TargetInfo.h

366

I've added _Float16 support in Clang commit r312794: "Add _Float16 as a C/C++ source language type" :-)

[t.p.northover]

include/clang/Basic/TargetInfo.h

366

Oh wow, that was ages ago! I have no idea how I still had a repo that old hanging around. Sorry about that.

[SjoerdMeijer]

Addressed comments: simplified the logic in GetNeonType.

[t.p.northover]

include/clang/Basic/TargetInfo.h

366

I'm afraid I still think this description is iffy. For a start it is actually still affecting __fp16 rather than Float16 (since that's what float16_t is in the NEON header). And it's still too vague about just what "support" implies.

lib/CodeGen/CGBuiltin.cpp

3444

Now that you set HasFloat16 in AArch64, I believe Arch is unused.

[SjoerdMeijer]

Removed unused function argument, and renamed HasFloat16

[t.p.northover]

Thanks. I think this looks reasonable now.

[SjoerdMeijer]

Thanks a lot for your help and reviews.

include/clang/Basic/TargetInfo.h

class TargetInfo : public RefCountedBase<TargetInfo> {
   llvm::Triple Triple;
 protected:
   // Target values set by the ctor of the actual target implementation.  Default
   // values are specified by the TargetInfo constructor.
   bool BigEndian;
   bool TLSSupported;
   bool VLASupported;
   bool NoAsmVariants;  // True if {|} are normal characters.
+  bool HasLegalHalfType; // True if the backend supports operations on the half
+                         // LLVM IR type.
   bool HasFloat128;
   unsigned char PointerWidth, PointerAlign;
   unsigned char BoolWidth, BoolAlign;
   unsigned char IntWidth, IntAlign;
   unsigned char HalfWidth, HalfAlign;
   unsigned char FloatWidth, FloatAlign;
   unsigned char DoubleWidth, DoubleAlign;
   unsigned char LongDoubleWidth, LongDoubleAlign, Float128Align;
public:
   unsigned getLongLongWidth() const { return LongLongWidth; }
   unsigned getLongLongAlign() const { return LongLongAlign; }
 
   /// \brief Determine whether the __int128 type is supported on this target.
   virtual bool hasInt128Type() const {
     return (getPointerWidth(0) >= 64) || getTargetOpts().ForceEnableInt128;
   } // FIXME
 
+  /// \brief Determine whether _Float16 is supported on this target.
+  virtual bool hasLegalHalfType() const { return HasLegalHalfType; }
+
   /// \brief Determine whether the __float128 type is supported on this target.
   virtual bool hasFloat128Type() const { return HasFloat128; }
 
   /// \brief Return the alignment that is suitable for storing any
   /// object with a fundamental alignment requirement.
   unsigned getSuitableAlign() const { return SuitableAlign; }
 
   /// \brief Return the default alignment for __attribute__((aligned)) on

lib/Basic/TargetInfo.cpp

 // TargetInfo Constructor.
 TargetInfo::TargetInfo(const llvm::Triple &T) : TargetOpts(), Triple(T) {
   // Set defaults.  Defaults are set for a 32-bit RISC platform, like PPC or
   // SPARC.  These should be overridden by concrete targets as needed.
   BigEndian = !T.isLittleEndian();
   TLSSupported = true;
   VLASupported = true;
   NoAsmVariants = false;
+  HasLegalHalfType = false;
   HasFloat128 = false;
   PointerWidth = PointerAlign = 32;
   BoolWidth = BoolAlign = 8;
   IntWidth = IntAlign = 32;
   LongWidth = LongAlign = 32;
   LongLongWidth = LongLongAlign = 64;
   SuitableAlign = 64;
   DefaultAlignForAttributeAligned = 128;

lib/Basic/Targets/AArch64.cpp

AArch64TargetInfo::AArch64TargetInfo(const llvm::Triple &Triple,
   } else {
     if (!getTriple().isOSDarwin())
       WCharType = UnsignedInt;
 
     Int64Type = SignedLong;
     IntMaxType = SignedLong;
   }
 
+  // All AArch64 implementations support ARMv8 FP, which makes half a legal type.
+  HasLegalHalfType = true;
 
   LongWidth = LongAlign = PointerWidth = PointerAlign = 64;
   MaxVectorAlign = 128;
   MaxAtomicInlineWidth = 128;
   MaxAtomicPromoteWidth = 128;
 
   LongDoubleWidth = LongDoubleAlign = SuitableAlign = 128;
   LongDoubleFormat = &llvm::APFloat::IEEEquad();

lib/Basic/Targets/ARM.h

class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo {
   // Initialized via features.
   unsigned SoftFloat : 1;
   unsigned SoftFloatABI : 1;
 
   unsigned CRC : 1;
   unsigned Crypto : 1;
   unsigned DSP : 1;
   unsigned Unaligned : 1;
-  unsigned HasFullFP16 : 1;
 
   enum {
     LDREX_B = (1 << 0), /// byte (8-bit)
     LDREX_H = (1 << 1), /// half (16-bit)
     LDREX_W = (1 << 2), /// word (32-bit)
     LDREX_D = (1 << 3), /// double (64-bit)
   };
 

lib/Basic/Targets/ARM.cpp

 bool ARMTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
                                          DiagnosticsEngine &Diags) {
   FPU = 0;
   CRC = 0;
   Crypto = 0;
   DSP = 0;
   Unaligned = 1;
   SoftFloat = SoftFloatABI = false;
   HWDiv = 0;
-  HasFullFP16 = 0;
 
   // This does not diagnose illegal cases like having both
   // "+vfpv2" and "+vfpv3" or having "+neon" and "+fp-only-sp".
   uint32_t HW_FP_remove = 0;
   for (const auto &Feature : Features) {
     if (Feature == "+soft-float") {
       SoftFloat = true;
     } else if (Feature == "+soft-float-abi") {
       SoftFloatABI = true;
if (Feature == "+soft-float") {
       DSP = 1;
     } else if (Feature == "+fp-only-sp") {
       HW_FP_remove |= HW_FP_DP;
     } else if (Feature == "+strict-align") {
       Unaligned = 0;
     } else if (Feature == "+fp16") {
       HW_FP |= HW_FP_HP;
     } else if (Feature == "+fullfp16") {
-      HasFullFP16 = 1;
+      HasLegalHalfType = true;
     }
   }
   HW_FP &= ~HW_FP_remove;
 
   switch (ArchVersion) {
   case 6:
     if (ArchProfile == llvm::ARM::ProfileKind::M)
       LDREX = 0;
void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   // ACLE 6.4.6 Q (saturation) flag
   if (DSP || SAT)
     Builder.defineMacro("__ARM_FEATURE_QBIT", "1");
 
   if (Opts.UnsafeFPMath)
     Builder.defineMacro("__ARM_FP_FAST", "1");
 
   // Armv8.2-A FP16 vector intrinsic
-  if ((FPU & NeonFPU) && HasFullFP16)
+  if ((FPU & NeonFPU) && HasLegalHalfType)
     Builder.defineMacro("__ARM_FEATURE_FP16_VECTOR_ARITHMETIC", "1");
 
   // Armv8.2-A FP16 scalar intrinsics
-  if (HasFullFP16)
+  if (HasLegalHalfType)
     Builder.defineMacro("__ARM_FEATURE_FP16_SCALAR_ARITHMETIC", "1");
 
 
   switch (ArchKind) {
   default:
     break;
   case llvm::ARM::ArchKind::ARMV8_1A:
     getTargetDefinesARMV81A(Opts, Builder);

lib/CodeGen/CGBuiltin.cpp

Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
   }
 
   return EmitTargetArchBuiltinExpr(this, BuiltinID, E,
                                    getTarget().getTriple().getArch());
 }
 
 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
                                      NeonTypeFlags TypeFlags,
-                                     llvm::Triple::ArchType Arch,
+                                     bool HasLegalHalfType=true,
                                      bool V1Ty=false) {
   int IsQuad = TypeFlags.isQuad();
   switch (TypeFlags.getEltType()) {
   case NeonTypeFlags::Int8:
   case NeonTypeFlags::Poly8:
     return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
   case NeonTypeFlags::Int16:
   case NeonTypeFlags::Poly16:
     return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
   case NeonTypeFlags::Float16:
-    // FIXME: Only AArch64 backend can so far properly handle half types.
-    // Remove else part once ARM backend support for half is complete.
-    if (Arch == llvm::Triple::aarch64)
+    if (HasLegalHalfType)
       return llvm::VectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
     else
       return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
   case NeonTypeFlags::Int32:
     return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
   case NeonTypeFlags::Int64:
   case NeonTypeFlags::Poly64:
     return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
   if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
     return nullptr;
 
   // Determine the type of this overloaded NEON intrinsic.
   NeonTypeFlags Type(NeonTypeConst.getZExtValue());
   bool Usgn = Type.isUnsigned();
   bool Quad = Type.isQuad();
+  const bool HasLegalHalfType = getTarget().hasLegalHalfType();
 
-  llvm::VectorType *VTy = GetNeonType(this, Type, Arch);
+  llvm::VectorType *VTy = GetNeonType(this, Type, HasLegalHalfType);
   llvm::Type *Ty = VTy;
   if (!Ty)
     return nullptr;
 
   auto getAlignmentValue32 = [&](Address addr) -> Value* {
     return Builder.getInt32(addr.getAlignment().getQuantity());
   };
 
Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
   case NEON::BI__builtin_neon_vclzq_v:
     // We generate target-independent intrinsic, which needs a second argument
     // for whether or not clz of zero is undefined; on ARM it isn't.
     Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
     break;
   case NEON::BI__builtin_neon_vcvt_f32_v:
   case NEON::BI__builtin_neon_vcvtq_f32_v:
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad), Arch);
+    Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
+                     HasLegalHalfType);
     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
   case NEON::BI__builtin_neon_vcvt_f16_v:
   case NEON::BI__builtin_neon_vcvtq_f16_v:
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad), Arch);
+    Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
+                     HasLegalHalfType);
     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
   case NEON::BI__builtin_neon_vcvt_n_f16_v:
   case NEON::BI__builtin_neon_vcvt_n_f32_v:
   case NEON::BI__builtin_neon_vcvt_n_f64_v:
   case NEON::BI__builtin_neon_vcvtq_n_f16_v:
   case NEON::BI__builtin_neon_vcvtq_n_f32_v:
   case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
     return Builder.CreateCall(F, Ops, "vcvtr");
   }
 
   // Determine the type of this overloaded NEON intrinsic.
   NeonTypeFlags Type(Result.getZExtValue());
   bool usgn = Type.isUnsigned();
   bool rightShift = false;
 
-  llvm::VectorType *VTy = GetNeonType(this, Type, Arch);
+  llvm::VectorType *VTy = GetNeonType(this, Type,
+                                      getTarget().hasLegalHalfType());
   llvm::Type *Ty = VTy;
   if (!Ty)
     return nullptr;
 
   // Many NEON builtins have identical semantics and uses in ARM and
   // AArch64. Emit these in a single function.
   auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
   const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
   // Get the last argument, which specifies the vector type.
   llvm::APSInt Result;
   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
   if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
     return nullptr;
 
   // Determine the type of this overloaded NEON intrinsic.
   NeonTypeFlags Type(Result.getZExtValue());
-  llvm::VectorType *Ty = GetNeonType(&CGF, Type, Arch);
+  llvm::VectorType *Ty = GetNeonType(&CGF, Type);
   if (!Ty)
     return nullptr;
 
   CodeGen::CGBuilderTy &Builder = CGF.Builder;
 
   // AArch64 scalar builtins are not overloaded, they do not have an extra
   // argument that specifies the vector type, need to handle each case.
   switch (BuiltinID) {
case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
     unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
                        BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
                           ? Intrinsic::aarch64_neon_sqadd
                           : Intrinsic::aarch64_neon_sqsub;
     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
   }
   }
 
-  llvm::VectorType *VTy = GetNeonType(this, Type, Arch);
+  llvm::VectorType *VTy = GetNeonType(this, Type);
   llvm::Type *Ty = VTy;
   if (!Ty)
     return nullptr;
 
   // Not all intrinsics handled by the common case work for AArch64 yet, so only
   // defer to common code if it's been added to our special map.
   Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID,
                                    AArch64SIMDIntrinsicsProvenSorted);

   }
   case NEON::BI__builtin_neon_vfma_laneq_v: {
     llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
     // v1f64 fma should be mapped to Neon scalar f64 fma
     if (VTy && VTy->getElementType() == DoubleTy) {
       Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
       Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
       llvm::Type *VTy = GetNeonType(this,
-        NeonTypeFlags(NeonTypeFlags::Float64, false, true), Arch);
+        NeonTypeFlags(NeonTypeFlags::Float64, false, true));
       Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
       Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
       Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
       Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
       return Builder.CreateBitCast(Result, Ty);
     }
     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
                                          ICmpInst::ICMP_SGT, "vcgtz");
   case NEON::BI__builtin_neon_vcltz_v:
   case NEON::BI__builtin_neon_vcltzq_v:
     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
                                          ICmpInst::ICMP_SLT, "vcltz");
   case NEON::BI__builtin_neon_vcvt_f64_v:
   case NEON::BI__builtin_neon_vcvtq_f64_v:
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad), Arch);
+    Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
   case NEON::BI__builtin_neon_vcvt_f64_f32: {
     assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
            "unexpected vcvt_f64_f32 builtin");
     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
-    Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag, Arch));
+    Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
 
     return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
   }
   case NEON::BI__builtin_neon_vcvt_f32_f64: {
     assert(Type.getEltType() == NeonTypeFlags::Float32 &&
            "unexpected vcvt_f32_f64 builtin");
     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
-    Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag, Arch));
+    Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
 
     return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
   }
   case NEON::BI__builtin_neon_vcvt_s32_v:
   case NEON::BI__builtin_neon_vcvt_u32_v:
   case NEON::BI__builtin_neon_vcvt_s64_v:
   case NEON::BI__builtin_neon_vcvt_u64_v:
   case NEON::BI__builtin_neon_vcvt_s16_v:

   case NEON::BI__builtin_neon_vmul_lane_v:
   case NEON::BI__builtin_neon_vmul_laneq_v: {
     // v1f64 vmul_lane should be mapped to Neon scalar mul lane
     bool Quad = false;
     if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
       Quad = true;
     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
     llvm::Type *VTy = GetNeonType(this,
-      NeonTypeFlags(NeonTypeFlags::Float64, false, Quad), Arch);
+      NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
     Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
     return Builder.CreateBitCast(Result, Ty);
   }
   case NEON::BI__builtin_neon_vnegd_s64:
     return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
   case NEON::BI__builtin_neon_vnegh_f16: