[Clang] Add lowering for _C complex arithmetic to complex intrinsics.

[jcranmer-intel]

Summary

This adds a new guard flag, -fuse-complex-intrinsics, that is defaulted to on
for the x86/x86-64 platforms, as they can be lowered to complex intrinsics
through the entire pipeline correctly. Other platforms could be added, but it
requires auditing their support correctly in the backend, and I don't have as
much familiarity with their ABIs as I do for x86/x86-64. Help would be
appreciated.

Future patches will build on top of this support to enable more complex limited
range support, but only when lowering to intrinsics.

Depends on D119288

clang/include/clang/Basic/CodeGenOptions.def

 CODEGENOPT(UniqueInternalLinkageNames, 1, 0) ///< Internal Linkage symbols get unique names.
 CODEGENOPT(SplitMachineFunctions, 1, 0) ///< Split machine functions using profile information.
 
 /// When false, this attempts to generate code as if the result of an
 /// overflowing conversion matches the overflowing behavior of a target's native
 /// float-to-int conversion instructions.
 CODEGENOPT(StrictFloatCastOverflow, 1, 1)
 
+CODEGENOPT(UseComplexIntrinsics, 1, 0) ///< Use LLVM complex intrinsics
+
 CODEGENOPT(UniformWGSize     , 1, 0) ///< -cl-uniform-work-group-size
 CODEGENOPT(NoZeroInitializedInBSS , 1, 0) ///< -fno-zero-initialized-in-bss.
 /// Method of Objective-C dispatch to use.
 ENUM_CODEGENOPT(ObjCDispatchMethod, ObjCDispatchMethodKind, 2, Legacy)
 /// Replace certain message sends with calls to ObjC runtime entrypoints
 CODEGENOPT(ObjCConvertMessagesToRuntimeCalls , 1, 1)
 CODEGENOPT(ObjCAvoidHeapifyLocalBlocks, 1, 0)
 

clang/include/clang/Basic/FPOptions.def

 #  error Define the OPTION macro to handle floating point language options
 #endif
 
 // OPTION(name, type, width, previousName)
 OPTION(FPContractMode, LangOptions::FPModeKind, 2, First)
 OPTION(RoundingMode, LangOptions::RoundingMode, 3, FPContractMode)
 OPTION(FPExceptionMode, LangOptions::FPExceptionModeKind, 2, RoundingMode)
 OPTION(AllowFEnvAccess, bool, 1, FPExceptionMode)
-OPTION(AllowFPReassociate, bool, 1, AllowFEnvAccess)
+OPTION(ComplexRange, LangOptions::ComplexRangeKind, 2, AllowFEnvAccess)
+OPTION(AllowFPReassociate, bool, 1, ComplexRange)
 OPTION(NoHonorNaNs, bool, 1, AllowFPReassociate)
 OPTION(NoHonorInfs, bool, 1, NoHonorNaNs)
 OPTION(NoSignedZero, bool, 1, NoHonorInfs)
 OPTION(AllowReciprocal, bool, 1, NoSignedZero)
 OPTION(AllowApproxFunc, bool, 1, AllowReciprocal)
 #undef OPTION

clang/include/clang/Basic/LangOptions.h

enum FPExceptionModeKind {
     FPE_MayTrap,
     /// Strictly preserve the floating-point exception semantics.
     FPE_Strict
   };
 
   /// Possible exception handling behavior.
   enum class ExceptionHandlingKind { None, SjLj, WinEH, DwarfCFI, Wasm };
 
+  /// Controls for the complex arithmetic range rules.
+  enum ComplexRangeKind {
+    /// Disable both NaN checks and scaling
+    CX_Limited,
+
+    /// Disable only NaN checks
+    CX_NoNan,
+
+    /// Require NaN checks and scaling,
+    CX_Full
+  };
+
   enum class LaxVectorConversionKind {
     /// Permit no implicit vector bitcasts.
     None,
     /// Permit vector bitcasts between integer vectors with different numbers
     /// of elements but the same total bit-width.
     Integer,
     /// Permit vector bitcasts between all vectors with the same total
     /// bit-width.
#include "clang/Basic/LangOptions.def"
   void remapPathPrefix(SmallString<256> &Path) const;
 };
 
 /// Floating point control options
 class FPOptionsOverride;
 class FPOptions {
 public:
   // We start by defining the layout.
-  using storage_type = uint16_t;
+  using storage_type = uint32_t;
 
   using RoundingMode = llvm::RoundingMode;
 
   static constexpr unsigned StorageBitSize = 8 * sizeof(storage_type);
 
   // Define a fake option named "First" so that we have a PREVIOUS even for the
   // real first option.
   static constexpr storage_type FirstShift = 0, FirstWidth = 0;
explicit FPOptions(const LangOptions &LO) {
     // frontend.
     auto LangOptContractMode = LO.getDefaultFPContractMode();
     if (LangOptContractMode == LangOptions::FPM_FastHonorPragmas)
       LangOptContractMode = LangOptions::FPM_Fast;
     setFPContractMode(LangOptContractMode);
     setRoundingMode(LO.getFPRoundingMode());
     setFPExceptionMode(LO.getFPExceptionMode());
     setAllowFPReassociate(LO.AllowFPReassoc);
+    setComplexRange(LO.getComplexRange());
     setNoHonorNaNs(LO.NoHonorNaNs);
     setNoHonorInfs(LO.NoHonorInfs);
     setNoSignedZero(LO.NoSignedZero);
     setAllowReciprocal(LO.AllowRecip);
     setAllowApproxFunc(LO.ApproxFunc);
     if (getFPContractMode() == LangOptions::FPM_On &&
         getRoundingMode() == llvm::RoundingMode::Dynamic &&
         getFPExceptionMode() == LangOptions::FPE_Strict)
class FPOptionsOverride {
   FPOptions Options = FPOptions::getFromOpaqueInt(0);
   FPOptions::storage_type OverrideMask = 0;
 
 public:
   using RoundingMode = llvm::RoundingMode;
 
   /// The type suitable for storing values of FPOptionsOverride. Must be twice
   /// as wide as bit size of FPOption.
-  using storage_type = uint32_t;
+  using storage_type = uint64_t;
   static_assert(sizeof(storage_type) >= 2 * sizeof(FPOptions::storage_type),
                 "Too short type for FPOptionsOverride");
 
   /// Bit mask selecting bits of OverrideMask in serialized representation of
   /// FPOptionsOverride.
   static constexpr storage_type OverrideMaskBits =
       (static_cast<storage_type>(1) << FPOptions::StorageBitSize) - 1;
 

clang/include/clang/Basic/LangOptions.def

 COMPATIBLE_LANGOPT(ProtectParens     , 1, 0, "optimizer honors parentheses "
                    "when floating-point expressions are evaluated")
 BENIGN_LANGOPT(AllowFPReassoc    , 1, 0, "Permit Floating Point reassociation")
 BENIGN_LANGOPT(NoHonorNaNs       , 1, 0, "Permit Floating Point optimization without regard to NaN")
 BENIGN_LANGOPT(NoHonorInfs       , 1, 0, "Permit Floating Point optimization without regard to infinities")
 BENIGN_LANGOPT(NoSignedZero      , 1, 0, "Permit Floating Point optimization without regard to signed zeros")
 BENIGN_LANGOPT(AllowRecip        , 1, 0, "Permit Floating Point reciprocal")
 BENIGN_LANGOPT(ApproxFunc        , 1, 0, "Permit Floating Point approximation")
+BENIGN_ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 2, CX_Full,
+                    "Range rules for complex multiplications and divisions")
 
 BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars")
 
 BENIGN_LANGOPT(AccessControl     , 1, 1, "C++ access control")
 LANGOPT(CharIsSigned      , 1, 1, "signed char")
 LANGOPT(WCharSize         , 4, 0, "width of wchar_t")
 LANGOPT(WCharIsSigned        , 1, 0, "signed or unsigned wchar_t")
 ENUM_LANGOPT(MSPointerToMemberRepresentationMethod, PragmaMSPointersToMembersKind, 2, PPTMK_BestCase, "member-pointer representation method")

clang/include/clang/Driver/Options.td

 
 defm protect_parens : BoolFOption<"protect-parens",
   LangOpts<"ProtectParens">, DefaultFalse,
   PosFlag<SetTrue, [CoreOption, CC1Option],
           "Determines whether the optimizer honors parentheses when "
           "floating-point expressions are evaluated">,
   NegFlag<SetFalse>>;
 
+defm use_complex_intrinsics : BoolFOption<"use-complex-intrinsics",
+  CodeGenOpts<"UseComplexIntrinsics">, DefaultFalse,
+  PosFlag<SetTrue, [CoreOption, CC1Option],
+          "Determines whether to lower _Complex operations to LLVM complex "
+          "intrinsics">,
+  NegFlag<SetFalse>>;
+
 def ffor_scope : Flag<["-"], "ffor-scope">, Group<f_Group>;
 def fno_for_scope : Flag<["-"], "fno-for-scope">, Group<f_Group>;
 
 defm rewrite_imports : BoolFOption<"rewrite-imports",
   PreprocessorOutputOpts<"RewriteImports">, DefaultFalse,
   PosFlag<SetTrue, [CC1Option]>, NegFlag<SetFalse>>;
 defm rewrite_includes : BoolFOption<"rewrite-includes",
   PreprocessorOutputOpts<"RewriteIncludes">, DefaultFalse,

clang/lib/CodeGen/CGExprComplex.cpp

public:
 
   /// Emit a cast from complex value Val to DestType.
   ComplexPairTy EmitComplexToComplexCast(ComplexPairTy Val, QualType SrcType,
                                          QualType DestType, SourceLocation Loc);
   /// Emit a cast from scalar value Val to DestType.
   ComplexPairTy EmitScalarToComplexCast(llvm::Value *Val, QualType SrcType,
                                         QualType DestType, SourceLocation Loc);
 
+  /// Convert a LLVM complex type representation into a pair of values
+  ComplexPairTy BreakLLVMComplexType(llvm::Value *Val) {
+    llvm::Value *Real = Builder.CreateExtractElement(Val, uint64_t(0));
+    llvm::Value *Imag = Builder.CreateExtractElement(Val, uint64_t(1));
+    return ComplexPairTy(Real, Imag);
+  }
+
   //===--------------------------------------------------------------------===//
   //                            Visitor Methods
   //===--------------------------------------------------------------------===//
 
   ComplexPairTy Visit(Expr *E) {
     ApplyDebugLocation DL(CGF, E);
     return StmtVisitor<ComplexExprEmitter, ComplexPairTy>::Visit(E);
   }
llvm::Constant *Null =
                        llvm::Constant::getNullValue(CGF.ConvertType(Elem));
     return ComplexPairTy(Null, Null);
   }
 
   struct BinOpInfo {
     ComplexPairTy LHS;
     ComplexPairTy RHS;
     QualType Ty;  // Computation Type.
+    FPOptions FPFeatures;
   };
 
   BinOpInfo EmitBinOps(const BinaryOperator *E);
   LValue EmitCompoundAssignLValue(const CompoundAssignOperator *E,
                                   ComplexPairTy (ComplexExprEmitter::*Func)
                                   (const BinOpInfo &),
                                   RValue &Val);
   ComplexPairTy EmitCompoundAssign(const CompoundAssignOperator *E,
if (Op.LHS.first->getType()->isFloatingPointTy()) {
     // (a + ib) * (c + id) = (a * c - b * d) + i(a * d + b * c)
     //
     // But we can fold away components which would be zero due to a real
     // operand according to C11 Annex G.5.1p2.
     // FIXME: C11 also provides for imaginary types which would allow folding
     // still more of this within the type system.
 
     if (Op.LHS.second && Op.RHS.second) {
+      if (CGF.CGM.getCodeGenOpts().UseComplexIntrinsics) {
+        Value *Op0 = Builder.CreateComplexValue(Op.LHS.first, Op.LHS.second);
+        Value *Op1 = Builder.CreateComplexValue(Op.RHS.first, Op.RHS.second);
+        Value *Result = Builder.CreateComplexMul(Op0, Op1,
+          Op.FPFeatures.getComplexRange() != LangOptions::CX_Full);
+        return BreakLLVMComplexType(Result);
+      }
+
       // If both operands are complex, emit the core math directly, and then
       // test for NaNs. If we find NaNs in the result, we delegate to a libcall
       // to carefully re-compute the correct infinity representation if
       // possible. The expectation is that the presence of NaNs here is
       // *extremely* rare, and so the cost of the libcall is almost irrelevant.
       // This is good, because the libcall re-computes the core multiplication
       // exactly the same as we do here and re-tests for NaNs in order to be
       // a generic complex*complex libcall.

 // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex
 // typed values.
 ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
   llvm::Value *LHSr = Op.LHS.first, *LHSi = Op.LHS.second;
   llvm::Value *RHSr = Op.RHS.first, *RHSi = Op.RHS.second;
 
   llvm::Value *DSTr, *DSTi;
   if (LHSr->getType()->isFloatingPointTy()) {
+    // If we are using complex intrinsics, do so whenever the right-hand side
+    // is complex, since no major simplification is possible in this scenario.
+    // (Simplifications are possible if the LHS is real or pure imaginary).
+    if (CGF.CGM.getCodeGenOpts().UseComplexIntrinsics && RHSi) {
+      llvm::Value *Op0 =
+          Builder.CreateComplexValue(Op.LHS.first, Op.LHS.second);
+      llvm::Value *Op1 =
+          Builder.CreateComplexValue(Op.RHS.first, Op.RHS.second);
+      llvm::Value *Result = Builder.CreateComplexDiv(Op0, Op1,
+          Op.FPFeatures.getComplexRange() != LangOptions::CX_Full,
+          Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited);
+      return BreakLLVMComplexType(Result);
+    }
+
     // If we have a complex operand on the RHS and FastMath is not allowed, we
     // delegate to a libcall to handle all of the complexities and minimize
     // underflow/overflow cases. When FastMath is allowed we construct the
     // divide inline using the same algorithm as for integer operands.
     //
     // FIXME: We would be able to avoid the libcall in many places if we
     // supported imaginary types in addition to complex types.
     if (RHSi && !CGF.getLangOpts().FastMath) {
ComplexExprEmitter::EmitBinOps(const BinaryOperator *E) {
   else
     Ops.LHS = Visit(E->getLHS());
   if (E->getRHS()->getType()->isRealFloatingType())
     Ops.RHS = ComplexPairTy(CGF.EmitScalarExpr(E->getRHS()), nullptr);
   else
     Ops.RHS = Visit(E->getRHS());
 
   Ops.Ty = E->getType();
+  Ops.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
   return Ops;
 }
 
 
 LValue ComplexExprEmitter::
 EmitCompoundAssignLValue(const CompoundAssignOperator *E,
           ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&),
                          RValue &Val) {

clang/lib/Driver/ToolChains/Clang.cpp

if (const Arg *A = Args.getLastArg(options::OPT_mfpmath_EQ)) {
     CmdArgs.push_back("-mfpmath");
     CmdArgs.push_back(A->getValue());
   }
 
   // Disable a codegen optimization for floating-point casts.
   if (Args.hasFlag(options::OPT_fno_strict_float_cast_overflow,
                    options::OPT_fstrict_float_cast_overflow, false))
     CmdArgs.push_back("-fno-strict-float-cast-overflow");
+
+  // For now, only enable complex intrinsics by default if we know we can lower
+  // the call correctly in ExpandComplex.
+  bool DefaultUseComplexIntrinsics = false;
+  switch (TC.getArch()) {
+  case llvm::Triple::x86:
+  case llvm::Triple::x86_64:
+    DefaultUseComplexIntrinsics = true;
+    break;
+  default:
+    DefaultUseComplexIntrinsics = false;
+    break;
+  }
+  if (Args.hasFlag(options::OPT_fuse_complex_intrinsics,
+                   options::OPT_fno_use_complex_intrinsics,
+                   DefaultUseComplexIntrinsics)) {
+    CmdArgs.push_back("-fuse-complex-intrinsics");
+  }
 }
 
 static void RenderAnalyzerOptions(const ArgList &Args, ArgStringList &CmdArgs,
                                   const llvm::Triple &Triple,
                                   const InputInfo &Input) {
   // Enable region store model by default.
   CmdArgs.push_back("-analyzer-store=region");
 

clang/test/CodeGen/complex-intrinsics.c

+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown -o - | FileCheck %s --check-prefix=NOINTRIN
+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-pc-win64 -o - | FileCheck %s --check-prefix=NOINTRIN
+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple i686-unknown-unknown -o - | FileCheck %s --check-prefix=NOINTRIN
+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple spir -o - | FileCheck %s --check-prefix=NOINTRIN
+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown -fuse-complex-intrinsics -o - | FileCheck %s --check-prefix=INTRIN
+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-pc-win64 -fuse-complex-intrinsics -o - | FileCheck %s --check-prefix=INTRIN
+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown -fuse-complex-intrinsics -DT=int -o - | FileCheck %s --check-prefix=INT
+// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown -DT=int -o - | FileCheck %s --check-prefix=INT
+
+// Check defaults for intrinsics:
+// RUN: %clang %s -O0 -S -emit-llvm -target x86_64-unknown-unknown -o - | FileCheck %s --check-prefix=INTRIN
+// RUN: %clang %s -O0 -S -emit-llvm -target x86_64-unknown-unknown -fuse-complex-intrinsics -o - | FileCheck %s --check-prefix=INTRIN
+// RUN: %clang %s -O0 -S -emit-llvm -target x86_64-unknown-unknown -fno-use-complex-intrinsics -o - | FileCheck %s --check-prefix=NOINTRIN
+// RUN: %clang %s -O0 -S -emit-llvm -target aarch64-unknown-unknown -o - | FileCheck %s --check-prefix=NOINTRIN
+// RUN: %clang %s -O0 -S -emit-llvm -target aarch64-unknown-unknown -fuse-complex-intrinsics -o - | FileCheck %s --check-prefix=INTRIN
+// RUN: %clang %s -O0 -S -emit-llvm -target aarch64-unknown-unknown -fno-use-complex-intrinsics -o - | FileCheck %s --check-prefix=NOINTRIN
+
+#ifndef T
+#  define T float
+#endif
+
+T check_var;
+// INTRIN: @check_var = {{.*}}global [[T:[a-z0-9]+]]
+// NOINTRIN: @check_var = {{.*}}global [[T:[a-z0-9]+]]
+// INT: @check_var = {{.*}}global [[T:i[0-9]+]]
+
+T _Complex add_rc(T a, T _Complex b) {
+  // INTRIN-LABEL: @add_rc(
+  // INTRIN-COUNT-1: fadd [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @add_rc(
+  // NOINTRIN-COUNT-1: fadd [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @add_rc(
+  // INT-COUNT-1: add [[T]]
+  // INT: ret
+  return a + b;
+}
+
+T _Complex add_cr(T _Complex a, T b) {
+  // INTRIN-LABEL: @add_cr(
+  // INTRIN-COUNT-1: fadd [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @add_cr(
+  // NOINTRIN-COUNT-1: fadd [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @add_cr(
+  // INT-COUNT-1: add [[T]]
+  // INT: ret
+  return a + b;
+}
+
+T _Complex add_cc(T _Complex a, T _Complex b) {
+  // INTRIN-LABEL: @add_cc(
+  // INTRIN-COUNT-2: fadd [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @add_cc(
+  // NOINTRIN-COUNT-2: fadd [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @add_cc(
+  // INT-COUNT-2: add [[T]]
+  // INT: ret
+  return a + b;
+}
+
+T _Complex sub_rc(T a, T _Complex b) {
+  // INTRIN-LABEL: @sub_rc(
+  // INTRIN: fsub [[T]]
+  // INTRIN: fneg [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @sub_rc(
+  // NOINTRIN: fsub [[T]]
+  // NOINTRIN: fneg [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @sub_rc(
+  // INT-COUNT-2: sub [[T]]
+  // INT: ret
+  return a - b;
+}
+
+T _Complex sub_cr(T _Complex a, T b) {
+  // INTRIN-LABEL: @sub_cr(
+  // INTRIN: fsub [[T]]
+  // INTRIN-NOT: fsub [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @sub_cr(
+  // NOINTRIN: fsub [[T]]
+  // NOINTRIN-NOT: fsub [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @sub_cr(
+  // INT-COUNT-2: sub [[T]]
+  // INT: ret
+  return a - b;
+}
+
+T _Complex sub_cc(T _Complex a, T _Complex b) {
+  // INTRIN-LABEL: @sub_cc(
+  // INTRIN-COUNT-2: fsub [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @sub_cc(
+  // NOINTRIN-COUNT-2: fsub [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @sub_cc(
+  // INT-COUNT-2: sub [[T]]
+  // INT: ret
+  return a - b;
+}
+
+T _Complex mul_rc(T a, T _Complex b) {
+  // INTRIN-LABEL: @mul_rc(
+  // INTRIN-COUNT-2: fmul [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @mul_rc(
+  // NOINTRIN-COUNT-2: fmul [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @mul_rc(
+  // INT-COUNT-4: mul [[T]]
+  // INT: ret
+  return a * b;
+}
+
+T _Complex mul_cr(T _Complex a, T b) {
+  // INTRIN-LABEL: @mul_cr(
+  // INTRIN-COUNT-2: fmul [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @mul_cr(
+  // NOINTRIN-COUNT-2: fmul [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @mul_cr(
+  // INT-COUNT-4: mul [[T]]
+  // INT: ret
+  return a * b;
+}
+
+T _Complex mul_cc(T _Complex a, T _Complex b) {
+  // INTRIN-LABEL: @mul_cc(
+  // INTRIN-NOT: fmul [[T]]
+  // INTRIN: call {{.*}} @llvm.experimental.complex.fmul
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @mul_cc(
+  // NOINTRIN-COUNT-4: fmul [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @mul_cc(
+  // INT-COUNT-4: mul [[T]]
+  // INT: ret
+  return a * b;
+}
+
+T _Complex div_rc(T a, T _Complex b) {
+  // INTRIN-LABEL: @div_rc(
+  // INTRIN-NOT: fdiv [[T]]
+  // INTRIN: call {{.*}} @llvm.experimental.complex.fdiv
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @div_rc(
+  // NOINTRIN: call {{.*}} @__div
+  // NOINTRIN: ret
+  // INT-LABEL: @div_rc(
+  // INT-COUNT-6: mul [[T]]
+  // INT: ret
+  return a / b;
+}
+
+T _Complex div_cr(T _Complex a, T b) {
+  // INTRIN-LABEL: @div_cr(
+  // INTRIN-COUNT-2: fdiv [[T]]
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @div_cr(
+  // NOINTRIN-COUNT-2: fdiv [[T]]
+  // NOINTRIN: ret
+  // INT-LABEL: @div_cr(
+  // INT-COUNT-5: mul [[T]]
+  // INT: ret
+  return a / b;
+}
+
+T _Complex div_cc(T _Complex a, T _Complex b) {
+  // INTRIN-LABEL: @div_cc(
+  // INTRIN-NOT: fdiv [[T]]
+  // INTRIN: call {{.*}} @llvm.experimental.complex.fdiv
+  // INTRIN: ret
+  // NOINTRIN-LABEL: @div_cc(
+  // NOINTRIN: call {{.*}} @__div
+  // NOINTRIN: ret
+  // INT-LABEL: @div_cc(
+  // INT-COUNT-6: mul [[T]]
+  // INT: ret
+  return a / b;
+}