[Complex] Don't use __div?c3 when building with fast-math.

[paulwalker-arm]

Summary

Plant an inline version of "((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))" instead.

[fhahn]

Looks good to me, with some nits. However it seems that we do not use the fast math flags anywhere else in Clang codegen, so it would be good to clarify if it is OK to use it here.

lib/CodeGen/CGExprComplex.cpp

765–766

Maybe we should update the comment that we do a similar simplification for floats with fast math enabled as for integers?

773

Would the following structure be slightly easier to read?

if (RHSi) {

if (FMF.isFast()) { simplify } else {libcall}

}

800

This is basically the same as the simplification for integers, unfortunate that we have to duplicate it (because of different instructions).

[hfinkel]

lib/CodeGen/CGExprComplex.cpp

773

I'd use CGF.getLangOpts().FastMath (instead of interrogating the implicit state stored in the IR builder).

test/CodeGen/complex-math.c

134

Please check the actual expansion here and below.

[paulwalker-arm]

Query LangOpts for FastMast rather than IRBuilder and fleshed out the tests.

[paulwalker-arm]

lib/CodeGen/CGExprComplex.cpp

773

Probably subjective but in this instance I preferred the look with fewer nested conditionals.

[hfinkel]

LGTM

[fhahn]

I'll commit this for you Paul

lib/CodeGen/CGExprComplex.cpp

 }
 
 // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex
 // typed values.
 ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
   llvm::Value *LHSr = Op.LHS.first, *LHSi = Op.LHS.second;
   llvm::Value *RHSr = Op.RHS.first, *RHSi = Op.RHS.second;
 
-
   llvm::Value *DSTr, *DSTi;
   if (LHSr->getType()->isFloatingPointTy()) {
-    // If we have a complex operand on the RHS, we delegate to a libcall to
-    // handle all of the complexities and minimize underflow/overflow cases.
+    // If we have a complex operand on the RHS and FastMath is not allowed, we
+    // delegate to a libcall to handle all of the complexities and minimize
+    // underflow/overflow cases. When FastMath is allowed we construct the
+    // divide inline using the same algorithm as for integer operands.
     //
     // FIXME: We would be able to avoid the libcall in many places if we
     // supported imaginary types in addition to complex types.
-    if (RHSi) {
+    if (RHSi && !CGF.getLangOpts().FastMath) {
       BinOpInfo LibCallOp = Op;
       // If LHS was a real, supply a null imaginary part.
       if (!LHSi)
         LibCallOp.LHS.second = llvm::Constant::getNullValue(LHSr->getType());
 
       switch (LHSr->getType()->getTypeID()) {
       default:
         llvm_unreachable("Unsupported floating point type!");
       case llvm::Type::HalfTyID:
         return EmitComplexBinOpLibCall("__divhc3", LibCallOp);
       case llvm::Type::FloatTyID:
         return EmitComplexBinOpLibCall("__divsc3", LibCallOp);
       case llvm::Type::DoubleTyID:
         return EmitComplexBinOpLibCall("__divdc3", LibCallOp);
       case llvm::Type::PPC_FP128TyID:
         return EmitComplexBinOpLibCall("__divtc3", LibCallOp);
       case llvm::Type::X86_FP80TyID:
         return EmitComplexBinOpLibCall("__divxc3", LibCallOp);
       case llvm::Type::FP128TyID:
         return EmitComplexBinOpLibCall("__divtc3", LibCallOp);
       }
-    }
+    } else if (RHSi) {
+      if (!LHSi)
+        LHSi = llvm::Constant::getNullValue(RHSi->getType());
+
+      // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
+      llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr); // a*c
+      llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi); // b*d
+      llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD); // ac+bd
+
+      llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr); // c*c
+      llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi); // d*d
+      llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD); // cc+dd
+
+      llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr); // b*c
+      llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi); // a*d
+      llvm::Value *BCmAD = Builder.CreateFSub(BC, AD); // bc-ad
+
+      DSTr = Builder.CreateFDiv(ACpBD, CCpDD);
+      DSTi = Builder.CreateFDiv(BCmAD, CCpDD);
+    } else {
       assert(LHSi && "Can have at most one non-complex operand!");
 
       DSTr = Builder.CreateFDiv(LHSr, RHSr);
       DSTi = Builder.CreateFDiv(LHSi, RHSr);
+    }
   } else {
     assert(Op.LHS.second && Op.RHS.second &&
            "Both operands of integer complex operators must be complex!");
     // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
     llvm::Value *Tmp1 = Builder.CreateMul(LHSr, RHSr); // a*c
     llvm::Value *Tmp2 = Builder.CreateMul(LHSi, RHSi); // b*d
     llvm::Value *Tmp3 = Builder.CreateAdd(Tmp1, Tmp2); // ac+bd
 

test/CodeGen/complex-math.c

 // RUN: %clang_cc1 %s -O1 -emit-llvm -triple x86_64-unknown-unknown -o - | FileCheck %s --check-prefix=X86
 // RUN: %clang_cc1 %s -O1 -emit-llvm -triple x86_64-pc-win64 -o - | FileCheck %s --check-prefix=X86
 // RUN: %clang_cc1 %s -O1 -emit-llvm -triple i686-unknown-unknown -o - | FileCheck %s --check-prefix=X86
 // RUN: %clang_cc1 %s -O1 -emit-llvm -triple powerpc-unknown-unknown -o - | FileCheck %s --check-prefix=PPC
 // RUN %clang_cc1 %s -O1 -emit-llvm -triple armv7-none-linux-gnueabi -o - | FileCheck %s --check-prefix=ARM
 // RUN: %clang_cc1 %s -O1 -emit-llvm -triple armv7-none-linux-gnueabihf -o - | FileCheck %s --check-prefix=ARMHF
 // RUN: %clang_cc1 %s -O1 -emit-llvm -triple thumbv7k-apple-watchos2.0 -o - -target-abi aapcs16 | FileCheck %s --check-prefix=ARM7K
+// RUN: %clang_cc1 %s -O1 -emit-llvm -triple aarch64-unknown-unknown -ffast-math -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH
 
 float _Complex add_float_rr(float a, float b) {
   // X86-LABEL: @add_float_rr(
   // X86: fadd
   // X86-NOT: fadd
   // X86: ret
   return a + b;
 }
float _Complex div_float_cr(float _Complex a, float b) {
   // X86: ret
   return a / b;
 }
 float _Complex div_float_rc(float a, float _Complex b) {
   // X86-LABEL: @div_float_rc(
   // X86-NOT: fdiv
   // X86: call {{.*}} @__divsc3(
   // X86: ret
+
+  // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
+  // AARCH64-FASTMATH-LABEL: @div_float_rc(float %a, [2 x float] %b.coerce)
+  // A = a
+  // B = 0
+  // AARCH64-FASTMATH: [[C:%.*]] = extractvalue [2 x float] %b.coerce, 0
+  // AARCH64-FASTMATH: [[D:%.*]] = extractvalue [2 x float] %b.coerce, 1
+  //
+  // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast float [[C]], %a
+  // BD = 0
+  // ACpBD = AC
+  //
+  // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast float [[C]], [[C]]
+  // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast float [[D]], [[D]]
+  // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast float [[CC]], [[DD]]
+  //
+  // BC = 0
+  // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float [[D]], %a
+  // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast float -0.000000e+00, [[AD]]
+  //
+  // AARCH64-FASTMATH: fdiv fast float [[AC]], [[CCpDD]]
+  // AARCH64-FASTMATH: fdiv fast float [[BCmAD]], [[CCpDD]]
+  // AARCH64-FASTMATH: ret
   return a / b;
 }
 float _Complex div_float_cc(float _Complex a, float _Complex b) {
   // X86-LABEL: @div_float_cc(
   // X86-NOT: fdiv
   // X86: call {{.*}} @__divsc3(
   // X86: ret
+
+  // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
+  // AARCH64-FASTMATH-LABEL: @div_float_cc([2 x float] %a.coerce, [2 x float] %b.coerce)
+  // AARCH64-FASTMATH: [[A:%.*]] = extractvalue [2 x float] %a.coerce, 0
+  // AARCH64-FASTMATH: [[B:%.*]] = extractvalue [2 x float] %a.coerce, 1
+  // AARCH64-FASTMATH: [[C:%.*]] = extractvalue [2 x float] %b.coerce, 0
+  // AARCH64-FASTMATH: [[D:%.*]] = extractvalue [2 x float] %b.coerce, 1
+  //
+  // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast float [[C]], [[A]]
+  // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast float [[D]], [[B]]
+  // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast float [[AC]], [[BD]]
+  //
+  // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast float [[C]], [[C]]
+  // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast float [[D]], [[D]]
+  // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast float [[CC]], [[DD]]
+  //
+  // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast float [[C]], [[B]]
+  // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float [[D]], [[A]]
+  // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast float [[BC]], [[AD]]
+  //
+  // AARCH64-FASTMATH: fdiv fast float [[ACpBD]], [[CCpDD]]
+  // AARCH64-FASTMATH: fdiv fast float [[BCmAD]], [[CCpDD]]
+  // AARCH64-FASTMATH: ret
   return a / b;
 }
 
 double _Complex add_double_rr(double a, double b) {
   // X86-LABEL: @add_double_rr(
   // X86: fadd
   // X86-NOT: fadd
   // X86: ret
double _Complex div_double_cr(double _Complex a, double b) {
   // X86: ret
   return a / b;
 }
 double _Complex div_double_rc(double a, double _Complex b) {
   // X86-LABEL: @div_double_rc(
   // X86-NOT: fdiv
   // X86: call {{.*}} @__divdc3(
   // X86: ret
+
+  // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
+  // AARCH64-FASTMATH-LABEL: @div_double_rc(double %a, [2 x double] %b.coerce)
+  // A = a
+  // B = 0
+  // AARCH64-FASTMATH: [[C:%.*]] = extractvalue [2 x double] %b.coerce, 0
+  // AARCH64-FASTMATH: [[D:%.*]] = extractvalue [2 x double] %b.coerce, 1
+  //
+  // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast double [[C]], %a
+  // BD = 0
+  // ACpBD = AC
+  //
+  // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast double [[C]], [[C]]
+  // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast double [[D]], [[D]]
+  // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast double [[CC]], [[DD]]
+  //
+  // BC = 0
+  // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double [[D]], %a
+  // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast double -0.000000e+00, [[AD]]
+  //
+  // AARCH64-FASTMATH: fdiv fast double [[AC]], [[CCpDD]]
+  // AARCH64-FASTMATH: fdiv fast double [[BCmAD]], [[CCpDD]]
+  // AARCH64-FASTMATH: ret
   return a / b;
 }
 double _Complex div_double_cc(double _Complex a, double _Complex b) {
   // X86-LABEL: @div_double_cc(
   // X86-NOT: fdiv
   // X86: call {{.*}} @__divdc3(
   // X86: ret
+
+  // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
+  // AARCH64-FASTMATH-LABEL: @div_double_cc([2 x double] %a.coerce, [2 x double] %b.coerce)
+  // AARCH64-FASTMATH: [[A:%.*]] = extractvalue [2 x double] %a.coerce, 0
+  // AARCH64-FASTMATH: [[B:%.*]] = extractvalue [2 x double] %a.coerce, 1
+  // AARCH64-FASTMATH: [[C:%.*]] = extractvalue [2 x double] %b.coerce, 0
+  // AARCH64-FASTMATH: [[D:%.*]] = extractvalue [2 x double] %b.coerce, 1
+  //
+  // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast double [[C]], [[A]]
+  // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast double [[D]], [[B]]
+  // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast double [[AC]], [[BD]]
+  //
+  // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast double [[C]], [[C]]
+  // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast double [[D]], [[D]]
+  // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast double [[CC]], [[DD]]
+  //
+  // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast double [[C]], [[B]]
+  // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double [[D]], [[A]]
+  // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast double [[BC]], [[AD]]
+  //
+  // AARCH64-FASTMATH: fdiv fast double [[ACpBD]], [[CCpDD]]
+  // AARCH64-FASTMATH: fdiv fast double [[BCmAD]], [[CCpDD]]
+  // AARCH64-FASTMATH: ret
   return a / b;
 }
 
 long double _Complex add_long_double_rr(long double a, long double b) {
   // X86-LABEL: @add_long_double_rr(
   // X86: fadd
   // X86-NOT: fadd
   // X86: ret
long double _Complex div_long_double_rc(long double a, long double _Complex b) {
   // X86-LABEL: @div_long_double_rc(
   // X86-NOT: fdiv
   // X86: call {{.*}} @__divxc3(
   // X86: ret
   // PPC-LABEL: @div_long_double_rc(
   // PPC-NOT: fdiv
   // PPC: call {{.*}} @__divtc3(
   // PPC: ret
+
+  // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
+  // AARCH64-FASTMATH-LABEL: @div_long_double_rc(fp128 %a, [2 x fp128] %b.coerce)
+  // A = a
+  // B = 0
+  // AARCH64-FASTMATH: [[C:%.*]] = extractvalue [2 x fp128] %b.coerce, 0
+  // AARCH64-FASTMATH: [[D:%.*]] = extractvalue [2 x fp128] %b.coerce, 1
+  //
+  // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast fp128 [[C]], %a
+  // BD = 0
+  // ACpBD = AC
+  //
+  // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast fp128 [[C]], [[C]]
+  // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast fp128 [[D]], [[D]]
+  // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast fp128 [[CC]], [[DD]]
+  //
+  // BC = 0
+  // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 [[D]], %a
+  // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast fp128 0xL00000000000000008000000000000000, [[AD]]
+  //
+  // AARCH64-FASTMATH: fdiv fast fp128 [[AC]], [[CCpDD]]
+  // AARCH64-FASTMATH: fdiv fast fp128 [[BCmAD]], [[CCpDD]]
+  // AARCH64-FASTMATH: ret
   return a / b;
 }
 long double _Complex div_long_double_cc(long double _Complex a, long double _Complex b) {
   // X86-LABEL: @div_long_double_cc(
   // X86-NOT: fdiv
   // X86: call {{.*}} @__divxc3(
   // X86: ret
   // PPC-LABEL: @div_long_double_cc(
   // PPC-NOT: fdiv
   // PPC: call {{.*}} @__divtc3(
   // PPC: ret
+
+  // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
+  // AARCH64-FASTMATH-LABEL: @div_long_double_cc([2 x fp128] %a.coerce, [2 x fp128] %b.coerce)
+  // AARCH64-FASTMATH: [[A:%.*]] = extractvalue [2 x fp128] %a.coerce, 0
+  // AARCH64-FASTMATH: [[B:%.*]] = extractvalue [2 x fp128] %a.coerce, 1
+  // AARCH64-FASTMATH: [[C:%.*]] = extractvalue [2 x fp128] %b.coerce, 0
+  // AARCH64-FASTMATH: [[D:%.*]] = extractvalue [2 x fp128] %b.coerce, 1
+  //
+  // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast fp128 [[C]], [[A]]
+  // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast fp128 [[D]], [[B]]
+  // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast fp128 [[AC]], [[BD]]
+  //
+  // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast fp128 [[C]], [[C]]
+  // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast fp128 [[D]], [[D]]
+  // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast fp128 [[CC]], [[DD]]
+  //
+  // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast fp128 [[C]], [[B]]
+  // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 [[D]], [[A]]
+  // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast fp128 [[BC]], [[AD]]
+  //
+  // AARCH64-FASTMATH: fdiv fast fp128 [[ACpBD]], [[CCpDD]]
+  // AARCH64-FASTMATH: fdiv fast fp128 [[BCmAD]], [[CCpDD]]
+  // AARCH64-FASTMATH: ret
   return a / b;
 }
 
 // Comparison operators don't rely on library calls or have interseting math
 // properties, but test that mixed types work correctly here.
 _Bool eq_float_cr(float _Complex a, float b) {
   // X86-LABEL: @eq_float_cr(
   // X86: fcmp oeq