Index: llvm/include/llvm/Target/TargetLowering.h =================================================================== --- llvm/include/llvm/Target/TargetLowering.h +++ llvm/include/llvm/Target/TargetLowering.h @@ -2951,19 +2951,23 @@ /// Hooks for building estimates in place of slower divisions and square /// roots. - /// Return a reciprocal square root estimate value for the input operand. - /// The RefinementSteps output is the number of Newton-Raphson refinement - /// iterations required to generate a sufficient (though not necessarily - /// IEEE-754 compliant) estimate for the value type. - /// The boolean UseOneConstNR output is used to select a Newton-Raphson - /// algorithm implementation that uses one constant or two constants. - /// A target may choose to implement its own refinement within this function. - /// If that's true, then return '0' as the number of RefinementSteps to avoid - /// any further refinement of the estimate. + /// Return initial estimate of the reciprocal square root or an approximate + /// square root or its reciprocal for the input operand. + /// The target has the choice to use the generic approximation algorithm, + /// when it should return the initial estimate of the reciprocal square root + /// and the number of extra refinement iterations to generate a sufficient + /// (though not necessarily IEEE-754 compliant) approximation in + /// RefinementSteps, or to approximate square root or its reciprocal itself, + /// when it should return the approximation and '0' in RefinementSteps. + /// Reciprocal indicates whether the square root or its reciprocal is being + /// approximated, either by the target itself or by the generic algorithm. + /// The boolean UseOneConstNR output is used to select between the generic + /// Newton-Raphson algorithms for the refinement iterations that use one or + /// two constants, in case the target provides just the initial estimate. /// An empty SDValue return means no estimate sequence can be created. - virtual SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const { + virtual SDValue getSqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, + unsigned &RefinementSteps, + bool &UseOneConstNR, bool Reciprocal) const { return SDValue(); } Index: llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -14497,34 +14497,33 @@ return S; } -SDValue DAGCombiner::BuildReciprocalEstimate(SDValue Op, SDNodeFlags *Flags) { +/// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i) +/// For the reciprocal, we need to find the zero of the function: +/// F(X) = A X - 1 [which has a zero at X = 1/A] +/// => +/// X_{i+1} = X_i (2 - A X_i) = X_i + X_i (1 - A X_i) [this second form +/// does not require additional intermediate precision] +SDValue DAGCombiner::BuildReciprocalEstimate(SDValue Arg, SDNodeFlags *Flags) { if (Level >= AfterLegalizeDAG) return SDValue(); // Expose the DAG combiner to the target combiner implementations. TargetLowering::DAGCombinerInfo DCI(DAG, Level, false, this); - unsigned Iterations = 0; - if (SDValue Est = TLI.getRecipEstimate(Op, DCI, Iterations)) { + if (SDValue Est = TLI.getRecipEstimate(Arg, DCI, Iterations)) { if (Iterations) { - // Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i) - // For the reciprocal, we need to find the zero of the function: - // F(X) = A X - 1 [which has a zero at X = 1/A] - // => - // X_{i+1} = X_i (2 - A X_i) = X_i + X_i (1 - A X_i) [this second form - // does not require additional intermediate precision] - EVT VT = Op.getValueType(); - SDLoc DL(Op); - SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); + EVT VT = Arg.getValueType(); + SDLoc DL(Arg); + SDValue One = DAG.getConstantFP(1.0, DL, VT); AddToWorklist(Est.getNode()); // Newton iterations: Est = Est + Est (1 - Arg * Est) for (unsigned i = 0; i < Iterations; ++i) { - SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, Est, Flags); + SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); AddToWorklist(NewEst.getNode()); - NewEst = DAG.getNode(ISD::FSUB, DL, VT, FPOne, NewEst, Flags); + NewEst = DAG.getNode(ISD::FSUB, DL, VT, One, NewEst, Flags); AddToWorklist(NewEst.getNode()); NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); @@ -14646,13 +14645,32 @@ TargetLowering::DAGCombinerInfo DCI(DAG, Level, false, this); unsigned Iterations = 0; bool UseOneConstNR = false; - if (SDValue Est = TLI.getRsqrtEstimate(Op, DCI, Iterations, UseOneConstNR)) { + if (SDValue Est = TLI.getSqrtEstimate(Op, DCI, Iterations, UseOneConstNR, + Reciprocal)) { AddToWorklist(Est.getNode()); + if (Iterations) { Est = UseOneConstNR - ? buildSqrtNROneConst(Op, Est, Iterations, Flags, Reciprocal) - : buildSqrtNRTwoConst(Op, Est, Iterations, Flags, Reciprocal); + ? buildSqrtNROneConst(Op, Est, Iterations, Flags, Reciprocal) + : buildSqrtNRTwoConst(Op, Est, Iterations, Flags, Reciprocal); + + if (!Reciprocal) { + // Unfortunately, Est is now NaN if the input was exactly 0. + // Select out this case and force the answer to 0. + EVT VT = Op.getValueType(); + SDLoc DL(Op); + + SDValue Zero = DAG.getConstantFP(0.0, DL, VT); + EVT CCVT = getSetCCResultType(VT); + SDValue ZeroCmp = DAG.getSetCC(DL, CCVT, Op, Zero, ISD::SETEQ); + AddToWorklist(ZeroCmp.getNode()); + + Est = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, + ZeroCmp, Zero, Est); + AddToWorklist(Est.getNode()); + } } + return Est; } @@ -14664,23 +14682,7 @@ } SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags *Flags) { - SDValue Est = buildSqrtEstimateImpl(Op, Flags, false); - if (!Est) - return SDValue(); - - // Unfortunately, Est is now NaN if the input was exactly 0. - // Select out this case and force the answer to 0. - EVT VT = Est.getValueType(); - SDLoc DL(Op); - SDValue Zero = DAG.getConstantFP(0.0, DL, VT); - EVT CCVT = getSetCCResultType(VT); - SDValue ZeroCmp = DAG.getSetCC(DL, CCVT, Op, Zero, ISD::SETEQ); - AddToWorklist(ZeroCmp.getNode()); - - Est = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, ZeroCmp, - Zero, Est); - AddToWorklist(Est.getNode()); - return Est; + return buildSqrtEstimateImpl(Op, Flags, false); } /// Return true if base is a frame index, which is known not to alias with Index: llvm/lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -187,9 +187,9 @@ SMULL, UMULL, - // Reciprocal estimates. - FRECPE, - FRSQRTE, + // Reciprocal estimates and steps. + FRECPE, FRECPS, + FRSQRTE, FRSQRTS, // NEON Load/Store with post-increment base updates LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE, @@ -520,9 +520,9 @@ SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, std::vector *Created) const override; - SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const override; + SDValue getSqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, + unsigned &RefinementSteps, bool &UseOneConstNR, + bool Reciprocal) const override; SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI, unsigned &RefinementSteps) const override; unsigned combineRepeatedFPDivisors() const override; Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -953,8 +953,10 @@ case AArch64ISD::ST4LANEpost: return "AArch64ISD::ST4LANEpost"; case AArch64ISD::SMULL: return "AArch64ISD::SMULL"; case AArch64ISD::UMULL: return "AArch64ISD::UMULL"; - case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE"; case AArch64ISD::FRECPE: return "AArch64ISD::FRECPE"; + case AArch64ISD::FRECPS: return "AArch64ISD::FRECPS"; + case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE"; + case AArch64ISD::FRSQRTS: return "AArch64ISD::FRSQRTS"; } return nullptr; } @@ -4581,15 +4583,66 @@ return DCI.DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); } -SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand, +SDValue AArch64TargetLowering::getRecipEstimate(SDValue Arg, DAGCombinerInfo &DCI, unsigned &ExtraSteps) const { - return getEstimate(*Subtarget, DCI, AArch64ISD::FRECPE, Operand, ExtraSteps); + if (SDValue Est = getEstimate(*Subtarget, DCI, AArch64ISD::FRECPE, Arg, + ExtraSteps)) { + SDLoc DL(Arg); + EVT VT = Arg.getValueType(); + + SDNodeFlags Flags; + Flags.setUnsafeAlgebra(true); + + // Newton reciprocal iteration: Est * (2 - Arg * Est) + // AArch64 reciprocal iteration instruction: (2 - M * N) + for (unsigned i = ExtraSteps; i > 0; --i) { + SDValue Step = DCI.DAG.getNode(AArch64ISD::FRECPS, DL, VT, Arg, Est, + &Flags); + Est = DCI.DAG.getNode(ISD::FMUL, DL, VT, Est, Step, &Flags); + } + + ExtraSteps = 0; + return Est; + } + + return SDValue(); } -SDValue AArch64TargetLowering::getRsqrtEstimate(SDValue Operand, - DAGCombinerInfo &DCI, unsigned &ExtraSteps, bool &UseOneConst) const { - UseOneConst = true; - return getEstimate(*Subtarget, DCI, AArch64ISD::FRSQRTE, Operand, ExtraSteps); +SDValue AArch64TargetLowering::getSqrtEstimate + (SDValue Arg, DAGCombinerInfo &DCI, unsigned &ExtraSteps, bool &UseOneConst, + bool Reciprocal) const { + if (SDValue Est = getEstimate(*Subtarget, DCI, AArch64ISD::FRSQRTE, Arg, + ExtraSteps)) { + SDLoc DL(Arg); + EVT VT = Arg.getValueType(); + + SDNodeFlags Flags; + Flags.setUnsafeAlgebra(true); + + // Newton reciprocal square root iteration: Est * 0.5 * (3 - Arg * Est^2) + // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N) + for (unsigned i = ExtraSteps; i > 0; --i) { + SDValue Step = DCI.DAG.getNode(ISD::FMUL, DL, VT, Est, Est, &Flags); + Step = DCI.DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Arg, Step, &Flags); + Est = DCI.DAG.getNode(ISD::FMUL, DL, VT, Est, Step, &Flags); + } + + if (!Reciprocal) { + EVT CCVT = getSetCCResultType(DCI.DAG.getDataLayout(), + *DCI.DAG.getContext(), VT); + SDValue Zero = DCI.DAG.getConstantFP(0.0, DL, VT); + Zero = DCI.DAG.getSetCC(DL, CCVT, Arg, Zero, ISD::SETEQ); + + Est = DCI.DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, &Flags); + Est = DCI.DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, + DL, VT, Zero, Arg, Est); + } + + ExtraSteps = 0; + return Est; + } + + return SDValue(); } //===----------------------------------------------------------------------===// Index: llvm/lib/Target/AArch64/AArch64InstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -287,7 +287,9 @@ def AArch64umull : SDNode<"AArch64ISD::UMULL", SDT_AArch64mull>; def AArch64frecpe : SDNode<"AArch64ISD::FRECPE", SDTFPUnaryOp>; +def AArch64frecps : SDNode<"AArch64ISD::FRECPS", SDTFPBinOp>; def AArch64frsqrte : SDNode<"AArch64ISD::FRSQRTE", SDTFPUnaryOp>; +def AArch64frsqrts : SDNode<"AArch64ISD::FRSQRTS", SDTFPBinOp>; def AArch64saddv : SDNode<"AArch64ISD::SADDV", SDT_AArch64UnaryVec>; def AArch64uaddv : SDNode<"AArch64ISD::UADDV", SDT_AArch64UnaryVec>; @@ -3414,6 +3416,17 @@ def : Pat<(v2f64 (AArch64frecpe (v2f64 FPR128:$Rn))), (FRECPEv2f64 FPR128:$Rn)>; +def : Pat<(f32 (AArch64frecps (f32 FPR32:$Rn), (f32 FPR32:$Rm))), + (FRECPS32 FPR32:$Rn, FPR32:$Rm)>; +def : Pat<(v2f32 (AArch64frecps (v2f32 V64:$Rn), (v2f32 V64:$Rm))), + (FRECPSv2f32 V64:$Rn, V64:$Rm)>; +def : Pat<(v4f32 (AArch64frecps (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm))), + (FRECPSv4f32 FPR128:$Rn, FPR128:$Rm)>; +def : Pat<(f64 (AArch64frecps (f64 FPR64:$Rn), (f64 FPR64:$Rm))), + (FRECPS64 FPR64:$Rn, FPR64:$Rm)>; +def : Pat<(v2f64 (AArch64frecps (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))), + (FRECPSv2f64 FPR128:$Rn, FPR128:$Rm)>; + def : Pat<(f32 (int_aarch64_neon_frecpx (f32 FPR32:$Rn))), (FRECPXv1i32 FPR32:$Rn)>; def : Pat<(f64 (int_aarch64_neon_frecpx (f64 FPR64:$Rn))), @@ -3439,6 +3452,17 @@ def : Pat<(v2f64 (AArch64frsqrte (v2f64 FPR128:$Rn))), (FRSQRTEv2f64 FPR128:$Rn)>; +def : Pat<(f32 (AArch64frsqrts (f32 FPR32:$Rn), (f32 FPR32:$Rm))), + (FRSQRTS32 FPR32:$Rn, FPR32:$Rm)>; +def : Pat<(v2f32 (AArch64frsqrts (v2f32 V64:$Rn), (v2f32 V64:$Rm))), + (FRSQRTSv2f32 V64:$Rn, V64:$Rm)>; +def : Pat<(v4f32 (AArch64frsqrts (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm))), + (FRSQRTSv4f32 FPR128:$Rn, FPR128:$Rm)>; +def : Pat<(f64 (AArch64frsqrts (f64 FPR64:$Rn), (f64 FPR64:$Rm))), + (FRSQRTS64 FPR64:$Rn, FPR64:$Rm)>; +def : Pat<(v2f64 (AArch64frsqrts (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))), + (FRSQRTSv2f64 FPR128:$Rn, FPR128:$Rm)>; + // If an integer is about to be converted to a floating point value, // just load it on the floating point unit. // Here are the patterns for 8 and 16-bits to float. Index: llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h +++ llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -171,10 +171,11 @@ const char* getTargetNodeName(unsigned Opcode) const override; - SDValue getRsqrtEstimate(SDValue Operand, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const override; + SDValue getSqrtEstimate(SDValue Operand, + DAGCombinerInfo &DCI, + unsigned &RefinementSteps, + bool &UseOneConstNR, + bool Reciprocal) const override; SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI, unsigned &RefinementSteps) const override; Index: llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -2851,10 +2851,11 @@ return nullptr; } -SDValue AMDGPUTargetLowering::getRsqrtEstimate(SDValue Operand, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const { +SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand, + DAGCombinerInfo &DCI, + unsigned &RefinementSteps, + bool &UseOneConstNR, + bool Reciprocal) const { SelectionDAG &DAG = DCI.DAG; EVT VT = Operand.getValueType(); Index: llvm/lib/Target/PowerPC/PPCISelLowering.h =================================================================== --- llvm/lib/Target/PowerPC/PPCISelLowering.h +++ llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -927,9 +927,9 @@ SDValue DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const; - SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const override; + SDValue getSqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, + unsigned &RefinementSteps, bool &UseOneConstNR, + bool Reciprocal) const override; SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI, unsigned &RefinementSteps) const override; unsigned combineRepeatedFPDivisors() const override; Index: llvm/lib/Target/PowerPC/PPCISelLowering.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -9396,10 +9396,11 @@ return RecipOp; } -SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const { +SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, + DAGCombinerInfo &DCI, + unsigned &RefinementSteps, + bool &UseOneConstNR, + bool Reciprocal) const { EVT VT = Operand.getValueType(); if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || Index: llvm/lib/Target/X86/X86ISelLowering.h =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.h +++ llvm/lib/Target/X86/X86ISelLowering.h @@ -1220,9 +1220,9 @@ SDValue ConvertCmpIfNecessary(SDValue Cmp, SelectionDAG &DAG) const; /// Use rsqrt* to speed up sqrt calculations. - SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const override; + SDValue getSqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, + unsigned &RefinementSteps, bool &UseOneConstNR, + bool Reciprocal) const override; /// Use rcp* to speed up fdiv calculations. SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI, Index: llvm/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.cpp +++ llvm/lib/Target/X86/X86ISelLowering.cpp @@ -14945,10 +14945,11 @@ /// The minimum architected relative accuracy is 2^-12. We need one /// Newton-Raphson step to have a good float result (24 bits of precision). -SDValue X86TargetLowering::getRsqrtEstimate(SDValue Op, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const { +SDValue X86TargetLowering::getSqrtEstimate(SDValue Op, + DAGCombinerInfo &DCI, + unsigned &RefinementSteps, + bool &UseOneConstNR, + bool Reciprocal) const { EVT VT = Op.getValueType(); const char *RecipOp; Index: llvm/test/CodeGen/AArch64/recp-fastmath.ll =================================================================== --- llvm/test/CodeGen/AArch64/recp-fastmath.ll +++ llvm/test/CodeGen/AArch64/recp-fastmath.ll @@ -12,8 +12,8 @@ ; CHECK-LABEL: frecp: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: frecpe -; CHECK-NEXT: fmov +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[R:s[0-7]]] +; CHECK-NEXT: frecps{{[[:space:]]+s[0-7](, s[0-7])?}}, [[R]] } define <2 x float> @f2recp(<2 x float> %x) #0 { @@ -27,8 +27,8 @@ ; CHECK-LABEL: f2recp: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frecpe +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[R:v[0-7]\.2s]] +; CHECK-NEXT: frecps{{[[:space:]]+v[0-7]\.2s(, v[0-7].2s)?}}, [[R]] } define <4 x float> @f4recp(<4 x float> %x) #0 { @@ -42,8 +42,8 @@ ; CHECK-LABEL: f4recp: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frecpe +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[R:v[0-7]\.4s]] +; CHECK-NEXT: frecps{{[[:space:]]+v[0-7]\.4s(, v[0-7].4s)?}}, [[R]] } define <8 x float> @f8recp(<8 x float> %x) #0 { @@ -58,9 +58,10 @@ ; CHECK-LABEL: f8recp: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frecpe -; CHECK: frecpe +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[RA:v[0-7]\.4s]] +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[RB:v[0-7]\.4s]] +; CHECK-NEXT: frecps{{[[:space:]]+v[0-7]\.4s(, v[0-7].4s)?}}, [[RA]] +; CHECK: frecps{{[[:space:]]+v[0-7]\.4s(, v[0-7].4s)?}}, [[RB]] } define double @drecp(double %x) #0 { @@ -74,8 +75,8 @@ ; CHECK-LABEL: drecp: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: frecpe -; CHECK-NEXT: fmov +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[R:d[0-7]]] +; CHECK-NEXT: frecps{{[[:space:]]+d[0-7](, d[0-7])?}}, [[R]] } define <2 x double> @d2recp(<2 x double> %x) #0 { @@ -89,8 +90,8 @@ ; CHECK-LABEL: d2recp: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frecpe +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[R:v[0-7]\.2d]] +; CHECK-NEXT: frecps{{[[:space:]]+v[0-7]\.2d(, v[0-7].2d)?}}, [[R]] } define <4 x double> @d4recp(<4 x double> %x) #0 { @@ -105,9 +106,10 @@ ; CHECK-LABEL: d4recp: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frecpe -; CHECK: frecpe +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[RA:v[0-7]\.2d]] +; CHECK-NEXT: frecpe{{[[:space:]]+}}[[RB:v[0-7]\.2d]] +; CHECK-NEXT: frecps{{[[:space:]]+v[0-7]\.2d(, v[0-7].2d)?}}, [[RA]] +; CHECK: frecps{{[[:space:]]+v[0-7]\.2d(, v[0-7].2d)?}}, [[RB]] } attributes #0 = { nounwind "unsafe-fp-math"="true" } Index: llvm/test/CodeGen/AArch64/sqrt-fastmath.ll =================================================================== --- llvm/test/CodeGen/AArch64/sqrt-fastmath.ll +++ llvm/test/CodeGen/AArch64/sqrt-fastmath.ll @@ -21,8 +21,10 @@ ; CHECK-LABEL: fsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:s[0-7]]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:s[0-7]]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+s[0-7](, s[0-7])?}}, [[RB]] +; CHECK: fcmp{{[[:space:]]+}}s0, #0 } define <2 x float> @f2sqrt(<2 x float> %a) #0 { @@ -35,9 +37,10 @@ ; CHECK-LABEL: f2sqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: mov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.2s]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:v[0-7]\.2s]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+v[0-7]\.2s(, v[0-7]\.2s)?}}, [[RB]] +; CHECK: fcmeq{{[[:space:]]+v[0-7]\.2s, v0\.2s}}, #0 } define <4 x float> @f4sqrt(<4 x float> %a) #0 { @@ -50,9 +53,10 @@ ; CHECK-LABEL: f4sqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: mov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.4s]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:v[0-7]\.4s]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]] +; CHECK: fcmeq{{[[:space:]]+v[0-7]\.4s, v0\.4s}}, #0 } define <8 x float> @f8sqrt(<8 x float> %a) #0 { @@ -66,10 +70,10 @@ ; CHECK-LABEL: f8sqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: mov -; CHECK-NEXT: frsqrte -; CHECK: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.4s]] +; CHECK: fmul{{[[:space:]]+}}[[RB:v[0-7]\.4s]], [[RA]], [[RA]] +; CHECK: frsqrts{{[[:space:]]+v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]] +; CHECK: fcmeq{{[[:space:]]+v[0-7]\.4s, v[0-1]\.4s}}, #0 } define double @dsqrt(double %a) #0 { @@ -82,8 +86,10 @@ ; CHECK-LABEL: dsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:d[0-7]]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:d[0-7]]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+d[0-7](, d[0-7])?}}, [[RB]] +; CHECK: fcmp{{[[:space:]]+}}d0, #0 } define <2 x double> @d2sqrt(<2 x double> %a) #0 { @@ -96,9 +102,10 @@ ; CHECK-LABEL: d2sqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: mov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.2d]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:v[0-7]\.2d]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]] +; CHECK: fcmeq{{[[:space:]]+v[0-7]\.2d, v0\.2d}}, #0 } define <4 x double> @d4sqrt(<4 x double> %a) #0 { @@ -112,10 +119,10 @@ ; CHECK-LABEL: d4sqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: mov -; CHECK-NEXT: frsqrte -; CHECK: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.2d]] +; CHECK: fmul{{[[:space:]]+}}[[RB:v[0-7]\.2d]], [[RA]], [[RA]] +; CHECK: frsqrts{{[[:space:]]+v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]] +; CHECK: fcmeq{{[[:space:]]+v[0-7]\.2d, v[0-1]\.2d}}, #0 } define float @frsqrt(float %a) #0 { @@ -129,8 +136,10 @@ ; CHECK-LABEL: frsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:s[0-7]]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:s[0-7]]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+s[0-7](, s[0-7])?}}, [[RB]] +; CHECK-NOT: fcmp{{[[:space:]]+s[0-7]}}, #0 } define <2 x float> @f2rsqrt(<2 x float> %a) #0 { @@ -144,8 +153,10 @@ ; CHECK-LABEL: f2rsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.2s]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:v[0-7]\.2s]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+v[0-7]\.2s(, v[0-7]\.2s)?}}, [[RB]] +; CHECK-NOT: fcmeq{{[[:space:]]+v[0-7]\.2s, v0\.2s}}, #0 } define <4 x float> @f4rsqrt(<4 x float> %a) #0 { @@ -159,8 +170,10 @@ ; CHECK-LABEL: f4rsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.4s]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:v[0-7]\.4s]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]] +; CHECK-NOT: fcmeq{{[[:space:]]+v[0-7]\.4s, v0\.4s}}, #0 } define <8 x float> @f8rsqrt(<8 x float> %a) #0 { @@ -175,9 +188,10 @@ ; CHECK-LABEL: f8rsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte -; CHECK: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.4s]] +; CHECK: fmul{{[[:space:]]+}}[[RB:v[0-7]\.4s]], [[RA]], [[RA]] +; CHECK: frsqrts{{[[:space:]]+v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]] +; CHECK-NOT: fcmeq{{[[:space:]]+v[0-7]\.4s, v0\.4s}}, #0 } define double @drsqrt(double %a) #0 { @@ -191,8 +205,10 @@ ; CHECK-LABEL: drsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:d[0-7]]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:d[0-7]]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+d[0-7](, d[0-7])?}}, [[RB]] +; CHECK-NOT: fcmp{{[[:space:]]+}}d0, #0 } define <2 x double> @d2rsqrt(<2 x double> %a) #0 { @@ -206,8 +222,10 @@ ; CHECK-LABEL: d2rsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.2d]] +; CHECK-NEXT: fmul{{[[:space:]]+}}[[RB:v[0-7]\.2d]], [[RA]], [[RA]] +; CHECK-NEXT: frsqrts{{[[:space:]]+v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]] +; CHECK-NOT: fcmeq{{[[:space:]]+v[0-7]\.2d, v0\.2d}}, #0 } define <4 x double> @d4rsqrt(<4 x double> %a) #0 { @@ -222,9 +240,10 @@ ; CHECK-LABEL: d4rsqrt: ; CHECK-NEXT: BB#0 -; CHECK-NEXT: fmov -; CHECK-NEXT: frsqrte -; CHECK: frsqrte +; CHECK-NEXT: frsqrte{{[[:space:]]+}}[[RA:v[0-7]\.2d]] +; CHECK: fmul{{[[:space:]]+}}[[RB:v[0-7]\.2d]], [[RA]], [[RA]] +; CHECK: frsqrts{{[[:space:]]+v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]] +; CHECK-NOT: fcmeq{{[[:space:]]+v[0-7]\.2d, v0\.2d}}, #0 } attributes #0 = { nounwind "unsafe-fp-math"="true" } Index: llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll =================================================================== --- llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll +++ llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll @@ -19,7 +19,6 @@ ; CHECK: %12 = VMULSSrr killed %11, killed %10 ; CHECK: %13 = FsFLD0SS ; CHECK: %14 = VCMPSSrr %0, killed %13, 0 -; CHECK: %15 = VFsANDNPSrr killed %14, killed %12 ; CHECK: %xmm0 = COPY %15 ; CHECK: RET 0, %xmm0 %call = tail call float @llvm.sqrt.f32(float %f) #1