[AArch64] Fix the number of iterations for the Newton series

[evandro]

Summary

The number of iterations was incorrectly determined for DP FP vector types and the tests were insufficient to flag this issue.

Probably the releases 4 and 5 of LLVM should also be fixed.

[javed.absar]

Full context is missing (seems you forgot diff .... -U9999)

[fhahn]

LGTM. This is a change I wanted to make soon myself, but never got around to it yet. It would be great if you could upload a full-context diff and wait a bit with committing, to give other people a chance to raise concerns.

[evandro]

Sorry about that.

[fhahn]

Thanks Evandro. Out of curiosity, were there any tests/benchmarks, where this caused failures/wrong results?

[evandro]

In D39507#914909, @fhahn wrote:

Thanks Evandro. Out of curiosity, were there any tests/benchmarks, where this caused failures/wrong results?

Yes, with the series expansion enabled, neither CPU2000 nor CPU2006 would validate. With this fix, expanding sqrt() with the series both validate, but not when expanding division.

I suspect that the initial estimate for division is insufficient for DP, failing to converge even with additional iterations.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (ST->hasNEON() &&
        VT == MVT::f32 || VT == MVT::v1f32 ||
        VT == MVT::v2f32 || VT == MVT::v4f32)) {
     if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
       // For the reciprocal estimates, convergence is quadratic, so the number
       // of digits is doubled after each iteration.  In ARMv8, the accuracy of
       // the initial estimate is 2^-8.  Thus the number of extra steps to refine
       // the result for float (23 mantissa bits) is 2 and for double (52
       // mantissa bits) is 3.
-      ExtraSteps = VT == MVT::f64 ? 3 : 2;
+      ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
 
     return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
   }
 
   return SDValue();
 }
 
 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,

llvm/test/CodeGen/AArch64/recp-fastmath.ll

 define float @frecp1(float %x) #1 {
   %div = fdiv fast float 1.0, %x
   ret float %div
 
 ; CHECK-LABEL: frecp1:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frecpe [[R:s[0-7]]]
 ; CHECK-NEXT: frecps {{s[0-7](, s[0-7])?}}, [[R]]
+; CHECK: frecps {{s[0-7]}}, {{s[0-7]}}, {{s[0-7]}}
+; CHECK-NOT: frecps {{s[0-7]}}, {{s[0-7]}}, {{s[0-7]}}
 }
 
 define <2 x float> @f2recp0(<2 x float> %x) #0 {
   %div = fdiv fast <2 x float> <float 1.0, float 1.0>, %x
   ret <2 x float> %div
 
 ; CHECK-LABEL: f2recp0:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: fmov
 ; CHECK-NEXT: fdiv
 }
 
 define <2 x float> @f2recp1(<2 x float> %x) #1 {
   %div = fdiv fast <2 x float> <float 1.0, float 1.0>, %x
   ret <2 x float> %div
 
 ; CHECK-LABEL: f2recp1:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frecpe [[R:v[0-7]\.2s]]
 ; CHECK-NEXT: frecps {{v[0-7]\.2s(, v[0-7].2s)?}}, [[R]]
+; CHECK: frecps {{v[0-7]\.2s}}, {{v[0-7]\.2s}}, {{v[0-7]\.2s}}
+; CHECK-NOT: frecps {{v[0-7]\.2s}}, {{v[0-7]\.2s}}, {{v[0-7]\.2s}}
 }
 
 define <4 x float> @f4recp0(<4 x float> %x) #0 {
   %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <4 x float> %div
 
 ; CHECK-LABEL: f4recp0:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: fmov
 ; CHECK-NEXT: fdiv
 }
 
 define <4 x float> @f4recp1(<4 x float> %x) #1 {
   %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <4 x float> %div
 
 ; CHECK-LABEL: f4recp1:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frecpe [[R:v[0-7]\.4s]]
 ; CHECK-NEXT: frecps {{v[0-7]\.4s(, v[0-7].4s)?}}, [[R]]
+; CHECK: frecps {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK-NOT: frecps {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
 }
 
 define <8 x float> @f8recp0(<8 x float> %x) #0 {
   %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <8 x float> %div
 
 ; CHECK-LABEL: f8recp0:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: fmov
 ; CHECK-NEXT: fdiv
 ; CHECK-NEXT: fdiv
 }
 
 define <8 x float> @f8recp1(<8 x float> %x) #1 {
   %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <8 x float> %div
 
 ; CHECK-LABEL: f8recp1:
 ; CHECK-NEXT: BB#0
-; CHECK-NEXT: frecpe [[RA:v[0-7]\.4s]]
-; CHECK-NEXT: frecpe [[RB:v[0-7]\.4s]]
-; CHECK-NEXT: frecps {{v[0-7]\.4s(, v[0-7].4s)?}}, [[RA]]
-; CHECK: frecps {{v[0-7]\.4s(, v[0-7].4s)?}}, [[RB]]
+; CHECK-NEXT: frecpe [[R:v[0-7]\.4s]]
+; CHECK: frecps {{v[0-7]\.4s(, v[0-7].4s)?}}, [[R]]
+; CHECK: frecps {{v[0-7]\.4s(, v[0-7].4s)?}}, {{v[0-7]\.4s}}
+; CHECK: frecps {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK: frecps {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK-NOT: frecps {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
 }
 
 define double @drecp0(double %x) #0 {
   %div = fdiv fast double 1.0, %x
   ret double %div
 
 ; CHECK-LABEL: drecp0:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: fmov
 ; CHECK-NEXT: fdiv
 }
 
 define double @drecp1(double %x) #1 {
   %div = fdiv fast double 1.0, %x
   ret double %div
 
 ; CHECK-LABEL: drecp1:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frecpe [[R:d[0-7]]]
 ; CHECK-NEXT: frecps {{d[0-7](, d[0-7])?}}, [[R]]
+; CHECK: frecps {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
+; CHECK: frecps {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
+; CHECK-NOT: frecps {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
 }
 
 define <2 x double> @d2recp0(<2 x double> %x) #0 {
   %div = fdiv fast <2 x double> <double 1.0, double 1.0>, %x
   ret <2 x double> %div
 
 ; CHECK-LABEL: d2recp0:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: fmov
 ; CHECK-NEXT: fdiv
 }
 
 define <2 x double> @d2recp1(<2 x double> %x) #1 {
   %div = fdiv fast <2 x double> <double 1.0, double 1.0>, %x
   ret <2 x double> %div
 
 ; CHECK-LABEL: d2recp1:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frecpe [[R:v[0-7]\.2d]]
 ; CHECK-NEXT: frecps {{v[0-7]\.2d(, v[0-7].2d)?}}, [[R]]
+; CHECK: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK-NOT: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
 }
 
 define <4 x double> @d4recp0(<4 x double> %x) #0 {
   %div = fdiv fast <4 x double> <double 1.0, double 1.0, double 1.0, double 1.0>, %x
   ret <4 x double> %div
 
 ; CHECK-LABEL: d4recp0:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: fmov
 ; CHECK-NEXT: fdiv
 ; CHECK-NEXT: fdiv
 }
 
 define <4 x double> @d4recp1(<4 x double> %x) #1 {
   %div = fdiv fast <4 x double> <double 1.0, double 1.0, double 1.0, double 1.0>, %x
   ret <4 x double> %div
 
 ; CHECK-LABEL: d4recp1:
 ; CHECK-NEXT: BB#0
-; CHECK-NEXT: frecpe [[RA:v[0-7]\.2d]]
-; CHECK-NEXT: frecpe [[RB:v[0-7]\.2d]]
-; CHECK-NEXT: frecps {{v[0-7]\.2d(, v[0-7].2d)?}}, [[RA]]
-; CHECK: frecps {{v[0-7]\.2d(, v[0-7].2d)?}}, [[RB]]
+; CHECK-NEXT: frecpe [[R:v[0-7]\.2d]]
+; CHECK: frecps {{v[0-7]\.2d(, v[0-7].2d)?}}, [[R]]
+; CHECK: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK-NOT: frecps {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
 }
 
 attributes #0 = { nounwind "unsafe-fp-math"="true" }
 attributes #1 = { nounwind "unsafe-fp-math"="true" "reciprocal-estimates"="div,vec-div" }

llvm/test/CodeGen/AArch64/sqrt-fastmath.ll

 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: fsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:s[0-7]]]
 ; CHECK-NEXT: fmul [[RB:s[0-7]]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{s[0-7](, s[0-7])?}}, [[RB]]
-; CHECK: fcmp s0, #0
+; CHECK: frsqrts {{s[0-7]}}, {{s[0-7]}}, {{s[0-7]}}
+; CHECK-NOT: frsqrts {{s[0-7]}}, {{s[0-7]}}, {{s[0-7]}}
+; CHECK: fcmp {{s[0-7]}}, #0
 }
 
 define <2 x float> @f2sqrt(<2 x float> %a) #0 {
   %1 = tail call fast <2 x float> @llvm.sqrt.v2f32(<2 x float> %a)
   ret <2 x float> %1
 
 ; FAULT-LABEL: f2sqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: f2sqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.2s]]
 ; CHECK-NEXT: fmul [[RB:v[0-7]\.2s]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{v[0-7]\.2s(, v[0-7]\.2s)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.2s}}, {{v[0-7]\.2s}}, {{v[0-7]\.2s}}
+; CHECK-NOT: frsqrts {{v[0-7]\.2s}}, {{v[0-7]\.2s}}, {{v[0-7]\.2s}}
 ; CHECK: fcmeq {{v[0-7]\.2s, v0\.2s}}, #0
 }
 
 define <4 x float> @f4sqrt(<4 x float> %a) #0 {
   %1 = tail call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
   ret <4 x float> %1
 
 ; FAULT-LABEL: f4sqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: f4sqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.4s]]
 ; CHECK-NEXT: fmul [[RB:v[0-7]\.4s]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK-NOT: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
 ; CHECK: fcmeq {{v[0-7]\.4s, v0\.4s}}, #0
 }
 
 define <8 x float> @f8sqrt(<8 x float> %a) #0 {
   %1 = tail call fast <8 x float> @llvm.sqrt.v8f32(<8 x float> %a)
   ret <8 x float> %1
 
 ; FAULT-LABEL: f8sqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: f8sqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.4s]]
-; CHECK: fmul [[RB:v[0-7]\.4s]], [[RA]], [[RA]]
-; CHECK: frsqrts {{v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]]
+; CHECK-NEXT: fmul [[RB:v[0-7]\.4s]], [[RA]], [[RA]]
+; CHECK-NEXT: frsqrts {{v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK: fcmeq {{v[0-7]\.4s, v[0-1]\.4s}}, #0
+; CHECK: frsqrte [[RC:v[0-7]\.4s]]
+; CHECK-NEXT: fmul [[RD:v[0-7]\.4s]], [[RC]], [[RC]]
+; CHECK-NEXT: frsqrts {{v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RD]]
+; CHECK: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK-NOT: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
 ; CHECK: fcmeq {{v[0-7]\.4s, v[0-1]\.4s}}, #0
 }
 
 define double @dsqrt(double %a) #0 {
   %1 = tail call fast double @llvm.sqrt.f64(double %a)
   ret double %1
 
 ; FAULT-LABEL: dsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: dsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:d[0-7]]]
 ; CHECK-NEXT: fmul [[RB:d[0-7]]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{d[0-7](, d[0-7])?}}, [[RB]]
-; CHECK: fcmp d0, #0
+; CHECK: frsqrts {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
+; CHECK: frsqrts {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
+; CHECK-NOT: frsqrts {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
+; CHECK: fcmp {{d[0-7]}}, #0
 }
 
 define <2 x double> @d2sqrt(<2 x double> %a) #0 {
   %1 = tail call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> %a)
   ret <2 x double> %1
 
 ; FAULT-LABEL: d2sqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: d2sqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.2d]]
 ; CHECK-NEXT: fmul [[RB:v[0-7]\.2d]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK-NOT: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
 ; CHECK: fcmeq {{v[0-7]\.2d, v0\.2d}}, #0
 }
 
 define <4 x double> @d4sqrt(<4 x double> %a) #0 {
   %1 = tail call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
   ret <4 x double> %1
 
 ; FAULT-LABEL: d4sqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: d4sqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.2d]]
-; CHECK: fmul [[RB:v[0-7]\.2d]], [[RA]], [[RA]]
-; CHECK: frsqrts {{v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]]
+; CHECK-NEXT: fmul [[RB:v[0-7]\.2d]], [[RA]], [[RA]]
+; CHECK-NEXT: frsqrts {{v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK-NOT: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrte [[RC:v[0-7]\.2d]]
+; CHECK-NEXT: fmul [[RD:v[0-7]\.2d]], [[RC]], [[RC]]
+; CHECK-NEXT: frsqrts {{v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RD]]
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK-NOT: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
 ; CHECK: fcmeq {{v[0-7]\.2d, v[0-1]\.2d}}, #0
 }
 
 define float @frsqrt(float %a) #0 {
   %1 = tail call fast float @llvm.sqrt.f32(float %a)
   %2 = fdiv fast float 1.000000e+00, %1
   ret float %2
 
 ; FAULT-LABEL: frsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: frsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:s[0-7]]]
 ; CHECK-NEXT: fmul [[RB:s[0-7]]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{s[0-7](, s[0-7])?}}, [[RB]]
+; CHECK: frsqrts {{s[0-7]}}, {{s[0-7]}}, {{s[0-7]}}
+; CHECK-NOT: frsqrts {{s[0-7]}}, {{s[0-7]}}, {{s[0-7]}}
 ; CHECK-NOT: fcmp {{s[0-7]}}, #0
 }
 
 define <2 x float> @f2rsqrt(<2 x float> %a) #0 {
   %1 = tail call fast <2 x float> @llvm.sqrt.v2f32(<2 x float> %a)
   %2 = fdiv fast <2 x float> <float 1.000000e+00, float 1.000000e+00>, %1
   ret <2 x float> %2
 
 ; FAULT-LABEL: f2rsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: f2rsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.2s]]
 ; CHECK-NEXT: fmul [[RB:v[0-7]\.2s]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{v[0-7]\.2s(, v[0-7]\.2s)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.2s}}, {{v[0-7]\.2s}}, {{v[0-7]\.2s}}
+; CHECK-NOT: frsqrts {{v[0-7]\.2s}}, {{v[0-7]\.2s}}, {{v[0-7]\.2s}}
 ; CHECK-NOT: fcmeq {{v[0-7]\.2s, v0\.2s}}, #0
 }
 
 define <4 x float> @f4rsqrt(<4 x float> %a) #0 {
   %1 = tail call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
   %2 = fdiv fast <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %1
   ret <4 x float> %2
 
 ; FAULT-LABEL: f4rsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: f4rsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.4s]]
 ; CHECK-NEXT: fmul [[RB:v[0-7]\.4s]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK-NOT: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
 ; CHECK-NOT: fcmeq {{v[0-7]\.4s, v0\.4s}}, #0
 }
 
 define <8 x float> @f8rsqrt(<8 x float> %a) #0 {
   %1 = tail call fast <8 x float> @llvm.sqrt.v8f32(<8 x float> %a)
   %2 = fdiv fast <8 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %1
   ret <8 x float> %2
 
 ; FAULT-LABEL: f8rsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: f8rsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.4s]]
 ; CHECK: fmul [[RB:v[0-7]\.4s]], [[RA]], [[RA]]
 ; CHECK: frsqrts {{v[0-7]\.4s(, v[0-7]\.4s)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
+; CHECK-NOT: frsqrts {{v[0-7]\.4s}}, {{v[0-7]\.4s}}, {{v[0-7]\.4s}}
 ; CHECK-NOT: fcmeq {{v[0-7]\.4s, v0\.4s}}, #0
 }
 
 define double @drsqrt(double %a) #0 {
   %1 = tail call fast double @llvm.sqrt.f64(double %a)
   %2 = fdiv fast double 1.000000e+00, %1
   ret double %2
 
 ; FAULT-LABEL: drsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: drsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:d[0-7]]]
 ; CHECK-NEXT: fmul [[RB:d[0-7]]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{d[0-7](, d[0-7])?}}, [[RB]]
+; CHECK: frsqrts {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
+; CHECK: frsqrts {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
+; CHECK-NOT: frsqrts {{d[0-7]}}, {{d[0-7]}}, {{d[0-7]}}
 ; CHECK-NOT: fcmp d0, #0
 }
 
 define <2 x double> @d2rsqrt(<2 x double> %a) #0 {
   %1 = tail call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> %a)
   %2 = fdiv fast <2 x double> <double 1.000000e+00, double 1.000000e+00>, %1
   ret <2 x double> %2
 
 ; FAULT-LABEL: d2rsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: d2rsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.2d]]
 ; CHECK-NEXT: fmul [[RB:v[0-7]\.2d]], [[RA]], [[RA]]
 ; CHECK-NEXT: frsqrts {{v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK-NOT: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
 ; CHECK-NOT: fcmeq {{v[0-7]\.2d, v0\.2d}}, #0
 }
 
 define <4 x double> @d4rsqrt(<4 x double> %a) #0 {
   %1 = tail call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
   %2 = fdiv fast <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, %1
   ret <4 x double> %2
 
 ; FAULT-LABEL: d4rsqrt:
 ; FAULT-NEXT: BB#0
 ; FAULT-NEXT: fsqrt
 ; FAULT-NEXT: fsqrt
 
 ; CHECK-LABEL: d4rsqrt:
 ; CHECK-NEXT: BB#0
 ; CHECK-NEXT: frsqrte [[RA:v[0-7]\.2d]]
 ; CHECK: fmul [[RB:v[0-7]\.2d]], [[RA]], [[RA]]
 ; CHECK: frsqrts {{v[0-7]\.2d(, v[0-7]\.2d)?}}, [[RB]]
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
+; CHECK-NOT: frsqrts {{v[0-7]\.2d}}, {{v[0-7]\.2d}}, {{v[0-7]\.2d}}
 ; CHECK-NOT: fcmeq {{v[0-7]\.2d, v0\.2d}}, #0
 }
 
 attributes #0 = { nounwind "unsafe-fp-math"="true" }