[Arm] Do not lower vmax/vmin to Neon instructions

[jgreenhalgh]

Summary

FeatureSplatVFPToNeon is on for Cortex-A15 and Exynos. The documentation for Cortex-A57 [1] and Cortex-A72 [2] suggests it would also be beneficial there. I put together a patch taking the obvious route towards adding FeatureSplatVFPToNeon to the features for those cores, but in looking at the root-cause of code generation that requires that pass to be enabled, came to a less intrusive solution, that looks more suitable for generic compilation.

The pass attempts to remove instances where a VFP register is written an an S register, and read as a D register (see the section Register Forwarding Hazards in the linked software optimisation guides).

By far the biggest contributor to instances of "write S, read D" is an optimisation applied to run VMAX through Neon; as so:

define float @max_f32(float, float) {
    %3 = call nnan float @llvm.maxnum.f32(float %1, float %0)
    ret float %3
}

max_f32:
    vmov.f32        s2, s1
    vmax.f32        d0, d1, d0

Rather than propose FeatureSplatVFPToNeon for generic to work around this codegen, I'd instead like to ask whether we should just avoid this codegen in the first place, by disabling the lowering to Neon unless we're under UseNEONForSinglePrecisionFP.

Note that this patch is only applicable to Armv7-A 32-bit targets; when Armv-8-A is enabled, the single precision VMAXNM instruction can be used.

This patch implements that for 32-bit floats, but leaves 16-bit floats alone - they exist after Armv8.2-A, which none of Cortex-A15, Cortex-A57 or Cortex-A72 implement.

I've validated that this gives performance improvements on Cortex-A57 and Cortex-A72 similar to that you get by turning on FeatureSplatVFPToNeon, and also validated this change against Cortex-A53, where I saw no performance difference. Across a larger range of benchmarks performance came out even on Cortex-A76, with one >5% regression.

If this looks Ok, I'd appreciate someone applying it for me, as I have no commit rights.

---

[1]: http://infocenter.arm.com/help/topic/com.arm.doc.uan0015b/Cortex_A57_Software_Optimization_Guide_external.pdf
[2]: https://static.docs.arm.com/uan0016/a/cortex_a72_software_optimization_guide_external.pdf

[dmgreen]

I think this is probably OK, I'm not against it, but you could easily argue it either way. If we have an option called useNEONForSinglePrecisionFP, we should probably stick to it. Even if it's not universally applicable. So long as not-one else objects.

We should keep it consistent for FP16 as well though.

llvm/lib/Target/ARM/ARMISelLowering.cpp

1422–1423

If we do this for f32, we should presumably do the same for f16.

[efriedma]

Is there also a correctness issue here? I just briefly looked at the ARM manual, and I think vmax flushes denormals.

[jgreenhalgh]

Update to also cover float16 as per David's request.

[dmgreen]

Thanks. LGTM, as as you said I will commit this shortly.

llvm/lib/Target/ARM/ARMISelLowering.cpp

if (Subtarget->hasFullFP16()) {
     setOperationAction(ISD::FEXP2, MVT::f16, Promote);
     setOperationAction(ISD::FLOG, MVT::f16, Promote);
     setOperationAction(ISD::FLOG10, MVT::f16, Promote);
     setOperationAction(ISD::FLOG2, MVT::f16, Promote);
 
     setOperationAction(ISD::FROUND, MVT::f16, Legal);
   }
 
   if (Subtarget->hasNEON()) {
-    // vmin and vmax aren't available in a scalar form, so we use
-    // a NEON instruction with an undef lane instead.
-    setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
-    setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
+    // vmin and vmax aren't available in a scalar form, so we can use
+    // a NEON instruction with an undef lane instead.  This has a performance
+    // penalty on some cores, so we don't do this unless we have been
+    // asked to by the core tuning model.
+    if (Subtarget->useNEONForSinglePrecisionFP()) {
       setOperationAction(ISD::FMINIMUM, MVT::f32, Legal);
       setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal);
+      setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
+      setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
+    }
     setOperationAction(ISD::FMINIMUM, MVT::v2f32, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::v2f32, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
 
     if (Subtarget->hasFullFP16()) {
       setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal);
       setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal);

llvm/test/CodeGen/ARM/fp16-fullfp16.ll

; CHECK-NEXT:    bx lr
   store half %r, half* %p
   ret void
 }
 
 define void @test_minimum(half* %p) {
 ; CHECK-LABEL: test_minimum:
 ; CHECK:         vldr.16 s2, [r0]
 ; CHECK-NEXT:    vmov.f16 s0, #1.000000e+00
-; CHECK-NEXT:    vmin.f16 d0, d1, d0
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s0, s2
 ; CHECK-NEXT:    vstr.16 s0, [r0]
 ; CHECK-NEXT:    bx lr
   %a = load half, half* %p, align 2
   %c = fcmp ult half %a, 1.0
   %r = select i1 %c, half %a, half 1.0
   store half %r, half* %p
   ret void
 }
 
 define void @test_maximum(half* %p) {
 ; CHECK-LABEL: test_maximum:
 ; CHECK:         vldr.16 s2, [r0]
 ; CHECK-NEXT:    vmov.f16 s0, #1.000000e+00
-; CHECK-NEXT:    vmax.f16 d0, d1, d0
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s0, s2
 ; CHECK-NEXT:    vstr.16 s0, [r0]
 ; CHECK-NEXT:    bx lr
   %a = load half, half* %p, align 2
   %c = fcmp ugt half %a, 1.0
   %r = select i1 %c, half %a, half 1.0
   store half %r, half* %p
   ret void
 }

llvm/test/CodeGen/ARM/fp16-promote.ll

 }
 
 ; CHECK-ALL-LABEL: test_minimum:
 ; CHECK-FP16: vmov.f32 s0, #1.000000e+00
 ; CHECK-FP16: vcvtb.f32.f16
 ; CHECK-LIBCALL: bl __aeabi_h2f
 ; CHECK-LIBCALL-VFP: vmov.f32 s{{[0-9]+}}, #1.000000e+00
 ; CHECK-NOVFP: mov r{{[0-9]+}}, #1065353216
-; CHECK-VFP: vmin.f32
+; CHECK-VFP: vcmp.f32
+; CHECK-VFP: vmrs
+; CHECK-VFP: vmovlt.f32
 ; CHECK-NOVFP: bl __aeabi_fcmpge
 ; CHECK-FP16: vcvtb.f16.f32
 ; CHECK-LIBCALL: bl __aeabi_f2h
 define void @test_minimum(half* %p) #0 {
   %a = load half, half* %p, align 2
   %c = fcmp ult half %a, 1.0
   %r = select i1 %c, half %a, half 1.0
   store half %r, half* %p
   ret void
 }
 
 ; CHECK-ALL-LABEL: test_maximum:
 ; CHECK-FP16: vmov.f32 s0, #1.000000e+00
 ; CHECK-FP16: vcvtb.f32.f16
 ; CHECK-LIBCALL: bl __aeabi_h2f
 ; CHECK-LIBCALL-VFP: vmov.f32 s0, #1.000000e+00
 ; CHECK-NOVFP: mov r{{[0-9]+}}, #1065353216
-; CHECK-VFP: vmax.f32
+; CHECK-VFP: vcmp.f32
+; CHECK-VFP: vmrs
+; CHECK-VFP: vmovhi.f32
 ; CHECK-NOVFP: bl __aeabi_fcmple
 ; CHECK-FP16: vcvtb.f16.f32
 ; CHECK-LIBCALL: bl __aeabi_f2h
 define void @test_maximum(half* %p) #0 {
   %a = load half, half* %p, align 2
   %c = fcmp ugt half %a, 1.0
   %r = select i1 %c, half %a, half 1.0
   store half %r, half* %p

llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=armv8-eabi -mattr=+fullfp16 | FileCheck %s
 ; RUN: llc < %s -mtriple thumbv7a -mattr=+fullfp16 | FileCheck %s
 
 ; TODO: we can't pass half-precision arguments as "half" types yet. We do
 ; that for the time being by passing "float %f.coerce" and the necessary
 ; bitconverts/truncates. In these tests we pass i16 and use 1 bitconvert, which
 ; is the shortest way to get a half type. But when we can pass half types, we
 ; want to use that here.
 
 define half @fp16_vminnm_o(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vminnm_o:
-; CHECK-NOT: vminnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r2
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp olt half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vminnm_o_rev(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vminnm_o_rev:
-; CHECK-NOT: vminnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r2
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ogt half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vminnm_u(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vminnm_u:
-; CHECK-NOT: vminnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, r2
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ult half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vminnm_ule(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vminnm_ule:
-; CHECK-NOT: vminnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, r2
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ule half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vminnm_u_rev(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vminnm_u_rev:
-; CHECK-NOT: vminnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r2
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ugt half %0, %1
   %cond = select i1 %cmp, half %1, half %0
   ret half %cond
 }
 
 define half @fp16_vmaxnm_o(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vmaxnm_o:
-; CHECK-NOT: vmaxnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r2
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ogt half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vmaxnm_oge(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vmaxnm_oge:
-; CHECK-NOT: vmaxnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r2
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp oge half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vmaxnm_o_rev(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vmaxnm_o_rev:
-; CHECK-NOT: vmaxnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, r2
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp olt half %0, %1
   %cond = select i1 %cmp, half %1, half %0
   ret half %cond
 }
 
 define half @fp16_vmaxnm_ole_rev(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vmaxnm_ole_rev:
-; CHECK-NOT: vmaxnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, r2
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ole half %0, %1
   %cond = select i1 %cmp, half %1, half %0
   ret half %cond
 }
 
 define half @fp16_vmaxnm_u(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vmaxnm_u:
-; CHECK-NOT: vmaxnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, r2
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ugt half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vmaxnm_uge(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vmaxnm_uge:
-; CHECK-NOT: vmaxnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, r2
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp uge half %0, %1
   %cond = select i1 %cmp, half %0, half %1
   ret half %cond
 }
 
 define half @fp16_vmaxnm_u_rev(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: fp16_vmaxnm_u_rev:
-; CHECK-NOT: vmaxnm.f16
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r2
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp ult half %0, %1
   %cond = select i1 %cmp, half %1, half %0
   ret half %cond
 }
 
 ; known non-NaNs
 
 define half @fp16_vminnm_NNNo(i16 signext %a) {
-; CHECK-LABEL:  fp16_vminnm_NNNo:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], #1.200000e+01
-; CHECK:  vmov.f16  [[S4:s[0-9]]], r{{.}}
-; CHECK:  vminnm.f16  s2, [[S4]], [[S2]]
-; CHECK:  vmin.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vminnm_NNNo:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, #1.200000e+01
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, .LCPI12_0
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI12_0:
+; CHECK-NEXT:    .short 0x5040 @ half 34
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp olt half %0, 12.
   %cond1 = select i1 %cmp1, half %0, half 12.
   %cmp2 = fcmp olt half 34., %cond1
   %cond2 = select i1 %cmp2, half 34., half %cond1
   ret half %cond2
 }
 
 define half @fp16_vminnm_NNNo_rev(i16 signext %a) {
-; CHECK-LABEL:  fp16_vminnm_NNNo_rev:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], r{{.}}
-; CHECK:  vmin.f16  d0, d1, d0
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vminnm.f16  s0, [[S0]], [[S2]]
+; CHECK-LABEL: fp16_vminnm_NNNo_rev:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s2, .LCPI13_0
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vldr.16 s2, .LCPI13_1
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI13_0:
+; CHECK-NEXT:    .short 0x5300 @ half 56
+; CHECK-NEXT:  .LCPI13_1:
+; CHECK-NEXT:    .short 0x54e0 @ half 78
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp ogt half %0, 56.
   %cond1 = select i1 %cmp1, half 56., half %0
   %cmp2 = fcmp ogt half 78., %cond1
   %cond2 = select i1 %cmp2, half %cond1, half 78.
   ret half %cond2
 }
 
 define half @fp16_vminnm_NNNu(i16 signext %b) {
 ; CHECK-LABEL: fp16_vminnm_NNNu:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], #1.200000e+01
-; CHECK:  vmov.f16  [[S4:s[0-9]]], r{{.}}
-; CHECK:  vminnm.f16  s2, [[S4]], [[S2]]
-; CHECK:  vmin.f16  d0, d1, d0
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, #1.200000e+01
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, .LCPI14_0
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI14_0:
+; CHECK-NEXT:    .short 0x5040 @ half 34
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp ult half 12., %0
   %cond1 = select i1 %cmp1, half 12., half %0
   %cmp2 = fcmp ult half %cond1, 34.
   %cond2 = select i1 %cmp2, half %cond1, half 34.
   ret half %cond2
 }
 
 define half @fp16_vminnm_NNNule(i16 signext %b) {
-; CHECK-LABEL:  fp16_vminnm_NNNule:
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S4:s[0-9]]], r{{.}}
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vminnm.f16  s2, [[S4]], [[S2]]
-; CHECK:  vmin.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vminnm_NNNule:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s2, .LCPI15_0
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, .LCPI15_1
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI15_0:
+; CHECK-NEXT:    .short 0x5040 @ half 34
+; CHECK-NEXT:  .LCPI15_1:
+; CHECK-NEXT:    .short 0x5300 @ half 56
 
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp ule half 34., %0
   %cond1 = select i1 %cmp1, half 34., half %0
   %cmp2 = fcmp ule half %cond1, 56.
   %cond2 = select i1 %cmp2, half %cond1, half 56.
   ret half %cond2
 }
 
 define half @fp16_vminnm_NNNu_rev(i16 signext %b) {
-; CHECK-LABEL:  fp16_vminnm_NNNu_rev:
+; CHECK-LABEL: fp16_vminnm_NNNu_rev:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s2, .LCPI16_0
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vldr.16 s2, .LCPI16_1
+; CHECK-NEXT:    vminnm.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI16_0:
+; CHECK-NEXT:    .short 0x5300 @ half 56
+; CHECK-NEXT:  .LCPI16_1:
+; CHECK-NEXT:    .short 0x54e0 @ half 78
 
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], r{{.}}
-; CHECK:  vmin.f16  d0, d1, d0
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vminnm.f16  s0, [[S0]], [[S2]]
 
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp ugt half 56., %0
   %cond1 = select i1 %cmp1, half %0, half 56.
   %cmp2 = fcmp ugt half %cond1, 78.
   %cond2 = select i1 %cmp2, half 78., half %cond1
   ret half %cond2
 }
 
 define half @fp16_vmaxnm_NNNo(i16 signext %a) {
-; CHECK-LABEL:  fp16_vmaxnm_NNNo:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], #1.200000e+01
-; CHECK:  vmov.f16  [[S4:s[0-9]]], r{{.}}
-; CHECK:  vmaxnm.f16  s2, [[S4]], [[S2]]
-; CHECK:  vmax.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vmaxnm_NNNo:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, #1.200000e+01
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, .LCPI17_0
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI17_0:
+; CHECK-NEXT:    .short 0x5040 @ half 34
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp ogt half %0, 12.
   %cond1 = select i1 %cmp1, half %0, half 12.
   %cmp2 = fcmp ogt half 34., %cond1
   %cond2 = select i1 %cmp2, half 34., half %cond1
   ret half %cond2
 }
 
 define half @fp16_vmaxnm_NNNoge(i16 signext %a) {
-; CHECK-LABEL:  fp16_vmaxnm_NNNoge:
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S4:s[0-9]]], r{{.}}
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmaxnm.f16  s2, [[S4]], [[S2]]
-; CHECK:  vmax.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vmaxnm_NNNoge:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s2, .LCPI18_0
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, .LCPI18_1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI18_0:
+; CHECK-NEXT:    .short 0x5040 @ half 34
+; CHECK-NEXT:  .LCPI18_1:
+; CHECK-NEXT:    .short 0x5300 @ half 56
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp oge half %0, 34.
   %cond1 = select i1 %cmp1, half %0, half 34.
   %cmp2 = fcmp oge half 56., %cond1
   %cond2 = select i1 %cmp2, half 56., half %cond1
   ret half %cond2
 }
 
 define half @fp16_vmaxnm_NNNo_rev(i16 signext %a) {
-; CHECK-LABEL:  fp16_vmaxnm_NNNo_rev:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], r{{.}}
-; CHECK:  vmax.f16  d0, d1, d0
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmaxnm.f16  s0, [[S0]], [[S2]]
+; CHECK-LABEL: fp16_vmaxnm_NNNo_rev:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s2, .LCPI19_0
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vldr.16 s2, .LCPI19_1
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI19_0:
+; CHECK-NEXT:    .short 0x5300 @ half 56
+; CHECK-NEXT:  .LCPI19_1:
+; CHECK-NEXT:    .short 0x54e0 @ half 78
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp olt half %0, 56.
   %cond1 = select i1 %cmp1, half 56., half %0
   %cmp2 = fcmp olt half 78., %cond1
   %cond2 = select i1 %cmp2, half %cond1, half 78.
   ret half %cond2
 }
 
 define half @fp16_vmaxnm_NNNole_rev(i16 signext %a) {
-; CHECK-LABEL:  fp16_vmaxnm_NNNole_rev:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], r{{.}}
-; CHECK:  vmax.f16  d0, d1, d0
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmaxnm.f16  s0, [[S0]], [[S2]]
+; CHECK-LABEL: fp16_vmaxnm_NNNole_rev:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s2, .LCPI20_0
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vldr.16 s2, .LCPI20_1
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI20_0:
+; CHECK-NEXT:    .short 0x54e0 @ half 78
+; CHECK-NEXT:  .LCPI20_1:
+; CHECK-NEXT:    .short 0x55a0 @ half 90
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp ole half %0, 78.
   %cond1 = select i1 %cmp1, half 78., half %0
   %cmp2 = fcmp ole half 90., %cond1
   %cond2 = select i1 %cmp2, half %cond1, half 90.
   ret half %cond2
 }
 
 define half @fp16_vmaxnm_NNNu(i16 signext %b) {
-; CHECK-LABEL:  fp16_vmaxnm_NNNu:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], #1.200000e+01
-; CHECK:  vmov.f16  [[S4:s[0-9]]], r{{.}}
-; CHECK:  vmaxnm.f16  s2, [[S4]], [[S2]]
-; CHECK:  vmax.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vmaxnm_NNNu:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmov.f16 s2, #1.200000e+01
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, .LCPI21_0
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI21_0:
+; CHECK-NEXT:    .short 0x5040 @ half 34
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp ugt half 12., %0
   %cond1 = select i1 %cmp1, half 12., half %0
   %cmp2 = fcmp ugt half %cond1, 34.
   %cond2 = select i1 %cmp2, half %cond1, half 34.
   ret half %cond2
 }
 
 define half @fp16_vmaxnm_NNNuge(i16 signext %b) {
-; CHECK-LABEL:  fp16_vmaxnm_NNNuge:
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S4:s[0-9]]], r{{.}}
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmaxnm.f16  s2, [[S4]], [[S2]]
-; CHECK:  vmax.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vmaxnm_NNNuge:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s2, .LCPI22_0
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vldr.16 s2, .LCPI22_1
+; CHECK-NEXT:    vcmp.f16 s2, s0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselgt.f16 s0, s2, s0
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI22_0:
+; CHECK-NEXT:    .short 0x5040 @ half 34
+; CHECK-NEXT:  .LCPI22_1:
+; CHECK-NEXT:    .short 0x5300 @ half 56
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp uge half 34., %0
   %cond1 = select i1 %cmp1, half 34., half %0
   %cmp2 = fcmp uge half %cond1, 56.
   %cond2 = select i1 %cmp2, half %cond1, half 56.
   ret half %cond2
 }
 
 define half @fp16_vminmaxnm_neg0(i16 signext %a) {
-; CHECK-LABEL:  fp16_vminmaxnm_neg0:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], r{{.}}
-; CHECK:  vminnm.f16  s2, [[S2]], [[S0]]
-; CHECK:  vmax.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vminmaxnm_neg0:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s0, .LCPI23_0
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vminnm.f16 s2, s2, s0
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI23_0:
+; CHECK-NEXT:    .short 0x8000 @ half -0
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp olt half %0, -0.
   %cond1 = select i1 %cmp1, half %0, half -0.
   %cmp2 = fcmp ugt half %cond1, -0.
   %cond2 = select i1 %cmp2, half %cond1, half -0.
   ret half %cond2
 }
 
 define half @fp16_vminmaxnm_e_0(i16 signext %a) {
-; CHECK-LABEL:  fp16_vminmaxnm_e_0:
-; CHECK:  vldr.16 [[S2:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S0:s[0-9]]], r{{.}}
-; CHECK:  vmin.f16  d0, d0, d1
-; CHECK:  vmaxnm.f16  s0, [[S0]], [[S2]]
+; CHECK-LABEL: fp16_vminmaxnm_e_0:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 s0, r1
+; CHECK-NEXT:    vldr.16 s2, .LCPI24_0
+; CHECK-NEXT:    vcmp.f16 s0, #0
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s2, s0
+; CHECK-NEXT:    vmaxnm.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI24_0:
+; CHECK-NEXT:    .short 0x0000 @ half 0
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp nsz ole half 0., %0
   %cond1 = select i1 %cmp1, half 0., half %0
   %cmp2 = fcmp nsz uge half 0., %cond1
   %cond2 = select i1 %cmp2, half 0., half %cond1
   ret half %cond2
 }
 
 define half @fp16_vminmaxnm_e_neg0(i16 signext %a) {
-; CHECK-LABEL:  fp16_vminmaxnm_e_neg0:
-; CHECK:  vldr.16 [[S0:s[0-9]]], .LCPI{{.*}}
-; CHECK:  vmov.f16  [[S2:s[0-9]]], r{{.}}
-; CHECK:  vminnm.f16  s2, [[S2]], [[S0]]
-; CHECK:  vmax.f16  d0, d1, d0
+; CHECK-LABEL: fp16_vminmaxnm_e_neg0:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr.16 s0, .LCPI25_0
+; CHECK-NEXT:    vmov.f16 s2, r1
+; CHECK-NEXT:    vminnm.f16 s2, s2, s0
+; CHECK-NEXT:    vcmp.f16 s0, s2
+; CHECK-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEXT:    vselge.f16 s0, s0, s2
+; CHECK-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI25_0:
+; CHECK-NEXT:    .short 0x8000 @ half -0
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp nsz ule half -0., %0
   %cond1 = select i1 %cmp1, half -0., half %0
   %cmp2 = fcmp nsz oge half -0., %cond1
   %cond2 = select i1 %cmp2, half -0., half %cond1
   ret half %cond2
 }

llvm/test/CodeGen/ARM/lower-vmax.ll

+; RUN: llc -mtriple=arm-eabihf -mattr=+neon < %s | FileCheck -check-prefixes=CHECK-NO_NEON %s
+; RUN: llc -mtriple=arm-eabihf -mattr=+neon,+neonfp < %s | FileCheck -check-prefixes=CHECK-NEON %s
+
+define float @max_f32(float, float) {
+;CHECK-NEON: vmax.f32
+;CHECK-NO_NEON: vcmp.f32
+;CHECK-NO_NEON: vmrs
+;CHECK-NO_NEON: vmovgt.f32
+  %3 = call nnan float @llvm.maxnum.f32(float %1, float %0)
+  ret float %3
+}
+
+declare float @llvm.maxnum.f32(float, float) #1
+
+define float @min_f32(float, float) {
+;CHECK-NEON: vmin.f32
+;CHECK-NO_NEON: vcmp.f32
+;CHECK-NO_NEON: vmrs
+;CHECK-NO_NEON: vmovlt.f32
+  %3 = call nnan float @llvm.minnum.f32(float %1, float %0)
+  ret float %3
+}
+
+declare float @llvm.minnum.f32(float, float) #1
+

llvm/test/CodeGen/ARM/vminmax.ll

 
 declare <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float>, <4 x float>) nounwind readnone
 
 declare float @llvm.maxnum.f32(float %a, float %b)
 declare float @llvm.minnum.f32(float %a, float %b)
 
 define float @maxnum(float %a, float %b) {
 ;CHECK-LABEL: maxnum:
-;CHECK: vmax.f32
+;CHECK: vcmp.f32
+;CHECK-NEXT: vmrs
+;CHECK-NEXT: vmovgt.f32
   %r = call nnan float @llvm.maxnum.f32(float %a, float %b)
   ret float %r
 }
 
 define float @minnum(float %a, float %b) {
 ;CHECK-LABEL: minnum:
-;CHECK: vmin.f32
+;CHECK: vcmp.f32
+;CHECK-NEXT: vmrs
+;CHECK-NEXT: vmovlt.f32
   %r = call nnan float @llvm.minnum.f32(float %a, float %b)
   ret float %r
 }