[AArch64][GISel] Additional FPExt vector lowering

[dmgreen]

Summary

Similar to D155311, this adds lowering for more vector cases for FPExt

[arsenm]

If you also test bfloat you'll find it miscompiles

[tschuett]

llvm/test/CodeGen/AArch64/fpext.ll

204 ↗

(On Diff #541535)

What is the trick behind the difference?

[arsenm]

llvm/test/CodeGen/AArch64/fpext.ll

204 ↗

(On Diff #541535)

the dag widened the vector and gisel scalarized it

[tschuett]

llvm/test/CodeGen/AArch64/fpext.ll

204 ↗

(On Diff #541535)

I missed the *3*.

[tschuett]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

545

.clampMinNumElements(0, s32, 2)
.clampMaxNumElements(0, s32, 4)
.clampMinNumElements(0, s64, 1)
.clampMaxNumElements(0, s64, 2)

?

[tschuett]

and
.widenScalarOrEltToNextPow2(0, /*Min */)
. moreElementsToNextPow2(0 , /*Min*/);

[dmgreen]

In D155601#4510788, @arsenm wrote:

If you also test bfloat you'll find it miscompiles

Yeah. My understanding is that either the s16 type needs to distinguish between different float representations, or the G_FPEXT operation would need to, either with a different G_ opcode or some sort of flag.

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

545

The result type of an fpext needs to be a v4s32 or v2s64, to match a fcvtl. There are no v2f16->v2f32 instructions (unless you count the lower 2 lanes of a v4f16->v4f32 fcvtl, and it would seem better to legalize in the legalize step so we only have to deal with legal operation later, and reuse all the tablegen patterns).

llvm/test/CodeGen/AArch64/fpext.ll

204 ↗

(On Diff #541535)

Yeah the 3 is awkward. This comes from the expansion, adding undef lanes that are not yet cleaned up properly. It should be possible to improve it, and we can get back to the same codegen as SDAG

[dmgreen]

Update and rebase.

[dmgreen]

^ Oops. That was somehow attached to the wrong patch.

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_FPTRUNC)
       .clampNumElements(0, v4s16, v4s16)
       .clampNumElements(0, v2s32, v2s32)
       .scalarize(0);
 
   getActionDefinitionsBuilder(G_FPEXT)
       .legalFor(
           {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
       .clampNumElements(0, v4s32, v4s32)
       .clampNumElements(0, v2s64, v2s64)
       .scalarize(0);
 
   // Conversions
   getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
       .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
       .widenScalarToNextPow2(0)
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(1)
CTPOPActions
       .clampScalar(0, s32, s128)
       .widenScalarToNextPow2(0)
       .minScalarEltSameAsIf(always, 1, 0)
       .maxScalarEltSameAsIf(always, 1, 0);
 
   // TODO: Vector types.
   getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT}).lowerIf(isScalar(0));
 
-  // TODO: Vector types.
   getActionDefinitionsBuilder({G_FMAXNUM, G_FMINNUM})
-      .legalFor({MinFPScalar, s32, s64})
+      .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
+      .legalIf([=](const LegalityQuery &Query) {
+        const auto &Ty = Query.Types[0];
+        return (Ty == v8s16 || Ty == v4s16) && HasFP16;
+      })
       .libcallFor({s128})
-      .minScalar(0, MinFPScalar);
+      .minScalarOrElt(0, MinFPScalar)
+      .clampNumElements(0, v4s16, v8s16)
+      .clampNumElements(0, v2s32, v4s32)
+      .clampNumElements(0, v2s64, v2s64)
+      .moreElementsToNextPow2(0);
 
   getActionDefinitionsBuilder({G_FMAXIMUM, G_FMINIMUM})
       .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
       .legalIf([=](const LegalityQuery &Query) {
         const auto &Ty = Query.Types[0];
         return (Ty == v8s16 || Ty == v4s16) && HasFP16;
       })
       .minScalar(0, MinFPScalar)

llvm/test/CodeGen/AArch64/GlobalISel/legalize-fminnum.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=legalizer -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=FP16
-# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=NO-FP16
+# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=legalizer -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
+# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16
 ...
 ---
 name:            s16_legal_with_full_fp16
 alignment:       4
 body:             |
   bb.0:
     liveins: $h0, $h1
-    ; FP16-LABEL: name: s16_legal_with_full_fp16
-    ; FP16: %a:_(s16) = COPY $h0
-    ; FP16: %b:_(s16) = COPY $h1
-    ; FP16: %minnum:_(s16) = G_FMINNUM %a, %b
-    ; FP16: $h0 = COPY %minnum(s16)
-    ; FP16: RET_ReallyLR implicit $h0
-    ; NO-FP16-LABEL: name: s16_legal_with_full_fp16
-    ; NO-FP16: %a:_(s16) = COPY $h0
-    ; NO-FP16: %b:_(s16) = COPY $h1
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %a(s16)
-    ; NO-FP16: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %b(s16)
-    ; NO-FP16: [[FMINNUM:%[0-9]+]]:_(s32) = G_FMINNUM [[FPEXT]], [[FPEXT1]]
-    ; NO-FP16: %minnum:_(s16) = G_FPTRUNC [[FMINNUM]](s32)
-    ; NO-FP16: $h0 = COPY %minnum(s16)
-    ; NO-FP16: RET_ReallyLR implicit $h0
+    ; CHECK-FP16-LABEL: name: s16_legal_with_full_fp16
+    ; CHECK-FP16: liveins: $h0, $h1
+    ; CHECK-FP16-NEXT: {{  $}}
+    ; CHECK-FP16-NEXT: %a:_(s16) = COPY $h0
+    ; CHECK-FP16-NEXT: %b:_(s16) = COPY $h1
+    ; CHECK-FP16-NEXT: %minnum:_(s16) = G_FMINNUM %a, %b
+    ; CHECK-FP16-NEXT: $h0 = COPY %minnum(s16)
+    ; CHECK-FP16-NEXT: RET_ReallyLR implicit $h0
+    ;
+    ; CHECK-NOFP16-LABEL: name: s16_legal_with_full_fp16
+    ; CHECK-NOFP16: liveins: $h0, $h1
+    ; CHECK-NOFP16-NEXT: {{  $}}
+    ; CHECK-NOFP16-NEXT: %a:_(s16) = COPY $h0
+    ; CHECK-NOFP16-NEXT: %b:_(s16) = COPY $h1
+    ; CHECK-NOFP16-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %a(s16)
+    ; CHECK-NOFP16-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %b(s16)
+    ; CHECK-NOFP16-NEXT: [[FMINNUM:%[0-9]+]]:_(s32) = G_FMINNUM [[FPEXT]], [[FPEXT1]]
+    ; CHECK-NOFP16-NEXT: %minnum:_(s16) = G_FPTRUNC [[FMINNUM]](s32)
+    ; CHECK-NOFP16-NEXT: $h0 = COPY %minnum(s16)
+    ; CHECK-NOFP16-NEXT: RET_ReallyLR implicit $h0
     %a:_(s16) = COPY $h0
     %b:_(s16) = COPY $h1
     %minnum:_(s16) = G_FMINNUM %a, %b
     $h0 = COPY %minnum(s16)
     RET_ReallyLR implicit $h0
 
 ...
 ---
 name:            s32_legal
 alignment:       4
 body:             |
   bb.0:
     liveins: $s0, $s1
-    ; FP16-LABEL: name: s32_legal
-    ; FP16: %a:_(s32) = COPY $s0
-    ; FP16: %b:_(s32) = COPY $s1
-    ; FP16: %minnum:_(s32) = G_FMINNUM %a, %b
-    ; FP16: $s0 = COPY %minnum(s32)
-    ; FP16: RET_ReallyLR implicit $s0
-    ; NO-FP16-LABEL: name: s32_legal
-    ; NO-FP16: %a:_(s32) = COPY $s0
-    ; NO-FP16: %b:_(s32) = COPY $s1
-    ; NO-FP16: %minnum:_(s32) = G_FMINNUM %a, %b
-    ; NO-FP16: $s0 = COPY %minnum(s32)
-    ; NO-FP16: RET_ReallyLR implicit $s0
+    ; CHECK-LABEL: name: s32_legal
+    ; CHECK: liveins: $s0, $s1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %a:_(s32) = COPY $s0
+    ; CHECK-NEXT: %b:_(s32) = COPY $s1
+    ; CHECK-NEXT: %minnum:_(s32) = G_FMINNUM %a, %b
+    ; CHECK-NEXT: $s0 = COPY %minnum(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $s0
     %a:_(s32) = COPY $s0
     %b:_(s32) = COPY $s1
     %minnum:_(s32) = G_FMINNUM %a, %b
     $s0 = COPY %minnum(s32)
     RET_ReallyLR implicit $s0
 
 ...
 ---
 name:            s64_legal
 alignment:       4
 body:             |
   bb.0:
     liveins: $d0, $d1
-    ; FP16-LABEL: name: s64_legal
-    ; FP16: %a:_(s64) = COPY $d0
-    ; FP16: %b:_(s64) = COPY $d1
-    ; FP16: %minnum:_(s64) = G_FMINNUM %a, %b
-    ; FP16: $d0 = COPY %minnum(s64)
-    ; FP16: RET_ReallyLR implicit $d0
-    ; NO-FP16-LABEL: name: s64_legal
-    ; NO-FP16: %a:_(s64) = COPY $d0
-    ; NO-FP16: %b:_(s64) = COPY $d1
-    ; NO-FP16: %minnum:_(s64) = G_FMINNUM %a, %b
-    ; NO-FP16: $d0 = COPY %minnum(s64)
-    ; NO-FP16: RET_ReallyLR implicit $d0
+    ; CHECK-LABEL: name: s64_legal
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %a:_(s64) = COPY $d0
+    ; CHECK-NEXT: %b:_(s64) = COPY $d1
+    ; CHECK-NEXT: %minnum:_(s64) = G_FMINNUM %a, %b
+    ; CHECK-NEXT: $d0 = COPY %minnum(s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
     %a:_(s64) = COPY $d0
     %b:_(s64) = COPY $d1
     %minnum:_(s64) = G_FMINNUM %a, %b
     $d0 = COPY %minnum(s64)
     RET_ReallyLR implicit $d0
 
 ...
 ---
 name:            s128_libcall
 alignment:       4
 body:             |
   bb.0:
     liveins: $q0, $q1
-    ; FP16-LABEL: name: s128_libcall
-    ; FP16: %a:_(s128) = COPY $q0
-    ; FP16: %b:_(s128) = COPY $q1
-    ; FP16: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
-    ; FP16: $q0 = COPY %a(s128)
-    ; FP16: $q1 = COPY %b(s128)
-    ; FP16: BL &fminl, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $q0, implicit $q1, implicit-def $q0
-    ; FP16: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
-    ; FP16: %minnum:_(s128) = COPY $q0
-    ; FP16: $q0 = COPY %minnum(s128)
-    ; FP16: RET_ReallyLR implicit $q0
-    ; NO-FP16-LABEL: name: s128_libcall
-    ; NO-FP16: %a:_(s128) = COPY $q0
-    ; NO-FP16: %b:_(s128) = COPY $q1
-    ; NO-FP16: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
-    ; NO-FP16: $q0 = COPY %a(s128)
-    ; NO-FP16: $q1 = COPY %b(s128)
-    ; NO-FP16: BL &fminl, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $q0, implicit $q1, implicit-def $q0
-    ; NO-FP16: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
-    ; NO-FP16: %minnum:_(s128) = COPY $q0
-    ; NO-FP16: $q0 = COPY %minnum(s128)
-    ; NO-FP16: RET_ReallyLR implicit $q0
+    ; CHECK-LABEL: name: s128_libcall
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %a:_(s128) = COPY $q0
+    ; CHECK-NEXT: %b:_(s128) = COPY $q1
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: $q0 = COPY %a(s128)
+    ; CHECK-NEXT: $q1 = COPY %b(s128)
+    ; CHECK-NEXT: BL &fminl, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $q0, implicit $q1, implicit-def $q0
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK-NEXT: %minnum:_(s128) = COPY $q0
+    ; CHECK-NEXT: $q0 = COPY %minnum(s128)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %a:_(s128) = COPY $q0
     %b:_(s128) = COPY $q1
     %minnum:_(s128) = G_FMINNUM %a, %b
     $q0 = COPY %minnum(s128)
     RET_ReallyLR implicit $q0
+
+...
+---
+name:            v4s32_legal
+alignment:       4
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: v4s32_legal
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %a:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: %b:_(<4 x s32>) = COPY $q1
+    ; CHECK-NEXT: %minnum:_(<4 x s32>) = G_FMINNUM %a, %b
+    ; CHECK-NEXT: $q0 = COPY %minnum(<4 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %a:_(<4 x s32>) = COPY $q0
+    %b:_(<4 x s32>) = COPY $q1
+    %minnum:_(<4 x s32>) = G_FMINNUM %a, %b
+    $q0 = COPY %minnum(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            v3s32_widen
+alignment:       4
+body:             |
+  bb.1.entry:
+    liveins: $q0, $q1
+    %2:_(<2 x s64>) = COPY $q0
+    %3:_(<4 x s32>) = G_BITCAST %2:_(<2 x s64>)
+    %4:_(s32), %5:_(s32), %6:_(s32), %7:_(s32) = G_UNMERGE_VALUES %3:_(<4 x s32>)
+    %0:_(<3 x s32>) = G_BUILD_VECTOR %4:_(s32), %5:_(s32), %6:_(s32)
+    %8:_(<2 x s64>) = COPY $q1
+    %9:_(<4 x s32>) = G_BITCAST %8:_(<2 x s64>)
+    %10:_(s32), %11:_(s32), %12:_(s32), %13:_(s32) = G_UNMERGE_VALUES %9:_(<4 x s32>)
+    %1:_(<3 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_(s32), %12:_(s32)
+    %14:_(<3 x s32>) = G_FMINNUM %0:_, %1:_
+    %15:_(s32), %16:_(s32), %17:_(s32) = G_UNMERGE_VALUES %14:_(<3 x s32>)
+    %18:_(s32) = G_IMPLICIT_DEF
+    %19:_(<4 x s32>) = G_BUILD_VECTOR %15:_(s32), %16:_(s32), %17:_(s32), %18:_(s32)
+    $q0 = COPY %19:_(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...

llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir

 # DEBUG-NEXT: G_IS_FPCLASS (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FCANONICALIZE (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FMINNUM (opcode {{[0-9]+}}): 1 type index
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_FMAXNUM (opcode {{[0-9]+}}): 1 type index
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_FMINNUM_IEEE (opcode {{[0-9]+}}): 1 type index
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FMAXNUM_IEEE (opcode {{[0-9]+}}): 1 type index
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FMINIMUM (opcode {{[0-9]+}}): 1 type index
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}

llvm/test/CodeGen/AArch64/fminmax.ll

-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16
-; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-NOFP16-SD
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16-SD
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-NOFP16-GI
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16-GI
 
 define double @min_f64(double %a, double %b) {
 ; CHECK-LABEL: min_f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    fminnm d0, d0, d1
 ; CHECK-NEXT:    ret
 entry:
   %c = call double @llvm.minnum.f64(double %a, double %b)

 ; CHECK-NEXT:    fmaxnm s0, s0, s1
 ; CHECK-NEXT:    ret
 entry:
   %c = call float @llvm.maxnum.f32(float %a, float %b)
   ret float %c
 }
 
 define half @min_f16(half %a, half %b) {
-; CHECK-NOFP16-LABEL: min_f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    fminnm s0, s0, s1
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: min_f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fminnm h0, h0, h1
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: min_f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fminnm s0, s0, s1
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: min_f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fminnm h0, h0, h1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: min_f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-GI-NEXT:    fminnm s0, s0, s1
+; CHECK-NOFP16-GI-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: min_f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fminnm h0, h0, h1
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call half @llvm.minnum.f16(half %a, half %b)
   ret half %c
 }
 
 define half @max_f16(half %a, half %b) {
-; CHECK-NOFP16-LABEL: max_f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    fmaxnm s0, s0, s1
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: max_f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fmaxnm h0, h0, h1
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: max_f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s0, s0, s1
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: max_f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmaxnm h0, h0, h1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: max_f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-GI-NEXT:    fmaxnm s0, s0, s1
+; CHECK-NOFP16-GI-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: max_f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmaxnm h0, h0, h1
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call half @llvm.maxnum.f16(half %a, half %b)
   ret half %c
 }
 
 define <2 x double> @min_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: min_v2f64:
 ; CHECK:       // %bb.0: // %entry

 ; CHECK-NEXT:    fmaxnm v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
 entry:
   %c = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %a, <2 x double> %b)
   ret <2 x double> %c
 }
 
 define <3 x double> @min_v3f64(<3 x double> %a, <3 x double> %b) {
-; CHECK-LABEL: min_v3f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d3 killed $d3 def $q3
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $d4 killed $d4 def $q4
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-NEXT:    // kill: def $d5 killed $d5 def $q5
-; CHECK-NEXT:    mov v3.d[1], v4.d[0]
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    fminnm v2.2d, v2.2d, v5.2d
-; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-NEXT:    fminnm v0.2d, v0.2d, v3.2d
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: min_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    fminnm v2.2d, v2.2d, v5.2d
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    fminnm v0.2d, v0.2d, v3.2d
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: min_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-GI-NEXT:    fminnm d2, d2, d5
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT:    fminnm v0.2d, v0.2d, v3.2d
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %c = call <3 x double> @llvm.minnum.v3f64(<3 x double> %a, <3 x double> %b)
   ret <3 x double> %c
 }
 
 define <3 x double> @max_v3f64(<3 x double> %a, <3 x double> %b) {
-; CHECK-LABEL: max_v3f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d3 killed $d3 def $q3
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $d4 killed $d4 def $q4
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-NEXT:    // kill: def $d5 killed $d5 def $q5
-; CHECK-NEXT:    mov v3.d[1], v4.d[0]
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    fmaxnm v2.2d, v2.2d, v5.2d
-; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-NEXT:    fmaxnm v0.2d, v0.2d, v3.2d
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: max_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    fmaxnm v2.2d, v2.2d, v5.2d
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    fmaxnm v0.2d, v0.2d, v3.2d
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: max_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-GI-NEXT:    fmaxnm d2, d2, d5
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT:    fmaxnm v0.2d, v0.2d, v3.2d
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %c = call <3 x double> @llvm.maxnum.v3f64(<3 x double> %a, <3 x double> %b)
   ret <3 x double> %c
 }
 
 define <4 x double> @min_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: min_v4f64:
 ; CHECK:       // %bb.0: // %entry

 ; CHECK-NEXT:    fmaxnm v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
 entry:
   %c = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %a, <2 x float> %b)
   ret <2 x float> %c
 }
 
 define <3 x float> @min_v3f32(<3 x float> %a, <3 x float> %b) {
-; CHECK-LABEL: min_v3f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fminnm v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: min_v3f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fminnm v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: min_v3f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s2, v0.s[1]
+; CHECK-GI-NEXT:    mov s3, v1.s[1]
+; CHECK-GI-NEXT:    mov s4, v0.s[2]
+; CHECK-GI-NEXT:    mov s5, v1.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT:    mov v1.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v4.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v5.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-GI-NEXT:    fminnm v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %c = call <3 x float> @llvm.minnum.v3f32(<3 x float> %a, <3 x float> %b)
   ret <3 x float> %c
 }
 
 define <3 x float> @max_v3f32(<3 x float> %a, <3 x float> %b) {
-; CHECK-LABEL: max_v3f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: max_v3f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: max_v3f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s2, v0.s[1]
+; CHECK-GI-NEXT:    mov s3, v1.s[1]
+; CHECK-GI-NEXT:    mov s4, v0.s[2]
+; CHECK-GI-NEXT:    mov s5, v1.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT:    mov v1.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v4.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v5.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-GI-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %c = call <3 x float> @llvm.maxnum.v3f32(<3 x float> %a, <3 x float> %b)
   ret <3 x float> %c
 }
 
 define <4 x float> @min_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: min_v4f32:
 ; CHECK:       // %bb.0: // %entry

 ; CHECK-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
 entry:
   %c = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %a, <4 x float> %b)
   ret <4 x float> %c
 }
 
 define <7 x float> @min_v7f32(<7 x float> %a, <7 x float> %b) {
-; CHECK-LABEL: min_v7f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    // kill: def $s7 killed $s7 def $q7
-; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-NEXT:    // kill: def $s1 killed $s1 def $q1
-; CHECK-NEXT:    add x9, sp, #32
-; CHECK-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-NEXT:    ldr s1, [sp, #24]
-; CHECK-NEXT:    // kill: def $s4 killed $s4 def $q4
-; CHECK-NEXT:    // kill: def $s5 killed $s5 def $q5
-; CHECK-NEXT:    // kill: def $s2 killed $s2 def $q2
-; CHECK-NEXT:    // kill: def $s6 killed $s6 def $q6
-; CHECK-NEXT:    // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT:    ld1 { v7.s }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #8
-; CHECK-NEXT:    ld1 { v1.s }[1], [x9]
-; CHECK-NEXT:    mov v0.s[2], v2.s[0]
-; CHECK-NEXT:    mov v4.s[1], v5.s[0]
-; CHECK-NEXT:    ld1 { v7.s }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #16
-; CHECK-NEXT:    mov v0.s[3], v3.s[0]
-; CHECK-NEXT:    mov v4.s[2], v6.s[0]
-; CHECK-NEXT:    ld1 { v7.s }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    ld1 { v1.s }[2], [x8]
-; CHECK-NEXT:    fminnm v0.4s, v0.4s, v7.4s
-; CHECK-NEXT:    fminnm v4.4s, v4.4s, v1.4s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    mov s2, v0.s[2]
-; CHECK-NEXT:    mov s3, v0.s[3]
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    mov s5, v4.s[1]
-; CHECK-NEXT:    mov s6, v4.s[2]
-; CHECK-NEXT:    // kill: def $s4 killed $s4 killed $q4
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: min_v7f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, sp
+; CHECK-SD-NEXT:    // kill: def $s7 killed $s7 def $q7
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-SD-NEXT:    add x9, sp, #32
+; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-SD-NEXT:    ldr s1, [sp, #24]
+; CHECK-SD-NEXT:    // kill: def $s4 killed $s4 def $q4
+; CHECK-SD-NEXT:    // kill: def $s5 killed $s5 def $q5
+; CHECK-SD-NEXT:    // kill: def $s2 killed $s2 def $q2
+; CHECK-SD-NEXT:    // kill: def $s6 killed $s6 def $q6
+; CHECK-SD-NEXT:    // kill: def $s3 killed $s3 def $q3
+; CHECK-SD-NEXT:    ld1 { v7.s }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #8
+; CHECK-SD-NEXT:    ld1 { v1.s }[1], [x9]
+; CHECK-SD-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-SD-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-SD-NEXT:    ld1 { v7.s }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #16
+; CHECK-SD-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-SD-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-SD-NEXT:    ld1 { v7.s }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    ld1 { v1.s }[2], [x8]
+; CHECK-SD-NEXT:    fminnm v0.4s, v0.4s, v7.4s
+; CHECK-SD-NEXT:    fminnm v4.4s, v4.4s, v1.4s
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    mov s2, v0.s[2]
+; CHECK-SD-NEXT:    mov s3, v0.s[3]
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-SD-NEXT:    mov s5, v4.s[1]
+; CHECK-SD-NEXT:    mov s6, v4.s[2]
+; CHECK-SD-NEXT:    // kill: def $s4 killed $s4 killed $q4
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: min_v7f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-GI-NEXT:    ldr s16, [sp]
+; CHECK-GI-NEXT:    // kill: def $s4 killed $s4 def $q4
+; CHECK-GI-NEXT:    // kill: def $s7 killed $s7 def $q7
+; CHECK-GI-NEXT:    // kill: def $s5 killed $s5 def $q5
+; CHECK-GI-NEXT:    // kill: def $s2 killed $s2 def $q2
+; CHECK-GI-NEXT:    // kill: def $s6 killed $s6 def $q6
+; CHECK-GI-NEXT:    // kill: def $s3 killed $s3 def $q3
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    ldr s19, [sp, #24]
+; CHECK-GI-NEXT:    ldr s20, [sp, #32]
+; CHECK-GI-NEXT:    mov v7.s[1], v16.s[0]
+; CHECK-GI-NEXT:    ldr s17, [sp, #8]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ldr s21, [sp, #40]
+; CHECK-GI-NEXT:    mov v19.s[1], v20.s[0]
+; CHECK-GI-NEXT:    ldr s18, [sp, #16]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], v17.s[0]
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v19.s[2], v21.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], v18.s[0]
+; CHECK-GI-NEXT:    mov v4.s[3], v0.s[0]
+; CHECK-GI-NEXT:    mov v19.s[3], v0.s[0]
+; CHECK-GI-NEXT:    fminnm v0.4s, v0.4s, v7.4s
+; CHECK-GI-NEXT:    fminnm v4.4s, v4.4s, v19.4s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov s3, v0.s[3]
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-GI-NEXT:    mov s5, v4.s[1]
+; CHECK-GI-NEXT:    mov s6, v4.s[2]
+; CHECK-GI-NEXT:    // kill: def $s4 killed $s4 killed $q4
+; CHECK-GI-NEXT:    ret
 entry:
   %c = call <7 x float> @llvm.minnum.v7f32(<7 x float> %a, <7 x float> %b)
   ret <7 x float> %c
 }
 
 define <7 x float> @max_v7f32(<7 x float> %a, <7 x float> %b) {
-; CHECK-LABEL: max_v7f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    // kill: def $s7 killed $s7 def $q7
-; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-NEXT:    // kill: def $s1 killed $s1 def $q1
-; CHECK-NEXT:    add x9, sp, #32
-; CHECK-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-NEXT:    ldr s1, [sp, #24]
-; CHECK-NEXT:    // kill: def $s4 killed $s4 def $q4
-; CHECK-NEXT:    // kill: def $s5 killed $s5 def $q5
-; CHECK-NEXT:    // kill: def $s2 killed $s2 def $q2
-; CHECK-NEXT:    // kill: def $s6 killed $s6 def $q6
-; CHECK-NEXT:    // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT:    ld1 { v7.s }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #8
-; CHECK-NEXT:    ld1 { v1.s }[1], [x9]
-; CHECK-NEXT:    mov v0.s[2], v2.s[0]
-; CHECK-NEXT:    mov v4.s[1], v5.s[0]
-; CHECK-NEXT:    ld1 { v7.s }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #16
-; CHECK-NEXT:    mov v0.s[3], v3.s[0]
-; CHECK-NEXT:    mov v4.s[2], v6.s[0]
-; CHECK-NEXT:    ld1 { v7.s }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    ld1 { v1.s }[2], [x8]
-; CHECK-NEXT:    fmaxnm v0.4s, v0.4s, v7.4s
-; CHECK-NEXT:    fmaxnm v4.4s, v4.4s, v1.4s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    mov s2, v0.s[2]
-; CHECK-NEXT:    mov s3, v0.s[3]
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    mov s5, v4.s[1]
-; CHECK-NEXT:    mov s6, v4.s[2]
-; CHECK-NEXT:    // kill: def $s4 killed $s4 killed $q4
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: max_v7f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, sp
+; CHECK-SD-NEXT:    // kill: def $s7 killed $s7 def $q7
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-SD-NEXT:    add x9, sp, #32
+; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-SD-NEXT:    ldr s1, [sp, #24]
+; CHECK-SD-NEXT:    // kill: def $s4 killed $s4 def $q4
+; CHECK-SD-NEXT:    // kill: def $s5 killed $s5 def $q5
+; CHECK-SD-NEXT:    // kill: def $s2 killed $s2 def $q2
+; CHECK-SD-NEXT:    // kill: def $s6 killed $s6 def $q6
+; CHECK-SD-NEXT:    // kill: def $s3 killed $s3 def $q3
+; CHECK-SD-NEXT:    ld1 { v7.s }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #8
+; CHECK-SD-NEXT:    ld1 { v1.s }[1], [x9]
+; CHECK-SD-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-SD-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-SD-NEXT:    ld1 { v7.s }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #16
+; CHECK-SD-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-SD-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-SD-NEXT:    ld1 { v7.s }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    ld1 { v1.s }[2], [x8]
+; CHECK-SD-NEXT:    fmaxnm v0.4s, v0.4s, v7.4s
+; CHECK-SD-NEXT:    fmaxnm v4.4s, v4.4s, v1.4s
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    mov s2, v0.s[2]
+; CHECK-SD-NEXT:    mov s3, v0.s[3]
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-SD-NEXT:    mov s5, v4.s[1]
+; CHECK-SD-NEXT:    mov s6, v4.s[2]
+; CHECK-SD-NEXT:    // kill: def $s4 killed $s4 killed $q4
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: max_v7f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-GI-NEXT:    ldr s16, [sp]
+; CHECK-GI-NEXT:    // kill: def $s4 killed $s4 def $q4
+; CHECK-GI-NEXT:    // kill: def $s7 killed $s7 def $q7
+; CHECK-GI-NEXT:    // kill: def $s5 killed $s5 def $q5
+; CHECK-GI-NEXT:    // kill: def $s2 killed $s2 def $q2
+; CHECK-GI-NEXT:    // kill: def $s6 killed $s6 def $q6
+; CHECK-GI-NEXT:    // kill: def $s3 killed $s3 def $q3
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    ldr s19, [sp, #24]
+; CHECK-GI-NEXT:    ldr s20, [sp, #32]
+; CHECK-GI-NEXT:    mov v7.s[1], v16.s[0]
+; CHECK-GI-NEXT:    ldr s17, [sp, #8]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ldr s21, [sp, #40]
+; CHECK-GI-NEXT:    mov v19.s[1], v20.s[0]
+; CHECK-GI-NEXT:    ldr s18, [sp, #16]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], v17.s[0]
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v19.s[2], v21.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], v18.s[0]
+; CHECK-GI-NEXT:    mov v4.s[3], v0.s[0]
+; CHECK-GI-NEXT:    mov v19.s[3], v0.s[0]
+; CHECK-GI-NEXT:    fmaxnm v0.4s, v0.4s, v7.4s
+; CHECK-GI-NEXT:    fmaxnm v4.4s, v4.4s, v19.4s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov s3, v0.s[3]
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-GI-NEXT:    mov s5, v4.s[1]
+; CHECK-GI-NEXT:    mov s6, v4.s[2]
+; CHECK-GI-NEXT:    // kill: def $s4 killed $s4 killed $q4
+; CHECK-GI-NEXT:    ret
 entry:
   %c = call <7 x float> @llvm.maxnum.v7f32(<7 x float> %a, <7 x float> %b)
   ret <7 x float> %c
 }
 
 define <8 x float> @min_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: min_v8f32:
 ; CHECK:       // %bb.0: // %entry

 ; CHECK-NEXT:    fmaxnm v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    ret
 entry:
   %c = call <8 x float> @llvm.maxnum.v8f32(<8 x float> %a, <8 x float> %b)
   ret <8 x float> %c
 }
 
 define <4 x half> @min_v4f16(<4 x half> %a, <4 x half> %b) {
-; CHECK-NOFP16-LABEL: min_v4f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fminnm s4, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s5, h7
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fminnm s2, s3, s2
-; CHECK-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-NOFP16-NEXT:    fcvt h0, s4
-; CHECK-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-NOFP16-NEXT:    fminnm s3, s5, s3
-; CHECK-NOFP16-NEXT:    fcvt s4, h6
-; CHECK-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h2, s3
-; CHECK-NOFP16-NEXT:    fminnm s1, s4, s1
-; CHECK-NOFP16-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: min_v4f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fminnm v0.4h, v0.4h, v1.4h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: min_v4f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fminnm s4, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fminnm s2, s3, s2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h6
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s2
+; CHECK-NOFP16-SD-NEXT:    fminnm s3, s5, s3
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h6
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s3
+; CHECK-NOFP16-SD-NEXT:    fminnm s1, s4, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h1, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[3], v1.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: min_v4f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fminnm v0.4h, v0.4h, v1.4h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: min_v4f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fminnm v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: min_v4f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fminnm v0.4h, v0.4h, v1.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <4 x half> @llvm.minnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %c
 }
 
 define <4 x half> @max_v4f16(<4 x half> %a, <4 x half> %b) {
-; CHECK-NOFP16-LABEL: max_v4f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fmaxnm s4, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s5, h7
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fmaxnm s2, s3, s2
-; CHECK-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-NOFP16-NEXT:    fcvt h0, s4
-; CHECK-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-NOFP16-NEXT:    fmaxnm s3, s5, s3
-; CHECK-NOFP16-NEXT:    fcvt s4, h6
-; CHECK-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h2, s3
-; CHECK-NOFP16-NEXT:    fmaxnm s1, s4, s1
-; CHECK-NOFP16-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: max_v4f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: max_v4f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s4, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s2, s3, s2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h6
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s2
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s3, s5, s3
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h6
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s3
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s1, s4, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h1, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[3], v1.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: max_v4f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: max_v4f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: max_v4f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %c
 }
 
 define <7 x half> @min_v7f16(<7 x half> %a, <7 x half> %b) {
-; CHECK-NOFP16-LABEL: min_v7f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fminnm s4, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fminnm s3, s3, s2
-; CHECK-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-NOFP16-NEXT:    fminnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-NOFP16-NEXT:    fminnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-NOFP16-NEXT:    fminnm s5, s6, s5
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[6]
-; CHECK-NOFP16-NEXT:    mov v2.h[2], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-NOFP16-NEXT:    fcvt s4, h16
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[6]
-; CHECK-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-NOFP16-NEXT:    mov v2.h[3], v7.h[0]
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fminnm s3, s4, s3
-; CHECK-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    mov v2.h[4], v4.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    fminnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    fminnm s0, s0, s1
-; CHECK-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: min_v7f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fminnm v0.8h, v0.8h, v1.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: min_v7f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h16, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fminnm s4, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fminnm s3, s3, s2
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s4
+; CHECK-NOFP16-SD-NEXT:    fminnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[4]
+; CHECK-NOFP16-SD-NEXT:    fminnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[5]
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt h7, s7
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[1], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    mov h4, v1.h[5]
+; CHECK-NOFP16-SD-NEXT:    fminnm s5, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[2], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[7]
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h16
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[3], v7.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fminnm s3, s4, s3
+; CHECK-NOFP16-SD-NEXT:    fcvt h4, s5
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[4], v4.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    fminnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    fminnm s0, s0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[5], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[6], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[7], v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: min_v7f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: min_v7f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    mov h2, v0.h[4]
+; CHECK-NOFP16-GI-NEXT:    mov h3, v0.h[5]
+; CHECK-NOFP16-GI-NEXT:    mov h4, v1.h[4]
+; CHECK-NOFP16-GI-NEXT:    mov h5, v1.h[5]
+; CHECK-NOFP16-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-NOFP16-GI-NEXT:    mov h7, v1.h[6]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    mov v2.h[1], v3.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v4.h[1], v5.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.h[2], v6.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v4.h[2], v7.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.h[3], v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v4.h[3], v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v4.4h
+; CHECK-NOFP16-GI-NEXT:    mov s4, v2.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s6, v2.s[2]
+; CHECK-NOFP16-GI-NEXT:    mov s5, v3.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s7, v3.s[2]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[1], v4.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v3.s[1], v5.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[2], v6.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v3.s[2], v7.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[3], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v3.s[3], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fminnm v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fminnm v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    mov s1, v2.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s3, v2.s[2]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[1], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-GI-NEXT:    mov h4, v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov h5, v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[3], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], v4.h[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[3], v5.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[4], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov h1, v1.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[5], v2.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[6], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: min_v7f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h7, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov h16, v1.h[1]
+; CHECK-FP16-GI-NEXT:    mov h17, v1.h[2]
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-FP16-GI-NEXT:    mov h18, v1.h[3]
+; CHECK-FP16-GI-NEXT:    mov h19, v1.h[4]
+; CHECK-FP16-GI-NEXT:    mov h20, v1.h[5]
+; CHECK-FP16-GI-NEXT:    mov h21, v1.h[6]
+; CHECK-FP16-GI-NEXT:    mov v1.h[1], v16.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], v3.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[2], v17.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[3], v4.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[3], v18.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[4], v5.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[4], v19.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[5], v6.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[5], v20.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[6], v7.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[6], v21.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[7], v0.h[0]
+; CHECK-FP16-GI-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[3], v3.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[4], v4.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[5], v5.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[6], v6.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <7 x half> @llvm.minnum.v7f16(<7 x half> %a, <7 x half> %b)
   ret <7 x half> %c
 }
 
 define <7 x half> @max_v7f16(<7 x half> %a, <7 x half> %b) {
-; CHECK-NOFP16-LABEL: max_v7f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fmaxnm s4, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fmaxnm s3, s3, s2
-; CHECK-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-NOFP16-NEXT:    fmaxnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-NOFP16-NEXT:    fmaxnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-NOFP16-NEXT:    fmaxnm s5, s6, s5
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[6]
-; CHECK-NOFP16-NEXT:    mov v2.h[2], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-NOFP16-NEXT:    fcvt s4, h16
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[6]
-; CHECK-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-NOFP16-NEXT:    mov v2.h[3], v7.h[0]
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fmaxnm s3, s4, s3
-; CHECK-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    mov v2.h[4], v4.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    fmaxnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    fmaxnm s0, s0, s1
-; CHECK-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: max_v7f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: max_v7f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h16, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s4, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s3, s3, s2
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s4
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[4]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[5]
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt h7, s7
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[1], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    mov h4, v1.h[5]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s5, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[2], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[7]
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h16
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[3], v7.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s3, s4, s3
+; CHECK-NOFP16-SD-NEXT:    fcvt h4, s5
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[4], v4.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s0, s0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[5], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[6], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[7], v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: max_v7f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: max_v7f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    mov h2, v0.h[4]
+; CHECK-NOFP16-GI-NEXT:    mov h3, v0.h[5]
+; CHECK-NOFP16-GI-NEXT:    mov h4, v1.h[4]
+; CHECK-NOFP16-GI-NEXT:    mov h5, v1.h[5]
+; CHECK-NOFP16-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-NOFP16-GI-NEXT:    mov h7, v1.h[6]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    mov v2.h[1], v3.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v4.h[1], v5.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.h[2], v6.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v4.h[2], v7.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.h[3], v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v4.h[3], v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v4.4h
+; CHECK-NOFP16-GI-NEXT:    mov s4, v2.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s6, v2.s[2]
+; CHECK-NOFP16-GI-NEXT:    mov s5, v3.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s7, v3.s[2]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[1], v4.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v3.s[1], v5.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[2], v6.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v3.s[2], v7.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[3], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v3.s[3], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    mov s1, v2.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s3, v2.s[2]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[1], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-GI-NEXT:    mov h4, v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov h5, v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[3], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], v4.h[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[3], v5.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[4], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov h1, v1.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[5], v2.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[6], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: max_v7f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h7, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov h16, v1.h[1]
+; CHECK-FP16-GI-NEXT:    mov h17, v1.h[2]
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-FP16-GI-NEXT:    mov h18, v1.h[3]
+; CHECK-FP16-GI-NEXT:    mov h19, v1.h[4]
+; CHECK-FP16-GI-NEXT:    mov h20, v1.h[5]
+; CHECK-FP16-GI-NEXT:    mov h21, v1.h[6]
+; CHECK-FP16-GI-NEXT:    mov v1.h[1], v16.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], v3.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[2], v17.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[3], v4.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[3], v18.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[4], v5.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[4], v19.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[5], v6.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[5], v20.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[6], v7.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[6], v21.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-FP16-GI-NEXT:    mov v1.h[7], v0.h[0]
+; CHECK-FP16-GI-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[3], v3.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[4], v4.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[5], v5.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[6], v6.h[0]
+; CHECK-FP16-GI-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <7 x half> @llvm.maxnum.v7f16(<7 x half> %a, <7 x half> %b)
   ret <7 x half> %c
 }
 
 define <8 x half> @min_v8f16(<8 x half> %a, <8 x half> %b) {
-; CHECK-NOFP16-LABEL: min_v8f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fminnm s4, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fminnm s3, s3, s2
-; CHECK-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-NOFP16-NEXT:    fminnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-NOFP16-NEXT:    fminnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-NOFP16-NEXT:    fminnm s5, s6, s5
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[6]
-; CHECK-NOFP16-NEXT:    mov v2.h[2], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-NOFP16-NEXT:    fcvt s4, h16
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[6]
-; CHECK-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-NOFP16-NEXT:    mov v2.h[3], v7.h[0]
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fminnm s3, s4, s3
-; CHECK-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    mov v2.h[4], v4.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    fminnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    fminnm s0, s0, s1
-; CHECK-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: min_v8f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fminnm v0.8h, v0.8h, v1.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: min_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h16, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fminnm s4, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fminnm s3, s3, s2
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s4
+; CHECK-NOFP16-SD-NEXT:    fminnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[4]
+; CHECK-NOFP16-SD-NEXT:    fminnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[5]
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt h7, s7
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[1], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    mov h4, v1.h[5]
+; CHECK-NOFP16-SD-NEXT:    fminnm s5, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[2], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[7]
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h16
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[3], v7.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fminnm s3, s4, s3
+; CHECK-NOFP16-SD-NEXT:    fcvt h4, s5
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[4], v4.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    fminnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    fminnm s0, s0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[5], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[6], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[7], v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: min_v8f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: min_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fminnm v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fminnm v1.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: min_v8f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <8 x half> @llvm.minnum.v8f16(<8 x half> %a, <8 x half> %b)
   ret <8 x half> %c
 }
 
 define <8 x half> @max_v8f16(<8 x half> %a, <8 x half> %b) {
-; CHECK-NOFP16-LABEL: max_v8f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fmaxnm s4, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fmaxnm s3, s3, s2
-; CHECK-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-NOFP16-NEXT:    fmaxnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-NOFP16-NEXT:    fmaxnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-NOFP16-NEXT:    fmaxnm s5, s6, s5
-; CHECK-NOFP16-NEXT:    mov h6, v1.h[6]
-; CHECK-NOFP16-NEXT:    mov v2.h[2], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-NOFP16-NEXT:    fcvt s4, h16
-; CHECK-NOFP16-NEXT:    mov h16, v0.h[6]
-; CHECK-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-NOFP16-NEXT:    mov v2.h[3], v7.h[0]
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fmaxnm s3, s4, s3
-; CHECK-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    mov v2.h[4], v4.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-NOFP16-NEXT:    fmaxnm s4, s6, s5
-; CHECK-NOFP16-NEXT:    fmaxnm s0, s0, s1
-; CHECK-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: max_v8f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: max_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h16, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s4, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s3, s3, s2
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s4
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[4]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[5]
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt h7, s7
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[1], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    mov h4, v1.h[5]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s5, s6, s5
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[2], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[7]
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h16
+; CHECK-NOFP16-SD-NEXT:    mov h16, v0.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[3], v7.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s3, s4, s3
+; CHECK-NOFP16-SD-NEXT:    fcvt h4, s5
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[4], v4.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s3
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s4, s6, s5
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s0, s0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[5], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h3, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[6], v3.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v2.h[7], v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: max_v8f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: max_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v1.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: max_v8f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <8 x half> @llvm.maxnum.v8f16(<8 x half> %a, <8 x half> %b)
   ret <8 x half> %c
 }
 
 define <16 x half> @min_v16f16(<16 x half> %a, <16 x half> %b) {
-; CHECK-NOFP16-LABEL: min_v16f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    mov h4, v2.h[1]
-; CHECK-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s6, h2
-; CHECK-NOFP16-NEXT:    fcvt s7, h0
-; CHECK-NOFP16-NEXT:    mov h16, v2.h[2]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h18, v3.h[1]
-; CHECK-NOFP16-NEXT:    mov h19, v1.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-NOFP16-NEXT:    fminnm s6, s7, s6
-; CHECK-NOFP16-NEXT:    fcvt s20, h1
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    mov h17, v2.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-NOFP16-NEXT:    fminnm s5, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-NOFP16-NEXT:    fminnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fcvt s16, h3
-; CHECK-NOFP16-NEXT:    mov h21, v0.h[4]
-; CHECK-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-NOFP16-NEXT:    fminnm s18, s19, s18
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    mov h19, v3.h[2]
-; CHECK-NOFP16-NEXT:    fminnm s16, s20, s16
-; CHECK-NOFP16-NEXT:    mov h20, v1.h[2]
-; CHECK-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-NOFP16-NEXT:    mov v4.h[1], v5.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-NOFP16-NEXT:    mov h7, v2.h[4]
-; CHECK-NOFP16-NEXT:    fminnm s6, s6, s17
-; CHECK-NOFP16-NEXT:    mov h17, v3.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-NOFP16-NEXT:    mov v4.h[2], v5.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h5, s16
-; CHECK-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-NOFP16-NEXT:    fcvt h16, s18
-; CHECK-NOFP16-NEXT:    mov h18, v1.h[3]
-; CHECK-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-NOFP16-NEXT:    fminnm s7, s21, s7
-; CHECK-NOFP16-NEXT:    mov v5.h[1], v16.h[0]
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fminnm s17, s20, s19
-; CHECK-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-NOFP16-NEXT:    mov h19, v3.h[4]
-; CHECK-NOFP16-NEXT:    mov h20, v1.h[4]
-; CHECK-NOFP16-NEXT:    mov v4.h[3], v6.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-NOFP16-NEXT:    fminnm s7, s18, s16
-; CHECK-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-NOFP16-NEXT:    mov v4.h[4], v6.h[0]
-; CHECK-NOFP16-NEXT:    mov h19, v1.h[5]
-; CHECK-NOFP16-NEXT:    mov v5.h[2], v16.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-NOFP16-NEXT:    fminnm s7, s18, s17
-; CHECK-NOFP16-NEXT:    mov h16, v2.h[5]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[5]
-; CHECK-NOFP16-NEXT:    mov h18, v3.h[5]
-; CHECK-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-NOFP16-NEXT:    mov h19, v2.h[6]
-; CHECK-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-NOFP16-NEXT:    mov v5.h[4], v6.h[0]
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-NOFP16-NEXT:    fminnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fminnm s16, s18, s17
-; CHECK-NOFP16-NEXT:    mov h17, v3.h[6]
-; CHECK-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fminnm s6, s6, s19
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-NOFP16-NEXT:    fminnm s17, s18, s17
-; CHECK-NOFP16-NEXT:    mov v4.h[5], v7.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-NOFP16-NEXT:    fminnm s0, s0, s2
-; CHECK-NOFP16-NEXT:    fminnm s1, s1, s3
-; CHECK-NOFP16-NEXT:    mov v5.h[5], v16.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h2, s17
-; CHECK-NOFP16-NEXT:    mov v4.h[6], v6.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-NOFP16-NEXT:    mov v5.h[6], v2.h[0]
-; CHECK-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-NOFP16-NEXT:    mov v1.16b, v5.16b
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: min_v16f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fminnm v0.8h, v0.8h, v2.8h
-; CHECK-FP16-NEXT:    fminnm v1.8h, v1.8h, v3.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: min_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov h4, v2.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h5, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h0
+; CHECK-NOFP16-SD-NEXT:    mov h16, v2.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h18, v3.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h19, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fminnm s6, s7, s6
+; CHECK-NOFP16-SD-NEXT:    fcvt s20, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    mov h17, v2.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    fminnm s5, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt h4, s6
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fminnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h3
+; CHECK-NOFP16-SD-NEXT:    mov h21, v0.h[4]
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h17
+; CHECK-NOFP16-SD-NEXT:    fcvt h5, s5
+; CHECK-NOFP16-SD-NEXT:    fminnm s18, s19, s18
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    mov h19, v3.h[2]
+; CHECK-NOFP16-SD-NEXT:    fminnm s16, s20, s16
+; CHECK-NOFP16-SD-NEXT:    mov h20, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s21, h21
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[1], v5.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h5, s7
+; CHECK-NOFP16-SD-NEXT:    mov h7, v2.h[4]
+; CHECK-NOFP16-SD-NEXT:    fminnm s6, s6, s17
+; CHECK-NOFP16-SD-NEXT:    mov h17, v3.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s20, h20
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[2], v5.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h5, s16
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt h16, s18
+; CHECK-NOFP16-SD-NEXT:    mov h18, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s6
+; CHECK-NOFP16-SD-NEXT:    fminnm s7, s21, s7
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[1], v16.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fminnm s17, s20, s19
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    mov h19, v3.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov h20, v1.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[3], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s7
+; CHECK-NOFP16-SD-NEXT:    fminnm s7, s18, s16
+; CHECK-NOFP16-SD-NEXT:    fcvt h16, s17
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h20
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[4], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h19, v1.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[2], v16.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s7
+; CHECK-NOFP16-SD-NEXT:    fminnm s7, s18, s17
+; CHECK-NOFP16-SD-NEXT:    mov h16, v2.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov h18, v3.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[3], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h18
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h19
+; CHECK-NOFP16-SD-NEXT:    mov h19, v2.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v2.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[4], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[6]
+; CHECK-NOFP16-SD-NEXT:    fminnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fminnm s16, s18, s17
+; CHECK-NOFP16-SD-NEXT:    mov h17, v3.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h18, v1.h[6]
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v3.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[7]
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h17
+; CHECK-NOFP16-SD-NEXT:    fcvt h7, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fminnm s6, s6, s19
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt h16, s16
+; CHECK-NOFP16-SD-NEXT:    fminnm s17, s18, s17
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[5], v7.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s6
+; CHECK-NOFP16-SD-NEXT:    fminnm s0, s0, s2
+; CHECK-NOFP16-SD-NEXT:    fminnm s1, s1, s3
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[5], v16.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s17
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[6], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt h1, s1
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[6], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[7], v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[7], v1.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v4.16b
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v5.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: min_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fminnm v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    fminnm v1.8h, v1.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: min_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v5.4s, v2.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v6.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-NOFP16-GI-NEXT:    fminnm v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fminnm v5.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    fminnm v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fminnm v3.4s, v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: min_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fminnm v0.8h, v0.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    fminnm v1.8h, v1.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <16 x half> @llvm.minnum.v16f16(<16 x half> %a, <16 x half> %b)
   ret <16 x half> %c
 }
 
 define <16 x half> @max_v16f16(<16 x half> %a, <16 x half> %b) {
-; CHECK-NOFP16-LABEL: max_v16f16:
-; CHECK-NOFP16:       // %bb.0: // %entry
-; CHECK-NOFP16-NEXT:    mov h4, v2.h[1]
-; CHECK-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s6, h2
-; CHECK-NOFP16-NEXT:    fcvt s7, h0
-; CHECK-NOFP16-NEXT:    mov h16, v2.h[2]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[2]
-; CHECK-NOFP16-NEXT:    mov h18, v3.h[1]
-; CHECK-NOFP16-NEXT:    mov h19, v1.h[1]
-; CHECK-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-NOFP16-NEXT:    fmaxnm s6, s7, s6
-; CHECK-NOFP16-NEXT:    fcvt s20, h1
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    mov h17, v2.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-NOFP16-NEXT:    fmaxnm s5, s5, s4
-; CHECK-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-NOFP16-NEXT:    fmaxnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fcvt s16, h3
-; CHECK-NOFP16-NEXT:    mov h21, v0.h[4]
-; CHECK-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-NOFP16-NEXT:    fmaxnm s18, s19, s18
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    mov h19, v3.h[2]
-; CHECK-NOFP16-NEXT:    fmaxnm s16, s20, s16
-; CHECK-NOFP16-NEXT:    mov h20, v1.h[2]
-; CHECK-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-NOFP16-NEXT:    mov v4.h[1], v5.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-NOFP16-NEXT:    mov h7, v2.h[4]
-; CHECK-NOFP16-NEXT:    fmaxnm s6, s6, s17
-; CHECK-NOFP16-NEXT:    mov h17, v3.h[3]
-; CHECK-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-NOFP16-NEXT:    mov v4.h[2], v5.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h5, s16
-; CHECK-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-NOFP16-NEXT:    fcvt h16, s18
-; CHECK-NOFP16-NEXT:    mov h18, v1.h[3]
-; CHECK-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-NOFP16-NEXT:    fmaxnm s7, s21, s7
-; CHECK-NOFP16-NEXT:    mov v5.h[1], v16.h[0]
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fmaxnm s17, s20, s19
-; CHECK-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-NOFP16-NEXT:    mov h19, v3.h[4]
-; CHECK-NOFP16-NEXT:    mov h20, v1.h[4]
-; CHECK-NOFP16-NEXT:    mov v4.h[3], v6.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-NOFP16-NEXT:    fmaxnm s7, s18, s16
-; CHECK-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-NOFP16-NEXT:    mov v4.h[4], v6.h[0]
-; CHECK-NOFP16-NEXT:    mov h19, v1.h[5]
-; CHECK-NOFP16-NEXT:    mov v5.h[2], v16.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-NOFP16-NEXT:    fmaxnm s7, s18, s17
-; CHECK-NOFP16-NEXT:    mov h16, v2.h[5]
-; CHECK-NOFP16-NEXT:    mov h17, v0.h[5]
-; CHECK-NOFP16-NEXT:    mov h18, v3.h[5]
-; CHECK-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-NOFP16-NEXT:    mov h19, v2.h[6]
-; CHECK-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-NOFP16-NEXT:    mov v5.h[4], v6.h[0]
-; CHECK-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-NOFP16-NEXT:    fmaxnm s7, s16, s7
-; CHECK-NOFP16-NEXT:    fmaxnm s16, s18, s17
-; CHECK-NOFP16-NEXT:    mov h17, v3.h[6]
-; CHECK-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-NOFP16-NEXT:    fmaxnm s6, s6, s19
-; CHECK-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-NOFP16-NEXT:    fmaxnm s17, s18, s17
-; CHECK-NOFP16-NEXT:    mov v4.h[5], v7.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-NOFP16-NEXT:    fmaxnm s0, s0, s2
-; CHECK-NOFP16-NEXT:    fmaxnm s1, s1, s3
-; CHECK-NOFP16-NEXT:    mov v5.h[5], v16.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h2, s17
-; CHECK-NOFP16-NEXT:    mov v4.h[6], v6.h[0]
-; CHECK-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-NOFP16-NEXT:    mov v5.h[6], v2.h[0]
-; CHECK-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-NOFP16-NEXT:    mov v1.16b, v5.16b
-; CHECK-NOFP16-NEXT:    ret
-;
-; CHECK-FP16-LABEL: max_v16f16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fmaxnm v0.8h, v0.8h, v2.8h
-; CHECK-FP16-NEXT:    fmaxnm v1.8h, v1.8h, v3.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: max_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov h4, v2.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h5, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h0
+; CHECK-NOFP16-SD-NEXT:    mov h16, v2.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h18, v3.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h19, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s6, s7, s6
+; CHECK-NOFP16-SD-NEXT:    fcvt s20, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    mov h17, v2.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s5, s5, s4
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt h4, s6
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h3
+; CHECK-NOFP16-SD-NEXT:    mov h21, v0.h[4]
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h17
+; CHECK-NOFP16-SD-NEXT:    fcvt h5, s5
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s18, s19, s18
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    mov h19, v3.h[2]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s16, s20, s16
+; CHECK-NOFP16-SD-NEXT:    mov h20, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s21, h21
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[1], v5.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h5, s7
+; CHECK-NOFP16-SD-NEXT:    mov h7, v2.h[4]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s6, s6, s17
+; CHECK-NOFP16-SD-NEXT:    mov h17, v3.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s20, h20
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[2], v5.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h5, s16
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt h16, s18
+; CHECK-NOFP16-SD-NEXT:    mov h18, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s6
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s7, s21, s7
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[1], v16.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s17, s20, s19
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    mov h19, v3.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov h20, v1.h[4]
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[3], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s7
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s7, s18, s16
+; CHECK-NOFP16-SD-NEXT:    fcvt h16, s17
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h20
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[4], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h19, v1.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[2], v16.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s7
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s7, s18, s17
+; CHECK-NOFP16-SD-NEXT:    mov h16, v2.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v0.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov h18, v3.h[5]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[3], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h17
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h18
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h19
+; CHECK-NOFP16-SD-NEXT:    mov h19, v2.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v2.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[4], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v0.h[6]
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s7, s16, s7
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s16, s18, s17
+; CHECK-NOFP16-SD-NEXT:    mov h17, v3.h[6]
+; CHECK-NOFP16-SD-NEXT:    mov h18, v1.h[6]
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v3.h[7]
+; CHECK-NOFP16-SD-NEXT:    mov h1, v1.h[7]
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h17
+; CHECK-NOFP16-SD-NEXT:    fcvt h7, s7
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s6, s6, s19
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt h16, s16
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s17, s18, s17
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[5], v7.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h6, s6
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s0, s0, s2
+; CHECK-NOFP16-SD-NEXT:    fmaxnm s1, s1, s3
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[5], v16.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s17
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[6], v6.h[0]
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt h1, s1
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[6], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v4.h[7], v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v5.h[7], v1.h[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v4.16b
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v5.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: max_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmaxnm v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    fmaxnm v1.8h, v1.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: max_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v5.4s, v2.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v6.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v5.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fmaxnm v3.4s, v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: max_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmaxnm v0.8h, v0.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    fmaxnm v1.8h, v1.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = call <16 x half> @llvm.maxnum.v16f16(<16 x half> %a, <16 x half> %b)
   ret <16 x half> %c
 }
 
 declare <16 x half> @llvm.maxnum.v16f16(<16 x half>, <16 x half>)
 declare <16 x half> @llvm.minnum.v16f16(<16 x half>, <16 x half>)
 declare <2 x double> @llvm.maxnum.v2f64(<2 x double>, <2 x double>)