[AArch64][GlobalISel] Optimise lowering for some vector types for min/max

[Rin]

Summary

This patch optimises the lowering for some vector types for the min/max instruction.

[dmgreen]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

172

I'm not sure you have to clamp the elements on a v2s64, as they are going to be expanded anyway.

Should it be dealing with vXi16 and vXi8 though?

[Rin]

Add optimisation for vXi16 and vXi8 vector types

[Rin]

clang-format

[Rin]

Make min-max.ll test global-isel

[aemerson]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

171

You can use clampMin/MaxNumElts instead to do these.

[dmgreen]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

171

It looks like clampNumElements is a wrapper around clampMinNumElements(..).clampMaxNumElements(..);

Out of interest, what would make the individual methods better to use themselves? Is it something about how small types are widened?

172

It turns out that G_ICMP doesn't support splitting vectors yet, which is why v2s64 is here too. Otherwise it could be lowered.

(It might be worth added a FIXME to that extent.)

[aemerson]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

171

You're right, I had the which-implements-which relation swapped in my head.

[Rin]

Add FIXME comment

[aemerson]

LGTM.

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder({G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
       .lowerFor({s1, s8, s16, s32, s64});
 
   getActionDefinitionsBuilder({G_SMULO, G_UMULO}).lowerFor({{s64, s1}});
 
   getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64});
 
   getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
       .legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
+      .clampNumElements(0, v8s8, v16s8)
+      .clampNumElements(0, v4s16, v8s16)
+      .clampNumElements(0, v2s32, v4s32)
+      // FIXME: This sholdn't be needed as v2s64 types are going to
+      // be expanded anyway, but G_ICMP doesn't support splitting vectors yet
+      .clampNumElements(0, v2s64, v2s64)
       .lower();
 
   getActionDefinitionsBuilder(
       {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
       .legalFor({{s32, s1}, {s64, s1}})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0);
 

llvm/test/CodeGen/AArch64/GlobalISel/legalize-min-max.mir

bb.0:
     %vec:_(<16 x s8>) = G_IMPLICIT_DEF
     %vec1:_(<16 x s8>) = G_IMPLICIT_DEF
     %smin:_(<16 x s8>) = G_SMIN %vec, %vec1
     $q0 = COPY %smin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v32s8_smin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v32s8_smin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[SMIN:%[0-9]+]]:_(<16 x s8>) = G_SMIN [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[SMIN1:%[0-9]+]]:_(<16 x s8>) = G_SMIN [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[SMIN]](<16 x s8>), [[COPY]](p0) :: (store (<16 x s8>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[SMIN1]](<16 x s8>), [[PTR_ADD]](p0) :: (store (<16 x s8>) into unknown-address + 16)
+    %vec:_(<32 x s8>) = G_IMPLICIT_DEF
+    %vec1:_(<32 x s8>) = G_IMPLICIT_DEF
+    %smin:_(<32 x s8>) = G_SMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smin(<32 x s8>), %1(p0) :: (store (<32 x s8>))
+
+...
+---
 name:            v4s16_smin
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v4s16_smin
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<8 x s16>) = G_IMPLICIT_DEF
     %vec1:_(<8 x s16>) = G_IMPLICIT_DEF
     %smin:_(<8 x s16>) = G_SMIN %vec, %vec1
     $q0 = COPY %smin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v16s16_smin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v16s16_smin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[SMIN:%[0-9]+]]:_(<8 x s16>) = G_SMIN [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[SMIN1:%[0-9]+]]:_(<8 x s16>) = G_SMIN [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[SMIN]](<8 x s16>), [[COPY]](p0) :: (store (<8 x s16>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[SMIN1]](<8 x s16>), [[PTR_ADD]](p0) :: (store (<8 x s16>) into unknown-address + 16)
+    %vec:_(<16 x s16>) = G_IMPLICIT_DEF
+    %vec1:_(<16 x s16>) = G_IMPLICIT_DEF
+    %smin:_(<16 x s16>) = G_SMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smin(<16 x s16>), %1(p0) :: (store (<16 x s16>))
+
+...
+---
 name:            v2s32_smin
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v2s32_smin
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<4 x s32>) = G_IMPLICIT_DEF
     %vec1:_(<4 x s32>) = G_IMPLICIT_DEF
     %smin:_(<4 x s32>) = G_SMIN %vec, %vec1
     $q0 = COPY %smin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v8s32_smin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v8s32_smin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[SMIN:%[0-9]+]]:_(<4 x s32>) = G_SMIN [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[SMIN1:%[0-9]+]]:_(<4 x s32>) = G_SMIN [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[SMIN]](<4 x s32>), [[COPY]](p0) :: (store (<4 x s32>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[SMIN1]](<4 x s32>), [[PTR_ADD]](p0) :: (store (<4 x s32>) into unknown-address + 16)
+    %vec:_(<8 x s32>) = G_IMPLICIT_DEF
+    %vec1:_(<8 x s32>) = G_IMPLICIT_DEF
+    %smin:_(<8 x s32>) = G_SMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smin(<8 x s32>), %1(p0) :: (store (<8 x s32>))
+
+...
+---
 name:            v2s64_smin
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $q0
 
     ; CHECK-LABEL: name: v2s64_smin
     ; CHECK: liveins: $q0
bb.0:
     %vec:_(<2 x s64>) = G_IMPLICIT_DEF
     %vec1:_(<2 x s64>) = G_IMPLICIT_DEF
     %smin:_(<2 x s64>) = G_SMIN %vec, %vec1
     $q0 = COPY %smin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v4s64_smin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v4s64_smin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(slt), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP]](<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR]], [[BUILD_VECTOR1]]
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND1]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(slt), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP1]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR1]]
+    ; CHECK: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
+    ; CHECK: [[OR1:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND2]], [[AND3]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[OR]](<2 x s64>), [[COPY2]](p0) :: (store (<2 x s64>), align 32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s64)
+    ; CHECK: G_STORE [[OR1]](<2 x s64>), [[PTR_ADD]](p0) :: (store (<2 x s64>) into unknown-address + 16)
+    %vec:_(<4 x s64>) = G_IMPLICIT_DEF
+    %vec1:_(<4 x s64>) = G_IMPLICIT_DEF
+    %smin:_(<4 x s64>) = G_SMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smin(<4 x s64>), %1(p0) :: (store (<4 x s64>))
+
+...
+---
 name:            v8s8_umin
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v8s8_umin
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<16 x s8>) = G_IMPLICIT_DEF
     %vec1:_(<16 x s8>) = G_IMPLICIT_DEF
     %umin:_(<16 x s8>) = G_UMIN %vec, %vec1
     $q0 = COPY %umin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v32s8_umin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v32s8_umin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[UMIN:%[0-9]+]]:_(<16 x s8>) = G_UMIN [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[UMIN1:%[0-9]+]]:_(<16 x s8>) = G_UMIN [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[UMIN]](<16 x s8>), [[COPY]](p0) :: (store (<16 x s8>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[UMIN1]](<16 x s8>), [[PTR_ADD]](p0) :: (store (<16 x s8>) into unknown-address + 16)
+    %vec:_(<32 x s8>) = G_IMPLICIT_DEF
+    %vec1:_(<32 x s8>) = G_IMPLICIT_DEF
+    %umin:_(<32 x s8>) = G_UMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umin(<32 x s8>), %1(p0) :: (store (<32 x s8>))
+
+...
+---
 name:            v4s16_umin
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v4s16_umin
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<8 x s16>) = G_IMPLICIT_DEF
     %vec1:_(<8 x s16>) = G_IMPLICIT_DEF
     %umin:_(<8 x s16>) = G_UMIN %vec, %vec1
     $q0 = COPY %umin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v16s16_umin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v16s16_umin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[UMIN:%[0-9]+]]:_(<8 x s16>) = G_UMIN [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[UMIN1:%[0-9]+]]:_(<8 x s16>) = G_UMIN [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[UMIN]](<8 x s16>), [[COPY]](p0) :: (store (<8 x s16>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[UMIN1]](<8 x s16>), [[PTR_ADD]](p0) :: (store (<8 x s16>) into unknown-address + 16)
+    %vec:_(<16 x s16>) = G_IMPLICIT_DEF
+    %vec1:_(<16 x s16>) = G_IMPLICIT_DEF
+    %umin:_(<16 x s16>) = G_UMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umin(<16 x s16>), %1(p0) :: (store (<16 x s16>))
+
+...
+---
 name:            v2s32_umin
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v2s32_umin
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<4 x s32>) = G_IMPLICIT_DEF
     %vec1:_(<4 x s32>) = G_IMPLICIT_DEF
     %umin:_(<4 x s32>) = G_UMIN %vec, %vec1
     $q0 = COPY %umin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v8s32_umin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v8s32_umin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[UMIN:%[0-9]+]]:_(<4 x s32>) = G_UMIN [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[UMIN1:%[0-9]+]]:_(<4 x s32>) = G_UMIN [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[UMIN]](<4 x s32>), [[COPY]](p0) :: (store (<4 x s32>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[UMIN1]](<4 x s32>), [[PTR_ADD]](p0) :: (store (<4 x s32>) into unknown-address + 16)
+    %vec:_(<8 x s32>) = G_IMPLICIT_DEF
+    %vec1:_(<8 x s32>) = G_IMPLICIT_DEF
+    %umin:_(<8 x s32>) = G_UMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umin(<8 x s32>), %1(p0) :: (store (<8 x s32>))
+
+...
+---
 name:            v2s64_umin
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $q0
 
     ; CHECK-LABEL: name: v2s64_umin
     ; CHECK: liveins: $q0
bb.0:
     %vec:_(<2 x s64>) = G_IMPLICIT_DEF
     %vec1:_(<2 x s64>) = G_IMPLICIT_DEF
     %umin:_(<2 x s64>) = G_UMIN %vec, %vec1
     $q0 = COPY %umin
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v4s64_umin
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v4s64_umin
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ult), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP]](<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR]], [[BUILD_VECTOR1]]
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND1]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ult), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP1]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR1]]
+    ; CHECK: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
+    ; CHECK: [[OR1:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND2]], [[AND3]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[OR]](<2 x s64>), [[COPY2]](p0) :: (store (<2 x s64>), align 32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s64)
+    ; CHECK: G_STORE [[OR1]](<2 x s64>), [[PTR_ADD]](p0) :: (store (<2 x s64>) into unknown-address + 16)
+    %vec:_(<4 x s64>) = G_IMPLICIT_DEF
+    %vec1:_(<4 x s64>) = G_IMPLICIT_DEF
+    %umin:_(<4 x s64>) = G_UMIN %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umin(<4 x s64>), %1(p0) :: (store (<4 x s64>))
+
+...
+---
 name:            v8s8_smax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v8s8_smax
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<4 x s16>) = G_IMPLICIT_DEF
     %vec1:_(<4 x s16>) = G_IMPLICIT_DEF
     %smax:_(<4 x s16>) = G_SMAX %vec, %vec1
     $x0 = COPY %smax
     RET_ReallyLR implicit $x0
 
 ...
 ---
+name:            v32s8_smax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v32s8_smax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[SMAX:%[0-9]+]]:_(<16 x s8>) = G_SMAX [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[SMAX1:%[0-9]+]]:_(<16 x s8>) = G_SMAX [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[SMAX]](<16 x s8>), [[COPY]](p0) :: (store (<16 x s8>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[SMAX1]](<16 x s8>), [[PTR_ADD]](p0) :: (store (<16 x s8>) into unknown-address + 16)
+    %vec:_(<32 x s8>) = G_IMPLICIT_DEF
+    %vec1:_(<32 x s8>) = G_IMPLICIT_DEF
+    %smax:_(<32 x s8>) = G_SMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smax(<32 x s8>), %1(p0) :: (store (<32 x s8>))
+
+...
+---
 name:            v8s16_smax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $q0
 
     ; CHECK-LABEL: name: v8s16_smax
     ; CHECK: liveins: $q0
     ; CHECK: %vec:_(<8 x s16>) = G_IMPLICIT_DEF
     ; CHECK: %vec1:_(<8 x s16>) = G_IMPLICIT_DEF
     ; CHECK: %smax:_(<8 x s16>) = G_SMAX %vec, %vec1
     ; CHECK: $q0 = COPY %smax(<8 x s16>)
     ; CHECK: RET_ReallyLR implicit $q0
     %vec:_(<8 x s16>) = G_IMPLICIT_DEF
     %vec1:_(<8 x s16>) = G_IMPLICIT_DEF
     %smax:_(<8 x s16>) = G_SMAX %vec, %vec1
     $q0 = COPY %smax
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v16s16_smax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v16s16_smax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[SMAX:%[0-9]+]]:_(<8 x s16>) = G_SMAX [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[SMAX1:%[0-9]+]]:_(<8 x s16>) = G_SMAX [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[SMAX]](<8 x s16>), [[COPY]](p0) :: (store (<8 x s16>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[SMAX1]](<8 x s16>), [[PTR_ADD]](p0) :: (store (<8 x s16>) into unknown-address + 16)
+    %vec:_(<16 x s16>) = G_IMPLICIT_DEF
+    %vec1:_(<16 x s16>) = G_IMPLICIT_DEF
+    %smax:_(<16 x s16>) = G_SMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smax(<16 x s16>), %1(p0) :: (store (<16 x s16>))
+
+...
+---
 name:            v2s32_smax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v2s32_smax
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<4 x s32>) = G_IMPLICIT_DEF
     %vec1:_(<4 x s32>) = G_IMPLICIT_DEF
     %smax:_(<4 x s32>) = G_SMAX %vec, %vec1
     $q0 = COPY %smax
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v8s32_smax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v8s32_smax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[SMAX:%[0-9]+]]:_(<4 x s32>) = G_SMAX [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[SMAX1:%[0-9]+]]:_(<4 x s32>) = G_SMAX [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[SMAX]](<4 x s32>), [[COPY]](p0) :: (store (<4 x s32>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[SMAX1]](<4 x s32>), [[PTR_ADD]](p0) :: (store (<4 x s32>) into unknown-address + 16)
+    %vec:_(<8 x s32>) = G_IMPLICIT_DEF
+    %vec1:_(<8 x s32>) = G_IMPLICIT_DEF
+    %smax:_(<8 x s32>) = G_SMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smax(<8 x s32>), %1(p0) :: (store (<8 x s32>))
+
+...
+---
 name:            v2s64_smax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $q0
 
     ; CHECK-LABEL: name: v2s64_smax
     ; CHECK: liveins: $q0
bb.0:
     %vec:_(<2 x s64>) = G_IMPLICIT_DEF
     %vec1:_(<2 x s64>) = G_IMPLICIT_DEF
     %smax:_(<2 x s64>) = G_SMAX %vec, %vec1
     $q0 = COPY %smax
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v4s64_smax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v4s64_smax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(sgt), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP]](<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR]], [[BUILD_VECTOR1]]
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND1]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(sgt), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP1]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR1]]
+    ; CHECK: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
+    ; CHECK: [[OR1:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND2]], [[AND3]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[OR]](<2 x s64>), [[COPY2]](p0) :: (store (<2 x s64>), align 32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s64)
+    ; CHECK: G_STORE [[OR1]](<2 x s64>), [[PTR_ADD]](p0) :: (store (<2 x s64>) into unknown-address + 16)
+    %vec:_(<4 x s64>) = G_IMPLICIT_DEF
+    %vec1:_(<4 x s64>) = G_IMPLICIT_DEF
+    %smax:_(<4 x s64>) = G_SMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %smax(<4 x s64>), %1(p0) :: (store (<4 x s64>))
+
+...
+---
 name:            v8s8_umax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v8s8_umax
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<16 x s8>) = G_IMPLICIT_DEF
     %vec1:_(<16 x s8>) = G_IMPLICIT_DEF
     %umax:_(<16 x s8>) = G_UMAX %vec, %vec1
     $q0 = COPY %umax
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v32s8_umax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v32s8_umax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK: [[UMAX:%[0-9]+]]:_(<16 x s8>) = G_UMAX [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[UMAX1:%[0-9]+]]:_(<16 x s8>) = G_UMAX [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[UMAX]](<16 x s8>), [[COPY]](p0) :: (store (<16 x s8>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[UMAX1]](<16 x s8>), [[PTR_ADD]](p0) :: (store (<16 x s8>) into unknown-address + 16)
+    %vec:_(<32 x s8>) = G_IMPLICIT_DEF
+    %vec1:_(<32 x s8>) = G_IMPLICIT_DEF
+    %umax:_(<32 x s8>) = G_UMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umax(<32 x s8>), %1(p0) :: (store (<32 x s8>))
+
+...
+---
 name:            v4s16_umax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v4s16_umax
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<8 x s16>) = G_IMPLICIT_DEF
     %vec1:_(<8 x s16>) = G_IMPLICIT_DEF
     %umax:_(<8 x s16>) = G_UMAX %vec, %vec1
     $q0 = COPY %umax
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v16s16_umax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v16s16_umax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK: [[UMAX:%[0-9]+]]:_(<8 x s16>) = G_UMAX [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[UMAX1:%[0-9]+]]:_(<8 x s16>) = G_UMAX [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[UMAX]](<8 x s16>), [[COPY]](p0) :: (store (<8 x s16>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[UMAX1]](<8 x s16>), [[PTR_ADD]](p0) :: (store (<8 x s16>) into unknown-address + 16)
+    %vec:_(<16 x s16>) = G_IMPLICIT_DEF
+    %vec1:_(<16 x s16>) = G_IMPLICIT_DEF
+    %umax:_(<16 x s16>) = G_UMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umax(<16 x s16>), %1(p0) :: (store (<16 x s16>))
+
+...
+---
 name:            v2s32_umax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: v2s32_umax
     ; CHECK: liveins: $x0
bb.0:
     %vec:_(<4 x s32>) = G_IMPLICIT_DEF
     %vec1:_(<4 x s32>) = G_IMPLICIT_DEF
     %umax:_(<4 x s32>) = G_UMAX %vec, %vec1
     $q0 = COPY %umax
     RET_ReallyLR implicit $q0
 
 ...
 ---
+name:            v8s32_umax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v8s32_umax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
+    ; CHECK: [[UMAX:%[0-9]+]]:_(<4 x s32>) = G_UMAX [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+    ; CHECK: [[UMAX1:%[0-9]+]]:_(<4 x s32>) = G_UMAX [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[UMAX]](<4 x s32>), [[COPY]](p0) :: (store (<4 x s32>), align 32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[UMAX1]](<4 x s32>), [[PTR_ADD]](p0) :: (store (<4 x s32>) into unknown-address + 16)
+    %vec:_(<8 x s32>) = G_IMPLICIT_DEF
+    %vec1:_(<8 x s32>) = G_IMPLICIT_DEF
+    %umax:_(<8 x s32>) = G_UMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umax(<8 x s32>), %1(p0) :: (store (<8 x s32>))
+
+...
+---
 name:            v2s64_umax
 tracksRegLiveness: true
 body: |
   bb.0:
     liveins: $q0
 
     ; CHECK-LABEL: name: v2s64_umax
     ; CHECK: liveins: $q0
bb.0:
     ; CHECK: RET_ReallyLR implicit $q0
     %vec:_(<2 x s64>) = G_IMPLICIT_DEF
     %vec1:_(<2 x s64>) = G_IMPLICIT_DEF
     %umax:_(<2 x s64>) = G_UMAX %vec, %vec1
     $q0 = COPY %umax
     RET_ReallyLR implicit $q0
 
 ...
+---
+name:            v4s64_umax
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $q0, $q1
+
+    ; CHECK-LABEL: name: v4s64_umax
+    ; CHECK: liveins: $x0, $q0, $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ugt), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP]](<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL]], [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR]], [[BUILD_VECTOR1]]
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND1]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ugt), [[DEF]](<2 x s64>), [[DEF]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY [[ICMP1]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(<2 x s64>) = G_SHL [[COPY1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[SHL1]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
+    ; CHECK: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ASHR1]], [[BUILD_VECTOR3]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ASHR1]]
+    ; CHECK: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
+    ; CHECK: [[OR1:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND2]], [[AND3]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: G_STORE [[OR]](<2 x s64>), [[COPY2]](p0) :: (store (<2 x s64>), align 32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s64)
+    ; CHECK: G_STORE [[OR1]](<2 x s64>), [[PTR_ADD]](p0) :: (store (<2 x s64>) into unknown-address + 16)
+    %vec:_(<4 x s64>) = G_IMPLICIT_DEF
+    %vec1:_(<4 x s64>) = G_IMPLICIT_DEF
+    %umax:_(<4 x s64>) = G_UMAX %vec, %vec1
+    %1:_(p0) = COPY $x0
+    G_STORE %umax(<4 x s64>), %1(p0) :: (store (<4 x s64>))
+
+...
 

llvm/test/CodeGen/AArch64/min-max.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL
+; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL
+
 
 ; These tests just check that the plumbing is in place for @llvm.smax, @llvm.umax,
 ; @llvm.smin, @llvm.umin.
 
 declare i8 @llvm.smax.i8(i8 %a, i8 %b) readnone
 
 define i8 @smaxi8(i8 %a, i8 %b) {
-; CHECK-LABEL: smaxi8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sxtb w8, w1
-; CHECK-NEXT:    sxtb w9, w0
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, gt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smaxi8:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    sxtb w8, w1
+; CHECK-ISEL-NEXT:    sxtb w9, w0
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, gt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smaxi8:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    sxtb w8, w0
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxtb
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.smax.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.smax.i16(i16 %a, i16 %b) readnone
 
 define i16 @smaxi16(i16 %a, i16 %b) {
-; CHECK-LABEL: smaxi16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sxth w8, w1
-; CHECK-NEXT:    sxth w9, w0
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, gt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smaxi16:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    sxth w8, w1
+; CHECK-ISEL-NEXT:    sxth w9, w0
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, gt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smaxi16:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    sxth w8, w0
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxth
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.smax.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.smax.i32(i32 %a, i32 %b) readnone
 
 define i32 @smaxi32(i32 %a, i32 %b) {
 ; CHECK-LABEL: smaxi32:

 ; CHECK-LABEL: smax16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %c = call <16 x i8> @llvm.smax.v16i8(<16 x i8> %a, <16 x i8> %b)
   ret <16 x i8> %c
 }
 
+declare <32 x i8> @llvm.smax.v32i8(<32 x i8> %a, <32 x i8> %b) readnone
+
+define void @smax32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
+; CHECK-LABEL: smax32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smax v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    smax v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <32 x i8> @llvm.smax.v32i8(<32 x i8> %a, <32 x i8> %b)
+  store <32 x i8> %c, <32 x i8>* %p
+  ret void
+}
+
 declare <4 x i16> @llvm.smax.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @smax4i16(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: smax4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
   %c = call <4 x i16> @llvm.smax.v4i16(<4 x i16> %a, <4 x i16> %b)
   ret <4 x i16> %c
 }
 
 declare <8 x i16> @llvm.smax.v8i16(<8 x i16> %a, <8 x i16> %b) readnone
 
 define <8 x i16> @smax8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: smax8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %c = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %a, <8 x i16> %b)
   ret <8 x i16> %c
 }
 
+declare <16 x i16> @llvm.smax.v16i16(<16 x i16> %a, <16 x i16> %b) readnone
+
+define void @smax16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
+; CHECK-LABEL: smax16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smax v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    smax v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %a, <16 x i16> %b)
+  store <16 x i16> %c, <16 x i16>* %p
+  ret void
+}
+
 declare <2 x i32> @llvm.smax.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @smax2i32(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: smax2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %c = call <2 x i32> @llvm.smax.v2i32(<2 x i32> %a, <2 x i32> %b)
   ret <2 x i32> %c
 }
 
 declare <4 x i32> @llvm.smax.v4i32(<4 x i32> %a, <4 x i32> %b) readnone
 
 define <4 x i32> @smax4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: smax4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %c = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %a, <4 x i32> %b)
   ret <4 x i32> %c
 }
 
+declare <8 x i32> @llvm.smax.v8i32(<8 x i32> %a, <8 x i32> %b) readnone
+
+define void @smax8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
+; CHECK-LABEL: smax8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smax v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    smax v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <8 x i32>@llvm.smax.v8i32(<8 x i32> %a, <8 x i32> %b)
+  store <8 x i32> %c, <8 x i32>* %p
+  ret void
+}
+
 declare <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @smax1i64(<1 x i64> %a, <1 x i64> %b) {
-; CHECK-LABEL: smax1i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d1
-; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    cmp x9, x8
-; CHECK-NEXT:    csel x8, x9, x8, gt
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smax1i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-ISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-ISEL-NEXT:    fmov x8, d1
+; CHECK-ISEL-NEXT:    fmov x9, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, gt
+; CHECK-ISEL-NEXT:    fmov d0, x8
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smax1i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-NEXT:    cmp x8, x9
+; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, gt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.smax.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @smax2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: smax2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, v1.d[1]
-; CHECK-NEXT:    mov x9, v0.d[1]
-; CHECK-NEXT:    fmov x10, d1
-; CHECK-NEXT:    fmov x11, d0
-; CHECK-NEXT:    cmp x9, x8
-; CHECK-NEXT:    csel x8, x9, x8, gt
-; CHECK-NEXT:    cmp x11, x10
-; CHECK-NEXT:    csel x9, x11, x10, gt
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    mov v0.d[1], x8
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smax2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    mov x8, v1.d[1]
+; CHECK-ISEL-NEXT:    mov x9, v0.d[1]
+; CHECK-ISEL-NEXT:    fmov x10, d1
+; CHECK-ISEL-NEXT:    fmov x11, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, gt
+; CHECK-ISEL-NEXT:    cmp x11, x10
+; CHECK-ISEL-NEXT:    csel x9, x11, x10, gt
+; CHECK-ISEL-NEXT:    fmov d0, x9
+; CHECK-ISEL-NEXT:    mov v0.d[1], x8
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smax2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmgt v2.2d, v0.2d, v1.2d
+; CHECK-GLOBAL-NEXT:    shl v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
+declare <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
+
+define void @smax4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
+; CHECK-ISEL-LABEL: smax4i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    mov x8, v2.d[1]
+; CHECK-ISEL-NEXT:    mov x9, v0.d[1]
+; CHECK-ISEL-NEXT:    fmov x10, d2
+; CHECK-ISEL-NEXT:    fmov x11, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, gt
+; CHECK-ISEL-NEXT:    cmp x11, x10
+; CHECK-ISEL-NEXT:    mov x9, v3.d[1]
+; CHECK-ISEL-NEXT:    csel x10, x11, x10, gt
+; CHECK-ISEL-NEXT:    mov x11, v1.d[1]
+; CHECK-ISEL-NEXT:    cmp x11, x9
+; CHECK-ISEL-NEXT:    fmov d0, x10
+; CHECK-ISEL-NEXT:    fmov x10, d3
+; CHECK-ISEL-NEXT:    csel x9, x11, x9, gt
+; CHECK-ISEL-NEXT:    fmov x11, d1
+; CHECK-ISEL-NEXT:    cmp x11, x10
+; CHECK-ISEL-NEXT:    csel x10, x11, x10, gt
+; CHECK-ISEL-NEXT:    fmov d1, x10
+; CHECK-ISEL-NEXT:    mov v0.d[1], x8
+; CHECK-ISEL-NEXT:    mov v1.d[1], x9
+; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smax4i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-GLOBAL-NEXT:    cmgt v5.2d, v1.2d, v3.2d
+; CHECK-GLOBAL-NEXT:    shl v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    shl v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-NEXT:    ret
+  %c = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %b)
+  store <4 x i64> %c, <4 x i64>* %p
+  ret void
+}
+
 declare i8 @llvm.umax.i8(i8 %a, i8 %b) readnone
 
 define i8 @umaxi8(i8 %a, i8 %b) {
-; CHECK-LABEL: umaxi8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w1, #0xff
-; CHECK-NEXT:    and w9, w0, #0xff
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, hi
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umaxi8:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    and w8, w1, #0xff
+; CHECK-ISEL-NEXT:    and w9, w0, #0xff
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, hi
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umaxi8:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    and w8, w0, #0xff
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxtb
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.umax.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.umax.i16(i16 %a, i16 %b) readnone
 
 define i16 @umaxi16(i16 %a, i16 %b) {
-; CHECK-LABEL: umaxi16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w1, #0xffff
-; CHECK-NEXT:    and w9, w0, #0xffff
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, hi
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umaxi16:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    and w8, w1, #0xffff
+; CHECK-ISEL-NEXT:    and w9, w0, #0xffff
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, hi
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umaxi16:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    and w8, w0, #0xffff
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxth
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.umax.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.umax.i32(i32 %a, i32 %b) readnone
 
 define i32 @umaxi32(i32 %a, i32 %b) {
 ; CHECK-LABEL: umaxi32:

 ; CHECK-LABEL: umax16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %c = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %a, <16 x i8> %b)
   ret <16 x i8> %c
 }
 
+declare <32 x i8> @llvm.umax.v32i8(<32 x i8> %a, <32 x i8> %b) readnone
+
+define void @umax32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
+; CHECK-LABEL: umax32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umax v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    umax v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <32 x i8> @llvm.umax.v32i8(<32 x i8> %a, <32 x i8> %b)
+  store <32 x i8> %c, <32 x i8>* %p
+  ret void
+}
+
 declare <4 x i16> @llvm.umax.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @umax4i16(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: umax4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
   %c = call <4 x i16> @llvm.umax.v4i16(<4 x i16> %a, <4 x i16> %b)
   ret <4 x i16> %c
 }
 
 declare <8 x i16> @llvm.umax.v8i16(<8 x i16> %a, <8 x i16> %b) readnone
 
 define <8 x i16> @umax8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: umax8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %c = call <8 x i16> @llvm.umax.v8i16(<8 x i16> %a, <8 x i16> %b)
   ret <8 x i16> %c
 }
 
+declare <16 x i16> @llvm.umax.v16i16(<16 x i16> %a, <16 x i16> %b) readnone
+
+define void @umax16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
+; CHECK-LABEL: umax16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umax v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    umax v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <16 x i16> @llvm.umax.v16i16(<16 x i16> %a, <16 x i16> %b)
+  store <16 x i16> %c, <16 x i16>* %p
+  ret void
+}
+
 declare <2 x i32> @llvm.umax.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @umax2i32(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: umax2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %c = call <2 x i32> @llvm.umax.v2i32(<2 x i32> %a, <2 x i32> %b)
   ret <2 x i32> %c
 }
 
 declare <4 x i32> @llvm.umax.v4i32(<4 x i32> %a, <4 x i32> %b) readnone
 
 define <4 x i32> @umax4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: umax4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %c = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %a, <4 x i32> %b)
   ret <4 x i32> %c
 }
 
+declare <8 x i32> @llvm.umax.v8i32(<8 x i32> %a, <8 x i32> %b) readnone
+
+define void @umax8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
+; CHECK-LABEL: umax8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umax v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    umax v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <8 x i32>@llvm.umax.v8i32(<8 x i32> %a, <8 x i32> %b)
+  store <8 x i32> %c, <8 x i32>* %p
+  ret void
+}
+
 declare <1 x i64> @llvm.umax.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @umax1i64(<1 x i64> %a, <1 x i64> %b) {
-; CHECK-LABEL: umax1i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d1
-; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    cmp x9, x8
-; CHECK-NEXT:    csel x8, x9, x8, hi
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umax1i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-ISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-ISEL-NEXT:    fmov x8, d1
+; CHECK-ISEL-NEXT:    fmov x9, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, hi
+; CHECK-ISEL-NEXT:    fmov d0, x8
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umax1i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-NEXT:    cmp x8, x9
+; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, hi
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.umax.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.umax.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @umax2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: umax2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uqsub v1.2d, v1.2d, v0.2d
-; CHECK-NEXT:    add v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umax2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    uqsub v1.2d, v1.2d, v0.2d
+; CHECK-ISEL-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umax2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmhi v2.2d, v0.2d, v1.2d
+; CHECK-GLOBAL-NEXT:    shl v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <2 x i64> @llvm.umax.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
+declare <4 x i64> @llvm.umax.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
+
+define void @umax4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
+; CHECK-ISEL-LABEL: umax4i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    uqsub v2.2d, v2.2d, v0.2d
+; CHECK-ISEL-NEXT:    uqsub v3.2d, v3.2d, v1.2d
+; CHECK-ISEL-NEXT:    add v0.2d, v0.2d, v2.2d
+; CHECK-ISEL-NEXT:    add v1.2d, v1.2d, v3.2d
+; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umax4i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmhi v4.2d, v0.2d, v2.2d
+; CHECK-GLOBAL-NEXT:    cmhi v5.2d, v1.2d, v3.2d
+; CHECK-GLOBAL-NEXT:    shl v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    shl v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-NEXT:    ret
+  %c = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %a, <4 x i64> %b)
+  store <4 x i64> %c, <4 x i64>* %p
+  ret void
+}
+
 declare i8 @llvm.smin.i8(i8 %a, i8 %b) readnone
 
 define i8 @smini8(i8 %a, i8 %b) {
-; CHECK-LABEL: smini8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sxtb w8, w1
-; CHECK-NEXT:    sxtb w9, w0
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, lt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smini8:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    sxtb w8, w1
+; CHECK-ISEL-NEXT:    sxtb w9, w0
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, lt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smini8:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    sxtb w8, w0
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxtb
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.smin.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.smin.i16(i16 %a, i16 %b) readnone
 
 define i16 @smini16(i16 %a, i16 %b) {
-; CHECK-LABEL: smini16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sxth w8, w1
-; CHECK-NEXT:    sxth w9, w0
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, lt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smini16:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    sxth w8, w1
+; CHECK-ISEL-NEXT:    sxth w9, w0
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, lt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smini16:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    sxth w8, w0
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxth
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.smin.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.smin.i32(i32 %a, i32 %b) readnone
 
 define i32 @smini32(i32 %a, i32 %b) {
 ; CHECK-LABEL: smini32:

 ; CHECK-LABEL: smin16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %c = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %a, <16 x i8> %b)
   ret <16 x i8> %c
 }
 
+declare <32 x i8> @llvm.smin.v32i8(<32 x i8> %a, <32 x i8> %b) readnone
+
+define void @smin32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
+; CHECK-LABEL: smin32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smin v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    smin v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <32 x i8> @llvm.smin.v32i8(<32 x i8> %a, <32 x i8> %b)
+  store <32 x i8> %c, <32 x i8>* %p
+  ret void
+}
+
 declare <4 x i16> @llvm.smin.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @smin4i16(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: smin4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
   %c = call <4 x i16> @llvm.smin.v4i16(<4 x i16> %a, <4 x i16> %b)
   ret <4 x i16> %c
 }
 
 declare <8 x i16> @llvm.smin.v8i16(<8 x i16> %a, <8 x i16> %b) readnone
 
 define <8 x i16> @smin8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: smin8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %c = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %a, <8 x i16> %b)
   ret <8 x i16> %c
 }
 
+declare <16 x i16> @llvm.smin.v16i16(<16 x i16> %a, <16 x i16> %b) readnone
+
+define void @smin16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
+; CHECK-LABEL: smin16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smin v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    smin v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %a, <16 x i16> %b)
+  store <16 x i16> %c, <16 x i16>* %p
+  ret void
+}
+
 declare <2 x i32> @llvm.smin.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @smin2i32(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: smin2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %c = call <2 x i32> @llvm.smin.v2i32(<2 x i32> %a, <2 x i32> %b)
   ret <2 x i32> %c
 }
 
 declare <4 x i32> @llvm.smin.v4i32(<4 x i32> %a, <4 x i32> %b) readnone
 
 define <4 x i32> @smin4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: smin4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %c = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %a, <4 x i32> %b)
   ret <4 x i32> %c
 }
 
+declare <8 x i32> @llvm.smin.v8i32(<8 x i32> %a, <8 x i32> %b) readnone
+
+define void @smin8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
+; CHECK-LABEL: smin8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smin v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    smin v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <8 x i32>@llvm.smin.v8i32(<8 x i32> %a, <8 x i32> %b)
+  store <8 x i32> %c, <8 x i32>* %p
+  ret void
+}
+
 declare <1 x i64> @llvm.smin.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @smin1i64(<1 x i64> %a, <1 x i64> %b) {
-; CHECK-LABEL: smin1i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d1
-; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    cmp x9, x8
-; CHECK-NEXT:    csel x8, x9, x8, lt
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smin1i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-ISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-ISEL-NEXT:    fmov x8, d1
+; CHECK-ISEL-NEXT:    fmov x9, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, lt
+; CHECK-ISEL-NEXT:    fmov d0, x8
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smin1i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-NEXT:    cmp x8, x9
+; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, lt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.smin.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.smin.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @smin2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: smin2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, v1.d[1]
-; CHECK-NEXT:    mov x9, v0.d[1]
-; CHECK-NEXT:    fmov x10, d1
-; CHECK-NEXT:    fmov x11, d0
-; CHECK-NEXT:    cmp x9, x8
-; CHECK-NEXT:    csel x8, x9, x8, lt
-; CHECK-NEXT:    cmp x11, x10
-; CHECK-NEXT:    csel x9, x11, x10, lt
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    mov v0.d[1], x8
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smin2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    mov x8, v1.d[1]
+; CHECK-ISEL-NEXT:    mov x9, v0.d[1]
+; CHECK-ISEL-NEXT:    fmov x10, d1
+; CHECK-ISEL-NEXT:    fmov x11, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, lt
+; CHECK-ISEL-NEXT:    cmp x11, x10
+; CHECK-ISEL-NEXT:    csel x9, x11, x10, lt
+; CHECK-ISEL-NEXT:    fmov d0, x9
+; CHECK-ISEL-NEXT:    mov v0.d[1], x8
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smin2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmgt v2.2d, v1.2d, v0.2d
+; CHECK-GLOBAL-NEXT:    shl v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
+declare <4 x i64> @llvm.smin.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
+
+define void @smin4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
+; CHECK-ISEL-LABEL: smin4i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    mov x8, v2.d[1]
+; CHECK-ISEL-NEXT:    mov x9, v0.d[1]
+; CHECK-ISEL-NEXT:    fmov x10, d2
+; CHECK-ISEL-NEXT:    fmov x11, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, lt
+; CHECK-ISEL-NEXT:    cmp x11, x10
+; CHECK-ISEL-NEXT:    mov x9, v3.d[1]
+; CHECK-ISEL-NEXT:    csel x10, x11, x10, lt
+; CHECK-ISEL-NEXT:    mov x11, v1.d[1]
+; CHECK-ISEL-NEXT:    cmp x11, x9
+; CHECK-ISEL-NEXT:    fmov d0, x10
+; CHECK-ISEL-NEXT:    fmov x10, d3
+; CHECK-ISEL-NEXT:    csel x9, x11, x9, lt
+; CHECK-ISEL-NEXT:    fmov x11, d1
+; CHECK-ISEL-NEXT:    cmp x11, x10
+; CHECK-ISEL-NEXT:    csel x10, x11, x10, lt
+; CHECK-ISEL-NEXT:    fmov d1, x10
+; CHECK-ISEL-NEXT:    mov v0.d[1], x8
+; CHECK-ISEL-NEXT:    mov v1.d[1], x9
+; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smin4i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-GLOBAL-NEXT:    cmgt v5.2d, v3.2d, v1.2d
+; CHECK-GLOBAL-NEXT:    shl v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    shl v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-NEXT:    ret
+  %c = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %a, <4 x i64> %b)
+  store <4 x i64> %c, <4 x i64>* %p
+  ret void
+}
+
 declare i8 @llvm.umin.i8(i8 %a, i8 %b) readnone
 
 define i8 @umini8(i8 %a, i8 %b) {
-; CHECK-LABEL: umini8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w1, #0xff
-; CHECK-NEXT:    and w9, w0, #0xff
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, lo
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umini8:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    and w8, w1, #0xff
+; CHECK-ISEL-NEXT:    and w9, w0, #0xff
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, lo
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umini8:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    and w8, w0, #0xff
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxtb
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.umin.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.umin.i16(i16 %a, i16 %b) readnone
 
 define i16 @umini16(i16 %a, i16 %b) {
-; CHECK-LABEL: umini16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w1, #0xffff
-; CHECK-NEXT:    and w9, w0, #0xffff
-; CHECK-NEXT:    cmp w9, w8
-; CHECK-NEXT:    csel w0, w9, w8, lo
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umini16:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    and w8, w1, #0xffff
+; CHECK-ISEL-NEXT:    and w9, w0, #0xffff
+; CHECK-ISEL-NEXT:    cmp w9, w8
+; CHECK-ISEL-NEXT:    csel w0, w9, w8, lo
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umini16:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    and w8, w0, #0xffff
+; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxth
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.umin.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.umin.i32(i32 %a, i32 %b) readnone
 
 define i32 @umini32(i32 %a, i32 %b) {
 ; CHECK-LABEL: umini32:

 ; CHECK-LABEL: umin16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %c = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %a, <16 x i8> %b)
   ret <16 x i8> %c
 }
 
+declare <32 x i8> @llvm.umin.v32i8(<32 x i8> %a, <32 x i8> %b) readnone
+
+define void @umin32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
+; CHECK-LABEL: umin32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umin v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    umin v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <32 x i8> @llvm.umin.v32i8(<32 x i8> %a, <32 x i8> %b)
+  store <32 x i8> %c, <32 x i8>* %p
+  ret void
+}
+
 declare <4 x i16> @llvm.umin.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @umin4i16(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: umin4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
   %c = call <4 x i16> @llvm.umin.v4i16(<4 x i16> %a, <4 x i16> %b)
   ret <4 x i16> %c
 }
 
 declare <8 x i16> @llvm.umin.v8i16(<8 x i16> %a, <8 x i16> %b) readnone
 
 define <8 x i16> @umin8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: umin8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %c = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a, <8 x i16> %b)
   ret <8 x i16> %c
 }
 
+declare <16 x i16> @llvm.umin.v16i16(<16 x i16> %a, <16 x i16> %b) readnone
+
+define void @umin16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
+; CHECK-LABEL: umin16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umin v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    umin v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %a, <16 x i16> %b)
+  store <16 x i16> %c, <16 x i16>* %p
+  ret void
+}
+
 declare <2 x i32> @llvm.umin.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @umin2i32(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: umin2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
   %c = call <2 x i32> @llvm.umin.v2i32(<2 x i32> %a, <2 x i32> %b)
   ret <2 x i32> %c
 }
 
 declare <4 x i32> @llvm.umin.v4i32(<4 x i32> %a, <4 x i32> %b) readnone
 
 define <4 x i32> @umin4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: umin4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %c = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %a, <4 x i32> %b)
   ret <4 x i32> %c
 }
 
+declare <8 x i32> @llvm.umin.v8i32(<8 x i32> %a, <8 x i32> %b) readnone
+
+define void @umin8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
+; CHECK-LABEL: umin8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umin v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    umin v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %c = call <8 x i32>@llvm.umin.v8i32(<8 x i32> %a, <8 x i32> %b)
+  store <8 x i32> %c, <8 x i32>* %p
+  ret void
+}
+
 declare <1 x i64> @llvm.umin.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @umin1i64(<1 x i64> %a, <1 x i64> %b) {
-; CHECK-LABEL: umin1i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d1
-; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    cmp x9, x8
-; CHECK-NEXT:    csel x8, x9, x8, lo
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umin1i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-ISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-ISEL-NEXT:    fmov x8, d1
+; CHECK-ISEL-NEXT:    fmov x9, d0
+; CHECK-ISEL-NEXT:    cmp x9, x8
+; CHECK-ISEL-NEXT:    csel x8, x9, x8, lo
+; CHECK-ISEL-NEXT:    fmov d0, x8
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umin1i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-NEXT:    cmp x8, x9
+; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, lo
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.umin.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.umin.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @umin2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: umin2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uqsub v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    sub v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umin2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    uqsub v1.2d, v0.2d, v1.2d
+; CHECK-ISEL-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umin2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-GLOBAL-NEXT:    shl v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v2.2d, v2.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
+
+declare <4 x i64> @llvm.umin.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
+
+define void @umin4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
+; CHECK-ISEL-LABEL: umin4i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    uqsub v2.2d, v0.2d, v2.2d
+; CHECK-ISEL-NEXT:    uqsub v3.2d, v1.2d, v3.2d
+; CHECK-ISEL-NEXT:    sub v0.2d, v0.2d, v2.2d
+; CHECK-ISEL-NEXT:    sub v1.2d, v1.2d, v3.2d
+; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umin4i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmhi v4.2d, v2.2d, v0.2d
+; CHECK-GLOBAL-NEXT:    cmhi v5.2d, v3.2d, v1.2d
+; CHECK-GLOBAL-NEXT:    shl v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    shl v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v4.2d, v4.2d, #63
+; CHECK-GLOBAL-NEXT:    sshr v5.2d, v5.2d, #63
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-NEXT:    ret
+  %c = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %a, <4 x i64> %b)
+  store <4 x i64> %c, <4 x i64>* %p
+  ret void
+}