[AArch64][GlobalISel] implement GPR (U/S)(MIN/MAX) instr support

[stuij]

Summary

Lower umin, umax, smin, smax intrinsics to corresponding UMIN, UMAX, SMIN, SMAX
instructions when feat CSSC is available.

[stuij]

I have tried to collapse the to check test output for the different run configuration as we discussed on the SelDag ticket for this (D138813). However I could not wrangle the prefixes in a configuration that would make the update script output sensible checks that would cover all 4 cases. Hand-crafting is not a good route I think, especially since even though spelling out all 4 outputs is a bit verbose, it doesn't make things less clear, which should be the more important measure I think.

[paquette]

llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

954 ↗

(On Diff #480446)

Can you add a test for this function in llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp?

Then maybe split this into two patches:

Patch 1: Add minScalarIf
Patch 2: Add the new lowering

[stuij]

spun out minScalarIf definition into it's own patch

[stuij]

llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

954 ↗

(On Diff #480446)

thanks, I made a separate patch for minScalarIf.

[arsenm]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

197–202

Should prefer to check all legal cases first

[stuij]

flip around .legalIf and .minScalarIf in the action definition builder

[arsenm]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

194–202

You can also conditionally add the predicates based on the subtarget feature check. i.e. you don't need to check the predicates inside the lambda, you can append rules for different subtargets.

[stuij]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

194–202

Right. Indeed, I didn't think of that. Reviewing I still prefer the lambda-inlining. Maybe because I'm giddy to finally have an excuse to write lots of lambdas in C++. From your phrasing I think you're also ok with the current setup.

[arsenm]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

194–202

These are called a lot

[stuij]

unlambdify actions

[stuij]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

194–202

If you're happy with this, I'll do the same to the other outstanding patches, lambda's be damned.

[aemerson]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

194–202

I think Matt's right in general, although IMO if it becomes extremely unwieldy for some reason we can embed the checks into the predicates (its ultimately not a big deal with these uncommon operations).

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
getActionDefinitionsBuilder({G_SMULO, G_UMULO})
       .clampScalar(0, s32, s64)
       .lower();
 
   getActionDefinitionsBuilder({G_SMULH, G_UMULH})
       .legalFor({s64, v8s16, v16s8, v4s32})
       .lower();
 
   getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
-      .legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
+      // Making clamping conditional on CSSC extension as without legal types we
+      // lower to CMP which can fold one of the two sxtb's we'd otherwise need
+      // if we detect a type smaller than 32-bit.
+      .minScalarIf([=](const LegalityQuery &Query) {
+                     return HasCSSC;
+                   }, 0, s32)
+      .legalIf([=](const LegalityQuery &Query) {
+                 if (HasCSSC && typeInSet(0, {s32, s64})(Query))
+                   return true;
+                 return typeInSet(0, {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
+                     (Query);
+               })
       .clampNumElements(0, v8s8, v16s8)
       .clampNumElements(0, v4s16, v8s16)
       .clampNumElements(0, v2s32, v4s32)
       // FIXME: This sholdn't be needed as v2s64 types are going to
       // be expanded anyway, but G_ICMP doesn't support splitting vectors yet
       .clampNumElements(0, v2s64, v2s64)
       .lower();
 

llvm/test/CodeGen/AArch64/min-max.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL
 ; RUN: llc -mtriple=aarch64-eabi %s -o - -mattr=+cssc | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL-CSSC
 ; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL
+; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - -mattr=+cssc | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL-CSSC
 
 ; These tests just check that the plumbing is in place for @llvm.smax, @llvm.umax,
 ; @llvm.smin, @llvm.umin.
 
 declare i8 @llvm.smax.i8(i8 %a, i8 %b) readnone
 
 define i8 @smaxi8(i8 %a, i8 %b) {
 ; CHECK-ISEL-LABEL: smaxi8:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smaxi8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxtb w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smaxi8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    sxtb w8, w0
+; CHECK-GLOBAL-CSSC-NEXT:    sxtb w9, w1
+; CHECK-GLOBAL-CSSC-NEXT:    smax w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i8 @llvm.smax.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.smax.i16(i16 %a, i16 %b) readnone
 
 define i16 @smaxi16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: smaxi16:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smaxi16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxth w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smaxi16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    sxth w8, w0
+; CHECK-GLOBAL-CSSC-NEXT:    sxth w9, w1
+; CHECK-GLOBAL-CSSC-NEXT:    smax w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i16 @llvm.smax.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.smax.i32(i32 %a, i32 %b) readnone
 
 define i32 @smaxi32(i32 %a, i32 %b) {
 ; CHECK-ISEL-LABEL: smaxi32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp w0, w1
 ; CHECK-ISEL-NEXT:    csel w0, w0, w1, gt
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: smaxi32:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    smax w0, w0, w1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smaxi32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp w0, w1
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smaxi32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smax w0, w0, w1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i32 @llvm.smax.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.smax.i64(i64 %a, i64 %b) readnone
 
 define i64 @smaxi64(i64 %a, i64 %b) {
 ; CHECK-ISEL-LABEL: smaxi64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp x0, x1
 ; CHECK-ISEL-NEXT:    csel x0, x0, x1, gt
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: smaxi64:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    smax x0, x0, x1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smaxi64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp x0, x1
 ; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, gt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smaxi64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smax x0, x0, x1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i64 @llvm.smax.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.smax.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @smax8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: smax8i8:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smax32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smax v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    smax v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smax32i8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smax v0.16b, v0.16b, v2.16b
+; CHECK-GLOBAL-CSSC-NEXT:    smax v1.16b, v1.16b, v3.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <32 x i8> @llvm.smax.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p
   ret void
 }
 
 declare <4 x i16> @llvm.smax.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @smax4i16(<4 x i16> %a, <4 x i16> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smax16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smax v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    smax v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smax16i16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smax v0.8h, v0.8h, v2.8h
+; CHECK-GLOBAL-CSSC-NEXT:    smax v1.8h, v1.8h, v3.8h
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p
   ret void
 }
 
 declare <2 x i32> @llvm.smax.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @smax2i32(<2 x i32> %a, <2 x i32> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smax8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smax v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    smax v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smax8i32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smax v0.4s, v0.4s, v2.4s
+; CHECK-GLOBAL-CSSC-NEXT:    smax v1.4s, v1.4s, v3.4s
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <8 x i32>@llvm.smax.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @smax1i64(<1 x i64> %a, <1 x i64> %b) {

 ;
 ; CHECK-GLOBAL-LABEL: smax1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, gt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smax1i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-CSSC-NEXT:    smax x8, x8, x9
+; CHECK-GLOBAL-CSSC-NEXT:    fmov d0, x8
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.smax.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @smax2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-ISEL-LABEL: smax2i64:
-; CHECK-ISEL:       // %bb.0:
-; CHECK-ISEL-NEXT:    cmgt v2.2d, v0.2d, v1.2d
-; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-NEXT:    ret
-;
-; CHECK-ISEL-CSSC-LABEL: smax2i64:
-; CHECK-ISEL-CSSC:       // %bb.0:
-; CHECK-ISEL-CSSC-NEXT:    cmgt v2.2d, v0.2d, v1.2d
-; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-CSSC-NEXT:    ret
-;
-; CHECK-GLOBAL-LABEL: smax2i64:
-; CHECK-GLOBAL:       // %bb.0:
-; CHECK-GLOBAL-NEXT:    cmgt v2.2d, v0.2d, v1.2d
-; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-GLOBAL-NEXT:    ret
+; CHECK-LABEL: smax2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v2.2d, v0.2d, v1.2d
+; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
   %c = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @smax4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: smax4i64:

 ; CHECK-GLOBAL-LABEL: smax4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmgt v4.2d, v0.2d, v2.2d
 ; CHECK-GLOBAL-NEXT:    cmgt v5.2d, v1.2d, v3.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smax4i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-GLOBAL-CSSC-NEXT:    cmgt v5.2d, v1.2d, v3.2d
+; CHECK-GLOBAL-CSSC-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-CSSC-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }
 
 declare i8 @llvm.umax.i8(i8 %a, i8 %b) readnone
 
 define i8 @umaxi8(i8 %a, i8 %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umaxi8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umaxi8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    and w8, w0, #0xff
+; CHECK-GLOBAL-CSSC-NEXT:    and w9, w1, #0xff
+; CHECK-GLOBAL-CSSC-NEXT:    umax w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i8 @llvm.umax.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.umax.i16(i16 %a, i16 %b) readnone
 
 define i16 @umaxi16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: umaxi16:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umaxi16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xffff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umaxi16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    and w8, w0, #0xffff
+; CHECK-GLOBAL-CSSC-NEXT:    and w9, w1, #0xffff
+; CHECK-GLOBAL-CSSC-NEXT:    umax w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i16 @llvm.umax.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.umax.i32(i32 %a, i32 %b) readnone
 
 define i32 @umaxi32(i32 %a, i32 %b) {
 ; CHECK-ISEL-LABEL: umaxi32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp w0, w1
 ; CHECK-ISEL-NEXT:    csel w0, w0, w1, hi
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: umaxi32:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    umax w0, w0, w1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umaxi32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp w0, w1
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umaxi32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umax w0, w0, w1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i32 @llvm.umax.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.umax.i64(i64 %a, i64 %b) readnone
 
 define i64 @umaxi64(i64 %a, i64 %b) {
 ; CHECK-ISEL-LABEL: umaxi64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp x0, x1
 ; CHECK-ISEL-NEXT:    csel x0, x0, x1, hi
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: umaxi64:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    umax x0, x0, x1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umaxi64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp x0, x1
 ; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, hi
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umaxi64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umax x0, x0, x1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i64 @llvm.umax.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.umax.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @umax8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: umax8i8:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umax32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umax v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    umax v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umax32i8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umax v0.16b, v0.16b, v2.16b
+; CHECK-GLOBAL-CSSC-NEXT:    umax v1.16b, v1.16b, v3.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <32 x i8> @llvm.umax.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p
   ret void
 }
 
 declare <4 x i16> @llvm.umax.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @umax4i16(<4 x i16> %a, <4 x i16> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umax16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umax v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    umax v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umax16i16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umax v0.8h, v0.8h, v2.8h
+; CHECK-GLOBAL-CSSC-NEXT:    umax v1.8h, v1.8h, v3.8h
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <16 x i16> @llvm.umax.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p
   ret void
 }
 
 declare <2 x i32> @llvm.umax.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @umax2i32(<2 x i32> %a, <2 x i32> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umax8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umax v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    umax v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umax8i32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umax v0.4s, v0.4s, v2.4s
+; CHECK-GLOBAL-CSSC-NEXT:    umax v1.4s, v1.4s, v3.4s
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <8 x i32>@llvm.umax.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.umax.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @umax1i64(<1 x i64> %a, <1 x i64> %b) {

 ;
 ; CHECK-GLOBAL-LABEL: umax1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, hi
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umax1i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-CSSC-NEXT:    umax x8, x8, x9
+; CHECK-GLOBAL-CSSC-NEXT:    fmov d0, x8
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <1 x i64> @llvm.umax.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.umax.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @umax2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-ISEL-LABEL: umax2i64:
-; CHECK-ISEL:       // %bb.0:
-; CHECK-ISEL-NEXT:    cmhi v2.2d, v0.2d, v1.2d
-; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-NEXT:    ret
-;
-; CHECK-ISEL-CSSC-LABEL: umax2i64:
-; CHECK-ISEL-CSSC:       // %bb.0:
-; CHECK-ISEL-CSSC-NEXT:    cmhi v2.2d, v0.2d, v1.2d
-; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-CSSC-NEXT:    ret
-;
-; CHECK-GLOBAL-LABEL: umax2i64:
-; CHECK-GLOBAL:       // %bb.0:
-; CHECK-GLOBAL-NEXT:    cmhi v2.2d, v0.2d, v1.2d
-; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-GLOBAL-NEXT:    ret
+; CHECK-LABEL: umax2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v2.2d, v0.2d, v1.2d
+; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
   %c = call <2 x i64> @llvm.umax.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.umax.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @umax4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: umax4i64:

 ; CHECK-GLOBAL-LABEL: umax4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmhi v4.2d, v0.2d, v2.2d
 ; CHECK-GLOBAL-NEXT:    cmhi v5.2d, v1.2d, v3.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umax4i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    cmhi v4.2d, v0.2d, v2.2d
+; CHECK-GLOBAL-CSSC-NEXT:    cmhi v5.2d, v1.2d, v3.2d
+; CHECK-GLOBAL-CSSC-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-CSSC-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }
 
 declare i8 @llvm.smin.i8(i8 %a, i8 %b) readnone
 
 define i8 @smini8(i8 %a, i8 %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smini8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxtb w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smini8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    sxtb w8, w0
+; CHECK-GLOBAL-CSSC-NEXT:    sxtb w9, w1
+; CHECK-GLOBAL-CSSC-NEXT:    smin w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i8 @llvm.smin.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.smin.i16(i16 %a, i16 %b) readnone
 
 define i16 @smini16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: smini16:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smini16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxth w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smini16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    sxth w8, w0
+; CHECK-GLOBAL-CSSC-NEXT:    sxth w9, w1
+; CHECK-GLOBAL-CSSC-NEXT:    smin w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i16 @llvm.smin.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.smin.i32(i32 %a, i32 %b) readnone
 
 define i32 @smini32(i32 %a, i32 %b) {
 ; CHECK-ISEL-LABEL: smini32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp w0, w1
 ; CHECK-ISEL-NEXT:    csel w0, w0, w1, lt
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: smini32:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    smin w0, w0, w1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smini32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp w0, w1
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smini32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smin w0, w0, w1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i32 @llvm.smin.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.smin.i64(i64 %a, i64 %b) readnone
 
 define i64 @smini64(i64 %a, i64 %b) {
 ; CHECK-ISEL-LABEL: smini64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp x0, x1
 ; CHECK-ISEL-NEXT:    csel x0, x0, x1, lt
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: smini64:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    smin x0, x0, x1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smini64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp x0, x1
 ; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, lt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smini64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smin x0, x0, x1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i64 @llvm.smin.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.smin.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @smin8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: smin8i8:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smin32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smin v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    smin v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smin32i8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smin v0.16b, v0.16b, v2.16b
+; CHECK-GLOBAL-CSSC-NEXT:    smin v1.16b, v1.16b, v3.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <32 x i8> @llvm.smin.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p
   ret void
 }
 
 declare <4 x i16> @llvm.smin.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @smin4i16(<4 x i16> %a, <4 x i16> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smin16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smin v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    smin v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smin16i16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smin v0.8h, v0.8h, v2.8h
+; CHECK-GLOBAL-CSSC-NEXT:    smin v1.8h, v1.8h, v3.8h
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p
   ret void
 }
 
 declare <2 x i32> @llvm.smin.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @smin2i32(<2 x i32> %a, <2 x i32> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: smin8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smin v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    smin v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smin8i32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    smin v0.4s, v0.4s, v2.4s
+; CHECK-GLOBAL-CSSC-NEXT:    smin v1.4s, v1.4s, v3.4s
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <8 x i32>@llvm.smin.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.smin.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @smin1i64(<1 x i64> %a, <1 x i64> %b) {

 ;
 ; CHECK-GLOBAL-LABEL: smin1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, lt
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smin1i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-CSSC-NEXT:    smin x8, x8, x9
+; CHECK-GLOBAL-CSSC-NEXT:    fmov d0, x8
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <1 x i64> @llvm.smin.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.smin.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @smin2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-ISEL-LABEL: smin2i64:
-; CHECK-ISEL:       // %bb.0:
-; CHECK-ISEL-NEXT:    cmgt v2.2d, v1.2d, v0.2d
-; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-NEXT:    ret
-;
-; CHECK-ISEL-CSSC-LABEL: smin2i64:
-; CHECK-ISEL-CSSC:       // %bb.0:
-; CHECK-ISEL-CSSC-NEXT:    cmgt v2.2d, v1.2d, v0.2d
-; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-CSSC-NEXT:    ret
-;
-; CHECK-GLOBAL-LABEL: smin2i64:
-; CHECK-GLOBAL:       // %bb.0:
-; CHECK-GLOBAL-NEXT:    cmgt v2.2d, v1.2d, v0.2d
-; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-GLOBAL-NEXT:    ret
+; CHECK-LABEL: smin2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v2.2d, v1.2d, v0.2d
+; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
   %c = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.smin.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @smin4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: smin4i64:

 ; CHECK-GLOBAL-LABEL: smin4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmgt v4.2d, v2.2d, v0.2d
 ; CHECK-GLOBAL-NEXT:    cmgt v5.2d, v3.2d, v1.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: smin4i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-GLOBAL-CSSC-NEXT:    cmgt v5.2d, v3.2d, v1.2d
+; CHECK-GLOBAL-CSSC-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-CSSC-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }
 
 declare i8 @llvm.umin.i8(i8 %a, i8 %b) readnone
 
 define i8 @umini8(i8 %a, i8 %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umini8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umini8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    and w8, w0, #0xff
+; CHECK-GLOBAL-CSSC-NEXT:    and w9, w1, #0xff
+; CHECK-GLOBAL-CSSC-NEXT:    umin w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i8 @llvm.umin.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.umin.i16(i16 %a, i16 %b) readnone
 
 define i16 @umini16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: umini16:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umini16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xffff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umini16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    and w8, w0, #0xffff
+; CHECK-GLOBAL-CSSC-NEXT:    and w9, w1, #0xffff
+; CHECK-GLOBAL-CSSC-NEXT:    umin w0, w8, w9
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i16 @llvm.umin.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.umin.i32(i32 %a, i32 %b) readnone
 
 define i32 @umini32(i32 %a, i32 %b) {
 ; CHECK-ISEL-LABEL: umini32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp w0, w1
 ; CHECK-ISEL-NEXT:    csel w0, w0, w1, lo
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: umini32:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    umin w0, w0, w1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umini32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp w0, w1
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umini32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umin w0, w0, w1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i32 @llvm.umin.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.umin.i64(i64 %a, i64 %b) readnone
 
 define i64 @umini64(i64 %a, i64 %b) {
 ; CHECK-ISEL-LABEL: umini64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmp x0, x1
 ; CHECK-ISEL-NEXT:    csel x0, x0, x1, lo
 ; CHECK-ISEL-NEXT:    ret
 ;
 ; CHECK-ISEL-CSSC-LABEL: umini64:
 ; CHECK-ISEL-CSSC:       // %bb.0:
 ; CHECK-ISEL-CSSC-NEXT:    umin x0, x0, x1
 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umini64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp x0, x1
 ; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, lo
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umini64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umin x0, x0, x1
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call i64 @llvm.umin.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.umin.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @umin8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: umin8i8:

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umin32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umin v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    umin v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umin32i8:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umin v0.16b, v0.16b, v2.16b
+; CHECK-GLOBAL-CSSC-NEXT:    umin v1.16b, v1.16b, v3.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <32 x i8> @llvm.umin.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p
   ret void
 }
 
 declare <4 x i16> @llvm.umin.v4i16(<4 x i16> %a, <4 x i16> %b) readnone
 
 define <4 x i16> @umin4i16(<4 x i16> %a, <4 x i16> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umin16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umin v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    umin v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umin16i16:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umin v0.8h, v0.8h, v2.8h
+; CHECK-GLOBAL-CSSC-NEXT:    umin v1.8h, v1.8h, v3.8h
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p
   ret void
 }
 
 declare <2 x i32> @llvm.umin.v2i32(<2 x i32> %a, <2 x i32> %b) readnone
 
 define <2 x i32> @umin2i32(<2 x i32> %a, <2 x i32> %b) {

 ; CHECK-ISEL-CSSC-NEXT:    ret
 ;
 ; CHECK-GLOBAL-LABEL: umin8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umin v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    umin v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umin8i32:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    umin v0.4s, v0.4s, v2.4s
+; CHECK-GLOBAL-CSSC-NEXT:    umin v1.4s, v1.4s, v3.4s
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <8 x i32>@llvm.umin.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.umin.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @umin1i64(<1 x i64> %a, <1 x i64> %b) {

 ;
 ; CHECK-GLOBAL-LABEL: umin1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, lo
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umin1i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x8, d0
+; CHECK-GLOBAL-CSSC-NEXT:    fmov x9, d1
+; CHECK-GLOBAL-CSSC-NEXT:    umin x8, x8, x9
+; CHECK-GLOBAL-CSSC-NEXT:    fmov d0, x8
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <1 x i64> @llvm.umin.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.umin.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @umin2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-ISEL-LABEL: umin2i64:
-; CHECK-ISEL:       // %bb.0:
-; CHECK-ISEL-NEXT:    cmhi v2.2d, v1.2d, v0.2d
-; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-NEXT:    ret
-;
-; CHECK-ISEL-CSSC-LABEL: umin2i64:
-; CHECK-ISEL-CSSC:       // %bb.0:
-; CHECK-ISEL-CSSC-NEXT:    cmhi v2.2d, v1.2d, v0.2d
-; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-ISEL-CSSC-NEXT:    ret
-;
-; CHECK-GLOBAL-LABEL: umin2i64:
-; CHECK-GLOBAL:       // %bb.0:
-; CHECK-GLOBAL-NEXT:    cmhi v2.2d, v1.2d, v0.2d
-; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-GLOBAL-NEXT:    ret
+; CHECK-LABEL: umin2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
   %c = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.umin.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @umin4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: umin4i64:
 ; CHECK-ISEL:       // %bb.0:

 ; CHECK-GLOBAL-LABEL: umin4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmhi v4.2d, v2.2d, v0.2d
 ; CHECK-GLOBAL-NEXT:    cmhi v5.2d, v3.2d, v1.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
+;
+; CHECK-GLOBAL-CSSC-LABEL: umin4i64:
+; CHECK-GLOBAL-CSSC:       // %bb.0:
+; CHECK-GLOBAL-CSSC-NEXT:    cmhi v4.2d, v2.2d, v0.2d
+; CHECK-GLOBAL-CSSC-NEXT:    cmhi v5.2d, v3.2d, v1.2d
+; CHECK-GLOBAL-CSSC-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-GLOBAL-CSSC-NEXT:    bif v1.16b, v3.16b, v5.16b
+; CHECK-GLOBAL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-GLOBAL-CSSC-NEXT:    ret
   %c = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }