[AArch64] implement GPR (U/S)(MIN/MAX) instruction SDag support

[stuij]

Summary

Using SelectionDag, lower umin, umax, smin, smax intrinsics to corresponding
UMIN, UMAX, SMIN, SMAX instructions when feat CSSC is available.

See specs for corresponding immediate and register versions in:
https://developer.arm.com/documentation/ddi0602/2022-09/Base-Instructions/

[ktkachov]

llvm/test/CodeGen/AArch64/min-max.ll

142–150

I'm not very familiar with tests in LLVM, but you shouldn't need CSSC for the vector smax/smin. CSSC only adds GP-register variants.

[stuij]

llvm/test/CodeGen/AArch64/min-max.ll

142–150

Thanks for the comment :)
To be verbose: the CHECK-ISEL-CSSC prefix corresponds to a test run of this file with a particular configuration. This file also includes tests for the vector variants of the instructions and tests those with other configurations (specified by te RUN lines at the top of the file). What we're testing here is that the scalar codegen doesn't interfere with the vector codegen.

[lenary]

One Nit, otherwise I'm happy.

llvm/test/CodeGen/AArch64/min-max.ll

2–4

You might get better test check merging with this instead, which would show the cases where CSSC matters and where it doesn't better.

Most of the reason that GlobalISel isn't merging is because it emits the max/min in the opposite order sometimes :(.

[lenary]

(To be clear, you don't need to act on my nit)

[stuij]

llvm/test/CodeGen/AArch64/min-max.ll

2–4

I'll look into this when the gisel patch for this lands.

[stuij]

solve minor merge conflict

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasCSSC()) {
     setOperationAction(ISD::PARITY, MVT::i128, Expand);
 
     setOperationAction(ISD::CTTZ, MVT::i32, Legal);
     setOperationAction(ISD::CTTZ, MVT::i64, Legal);
     setOperationAction(ISD::CTTZ, MVT::i128, Expand);
 
     setOperationAction(ISD::ABS, MVT::i32, Legal);
     setOperationAction(ISD::ABS, MVT::i64, Legal);
+
+    setOperationAction(ISD::SMAX, MVT::i32, Legal);
+    setOperationAction(ISD::SMAX, MVT::i64, Legal);
+    setOperationAction(ISD::UMAX, MVT::i32, Legal);
+    setOperationAction(ISD::UMAX, MVT::i64, Legal);
+
+    setOperationAction(ISD::SMIN, MVT::i32, Legal);
+    setOperationAction(ISD::SMIN, MVT::i64, Legal);
+    setOperationAction(ISD::UMIN, MVT::i32, Legal);
+    setOperationAction(ISD::UMIN, MVT::i64, Legal);
   } else {
     setOperationAction(ISD::CTPOP, MVT::i32, Custom);
     setOperationAction(ISD::CTPOP, MVT::i64, Custom);
     setOperationAction(ISD::CTPOP, MVT::i128, Custom);
 
     setOperationAction(ISD::PARITY, MVT::i64, Custom);
     setOperationAction(ISD::PARITY, MVT::i128, Custom);
 

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 
 //===----------------------------------------------------------------------===//
 // General Data-Processing Instructions (FEAT_V94_DP)
 //===----------------------------------------------------------------------===//
 defm ABS : OneOperandData<0b001000, "abs", abs>, Requires<[HasCSSC]>;
 defm CNT : OneOperandData<0b000111, "cnt", ctpop>, Requires<[HasCSSC]>;
 defm CTZ : OneOperandData<0b000110, "ctz", cttz>, Requires<[HasCSSC]>;
 
-defm SMAX : ComparisonOp<0, 0, "smax">, Requires<[HasCSSC]>;
-defm SMIN : ComparisonOp<0, 1, "smin">, Requires<[HasCSSC]>;
-defm UMAX : ComparisonOp<1, 0, "umax">, Requires<[HasCSSC]>;
-defm UMIN : ComparisonOp<1, 1, "umin">, Requires<[HasCSSC]>;
+defm SMAX : ComparisonOp<0, 0, "smax", smax>, Requires<[HasCSSC]>;
+defm SMIN : ComparisonOp<0, 1, "smin", smin>, Requires<[HasCSSC]>;
+defm UMAX : ComparisonOp<1, 0, "umax", umax>, Requires<[HasCSSC]>;
+defm UMIN : ComparisonOp<1, 1, "umin", umin>, Requires<[HasCSSC]>;
 
 def RPRFM:
     I<(outs), (ins rprfop:$Rt, GPR64:$Rm, GPR64sp:$Rn),
       "rprfm", "\t$Rt, $Rm, [$Rn]", "", []>,
     Sched<[]> {
   bits<6> Rt;
   bits<5> Rn;
   bits<5> Rm;

llvm/test/CodeGen/AArch64/min-max.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL
+; RUN: llc -mtriple=aarch64-eabi %s -o - -mattr=+cssc | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL-CSSC
 ; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL
 
-
 ; These tests just check that the plumbing is in place for @llvm.smax, @llvm.umax,
 ; @llvm.smin, @llvm.umin.
 
 declare i8 @llvm.smax.i8(i8 %a, i8 %b) readnone
 
 define i8 @smaxi8(i8 %a, i8 %b) {
 ; CHECK-ISEL-LABEL: smaxi8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    sxtb w8, w1
 ; CHECK-ISEL-NEXT:    sxtb w9, w0
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, gt
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smaxi8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    sxtb w8, w1
+; CHECK-ISEL-CSSC-NEXT:    sxtb w9, w0
+; CHECK-ISEL-CSSC-NEXT:    smax w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smaxi8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxtb w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.smax.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.smax.i16(i16 %a, i16 %b) readnone
 
 define i16 @smaxi16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: smaxi16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    sxth w8, w1
 ; CHECK-ISEL-NEXT:    sxth w9, w0
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, gt
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smaxi16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    sxth w8, w1
+; CHECK-ISEL-CSSC-NEXT:    sxth w9, w0
+; CHECK-ISEL-CSSC-NEXT:    smax w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smaxi16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxth w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.smax.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.smax.i32(i32 %a, i32 %b) readnone
 
 define i32 @smaxi32(i32 %a, i32 %b) {
-; CHECK-LABEL: smaxi32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp w0, w1
-; CHECK-NEXT:    csel w0, w0, w1, gt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smaxi32:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp w0, w1
+; CHECK-ISEL-NEXT:    csel w0, w0, w1, gt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: smaxi32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smax w0, w0, w1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smaxi32:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp w0, w1
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, gt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i32 @llvm.smax.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.smax.i64(i64 %a, i64 %b) readnone
 
 define i64 @smaxi64(i64 %a, i64 %b) {
-; CHECK-LABEL: smaxi64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp x0, x1
-; CHECK-NEXT:    csel x0, x0, x1, gt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smaxi64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp x0, x1
+; CHECK-ISEL-NEXT:    csel x0, x0, x1, gt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: smaxi64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smax x0, x0, x1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smaxi64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp x0, x1
+; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, gt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i64 @llvm.smax.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.smax.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @smax8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: smax8i8:

 
 define void @smax32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
 ; CHECK-ISEL-LABEL: smax32i8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    smax v1.16b, v1.16b, v3.16b
 ; CHECK-ISEL-NEXT:    smax v0.16b, v0.16b, v2.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smax32i8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smax v1.16b, v1.16b, v3.16b
+; CHECK-ISEL-CSSC-NEXT:    smax v0.16b, v0.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smax32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smax v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    smax v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <32 x i8> @llvm.smax.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p
   ret void

 define void @smax16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
 ; CHECK-ISEL-LABEL: smax16i16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    smax v1.8h, v1.8h, v3.8h
 ; CHECK-ISEL-NEXT:    smax v0.8h, v0.8h, v2.8h
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smax16i16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smax v1.8h, v1.8h, v3.8h
+; CHECK-ISEL-CSSC-NEXT:    smax v0.8h, v0.8h, v2.8h
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smax16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smax v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    smax v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p

 define void @smax8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
 ; CHECK-ISEL-LABEL: smax8i32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    smax v1.4s, v1.4s, v3.4s
 ; CHECK-ISEL-NEXT:    smax v0.4s, v0.4s, v2.4s
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smax8i32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smax v1.4s, v1.4s, v3.4s
+; CHECK-ISEL-CSSC-NEXT:    smax v0.4s, v0.4s, v2.4s
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smax8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smax v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    smax v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <8 x i32>@llvm.smax.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @smax1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-ISEL-LABEL: smax1i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmgt d2, d0, d1
 ; CHECK-ISEL-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smax1i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmgt d2, d0, d1
+; CHECK-ISEL-CSSC-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smax1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, gt
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.smax.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @smax2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: smax2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmgt v2.2d, v0.2d, v1.2d
-; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smax2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmgt v2.2d, v0.2d, v1.2d
+; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: smax2i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmgt v2.2d, v0.2d, v1.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smax2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmgt v2.2d, v0.2d, v1.2d
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @smax4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: smax4i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmgt v4.2d, v1.2d, v3.2d
 ; CHECK-ISEL-NEXT:    cmgt v5.2d, v0.2d, v2.2d
 ; CHECK-ISEL-NEXT:    bif v1.16b, v3.16b, v4.16b
 ; CHECK-ISEL-NEXT:    bif v0.16b, v2.16b, v5.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smax4i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmgt v4.2d, v1.2d, v3.2d
+; CHECK-ISEL-CSSC-NEXT:    cmgt v5.2d, v0.2d, v2.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v1.16b, v3.16b, v4.16b
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v2.16b, v5.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smax4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmgt v4.2d, v0.2d, v2.2d
 ; CHECK-GLOBAL-NEXT:    cmgt v5.2d, v1.2d, v3.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }
 
 declare i8 @llvm.umax.i8(i8 %a, i8 %b) readnone
 
 define i8 @umaxi8(i8 %a, i8 %b) {
 ; CHECK-ISEL-LABEL: umaxi8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    and w8, w1, #0xff
 ; CHECK-ISEL-NEXT:    and w9, w0, #0xff
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, hi
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umaxi8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    and w8, w1, #0xff
+; CHECK-ISEL-CSSC-NEXT:    and w9, w0, #0xff
+; CHECK-ISEL-CSSC-NEXT:    umax w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umaxi8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.umax.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.umax.i16(i16 %a, i16 %b) readnone
 
 define i16 @umaxi16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: umaxi16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    and w8, w1, #0xffff
 ; CHECK-ISEL-NEXT:    and w9, w0, #0xffff
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, hi
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umaxi16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    and w8, w1, #0xffff
+; CHECK-ISEL-CSSC-NEXT:    and w9, w0, #0xffff
+; CHECK-ISEL-CSSC-NEXT:    umax w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umaxi16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xffff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.umax.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.umax.i32(i32 %a, i32 %b) readnone
 
 define i32 @umaxi32(i32 %a, i32 %b) {
-; CHECK-LABEL: umaxi32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp w0, w1
-; CHECK-NEXT:    csel w0, w0, w1, hi
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umaxi32:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp w0, w1
+; CHECK-ISEL-NEXT:    csel w0, w0, w1, hi
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: umaxi32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umax w0, w0, w1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umaxi32:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp w0, w1
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, hi
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i32 @llvm.umax.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.umax.i64(i64 %a, i64 %b) readnone
 
 define i64 @umaxi64(i64 %a, i64 %b) {
-; CHECK-LABEL: umaxi64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp x0, x1
-; CHECK-NEXT:    csel x0, x0, x1, hi
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umaxi64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp x0, x1
+; CHECK-ISEL-NEXT:    csel x0, x0, x1, hi
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: umaxi64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umax x0, x0, x1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umaxi64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp x0, x1
+; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, hi
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i64 @llvm.umax.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.umax.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @umax8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: umax8i8:

 define void @umax32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
 ; CHECK-ISEL-LABEL: umax32i8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    umax v1.16b, v1.16b, v3.16b
 ; CHECK-ISEL-NEXT:    umax v0.16b, v0.16b, v2.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umax32i8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umax v1.16b, v1.16b, v3.16b
+; CHECK-ISEL-CSSC-NEXT:    umax v0.16b, v0.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umax32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umax v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    umax v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <32 x i8> @llvm.umax.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p

 define void @umax16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
 ; CHECK-ISEL-LABEL: umax16i16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    umax v1.8h, v1.8h, v3.8h
 ; CHECK-ISEL-NEXT:    umax v0.8h, v0.8h, v2.8h
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umax16i16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umax v1.8h, v1.8h, v3.8h
+; CHECK-ISEL-CSSC-NEXT:    umax v0.8h, v0.8h, v2.8h
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umax16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umax v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    umax v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <16 x i16> @llvm.umax.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p

 define void @umax8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
 ; CHECK-ISEL-LABEL: umax8i32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    umax v1.4s, v1.4s, v3.4s
 ; CHECK-ISEL-NEXT:    umax v0.4s, v0.4s, v2.4s
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umax8i32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umax v1.4s, v1.4s, v3.4s
+; CHECK-ISEL-CSSC-NEXT:    umax v0.4s, v0.4s, v2.4s
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umax8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umax v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    umax v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <8 x i32>@llvm.umax.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.umax.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @umax1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-ISEL-LABEL: umax1i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmhi d2, d0, d1
 ; CHECK-ISEL-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umax1i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmhi d2, d0, d1
+; CHECK-ISEL-CSSC-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umax1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, hi
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.umax.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.umax.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @umax2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: umax2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmhi v2.2d, v0.2d, v1.2d
-; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umax2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmhi v2.2d, v0.2d, v1.2d
+; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: umax2i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmhi v2.2d, v0.2d, v1.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umax2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmhi v2.2d, v0.2d, v1.2d
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <2 x i64> @llvm.umax.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.umax.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @umax4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: umax4i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmhi v4.2d, v1.2d, v3.2d
 ; CHECK-ISEL-NEXT:    cmhi v5.2d, v0.2d, v2.2d
 ; CHECK-ISEL-NEXT:    bif v1.16b, v3.16b, v4.16b
 ; CHECK-ISEL-NEXT:    bif v0.16b, v2.16b, v5.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umax4i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmhi v4.2d, v1.2d, v3.2d
+; CHECK-ISEL-CSSC-NEXT:    cmhi v5.2d, v0.2d, v2.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v1.16b, v3.16b, v4.16b
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v2.16b, v5.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umax4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmhi v4.2d, v0.2d, v2.2d
 ; CHECK-GLOBAL-NEXT:    cmhi v5.2d, v1.2d, v3.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }
 
 declare i8 @llvm.smin.i8(i8 %a, i8 %b) readnone
 
 define i8 @smini8(i8 %a, i8 %b) {
 ; CHECK-ISEL-LABEL: smini8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    sxtb w8, w1
 ; CHECK-ISEL-NEXT:    sxtb w9, w0
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, lt
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smini8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    sxtb w8, w1
+; CHECK-ISEL-CSSC-NEXT:    sxtb w9, w0
+; CHECK-ISEL-CSSC-NEXT:    smin w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smini8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxtb w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.smin.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.smin.i16(i16 %a, i16 %b) readnone
 
 define i16 @smini16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: smini16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    sxth w8, w1
 ; CHECK-ISEL-NEXT:    sxth w9, w0
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, lt
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smini16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    sxth w8, w1
+; CHECK-ISEL-CSSC-NEXT:    sxth w9, w0
+; CHECK-ISEL-CSSC-NEXT:    smin w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smini16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxth w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, sxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.smin.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.smin.i32(i32 %a, i32 %b) readnone
 
 define i32 @smini32(i32 %a, i32 %b) {
-; CHECK-LABEL: smini32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp w0, w1
-; CHECK-NEXT:    csel w0, w0, w1, lt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smini32:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp w0, w1
+; CHECK-ISEL-NEXT:    csel w0, w0, w1, lt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: smini32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smin w0, w0, w1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smini32:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp w0, w1
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i32 @llvm.smin.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.smin.i64(i64 %a, i64 %b) readnone
 
 define i64 @smini64(i64 %a, i64 %b) {
-; CHECK-LABEL: smini64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp x0, x1
-; CHECK-NEXT:    csel x0, x0, x1, lt
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smini64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp x0, x1
+; CHECK-ISEL-NEXT:    csel x0, x0, x1, lt
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: smini64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smin x0, x0, x1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smini64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp x0, x1
+; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, lt
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i64 @llvm.smin.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.smin.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @smin8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: smin8i8:

 define void @smin32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
 ; CHECK-ISEL-LABEL: smin32i8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    smin v1.16b, v1.16b, v3.16b
 ; CHECK-ISEL-NEXT:    smin v0.16b, v0.16b, v2.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smin32i8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smin v1.16b, v1.16b, v3.16b
+; CHECK-ISEL-CSSC-NEXT:    smin v0.16b, v0.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smin32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smin v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    smin v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <32 x i8> @llvm.smin.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p

 define void @smin16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
 ; CHECK-ISEL-LABEL: smin16i16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    smin v1.8h, v1.8h, v3.8h
 ; CHECK-ISEL-NEXT:    smin v0.8h, v0.8h, v2.8h
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smin16i16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smin v1.8h, v1.8h, v3.8h
+; CHECK-ISEL-CSSC-NEXT:    smin v0.8h, v0.8h, v2.8h
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smin16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smin v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    smin v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p

 define void @smin8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
 ; CHECK-ISEL-LABEL: smin8i32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    smin v1.4s, v1.4s, v3.4s
 ; CHECK-ISEL-NEXT:    smin v0.4s, v0.4s, v2.4s
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smin8i32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    smin v1.4s, v1.4s, v3.4s
+; CHECK-ISEL-CSSC-NEXT:    smin v0.4s, v0.4s, v2.4s
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smin8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    smin v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    smin v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <8 x i32>@llvm.smin.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.smin.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @smin1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-ISEL-LABEL: smin1i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmgt d2, d1, d0
 ; CHECK-ISEL-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smin1i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmgt d2, d1, d0
+; CHECK-ISEL-CSSC-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smin1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, lt
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.smin.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.smin.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @smin2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: smin2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmgt v2.2d, v1.2d, v0.2d
-; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: smin2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmgt v2.2d, v1.2d, v0.2d
+; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: smin2i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmgt v2.2d, v1.2d, v0.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: smin2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmgt v2.2d, v1.2d, v0.2d
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   %c = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %a, <2 x i64> %b)
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.smin.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @smin4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: smin4i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmgt v4.2d, v3.2d, v1.2d
 ; CHECK-ISEL-NEXT:    cmgt v5.2d, v2.2d, v0.2d
 ; CHECK-ISEL-NEXT:    bif v1.16b, v3.16b, v4.16b
 ; CHECK-ISEL-NEXT:    bif v0.16b, v2.16b, v5.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: smin4i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmgt v4.2d, v3.2d, v1.2d
+; CHECK-ISEL-CSSC-NEXT:    cmgt v5.2d, v2.2d, v0.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v1.16b, v3.16b, v4.16b
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v2.16b, v5.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: smin4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmgt v4.2d, v2.2d, v0.2d
 ; CHECK-GLOBAL-NEXT:    cmgt v5.2d, v3.2d, v1.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }
 
 declare i8 @llvm.umin.i8(i8 %a, i8 %b) readnone
 
 define i8 @umini8(i8 %a, i8 %b) {
 ; CHECK-ISEL-LABEL: umini8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    and w8, w1, #0xff
 ; CHECK-ISEL-NEXT:    and w9, w0, #0xff
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, lo
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umini8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    and w8, w1, #0xff
+; CHECK-ISEL-CSSC-NEXT:    and w9, w0, #0xff
+; CHECK-ISEL-CSSC-NEXT:    umin w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umini8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxtb
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.umin.i8(i8 %a, i8 %b)
   ret i8 %c
 }
 
 declare i16 @llvm.umin.i16(i16 %a, i16 %b) readnone
 
 define i16 @umini16(i16 %a, i16 %b) {
 ; CHECK-ISEL-LABEL: umini16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    and w8, w1, #0xffff
 ; CHECK-ISEL-NEXT:    and w9, w0, #0xffff
 ; CHECK-ISEL-NEXT:    cmp w9, w8
 ; CHECK-ISEL-NEXT:    csel w0, w9, w8, lo
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umini16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    and w8, w1, #0xffff
+; CHECK-ISEL-CSSC-NEXT:    and w9, w0, #0xffff
+; CHECK-ISEL-CSSC-NEXT:    umin w0, w9, w8
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umini16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    and w8, w0, #0xffff
 ; CHECK-GLOBAL-NEXT:    cmp w8, w1, uxth
 ; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.umin.i16(i16 %a, i16 %b)
   ret i16 %c
 }
 
 declare i32 @llvm.umin.i32(i32 %a, i32 %b) readnone
 
 define i32 @umini32(i32 %a, i32 %b) {
-; CHECK-LABEL: umini32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp w0, w1
-; CHECK-NEXT:    csel w0, w0, w1, lo
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umini32:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp w0, w1
+; CHECK-ISEL-NEXT:    csel w0, w0, w1, lo
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: umini32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umin w0, w0, w1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umini32:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp w0, w1
+; CHECK-GLOBAL-NEXT:    csel w0, w0, w1, lo
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i32 @llvm.umin.i32(i32 %a, i32 %b)
   ret i32 %c
 }
 
 declare i64 @llvm.umin.i64(i64 %a, i64 %b) readnone
 
 define i64 @umini64(i64 %a, i64 %b) {
-; CHECK-LABEL: umini64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp x0, x1
-; CHECK-NEXT:    csel x0, x0, x1, lo
-; CHECK-NEXT:    ret
+; CHECK-ISEL-LABEL: umini64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmp x0, x1
+; CHECK-ISEL-NEXT:    csel x0, x0, x1, lo
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: umini64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umin x0, x0, x1
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umini64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmp x0, x1
+; CHECK-GLOBAL-NEXT:    csel x0, x0, x1, lo
+; CHECK-GLOBAL-NEXT:    ret
   %c = call i64 @llvm.umin.i64(i64 %a, i64 %b)
   ret i64 %c
 }
 
 declare <8 x i8> @llvm.umin.v8i8(<8 x i8> %a, <8 x i8> %b) readnone
 
 define <8 x i8> @umin8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: umin8i8:

 define void @umin32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %p) {
 ; CHECK-ISEL-LABEL: umin32i8:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    umin v1.16b, v1.16b, v3.16b
 ; CHECK-ISEL-NEXT:    umin v0.16b, v0.16b, v2.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umin32i8:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umin v1.16b, v1.16b, v3.16b
+; CHECK-ISEL-CSSC-NEXT:    umin v0.16b, v0.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umin32i8:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umin v0.16b, v0.16b, v2.16b
 ; CHECK-GLOBAL-NEXT:    umin v1.16b, v1.16b, v3.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <32 x i8> @llvm.umin.v32i8(<32 x i8> %a, <32 x i8> %b)
   store <32 x i8> %c, <32 x i8>* %p

 define void @umin16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16>* %p) {
 ; CHECK-ISEL-LABEL: umin16i16:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    umin v1.8h, v1.8h, v3.8h
 ; CHECK-ISEL-NEXT:    umin v0.8h, v0.8h, v2.8h
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umin16i16:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umin v1.8h, v1.8h, v3.8h
+; CHECK-ISEL-CSSC-NEXT:    umin v0.8h, v0.8h, v2.8h
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umin16i16:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umin v0.8h, v0.8h, v2.8h
 ; CHECK-GLOBAL-NEXT:    umin v1.8h, v1.8h, v3.8h
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %a, <16 x i16> %b)
   store <16 x i16> %c, <16 x i16>* %p

 define void @umin8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32>* %p) {
 ; CHECK-ISEL-LABEL: umin8i32:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    umin v1.4s, v1.4s, v3.4s
 ; CHECK-ISEL-NEXT:    umin v0.4s, v0.4s, v2.4s
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umin8i32:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    umin v1.4s, v1.4s, v3.4s
+; CHECK-ISEL-CSSC-NEXT:    umin v0.4s, v0.4s, v2.4s
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umin8i32:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    umin v0.4s, v0.4s, v2.4s
 ; CHECK-GLOBAL-NEXT:    umin v1.4s, v1.4s, v3.4s
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <8 x i32>@llvm.umin.v8i32(<8 x i32> %a, <8 x i32> %b)
   store <8 x i32> %c, <8 x i32>* %p
   ret void
 }
 
 declare <1 x i64> @llvm.umin.v1i64(<1 x i64> %a, <1 x i64> %b) readnone
 
 define <1 x i64> @umin1i64(<1 x i64> %a, <1 x i64> %b) {
 ; CHECK-ISEL-LABEL: umin1i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmhi d2, d1, d0
 ; CHECK-ISEL-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umin1i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmhi d2, d1, d0
+; CHECK-ISEL-CSSC-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umin1i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    fmov x8, d0
 ; CHECK-GLOBAL-NEXT:    fmov x9, d1
 ; CHECK-GLOBAL-NEXT:    cmp x8, x9
 ; CHECK-GLOBAL-NEXT:    fcsel d0, d0, d1, lo
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <1 x i64> @llvm.umin.v1i64(<1 x i64> %a, <1 x i64> %b)
   ret <1 x i64> %c
 }
 
 declare <2 x i64> @llvm.umin.v2i64(<2 x i64> %a, <2 x i64> %b) readnone
 
 define <2 x i64> @umin2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: umin2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmhi v2.2d, v1.2d, v0.2d
-; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-NEXT:    ret
-  %c = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %a, <2 x i64> %b)
+; CHECK-ISEL-LABEL: umin2i64:
+; CHECK-ISEL:       // %bb.0:
+; CHECK-ISEL-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-ISEL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-NEXT:    ret
+;
+; CHECK-ISEL-CSSC-LABEL: umin2i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
+; CHECK-GLOBAL-LABEL: umin2i64:
+; CHECK-GLOBAL:       // %bb.0:
+; CHECK-GLOBAL-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-GLOBAL-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GLOBAL-NEXT:    ret
   ret <2 x i64> %c
 }
 
 declare <4 x i64> @llvm.umin.v4i64(<4 x i64> %a, <4 x i64> %b) readnone
 
 define void @umin4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>* %p) {
 ; CHECK-ISEL-LABEL: umin4i64:
 ; CHECK-ISEL:       // %bb.0:
 ; CHECK-ISEL-NEXT:    cmhi v4.2d, v3.2d, v1.2d
 ; CHECK-ISEL-NEXT:    cmhi v5.2d, v2.2d, v0.2d
 ; CHECK-ISEL-NEXT:    bif v1.16b, v3.16b, v4.16b
 ; CHECK-ISEL-NEXT:    bif v0.16b, v2.16b, v5.16b
 ; CHECK-ISEL-NEXT:    stp q0, q1, [x0]
 ; CHECK-ISEL-NEXT:    ret
 ;
+; CHECK-ISEL-CSSC-LABEL: umin4i64:
+; CHECK-ISEL-CSSC:       // %bb.0:
+; CHECK-ISEL-CSSC-NEXT:    cmhi v4.2d, v3.2d, v1.2d
+; CHECK-ISEL-CSSC-NEXT:    cmhi v5.2d, v2.2d, v0.2d
+; CHECK-ISEL-CSSC-NEXT:    bif v1.16b, v3.16b, v4.16b
+; CHECK-ISEL-CSSC-NEXT:    bif v0.16b, v2.16b, v5.16b
+; CHECK-ISEL-CSSC-NEXT:    stp q0, q1, [x0]
+; CHECK-ISEL-CSSC-NEXT:    ret
+;
 ; CHECK-GLOBAL-LABEL: umin4i64:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmhi v4.2d, v2.2d, v0.2d
 ; CHECK-GLOBAL-NEXT:    cmhi v5.2d, v3.2d, v1.2d
 ; CHECK-GLOBAL-NEXT:    bif v0.16b, v2.16b, v4.16b
 ; CHECK-GLOBAL-NEXT:    bif v1.16b, v3.16b, v5.16b
 ; CHECK-GLOBAL-NEXT:    stp q0, q1, [x0]
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %a, <4 x i64> %b)
   store <4 x i64> %c, <4 x i64>* %p
   ret void
 }