[AArch64] Add `foldCSELOfCSEl` DAG combine

[samparker]

Maybe this should be done in InstCombine instead?

[Kmeakin]

In D125504#3701875, @samparker wrote:

Maybe this should be done in InstCombine instead?

It may be worth adding a similar transform to InstCombine. However InstCombine cannot catch all instances of CSEL, because some LLVM IR intrinsics (eg llvm.uadd.sat) expand to CSEL during the lowering from LLVM IR to SelectionDAG IR: too late to be handled by InstCombine.

[samparker]

Oh, okay, yeah, I hadn't noticed the types that you're dealing with here!

[samparker]

This looks okay to me.

[vitalybuka]

It fails on this bot: https://reviews.llvm.org/D125504
and some tests timeouts/hangs on this one https://lab.llvm.org/staging/#/builders/224/builds/6

[vitalybuka]

https://lab.llvm.org/staging/#/builders/224/builds/6/steps/10/logs/stdio

affected tests:
llvm/test/tools/dsymutil/X86/dwarf4-linetable.test
llvm/test/tools/dsymutil/X86/odr-fwd-declaration.cpp
llvm/test/tools/dsymutil/X86/generate-empty-CU.test
llvm/test/tools/dsymutil/X86/custom-line-table.test
llvm/test/tools/dsymutil/X86/modules.m
llvm/test/tools/dsymutil/X86/odr-anon-namespace.cpp
llvm/test/tools/dsymutil/X86/modules-empty.m
llvm/test/tools/dsymutil/X86/dwarf5-linetable.test
llvm/test/tools/dsymutil/X86/submodules.m
llvm/test/tools/dsymutil/X86/inlined-static-variable.cpp
llvm/test/tools/dsymutil/ARM/scattered.c
llvm/test/tools/dsymutil/X86/mismatch.m
llvm/test/tools/dsymutil/ARM/fat-arch-not-found.test
llvm/test/tools/dsymutil/X86/dead-stripped.cpp
llvm/test/tools/dsymutil/X86/fat-archive-input-i386.test
llvm/test/tools/dsymutil/X86/empty_range.s
llvm/test/tools/dsymutil/ARM/inlined-low_pc.c
llvm/test/tools/dsymutil/X86/odr-member-functions.cpp
llvm/test/tools/dsymutil/X86/fat-object-input-x86_64.test
llvm/test/tools/dsymutil/X86/fat-object-input-x86_64h.test
llvm/test/tools/dsymutil/X86/modules-pruning.cpp
llvm/test/tools/dsymutil/X86/label.test
llvm/test/tools/dsymutil/X86/odr-fwd-declaration2.cpp
llvm/test/tools/dsymutil/X86/odr-uniquing.cpp

reproducer:

# GCE T2A instance
# 5.15.0-1016-gcp #21-Ubuntu SMP Fri Aug 5 12:20:06 UTC 2022 aarch64 aarch64 aarch64 GNU/Linux
# deps from https://github.com/google/sanitizers/blob/master/buildbot/install_deps.sh

mkdir mybuild && cd mybuild
git clone https://github.com/llvm/llvm-project.git
(
  mkdir llvm_build0 && cd llvm_build0
  cmake -DLLVM_LIT_ARGS=-v -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_CCACHE_BUILD=ON ../llvm-project/llvm
ninja
)


# I don't know if this reproduces without instrumented libc++.
(
  mkdir libcxx_build_ubsan && cd libcxx_build_ubsan
  cmake -DLLVM_LIT_ARGS=-v -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DCMAKE_C_COMPILER=$(readlink -f ../llvm_build0/bin)/clang -DCMAKE_CXX_COMPILER=$(readlink -f ../llvm_build0/bin)/clang++ -DLLVM_USE_LINKER=lld '-DLLVM_ENABLE_PROJECTS=libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Undefined '-DCMAKE_C_FLAGS=-fsanitize=undefined ' '-DCMAKE_CXX_FLAGS=-fsanitize=undefined '  ../llvm-project/llvm
ninja cxx cxxabi
)

(
  mkdir llvm_build_ubsan && cd llvm_build_ubsan
  cmake -DLLVM_LIT_ARGS=-v -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DCMAKE_C_COMPILER=$(readlink -f ..)/llvm_build0/bin/clang -DCMAKE_CXX_COMPILER=$(readlink -f ..)/llvm_build0/bin/clang++ -DLLVM_USE_LINKER=lld '-DLLVM_ENABLE_PROJECTS='\''clang;lld;clang-tools-extra;mlir'\''' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Undefined -DLLVM_ENABLE_LIBCXX=ON '-DCMAKE_C_FLAGS=-nostdinc++ -isystem $(readlink -f ..)/libcxx_build_ubsan/include -isystem $(readlink -f ..)/libcxx_build_ubsan/include/c++/v1 -fsanitize=undefined -lc++abi -Wl,--rpath=$(readlink -f ..)/libcxx_build_ubsan/lib -L$(readlink -f ..)/libcxx_build_ubsan/lib -w' '-DCMAKE_CXX_FLAGS=-nostdinc++ -isystem $(readlink -f ..)/libcxx_build_ubsan/include -isystem $(readlink -f ..)/libcxx_build_ubsan/include/c++/v1 -fsanitize=undefined -lc++abi -Wl,--rpath=$(readlink -f ..)/libcxx_build_ubsan/lib -L$(readlink -f ..)/libcxx_build_ubsan/lib -w' '-DCMAKE_EXE_LINKER_FLAGS=-lc++abi -Wl,--rpath=$(readlink -f ..)/libcxx_build_ubsan/lib -L$(readlink -f ..)/libcxx_build_ubsan/lib' ../llvm-project/llvm
)

git -C llvm-project/ checkout 6f9423ef06926a70af84b77cb290c91214cf791a
ninja -C llvm_build0 && ninja -C llvm_build_ubsan/ -t clean && LIT_FILTER=fat-object-input-x86_64 ninja -C llvm_build_ubsan/ check-llvm
# RESULT: hangs

git -C llvm-project/ checkout 6f9423ef06926a70af84b77cb290c91214cf791a^
ninja -C llvm_build0 && ninja -C llvm_build_ubsan/ -t clean && LIT_FILTER=fat-object-input-x86_64 ninja -C llvm_build_ubsan/ check-llvm
# RESULT: pass
-- Testing: 2 of 44592 tests, 2 workers --
PASS: LLVM :: tools/dsymutil/X86/fat-object-input-x86_64h.test (1 of 2)
PASS: LLVM :: tools/dsymutil/X86/fat-object-input-x86_64.test (2 of 2)

git -C llvm-project/ checkout origin/main
git -C llvm-project/ revert 6f9423ef06926a70af84b77cb290c91214cf791a
ninja -C llvm_build0 && ninja -C llvm_build_ubsan/ -t clean && LIT_FILTER=fat-object-input-x86_64 ninja -C llvm_build_ubsan/ check-llvm
# RESULT: also pass

[mstorsjo]

I also ran into a case where this commit broke things. One of the miscompiled files is https://martin.st/temp/bprint-preproc.c, compiled with clang -target aarch64-linux-gnu bprint-preproc.c -O3, if that helps identifying what the issue is.

[Kmeakin]

In D125504#3728198, @mstorsjo wrote:

I also ran into a case where this commit broke things. One of the miscompiled files is https://martin.st/temp/bprint-preproc.c, compiled with clang -target aarch64-linux-gnu bprint-preproc.c -O3, if that helps identifying what the issue is.

Could you elaborate on what broke? What output were you expecting, and how did it change after this patch?

[mstorsjo]

In D125504#3728916, @Kmeakin wrote:

In D125504#3728198, @mstorsjo wrote:

I also ran into a case where this commit broke things. One of the miscompiled files is https://martin.st/temp/bprint-preproc.c, compiled with clang -target aarch64-linux-gnu bprint-preproc.c -O3, if that helps identifying what the issue is.

Could you elaborate on what broke? What output were you expecting, and how did it change after this patch?

I didn't dive into it in detail to see what actually behaves differently. But it's fairly straightforward to repro it on aarch64-linux like this:

$ git clone git://source.ffmpeg.org/ffmpeg
$ cd ffmpeg
# Build everything with a good clang, run tests which succeed:
$ ./configure --samples=../samples --cc=clang
$ make fate-rsync
$ make -j$(nproc) fate-fits-demux # succeeds
# Update clang in $PATH to one with this patch applied.
# Rebuild libavutil/bprint.o:
$ rm libavutil/bprint.o 
$ make V=1 -j$(nproc) fate-fits-demux # fails (with a segfault)

[Kmeakin]

Make foldCSELOfCSEL more conservative

[mstorsjo]

Thanks, this version seems to work for all the tests that were broken previously.

[chapuni]

I was investigating this.

(CSEL l r EQ (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond)

Could it be said as "Reduction of (CMP(CSEL)), regardless of outer CSEL"?

// Where x and y are constants

I am dubious prerequisites. Is it sufficient?

cc = ((cc2 ? x : y) == x)

This can be transformed to;

cc = (cc2 || y == x)

I am afraid if correlation between x and y would be missed.
I think such a transformation may be applied to (CMP(CSEL(CMP))).

[Kmeakin]

In D125504#3736490, @chapuni wrote:

I was investigating this.

(CSEL l r EQ (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond)

Could it be said as "Reduction of (CMP(CSEL)), regardless of outer CSEL"?

I suppose we could do that. Can you produce any tests that would benefit from a more generalised transform?

// Where x and y are constants

I am dubious prerequisites. Is it sufficient?

cc = ((cc2 ? x : y) == x)

This can be transformed to;

cc = (cc2 || y == x)

I am afraid if correlation between x and y would be missed.
I think such a transformation may be applied to (CMP(CSEL(CMP))).

I am not sure I understand what you mean by this. Could you give an example?

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

unsigned BitWidth = CTTZ.getOpcode() == ISD::TRUNCATE
                           ? CTTZ.getOperand(0).getValueSizeInBits()
                           : CTTZ.getValueSizeInBits();
   SDValue BitWidthMinusOne =
       DAG.getConstant(BitWidth - 1, SDLoc(N), CTTZ.getValueType());
   return DAG.getNode(ISD::AND, SDLoc(N), CTTZ.getValueType(), CTTZ,
                      BitWidthMinusOne);
 }
 
+// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond)
+// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) y)) => (CSEL l r !cc2 cond)
+// Where cc1 is any reflexive relation (eg EQ)
+
+// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) x)) => (CSEL l r !cc2 cond)
+// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) y)) => (CSEL l r cc2 cond)
+// Where cc1 is any irreflexive relation (eg NE)
+static SDValue foldCSELOfCSEL(SDNode *Op, SelectionDAG &DAG) {
+  SDValue L = Op->getOperand(0);
+  SDValue R = Op->getOperand(1);
+  AArch64CC::CondCode OpCC =
+      static_cast<AArch64CC::CondCode>(Op->getConstantOperandVal(2));
+
+  SDValue OpCmp = Op->getOperand(3);
+  if (!isCMP(OpCmp))
+    return SDValue();
+
+  SDValue CmpLHS = OpCmp.getOperand(0);
+  SDValue CmpRHS = OpCmp.getOperand(1);
+
+  if (CmpRHS.getOpcode() == AArch64ISD::CSEL)
+    std::swap(CmpLHS, CmpRHS);
+  else if (CmpLHS.getOpcode() != AArch64ISD::CSEL)
+    return SDValue();
+
+  SDValue X = CmpLHS->getOperand(0);
+  SDValue Y = CmpLHS->getOperand(1);
+  AArch64CC::CondCode CC =
+      static_cast<AArch64CC::CondCode>(CmpLHS->getConstantOperandVal(2));
+  SDValue Cond = CmpLHS->getOperand(3);
+
+  if (CmpRHS == Y)
+    CC = AArch64CC::getInvertedCondCode(CC);
+  else if (CmpRHS != X)
+    return SDValue();
+
+  if (AArch64CC::isIrreflexive(OpCC))
+    CC = AArch64CC::getInvertedCondCode(CC);
+  else if (!AArch64CC::isReflexive(OpCC))
+    return SDValue();
+
+  SDLoc DL(Op);
+  EVT VT = Op->getValueType(0);
+
+  SDValue CCValue = DAG.getConstant(CC, DL, MVT::i32);
+  return DAG.getNode(AArch64ISD::CSEL, DL, VT, L, R, CCValue, Cond);
+}
+
 // Optimize CSEL instructions
 static SDValue performCSELCombine(SDNode *N,
                                   TargetLowering::DAGCombinerInfo &DCI,
                                   SelectionDAG &DAG) {
   // CSEL x, x, cc -> x
   if (N->getOperand(0) == N->getOperand(1))
     return N->getOperand(0);
 
+  if (SDValue R = foldCSELOfCSEL(N, DAG))
+    return R;
+
   // CSEL 0, cttz(X), eq(X, 0) -> AND cttz bitwidth-1
   // CSEL cttz(X), 0, ne(X, 0) -> AND cttz bitwidth-1
   if (SDValue Folded = foldCSELofCTTZ(N, DAG))
     return Folded;
 
   return performCONDCombine(N, DCI, DAG, 2, 3);
 }
 

llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h

inline static unsigned getNZCVToSatisfyCondCode(CondCode Code) {
   case HI: return C; // C == 1 && Z == 0
   case LS: return 0; // C == 0 || Z == 1
   case GE: return 0; // N == V
   case LT: return N; // N != V
   case GT: return 0; // Z == 0 && N == V
   case LE: return Z; // Z == 1 || N != V
   }
 }
+
+/// Return true if Code is a reflexive relationship:
+/// forall x. (CSET Code (CMP x x)) == 1
+inline static bool isReflexive(CondCode Code) {
+  switch (Code) {
+  case EQ:
+  case HS:
+  case PL:
+  case LS:
+  case GE:
+  case LE:
+  case AL:
+  case NV:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/// Return true if Code is an irreflexive relationship:
+/// forall x. (CSET Code (CMP x x)) == 0
+inline static bool isIrreflexive(CondCode Code) {
+  switch (Code) {
+  case NE:
+  case LO:
+  case MI:
+  case HI:
+  case LT:
+  case GT:
+    return true;
+  default:
+    return false;
+  }
+}
+
 } // end namespace AArch64CC
 
 struct SysAlias {
   const char *Name;
   uint16_t Encoding;
   FeatureBitset FeaturesRequired;
 
   constexpr SysAlias(const char *N, uint16_t E) : Name(N), Encoding(E) {}

llvm/test/CodeGen/AArch64/fpclamptosat.ll

 define i64 @ustest_f64i64(double %x) {
 ; CHECK-LABEL: ustest_f64i64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    csel x8, x0, xzr, lt
 ; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    cset w10, ne
+; CHECK-NEXT:    csel x8, x0, xzr, lt
+; CHECK-NEXT:    cset w10, lt
 ; CHECK-NEXT:    cmp x9, #0
 ; CHECK-NEXT:    cset w9, gt
 ; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x0, x8, xzr, ne
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:

 define i64 @ustest_f32i64(float %x) {
 ; CHECK-LABEL: ustest_f32i64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    csel x8, x0, xzr, lt
 ; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    cset w10, ne
+; CHECK-NEXT:    csel x8, x0, xzr, lt
+; CHECK-NEXT:    cset w10, lt
 ; CHECK-NEXT:    cmp x9, #0
 ; CHECK-NEXT:    cset w9, gt
 ; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x0, x8, xzr, ne
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:

 define i64 @ustest_f16i64(half %x) {
 ; CHECK-LABEL: ustest_f16i64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __fixhfti
 ; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    csel x8, x0, xzr, lt
 ; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    cset w10, ne
+; CHECK-NEXT:    csel x8, x0, xzr, lt
+; CHECK-NEXT:    cset w10, lt
 ; CHECK-NEXT:    cmp x9, #0
 ; CHECK-NEXT:    cset w9, gt
 ; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x0, x8, xzr, ne
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:

llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll

 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w30, -32
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    mov d0, v0.d[1]
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    cmp x1, #1
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, lt
 ; CHECK-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-NEXT:    csel x8, x0, xzr, lt
+; CHECK-NEXT:    cset w10, lt
+; CHECK-NEXT:    cmp x9, #0
+; CHECK-NEXT:    cset w9, gt
+; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csel x10, x19, xzr, lt
 ; CHECK-NEXT:    csinc x11, x20, xzr, lt
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    csel x10, x19, xzr, lt
+; CHECK-NEXT:    cset w12, lt
 ; CHECK-NEXT:    cmp x11, #0
 ; CHECK-NEXT:    cset w11, gt
 ; CHECK-NEXT:    csel w11, w12, w11, eq
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    cset w12, ne
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    cset w9, gt
-; CHECK-NEXT:    csel w9, w12, w9, eq
+; CHECK-NEXT:    cmp w11, #0
+; CHECK-NEXT:    csel x10, x10, xzr, ne
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x8, x8, xzr, ne
-; CHECK-NEXT:    cmp w11, #0
-; CHECK-NEXT:    csel x9, x10, xzr, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x8
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
   %spec.store.select = select <2 x i1> %0, <2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>
   %1 = icmp sgt <2 x i128> %spec.store.select, zeroinitializer

 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w30, -32
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    cmp x1, #1
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csinc x8, x1, xzr, lt
-; CHECK-NEXT:    csel x9, x0, xzr, lt
+; CHECK-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-NEXT:    csel x8, x0, xzr, lt
+; CHECK-NEXT:    cset w10, lt
+; CHECK-NEXT:    cmp x9, #0
+; CHECK-NEXT:    cset w9, gt
+; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csel x10, x19, xzr, lt
 ; CHECK-NEXT:    csinc x11, x20, xzr, lt
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    csel x10, x19, xzr, lt
+; CHECK-NEXT:    cset w12, lt
 ; CHECK-NEXT:    cmp x11, #0
 ; CHECK-NEXT:    cset w11, gt
 ; CHECK-NEXT:    csel w11, w12, w11, eq
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    cset w12, ne
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    cset w8, gt
-; CHECK-NEXT:    csel w8, w12, w8, eq
-; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel x8, x9, xzr, ne
 ; CHECK-NEXT:    cmp w11, #0
-; CHECK-NEXT:    csel x9, x10, xzr, ne
+; CHECK-NEXT:    csel x10, x10, xzr, ne
+; CHECK-NEXT:    cmp w9, #0
+; CHECK-NEXT:    csel x8, x8, xzr, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x8
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
   %spec.store.select = select <2 x i1> %0, <2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>
   %1 = icmp sgt <2 x i128> %spec.store.select, zeroinitializer

 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w30, -32
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov h0, v0.h[1]
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
 ; CHECK-NEXT:    bl __fixhfti
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-NEXT:    mov h0, v0.h[1]
 ; CHECK-NEXT:    bl __fixhfti
 ; CHECK-NEXT:    cmp x1, #1
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csinc x8, x1, xzr, lt
-; CHECK-NEXT:    csel x9, x0, xzr, lt
+; CHECK-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-NEXT:    csel x8, x0, xzr, lt
+; CHECK-NEXT:    cset w10, lt
+; CHECK-NEXT:    cmp x9, #0
+; CHECK-NEXT:    cset w9, gt
+; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csel x10, x19, xzr, lt
 ; CHECK-NEXT:    csinc x11, x20, xzr, lt
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    csel x10, x19, xzr, lt
+; CHECK-NEXT:    cset w12, lt
 ; CHECK-NEXT:    cmp x11, #0
 ; CHECK-NEXT:    cset w11, gt
 ; CHECK-NEXT:    csel w11, w12, w11, eq
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    cset w12, ne
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    cset w8, gt
-; CHECK-NEXT:    csel w8, w12, w8, eq
-; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel x8, x9, xzr, ne
 ; CHECK-NEXT:    cmp w11, #0
-; CHECK-NEXT:    csel x9, x10, xzr, ne
+; CHECK-NEXT:    csel x10, x10, xzr, ne
+; CHECK-NEXT:    cmp w9, #0
+; CHECK-NEXT:    csel x8, x8, xzr, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x8
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
   %spec.store.select = select <2 x i1> %0, <2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>
   %1 = icmp sgt <2 x i128> %spec.store.select, zeroinitializer

llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll

 declare i128 @llvm.fptosi.sat.i128.f32(float)
 
 define i1 @test_signed_i1_f32(float %f) nounwind {
 ; CHECK-LABEL: test_signed_i1_f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcvtzs w8, s0
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel w8, w8, wzr, lt
-; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f32(float %f)
     ret i1 %x
 }
 
 define i8 @test_signed_i8_f32(float %f) nounwind {

 declare i128 @llvm.fptosi.sat.i128.f64(double)
 
 define i1 @test_signed_i1_f64(double %f) nounwind {
 ; CHECK-LABEL: test_signed_i1_f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcvtzs w8, d0
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel w8, w8, wzr, lt
-; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f64(double %f)
     ret i1 %x
 }
 
 define i8 @test_signed_i8_f64(double %f) nounwind {

 
 define i1 @test_signed_i1_f16(half %f) nounwind {
 ; CHECK-CVT-LABEL: test_signed_i1_f16:
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    fcvt s0, h0
 ; CHECK-CVT-NEXT:    fcvtzs w8, s0
 ; CHECK-CVT-NEXT:    cmp w8, #0
 ; CHECK-CVT-NEXT:    csel w8, w8, wzr, lt
-; CHECK-CVT-NEXT:    cmp w8, #0
 ; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-CVT-NEXT:    and w0, w8, #0x1
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_signed_i1_f16:
 ; CHECK-FP16:       // %bb.0:
 ; CHECK-FP16-NEXT:    fcvtzs w8, h0
 ; CHECK-FP16-NEXT:    cmp w8, #0
 ; CHECK-FP16-NEXT:    csel w8, w8, wzr, lt
-; CHECK-FP16-NEXT:    cmp w8, #0
 ; CHECK-FP16-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-FP16-NEXT:    and w0, w8, #0x1
 ; CHECK-FP16-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f16(half %f)
     ret i1 %x
 }
 
 define i8 @test_signed_i8_f16(half %f) nounwind {

llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll

 define <2 x i1> @test_signed_v2f64_v2i1(<2 x double> %f) {
 ; CHECK-LABEL: test_signed_v2f64_v2i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov d1, v0.d[1]
 ; CHECK-NEXT:    fcvtzs w9, d0
 ; CHECK-NEXT:    fcvtzs w8, d1
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel w8, w8, wzr, lt
-; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel w9, w9, wzr, lt
-; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csinv w9, w9, wzr, ge
 ; CHECK-NEXT:    fmov s0, w9
 ; CHECK-NEXT:    mov v0.s[1], w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
     %x = call <2 x i1> @llvm.fptosi.sat.v2f64.v2i1(<2 x double> %f)
     ret <2 x i1> %x
 }

 declare <8 x i128> @llvm.fptosi.sat.v8f16.v8i128(<8 x half>)
 
 define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
 ; CHECK-CVT-LABEL: test_signed_v8f16_v8i1:
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v0.8h
 ; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
 ; CHECK-CVT-NEXT:    mov s2, v1.s[1]
+; CHECK-CVT-NEXT:    mov s3, v1.s[2]
 ; CHECK-CVT-NEXT:    fcvtzs w9, s1
+; CHECK-CVT-NEXT:    mov s1, v1.s[3]
 ; CHECK-CVT-NEXT:    fcvtzs w13, s0
 ; CHECK-CVT-NEXT:    fcvtzs w8, s2
-; CHECK-CVT-NEXT:    mov s2, v1.s[2]
-; CHECK-CVT-NEXT:    mov s1, v1.s[3]
+; CHECK-CVT-NEXT:    mov s2, v0.s[1]
+; CHECK-CVT-NEXT:    fcvtzs w10, s3
+; CHECK-CVT-NEXT:    fcvtzs w11, s1
+; CHECK-CVT-NEXT:    mov s1, v0.s[2]
+; CHECK-CVT-NEXT:    mov s0, v0.s[3]
 ; CHECK-CVT-NEXT:    cmp w8, #0
 ; CHECK-CVT-NEXT:    csel w8, w8, wzr, lt
-; CHECK-CVT-NEXT:    fcvtzs w10, s2
-; CHECK-CVT-NEXT:    cmp w8, #0
-; CHECK-CVT-NEXT:    mov s2, v0.s[1]
+; CHECK-CVT-NEXT:    fcvtzs w12, s2
 ; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w9, #0
 ; CHECK-CVT-NEXT:    csel w9, w9, wzr, lt
-; CHECK-CVT-NEXT:    fcvtzs w11, s1
-; CHECK-CVT-NEXT:    cmp w9, #0
-; CHECK-CVT-NEXT:    mov s1, v0.s[2]
 ; CHECK-CVT-NEXT:    csinv w9, w9, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w10, #0
 ; CHECK-CVT-NEXT:    csel w10, w10, wzr, lt
-; CHECK-CVT-NEXT:    fcvtzs w12, s2
-; CHECK-CVT-NEXT:    cmp w10, #0
-; CHECK-CVT-NEXT:    mov s0, v0.s[3]
 ; CHECK-CVT-NEXT:    csinv w10, w10, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w11, #0
 ; CHECK-CVT-NEXT:    csel w11, w11, wzr, lt
 ; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    cmp w11, #0
 ; CHECK-CVT-NEXT:    csinv w11, w11, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w12, #0
 ; CHECK-CVT-NEXT:    csel w12, w12, wzr, lt
-; CHECK-CVT-NEXT:    cmp w12, #0
+; CHECK-CVT-NEXT:    fcvtzs w9, s1
 ; CHECK-CVT-NEXT:    csinv w12, w12, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w13, #0
 ; CHECK-CVT-NEXT:    csel w13, w13, wzr, lt
-; CHECK-CVT-NEXT:    cmp w13, #0
-; CHECK-CVT-NEXT:    csinv w9, w13, wzr, ge
-; CHECK-CVT-NEXT:    fcvtzs w13, s1
+; CHECK-CVT-NEXT:    csinv w13, w13, wzr, ge
+; CHECK-CVT-NEXT:    cmp w9, #0
 ; CHECK-CVT-NEXT:    mov v2.s[1], w8
-; CHECK-CVT-NEXT:    fmov s1, w9
-; CHECK-CVT-NEXT:    cmp w13, #0
-; CHECK-CVT-NEXT:    csel w8, w13, wzr, lt
+; CHECK-CVT-NEXT:    csel w8, w9, wzr, lt
 ; CHECK-CVT-NEXT:    fcvtzs w9, s0
-; CHECK-CVT-NEXT:    cmp w8, #0
-; CHECK-CVT-NEXT:    mov v1.s[1], w12
 ; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-CVT-NEXT:    fmov s1, w13
 ; CHECK-CVT-NEXT:    cmp w9, #0
-; CHECK-CVT-NEXT:    csel w9, w9, wzr, lt
+; CHECK-CVT-NEXT:    mov v1.s[1], w12
 ; CHECK-CVT-NEXT:    mov v2.s[2], w10
-; CHECK-CVT-NEXT:    cmp w9, #0
 ; CHECK-CVT-NEXT:    mov v1.s[2], w8
-; CHECK-CVT-NEXT:    csinv w8, w9, wzr, ge
+; CHECK-CVT-NEXT:    csel w8, w9, wzr, lt
+; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-CVT-NEXT:    mov v2.s[3], w11
 ; CHECK-CVT-NEXT:    mov v1.s[3], w8
 ; CHECK-CVT-NEXT:    uzp1 v0.8h, v1.8h, v2.8h
 ; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_signed_v8f16_v8i1:
 ; CHECK-FP16:       // %bb.0:

llvm/test/CodeGen/AArch64/i128-math.ll

; CHECK-NEXT:    ret
   ret { i128, i8 } %6
 }
 
 define i128 @u128_saturating_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_saturating_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x0, x2
 ; CHECK-NEXT:    adcs x9, x1, x3
-; CHECK-NEXT:    cset w10, hs
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csinv x0, x8, xzr, eq
-; CHECK-NEXT:    csinv x1, x9, xzr, eq
+; CHECK-NEXT:    csinv x0, x8, xzr, lo
+; CHECK-NEXT:    csinv x1, x9, xzr, lo
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.uadd.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @u128_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_sub:
 ; CHECK:       // %bb.0:
; CHECK-NEXT:    ret
   ret { i128, i8 } %6
 }
 
 define i128 @u128_saturating_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_saturating_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x0, x2
 ; CHECK-NEXT:    sbcs x9, x1, x3
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csel x0, xzr, x8, ne
-; CHECK-NEXT:    csel x1, xzr, x9, ne
+; CHECK-NEXT:    csel x0, xzr, x8, lo
+; CHECK-NEXT:    csel x1, xzr, x9, lo
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.usub.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @i128_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_add:
 ; CHECK:       // %bb.0:

 }
 
 define i128 @i128_saturating_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_saturating_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x0, x2
 ; CHECK-NEXT:    adcs x9, x1, x3
 ; CHECK-NEXT:    asr x10, x9, #63
-; CHECK-NEXT:    cset w11, vs
-; CHECK-NEXT:    cmp w11, #0
 ; CHECK-NEXT:    eor x11, x10, #0x8000000000000000
-; CHECK-NEXT:    csel x0, x10, x8, ne
-; CHECK-NEXT:    csel x1, x11, x9, ne
+; CHECK-NEXT:    csel x0, x10, x8, vs
+; CHECK-NEXT:    csel x1, x11, x9, vs
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.sadd.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @i128_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_sub:
 ; CHECK:       // %bb.0:

 }
 
 define i128 @i128_saturating_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_saturating_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x0, x2
 ; CHECK-NEXT:    sbcs x9, x1, x3
 ; CHECK-NEXT:    asr x10, x9, #63
-; CHECK-NEXT:    cset w11, vs
-; CHECK-NEXT:    cmp w11, #0
 ; CHECK-NEXT:    eor x11, x10, #0x8000000000000000
-; CHECK-NEXT:    csel x0, x10, x8, ne
-; CHECK-NEXT:    csel x1, x11, x9, ne
+; CHECK-NEXT:    csel x0, x10, x8, vs
+; CHECK-NEXT:    csel x1, x11, x9, vs
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.ssub.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @u128_mul(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_mul:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/AArch64/i256-math.ll

 
 define i256 @u256_saturating_add(i256 %x, i256 %y) {
 ; CHECK-LABEL: u256_saturating_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x0, x4
 ; CHECK-NEXT:    adcs x9, x1, x5
 ; CHECK-NEXT:    adcs x10, x2, x6
 ; CHECK-NEXT:    adcs x11, x3, x7
-; CHECK-NEXT:    cset w12, hs
-; CHECK-NEXT:    cmp w12, #0
-; CHECK-NEXT:    csinv x0, x8, xzr, eq
-; CHECK-NEXT:    csinv x1, x9, xzr, eq
-; CHECK-NEXT:    csinv x2, x10, xzr, eq
-; CHECK-NEXT:    csinv x3, x11, xzr, eq
+; CHECK-NEXT:    csinv x0, x8, xzr, lo
+; CHECK-NEXT:    csinv x1, x9, xzr, lo
+; CHECK-NEXT:    csinv x2, x10, xzr, lo
+; CHECK-NEXT:    csinv x3, x11, xzr, lo
 ; CHECK-NEXT:    ret
   %1 = tail call i256 @llvm.uadd.sat.i256(i256 %x, i256 %y)
   ret i256 %1
 }
 
 define i256 @u256_sub(i256 %x, i256 %y) {
 ; CHECK-LABEL: u256_sub:
 ; CHECK:       // %bb.0:

 
 define i256 @u256_saturating_sub(i256 %x, i256 %y) {
 ; CHECK-LABEL: u256_saturating_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x0, x4
 ; CHECK-NEXT:    sbcs x9, x1, x5
 ; CHECK-NEXT:    sbcs x10, x2, x6
 ; CHECK-NEXT:    sbcs x11, x3, x7
-; CHECK-NEXT:    cset w12, lo
-; CHECK-NEXT:    cmp w12, #0
-; CHECK-NEXT:    csel x0, xzr, x8, ne
-; CHECK-NEXT:    csel x1, xzr, x9, ne
-; CHECK-NEXT:    csel x2, xzr, x10, ne
-; CHECK-NEXT:    csel x3, xzr, x11, ne
+; CHECK-NEXT:    csel x0, xzr, x8, lo
+; CHECK-NEXT:    csel x1, xzr, x9, lo
+; CHECK-NEXT:    csel x2, xzr, x10, lo
+; CHECK-NEXT:    csel x3, xzr, x11, lo
 ; CHECK-NEXT:    ret
   %1 = tail call i256 @llvm.usub.sat.i256(i256 %x, i256 %y)
   ret i256 %1
 }
 
 define i256 @i256_add(i256 %x, i256 %y) {
 ; CHECK-LABEL: i256_add:
 ; CHECK:       // %bb.0:

 
 define i256 @i256_saturating_add(i256 %x, i256 %y) {
 ; CHECK-LABEL: i256_saturating_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x0, x4
 ; CHECK-NEXT:    adcs x9, x1, x5
 ; CHECK-NEXT:    adcs x10, x2, x6
 ; CHECK-NEXT:    adcs x11, x3, x7
-; CHECK-NEXT:    cset w12, vs
-; CHECK-NEXT:    asr x13, x11, #63
-; CHECK-NEXT:    cmp w12, #0
-; CHECK-NEXT:    csel x0, x13, x8, ne
-; CHECK-NEXT:    eor x8, x13, #0x8000000000000000
-; CHECK-NEXT:    csel x1, x13, x9, ne
-; CHECK-NEXT:    csel x2, x13, x10, ne
-; CHECK-NEXT:    csel x3, x8, x11, ne
+; CHECK-NEXT:    asr x12, x11, #63
+; CHECK-NEXT:    csel x0, x12, x8, vs
+; CHECK-NEXT:    eor x8, x12, #0x8000000000000000
+; CHECK-NEXT:    csel x1, x12, x9, vs
+; CHECK-NEXT:    csel x2, x12, x10, vs
+; CHECK-NEXT:    csel x3, x8, x11, vs
 ; CHECK-NEXT:    ret
   %1 = tail call i256 @llvm.sadd.sat.i256(i256 %x, i256 %y)
   ret i256 %1
 }
 
 define i256 @i256_sub(i256 %x, i256 %y) {
 ; CHECK-LABEL: i256_sub:
 ; CHECK:       // %bb.0:

 
 define i256 @i256_saturating_sub(i256 %x, i256 %y) {
 ; CHECK-LABEL: i256_saturating_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x0, x4
 ; CHECK-NEXT:    sbcs x9, x1, x5
 ; CHECK-NEXT:    sbcs x10, x2, x6
 ; CHECK-NEXT:    sbcs x11, x3, x7
-; CHECK-NEXT:    cset w12, vs
-; CHECK-NEXT:    asr x13, x11, #63
-; CHECK-NEXT:    cmp w12, #0
-; CHECK-NEXT:    csel x0, x13, x8, ne
-; CHECK-NEXT:    eor x8, x13, #0x8000000000000000
-; CHECK-NEXT:    csel x1, x13, x9, ne
-; CHECK-NEXT:    csel x2, x13, x10, ne
-; CHECK-NEXT:    csel x3, x8, x11, ne
+; CHECK-NEXT:    asr x12, x11, #63
+; CHECK-NEXT:    csel x0, x12, x8, vs
+; CHECK-NEXT:    eor x8, x12, #0x8000000000000000
+; CHECK-NEXT:    csel x1, x12, x9, vs
+; CHECK-NEXT:    csel x2, x12, x10, vs
+; CHECK-NEXT:    csel x3, x8, x11, vs
 ; CHECK-NEXT:    ret
   %1 = tail call i256 @llvm.ssub.sat.i256(i256 %x, i256 %y)
   ret i256 %1
 }

llvm/test/CodeGen/AArch64/sadd_sat_vec.ll

; CHECK-NEXT:    ret
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x2, x6
 ; CHECK-NEXT:    adcs x9, x3, x7
-; CHECK-NEXT:    cset w10, vs
-; CHECK-NEXT:    asr x11, x9, #63
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csel x2, x11, x8, ne
-; CHECK-NEXT:    eor x8, x11, #0x8000000000000000
-; CHECK-NEXT:    csel x3, x8, x9, ne
+; CHECK-NEXT:    asr x10, x9, #63
+; CHECK-NEXT:    csel x2, x10, x8, vs
+; CHECK-NEXT:    eor x8, x10, #0x8000000000000000
+; CHECK-NEXT:    csel x3, x8, x9, vs
 ; CHECK-NEXT:    adds x8, x0, x4
 ; CHECK-NEXT:    adcs x9, x1, x5
-; CHECK-NEXT:    cset w10, vs
-; CHECK-NEXT:    asr x11, x9, #63
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    eor x10, x11, #0x8000000000000000
-; CHECK-NEXT:    csel x8, x11, x8, ne
-; CHECK-NEXT:    csel x1, x10, x9, ne
+; CHECK-NEXT:    asr x10, x9, #63
+; CHECK-NEXT:    csel x8, x10, x8, vs
+; CHECK-NEXT:    eor x10, x10, #0x8000000000000000
+; CHECK-NEXT:    csel x1, x10, x9, vs
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/AArch64/ssub_sat_vec.ll

; CHECK-NEXT:    ret
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x2, x6
 ; CHECK-NEXT:    sbcs x9, x3, x7
-; CHECK-NEXT:    cset w10, vs
-; CHECK-NEXT:    asr x11, x9, #63
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csel x2, x11, x8, ne
-; CHECK-NEXT:    eor x8, x11, #0x8000000000000000
-; CHECK-NEXT:    csel x3, x8, x9, ne
+; CHECK-NEXT:    asr x10, x9, #63
+; CHECK-NEXT:    csel x2, x10, x8, vs
+; CHECK-NEXT:    eor x8, x10, #0x8000000000000000
+; CHECK-NEXT:    csel x3, x8, x9, vs
 ; CHECK-NEXT:    subs x8, x0, x4
 ; CHECK-NEXT:    sbcs x9, x1, x5
-; CHECK-NEXT:    cset w10, vs
-; CHECK-NEXT:    asr x11, x9, #63
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    eor x10, x11, #0x8000000000000000
-; CHECK-NEXT:    csel x8, x11, x8, ne
-; CHECK-NEXT:    csel x1, x10, x9, ne
+; CHECK-NEXT:    asr x10, x9, #63
+; CHECK-NEXT:    csel x8, x10, x8, vs
+; CHECK-NEXT:    eor x10, x10, #0x8000000000000000
+; CHECK-NEXT:    csel x1, x10, x9, vs
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/AArch64/uadd_sat_vec.ll

; CHECK-NEXT:    ret
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x2, x6
 ; CHECK-NEXT:    adcs x9, x3, x7
-; CHECK-NEXT:    cset w10, hs
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csinv x2, x8, xzr, eq
-; CHECK-NEXT:    csinv x3, x9, xzr, eq
+; CHECK-NEXT:    csinv x2, x8, xzr, lo
+; CHECK-NEXT:    csinv x3, x9, xzr, lo
 ; CHECK-NEXT:    adds x8, x0, x4
 ; CHECK-NEXT:    adcs x9, x1, x5
-; CHECK-NEXT:    cset w10, hs
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csinv x8, x8, xzr, eq
-; CHECK-NEXT:    csinv x1, x9, xzr, eq
+; CHECK-NEXT:    csinv x8, x8, xzr, lo
+; CHECK-NEXT:    csinv x1, x9, xzr, lo
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/AArch64/usub_sat_vec.ll

; CHECK-NEXT:    ret
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x2, x6
 ; CHECK-NEXT:    sbcs x9, x3, x7
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csel x2, xzr, x8, ne
-; CHECK-NEXT:    csel x3, xzr, x9, ne
+; CHECK-NEXT:    csel x2, xzr, x8, lo
+; CHECK-NEXT:    csel x3, xzr, x9, lo
 ; CHECK-NEXT:    subs x8, x0, x4
 ; CHECK-NEXT:    sbcs x9, x1, x5
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csel x8, xzr, x8, ne
-; CHECK-NEXT:    csel x1, xzr, x9, ne
+; CHECK-NEXT:    csel x8, xzr, x8, lo
+; CHECK-NEXT:    csel x1, xzr, x9, lo
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.usub.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }