[AArch64] Add `foldCSELOfCSEl` DAG combine

[samparker]

Maybe this should be done in InstCombine instead?

[Kmeakin]

In D125504#3701875, @samparker wrote:

Maybe this should be done in InstCombine instead?

It may be worth adding a similar transform to InstCombine. However InstCombine cannot catch all instances of CSEL, because some LLVM IR intrinsics (eg llvm.uadd.sat) expand to CSEL during the lowering from LLVM IR to SelectionDAG IR: too late to be handled by InstCombine.

[samparker]

Oh, okay, yeah, I hadn't noticed the types that you're dealing with here!

[samparker]

This looks okay to me.

[vitalybuka]

It fails on this bot: https://reviews.llvm.org/D125504
and some tests timeouts/hangs on this one https://lab.llvm.org/staging/#/builders/224/builds/6

[vitalybuka]

https://lab.llvm.org/staging/#/builders/224/builds/6/steps/10/logs/stdio

affected tests:
llvm/test/tools/dsymutil/X86/dwarf4-linetable.test
llvm/test/tools/dsymutil/X86/odr-fwd-declaration.cpp
llvm/test/tools/dsymutil/X86/generate-empty-CU.test
llvm/test/tools/dsymutil/X86/custom-line-table.test
llvm/test/tools/dsymutil/X86/modules.m
llvm/test/tools/dsymutil/X86/odr-anon-namespace.cpp
llvm/test/tools/dsymutil/X86/modules-empty.m
llvm/test/tools/dsymutil/X86/dwarf5-linetable.test
llvm/test/tools/dsymutil/X86/submodules.m
llvm/test/tools/dsymutil/X86/inlined-static-variable.cpp
llvm/test/tools/dsymutil/ARM/scattered.c
llvm/test/tools/dsymutil/X86/mismatch.m
llvm/test/tools/dsymutil/ARM/fat-arch-not-found.test
llvm/test/tools/dsymutil/X86/dead-stripped.cpp
llvm/test/tools/dsymutil/X86/fat-archive-input-i386.test
llvm/test/tools/dsymutil/X86/empty_range.s
llvm/test/tools/dsymutil/ARM/inlined-low_pc.c
llvm/test/tools/dsymutil/X86/odr-member-functions.cpp
llvm/test/tools/dsymutil/X86/fat-object-input-x86_64.test
llvm/test/tools/dsymutil/X86/fat-object-input-x86_64h.test
llvm/test/tools/dsymutil/X86/modules-pruning.cpp
llvm/test/tools/dsymutil/X86/label.test
llvm/test/tools/dsymutil/X86/odr-fwd-declaration2.cpp
llvm/test/tools/dsymutil/X86/odr-uniquing.cpp

reproducer:

# GCE T2A instance
# 5.15.0-1016-gcp #21-Ubuntu SMP Fri Aug 5 12:20:06 UTC 2022 aarch64 aarch64 aarch64 GNU/Linux
# deps from https://github.com/google/sanitizers/blob/master/buildbot/install_deps.sh

mkdir mybuild && cd mybuild
git clone https://github.com/llvm/llvm-project.git
(
  mkdir llvm_build0 && cd llvm_build0
  cmake -DLLVM_LIT_ARGS=-v -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_CCACHE_BUILD=ON ../llvm-project/llvm
ninja
)


# I don't know if this reproduces without instrumented libc++.
(
  mkdir libcxx_build_ubsan && cd libcxx_build_ubsan
  cmake -DLLVM_LIT_ARGS=-v -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DCMAKE_C_COMPILER=$(readlink -f ../llvm_build0/bin)/clang -DCMAKE_CXX_COMPILER=$(readlink -f ../llvm_build0/bin)/clang++ -DLLVM_USE_LINKER=lld '-DLLVM_ENABLE_PROJECTS=libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Undefined '-DCMAKE_C_FLAGS=-fsanitize=undefined ' '-DCMAKE_CXX_FLAGS=-fsanitize=undefined '  ../llvm-project/llvm
ninja cxx cxxabi
)

(
  mkdir llvm_build_ubsan && cd llvm_build_ubsan
  cmake -DLLVM_LIT_ARGS=-v -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DCMAKE_C_COMPILER=$(readlink -f ..)/llvm_build0/bin/clang -DCMAKE_CXX_COMPILER=$(readlink -f ..)/llvm_build0/bin/clang++ -DLLVM_USE_LINKER=lld '-DLLVM_ENABLE_PROJECTS='\''clang;lld;clang-tools-extra;mlir'\''' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Undefined -DLLVM_ENABLE_LIBCXX=ON '-DCMAKE_C_FLAGS=-nostdinc++ -isystem $(readlink -f ..)/libcxx_build_ubsan/include -isystem $(readlink -f ..)/libcxx_build_ubsan/include/c++/v1 -fsanitize=undefined -lc++abi -Wl,--rpath=$(readlink -f ..)/libcxx_build_ubsan/lib -L$(readlink -f ..)/libcxx_build_ubsan/lib -w' '-DCMAKE_CXX_FLAGS=-nostdinc++ -isystem $(readlink -f ..)/libcxx_build_ubsan/include -isystem $(readlink -f ..)/libcxx_build_ubsan/include/c++/v1 -fsanitize=undefined -lc++abi -Wl,--rpath=$(readlink -f ..)/libcxx_build_ubsan/lib -L$(readlink -f ..)/libcxx_build_ubsan/lib -w' '-DCMAKE_EXE_LINKER_FLAGS=-lc++abi -Wl,--rpath=$(readlink -f ..)/libcxx_build_ubsan/lib -L$(readlink -f ..)/libcxx_build_ubsan/lib' ../llvm-project/llvm
)

git -C llvm-project/ checkout 6f9423ef06926a70af84b77cb290c91214cf791a
ninja -C llvm_build0 && ninja -C llvm_build_ubsan/ -t clean && LIT_FILTER=fat-object-input-x86_64 ninja -C llvm_build_ubsan/ check-llvm
# RESULT: hangs

git -C llvm-project/ checkout 6f9423ef06926a70af84b77cb290c91214cf791a^
ninja -C llvm_build0 && ninja -C llvm_build_ubsan/ -t clean && LIT_FILTER=fat-object-input-x86_64 ninja -C llvm_build_ubsan/ check-llvm
# RESULT: pass
-- Testing: 2 of 44592 tests, 2 workers --
PASS: LLVM :: tools/dsymutil/X86/fat-object-input-x86_64h.test (1 of 2)
PASS: LLVM :: tools/dsymutil/X86/fat-object-input-x86_64.test (2 of 2)

git -C llvm-project/ checkout origin/main
git -C llvm-project/ revert 6f9423ef06926a70af84b77cb290c91214cf791a
ninja -C llvm_build0 && ninja -C llvm_build_ubsan/ -t clean && LIT_FILTER=fat-object-input-x86_64 ninja -C llvm_build_ubsan/ check-llvm
# RESULT: also pass

[mstorsjo]

I also ran into a case where this commit broke things. One of the miscompiled files is https://martin.st/temp/bprint-preproc.c, compiled with clang -target aarch64-linux-gnu bprint-preproc.c -O3, if that helps identifying what the issue is.

[Kmeakin]

In D125504#3728198, @mstorsjo wrote:

I also ran into a case where this commit broke things. One of the miscompiled files is https://martin.st/temp/bprint-preproc.c, compiled with clang -target aarch64-linux-gnu bprint-preproc.c -O3, if that helps identifying what the issue is.

Could you elaborate on what broke? What output were you expecting, and how did it change after this patch?

[mstorsjo]

In D125504#3728916, @Kmeakin wrote:

In D125504#3728198, @mstorsjo wrote:

I also ran into a case where this commit broke things. One of the miscompiled files is https://martin.st/temp/bprint-preproc.c, compiled with clang -target aarch64-linux-gnu bprint-preproc.c -O3, if that helps identifying what the issue is.

Could you elaborate on what broke? What output were you expecting, and how did it change after this patch?

I didn't dive into it in detail to see what actually behaves differently. But it's fairly straightforward to repro it on aarch64-linux like this:

$ git clone git://source.ffmpeg.org/ffmpeg
$ cd ffmpeg
# Build everything with a good clang, run tests which succeed:
$ ./configure --samples=../samples --cc=clang
$ make fate-rsync
$ make -j$(nproc) fate-fits-demux # succeeds
# Update clang in $PATH to one with this patch applied.
# Rebuild libavutil/bprint.o:
$ rm libavutil/bprint.o 
$ make V=1 -j$(nproc) fate-fits-demux # fails (with a segfault)

[Kmeakin]

Make foldCSELOfCSEL more conservative

[mstorsjo]

Thanks, this version seems to work for all the tests that were broken previously.

[chapuni]

I was investigating this.

(CSEL l r EQ (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond)

Could it be said as "Reduction of (CMP(CSEL)), regardless of outer CSEL"?

// Where x and y are constants

I am dubious prerequisites. Is it sufficient?

cc = ((cc2 ? x : y) == x)

This can be transformed to;

cc = (cc2 || y == x)

I am afraid if correlation between x and y would be missed.
I think such a transformation may be applied to (CMP(CSEL(CMP))).

[Kmeakin]

In D125504#3736490, @chapuni wrote:

I was investigating this.

(CSEL l r EQ (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond)

Could it be said as "Reduction of (CMP(CSEL)), regardless of outer CSEL"?

I suppose we could do that. Can you produce any tests that would benefit from a more generalised transform?

// Where x and y are constants

I am dubious prerequisites. Is it sufficient?

cc = ((cc2 ? x : y) == x)

This can be transformed to;

cc = (cc2 || y == x)

I am afraid if correlation between x and y would be missed.
I think such a transformation may be applied to (CMP(CSEL(CMP))).

I am not sure I understand what you mean by this. Could you give an example?

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

unsigned BitWidth = CTTZ.getOpcode() == ISD::TRUNCATE
                           ? CTTZ.getOperand(0).getValueSizeInBits()
                           : CTTZ.getValueSizeInBits();
   SDValue BitWidthMinusOne =
       DAG.getConstant(BitWidth - 1, SDLoc(N), CTTZ.getValueType());
   return DAG.getNode(ISD::AND, SDLoc(N), CTTZ.getValueType(), CTTZ,
                      BitWidthMinusOne);
 }
 
-// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond)
-// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) y)) => (CSEL l r !cc2 cond)
-// Where cc1 is any reflexive relation (eg EQ)
-
-// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) x)) => (CSEL l r !cc2 cond)
-// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) y)) => (CSEL l r cc2 cond)
-// Where cc1 is any irreflexive relation (eg NE)
+// (CSEL l r EQ (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond)
+// (CSEL l r EQ (CMP (CSEL x y cc2 cond) y)) => (CSEL l r !cc2 cond)
+// Where x and y are constants
+
+// (CSEL l r NE (CMP (CSEL x y cc2 cond) x)) => (CSEL l r !cc2 cond)
+// (CSEL l r NE (CMP (CSEL x y cc2 cond) y)) => (CSEL l r cc2 cond)
+// Where x and y are constants
 static SDValue foldCSELOfCSEL(SDNode *Op, SelectionDAG &DAG) {
   SDValue L = Op->getOperand(0);
   SDValue R = Op->getOperand(1);
   AArch64CC::CondCode OpCC =
       static_cast<AArch64CC::CondCode>(Op->getConstantOperandVal(2));
 
   SDValue OpCmp = Op->getOperand(3);
   if (!isCMP(OpCmp))
     return SDValue();
 
   SDValue CmpLHS = OpCmp.getOperand(0);
   SDValue CmpRHS = OpCmp.getOperand(1);
 
   if (CmpRHS.getOpcode() == AArch64ISD::CSEL)
     std::swap(CmpLHS, CmpRHS);
   else if (CmpLHS.getOpcode() != AArch64ISD::CSEL)
     return SDValue();
 
   SDValue X = CmpLHS->getOperand(0);
   SDValue Y = CmpLHS->getOperand(1);
+  if (!isa<ConstantSDNode>(X) || !isa<ConstantSDNode>(Y)) {
+    return SDValue();
+  }
+
   AArch64CC::CondCode CC =
       static_cast<AArch64CC::CondCode>(CmpLHS->getConstantOperandVal(2));
   SDValue Cond = CmpLHS->getOperand(3);
 
   if (CmpRHS == Y)
     CC = AArch64CC::getInvertedCondCode(CC);
   else if (CmpRHS != X)
     return SDValue();
 
-  if (AArch64CC::isIrreflexive(OpCC))
+  if (OpCC == AArch64CC::NE)
     CC = AArch64CC::getInvertedCondCode(CC);
-  else if (!AArch64CC::isReflexive(OpCC))
+  else if (OpCC != AArch64CC::EQ)
     return SDValue();
 
   SDLoc DL(Op);
   EVT VT = Op->getValueType(0);
 
   SDValue CCValue = DAG.getConstant(CC, DL, MVT::i32);
   return DAG.getNode(AArch64ISD::CSEL, DL, VT, L, R, CCValue, Cond);
 }

llvm/test/CodeGen/AArch64/fpclamptosat.ll

 define i64 @ustest_f64i64(double %x) {
 ; CHECK-LABEL: ustest_f64i64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
 ; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    cset w10, lt
+; CHECK-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w10, ne
 ; CHECK-NEXT:    cmp x9, #0
 ; CHECK-NEXT:    cset w9, gt
 ; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x0, x8, xzr, ne
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:

 define i64 @ustest_f32i64(float %x) {
 ; CHECK-LABEL: ustest_f32i64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
 ; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    cset w10, lt
+; CHECK-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w10, ne
 ; CHECK-NEXT:    cmp x9, #0
 ; CHECK-NEXT:    cset w9, gt
 ; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x0, x8, xzr, ne
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:

 define i64 @ustest_f16i64(half %x) {
 ; CHECK-LABEL: ustest_f16i64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __fixhfti
 ; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
 ; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    cset w10, lt
+; CHECK-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w10, ne
 ; CHECK-NEXT:    cmp x9, #0
 ; CHECK-NEXT:    cset w9, gt
 ; CHECK-NEXT:    csel w9, w10, w9, eq
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x0, x8, xzr, ne
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:

llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll

 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w30, -32
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    mov d0, v0.d[1]
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    bl __fixdfti
 ; CHECK-NEXT:    cmp x1, #1
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
 ; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    cset w10, lt
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    cset w9, gt
-; CHECK-NEXT:    csel w9, w10, w9, eq
+; CHECK-NEXT:    csinc x9, x1, xzr, lt
 ; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csinc x11, x20, xzr, lt
 ; CHECK-NEXT:    csel x10, x19, xzr, lt
-; CHECK-NEXT:    cset w12, lt
+; CHECK-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-NEXT:    cmp x10, #0
+; CHECK-NEXT:    cset w12, ne
 ; CHECK-NEXT:    cmp x11, #0
 ; CHECK-NEXT:    cset w11, gt
 ; CHECK-NEXT:    csel w11, w12, w11, eq
-; CHECK-NEXT:    cmp w11, #0
-; CHECK-NEXT:    csel x10, x10, xzr, ne
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    cmp x9, #0
+; CHECK-NEXT:    cset w9, gt
+; CHECK-NEXT:    csel w9, w12, w9, eq
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel x8, x8, xzr, ne
+; CHECK-NEXT:    cmp w11, #0
+; CHECK-NEXT:    csel x9, x10, xzr, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
   %spec.store.select = select <2 x i1> %0, <2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>
   %1 = icmp sgt <2 x i128> %spec.store.select, zeroinitializer

 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w30, -32
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl __fixsfti
 ; CHECK-NEXT:    cmp x1, #1
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    cset w10, lt
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    cset w9, gt
-; CHECK-NEXT:    csel w9, w10, w9, eq
+; CHECK-NEXT:    csinc x8, x1, xzr, lt
+; CHECK-NEXT:    csel x9, x0, xzr, lt
 ; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csinc x11, x20, xzr, lt
 ; CHECK-NEXT:    csel x10, x19, xzr, lt
-; CHECK-NEXT:    cset w12, lt
+; CHECK-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-NEXT:    cmp x10, #0
+; CHECK-NEXT:    cset w12, ne
 ; CHECK-NEXT:    cmp x11, #0
 ; CHECK-NEXT:    cset w11, gt
 ; CHECK-NEXT:    csel w11, w12, w11, eq
+; CHECK-NEXT:    cmp x9, #0
+; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w8, gt
+; CHECK-NEXT:    csel w8, w12, w8, eq
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel x8, x9, xzr, ne
 ; CHECK-NEXT:    cmp w11, #0
-; CHECK-NEXT:    csel x10, x10, xzr, ne
-; CHECK-NEXT:    cmp w9, #0
-; CHECK-NEXT:    csel x8, x8, xzr, ne
+; CHECK-NEXT:    csel x9, x10, xzr, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
   %spec.store.select = select <2 x i1> %0, <2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>
   %1 = icmp sgt <2 x i128> %spec.store.select, zeroinitializer

 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w30, -32
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-NEXT:    mov h0, v0.h[1]
 ; CHECK-NEXT:    bl __fixhfti
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov h0, v0.h[1]
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
 ; CHECK-NEXT:    bl __fixhfti
 ; CHECK-NEXT:    cmp x1, #1
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    cset w10, lt
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    cset w9, gt
-; CHECK-NEXT:    csel w9, w10, w9, eq
+; CHECK-NEXT:    csinc x8, x1, xzr, lt
+; CHECK-NEXT:    csel x9, x0, xzr, lt
 ; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csinc x11, x20, xzr, lt
 ; CHECK-NEXT:    csel x10, x19, xzr, lt
-; CHECK-NEXT:    cset w12, lt
+; CHECK-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-NEXT:    cmp x10, #0
+; CHECK-NEXT:    cset w12, ne
 ; CHECK-NEXT:    cmp x11, #0
 ; CHECK-NEXT:    cset w11, gt
 ; CHECK-NEXT:    csel w11, w12, w11, eq
+; CHECK-NEXT:    cmp x9, #0
+; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w8, gt
+; CHECK-NEXT:    csel w8, w12, w8, eq
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel x8, x9, xzr, ne
 ; CHECK-NEXT:    cmp w11, #0
-; CHECK-NEXT:    csel x10, x10, xzr, ne
-; CHECK-NEXT:    cmp w9, #0
-; CHECK-NEXT:    csel x8, x8, xzr, ne
+; CHECK-NEXT:    csel x9, x10, xzr, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
   %spec.store.select = select <2 x i1> %0, <2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>
   %1 = icmp sgt <2 x i128> %spec.store.select, zeroinitializer

llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll

 declare i128 @llvm.fptosi.sat.i128.f32(float)
 
 define i1 @test_signed_i1_f32(float %f) nounwind {
 ; CHECK-LABEL: test_signed_i1_f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcvtzs w8, s0
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel w8, w8, wzr, lt
+; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f32(float %f)
     ret i1 %x
 }
 
 define i8 @test_signed_i8_f32(float %f) nounwind {

 declare i128 @llvm.fptosi.sat.i128.f64(double)
 
 define i1 @test_signed_i1_f64(double %f) nounwind {
 ; CHECK-LABEL: test_signed_i1_f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcvtzs w8, d0
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel w8, w8, wzr, lt
+; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f64(double %f)
     ret i1 %x
 }
 
 define i8 @test_signed_i8_f64(double %f) nounwind {

 
 define i1 @test_signed_i1_f16(half %f) nounwind {
 ; CHECK-CVT-LABEL: test_signed_i1_f16:
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    fcvt s0, h0
 ; CHECK-CVT-NEXT:    fcvtzs w8, s0
 ; CHECK-CVT-NEXT:    cmp w8, #0
 ; CHECK-CVT-NEXT:    csel w8, w8, wzr, lt
+; CHECK-CVT-NEXT:    cmp w8, #0
 ; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-CVT-NEXT:    and w0, w8, #0x1
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_signed_i1_f16:
 ; CHECK-FP16:       // %bb.0:
 ; CHECK-FP16-NEXT:    fcvtzs w8, h0
 ; CHECK-FP16-NEXT:    cmp w8, #0
 ; CHECK-FP16-NEXT:    csel w8, w8, wzr, lt
+; CHECK-FP16-NEXT:    cmp w8, #0
 ; CHECK-FP16-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-FP16-NEXT:    and w0, w8, #0x1
 ; CHECK-FP16-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f16(half %f)
     ret i1 %x
 }
 
 define i8 @test_signed_i8_f16(half %f) nounwind {

llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll

 define <2 x i1> @test_signed_v2f64_v2i1(<2 x double> %f) {
 ; CHECK-LABEL: test_signed_v2f64_v2i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov d1, v0.d[1]
 ; CHECK-NEXT:    fcvtzs w9, d0
 ; CHECK-NEXT:    fcvtzs w8, d1
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel w8, w8, wzr, lt
+; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csel w9, w9, wzr, lt
+; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    csinv w9, w9, wzr, ge
 ; CHECK-NEXT:    fmov s0, w9
 ; CHECK-NEXT:    mov v0.s[1], w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
     %x = call <2 x i1> @llvm.fptosi.sat.v2f64.v2i1(<2 x double> %f)
     ret <2 x i1> %x
 }

 declare <8 x i128> @llvm.fptosi.sat.v8f16.v8i128(<8 x half>)
 
 define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
 ; CHECK-CVT-LABEL: test_signed_v8f16_v8i1:
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v0.8h
 ; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
 ; CHECK-CVT-NEXT:    mov s2, v1.s[1]
-; CHECK-CVT-NEXT:    mov s3, v1.s[2]
 ; CHECK-CVT-NEXT:    fcvtzs w9, s1
-; CHECK-CVT-NEXT:    mov s1, v1.s[3]
 ; CHECK-CVT-NEXT:    fcvtzs w13, s0
 ; CHECK-CVT-NEXT:    fcvtzs w8, s2
-; CHECK-CVT-NEXT:    mov s2, v0.s[1]
-; CHECK-CVT-NEXT:    fcvtzs w10, s3
-; CHECK-CVT-NEXT:    fcvtzs w11, s1
-; CHECK-CVT-NEXT:    mov s1, v0.s[2]
-; CHECK-CVT-NEXT:    mov s0, v0.s[3]
+; CHECK-CVT-NEXT:    mov s2, v1.s[2]
+; CHECK-CVT-NEXT:    mov s1, v1.s[3]
 ; CHECK-CVT-NEXT:    cmp w8, #0
 ; CHECK-CVT-NEXT:    csel w8, w8, wzr, lt
-; CHECK-CVT-NEXT:    fcvtzs w12, s2
+; CHECK-CVT-NEXT:    fcvtzs w10, s2
+; CHECK-CVT-NEXT:    cmp w8, #0
+; CHECK-CVT-NEXT:    mov s2, v0.s[1]
 ; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w9, #0
 ; CHECK-CVT-NEXT:    csel w9, w9, wzr, lt
+; CHECK-CVT-NEXT:    fcvtzs w11, s1
+; CHECK-CVT-NEXT:    cmp w9, #0
+; CHECK-CVT-NEXT:    mov s1, v0.s[2]
 ; CHECK-CVT-NEXT:    csinv w9, w9, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w10, #0
 ; CHECK-CVT-NEXT:    csel w10, w10, wzr, lt
+; CHECK-CVT-NEXT:    fcvtzs w12, s2
+; CHECK-CVT-NEXT:    cmp w10, #0
+; CHECK-CVT-NEXT:    mov s0, v0.s[3]
 ; CHECK-CVT-NEXT:    csinv w10, w10, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w11, #0
 ; CHECK-CVT-NEXT:    csel w11, w11, wzr, lt
 ; CHECK-CVT-NEXT:    fmov s2, w9
+; CHECK-CVT-NEXT:    cmp w11, #0
 ; CHECK-CVT-NEXT:    csinv w11, w11, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w12, #0
 ; CHECK-CVT-NEXT:    csel w12, w12, wzr, lt
-; CHECK-CVT-NEXT:    fcvtzs w9, s1
+; CHECK-CVT-NEXT:    cmp w12, #0
 ; CHECK-CVT-NEXT:    csinv w12, w12, wzr, ge
 ; CHECK-CVT-NEXT:    cmp w13, #0
 ; CHECK-CVT-NEXT:    csel w13, w13, wzr, lt
-; CHECK-CVT-NEXT:    csinv w13, w13, wzr, ge
-; CHECK-CVT-NEXT:    cmp w9, #0
+; CHECK-CVT-NEXT:    cmp w13, #0
+; CHECK-CVT-NEXT:    csinv w9, w13, wzr, ge
+; CHECK-CVT-NEXT:    fcvtzs w13, s1
 ; CHECK-CVT-NEXT:    mov v2.s[1], w8
-; CHECK-CVT-NEXT:    csel w8, w9, wzr, lt
+; CHECK-CVT-NEXT:    fmov s1, w9
+; CHECK-CVT-NEXT:    cmp w13, #0
+; CHECK-CVT-NEXT:    csel w8, w13, wzr, lt
 ; CHECK-CVT-NEXT:    fcvtzs w9, s0
+; CHECK-CVT-NEXT:    cmp w8, #0
+; CHECK-CVT-NEXT:    mov v1.s[1], w12
 ; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
-; CHECK-CVT-NEXT:    fmov s1, w13
 ; CHECK-CVT-NEXT:    cmp w9, #0
-; CHECK-CVT-NEXT:    mov v1.s[1], w12
+; CHECK-CVT-NEXT:    csel w9, w9, wzr, lt
 ; CHECK-CVT-NEXT:    mov v2.s[2], w10
+; CHECK-CVT-NEXT:    cmp w9, #0
 ; CHECK-CVT-NEXT:    mov v1.s[2], w8
-; CHECK-CVT-NEXT:    csel w8, w9, wzr, lt
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-CVT-NEXT:    csinv w8, w9, wzr, ge
 ; CHECK-CVT-NEXT:    mov v2.s[3], w11
 ; CHECK-CVT-NEXT:    mov v1.s[3], w8
 ; CHECK-CVT-NEXT:    uzp1 v0.8h, v1.8h, v2.8h
 ; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_signed_v8f16_v8i1:
 ; CHECK-FP16:       // %bb.0: