diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -18420,13 +18420,13 @@ BitWidthMinusOne); } -// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond) -// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) y)) => (CSEL l r !cc2 cond) -// Where cc1 is any reflexive relation (eg EQ) +// (CSEL l r EQ (CMP (CSEL x y cc2 cond) x)) => (CSEL l r cc2 cond) +// (CSEL l r EQ (CMP (CSEL x y cc2 cond) y)) => (CSEL l r !cc2 cond) +// Where x and y are constants -// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) x)) => (CSEL l r !cc2 cond) -// (CSEL l r cc1 (CMP (CSEL x y cc2 cond) y)) => (CSEL l r cc2 cond) -// Where cc1 is any irreflexive relation (eg NE) +// (CSEL l r NE (CMP (CSEL x y cc2 cond) x)) => (CSEL l r !cc2 cond) +// (CSEL l r NE (CMP (CSEL x y cc2 cond) y)) => (CSEL l r cc2 cond) +// Where x and y are constants static SDValue foldCSELOfCSEL(SDNode *Op, SelectionDAG &DAG) { SDValue L = Op->getOperand(0); SDValue R = Op->getOperand(1); @@ -18447,6 +18447,10 @@ SDValue X = CmpLHS->getOperand(0); SDValue Y = CmpLHS->getOperand(1); + if (!isa(X) || !isa(Y)) { + return SDValue(); + } + AArch64CC::CondCode CC = static_cast(CmpLHS->getConstantOperandVal(2)); SDValue Cond = CmpLHS->getOperand(3); @@ -18456,9 +18460,9 @@ else if (CmpRHS != X) return SDValue(); - if (AArch64CC::isIrreflexive(OpCC)) + if (OpCC == AArch64CC::NE) CC = AArch64CC::getInvertedCondCode(CC); - else if (!AArch64CC::isReflexive(OpCC)) + else if (OpCC != AArch64CC::EQ) return SDValue(); SDLoc DL(Op); diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat.ll b/llvm/test/CodeGen/AArch64/fpclamptosat.ll --- a/llvm/test/CodeGen/AArch64/fpclamptosat.ll +++ b/llvm/test/CodeGen/AArch64/fpclamptosat.ll @@ -396,9 +396,10 @@ ; CHECK-NEXT: .cfi_offset w30, -16 ; CHECK-NEXT: bl __fixdfti ; CHECK-NEXT: cmp x1, #1 -; CHECK-NEXT: csinc x9, x1, xzr, lt ; CHECK-NEXT: csel x8, x0, xzr, lt -; CHECK-NEXT: cset w10, lt +; CHECK-NEXT: csinc x9, x1, xzr, lt +; CHECK-NEXT: cmp x8, #0 +; CHECK-NEXT: cset w10, ne ; CHECK-NEXT: cmp x9, #0 ; CHECK-NEXT: cset w9, gt ; CHECK-NEXT: csel w9, w10, w9, eq @@ -458,9 +459,10 @@ ; CHECK-NEXT: .cfi_offset w30, -16 ; CHECK-NEXT: bl __fixsfti ; CHECK-NEXT: cmp x1, #1 -; CHECK-NEXT: csinc x9, x1, xzr, lt ; CHECK-NEXT: csel x8, x0, xzr, lt -; CHECK-NEXT: cset w10, lt +; CHECK-NEXT: csinc x9, x1, xzr, lt +; CHECK-NEXT: cmp x8, #0 +; CHECK-NEXT: cset w10, ne ; CHECK-NEXT: cmp x9, #0 ; CHECK-NEXT: cset w9, gt ; CHECK-NEXT: csel w9, w10, w9, eq @@ -526,9 +528,10 @@ ; CHECK-NEXT: .cfi_offset w30, -16 ; CHECK-NEXT: bl __fixhfti ; CHECK-NEXT: cmp x1, #1 -; CHECK-NEXT: csinc x9, x1, xzr, lt ; CHECK-NEXT: csel x8, x0, xzr, lt -; CHECK-NEXT: cset w10, lt +; CHECK-NEXT: csinc x9, x1, xzr, lt +; CHECK-NEXT: cmp x8, #0 +; CHECK-NEXT: cset w10, ne ; CHECK-NEXT: cmp x9, #0 ; CHECK-NEXT: cset w9, gt ; CHECK-NEXT: csel w9, w10, w9, eq diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll --- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll @@ -397,35 +397,37 @@ ; CHECK-NEXT: .cfi_offset w20, -16 ; CHECK-NEXT: .cfi_offset w30, -32 ; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill -; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: mov d0, v0.d[1] ; CHECK-NEXT: bl __fixdfti ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: mov x19, x0 ; CHECK-NEXT: mov x20, x1 -; CHECK-NEXT: mov d0, v0.d[1] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 ; CHECK-NEXT: bl __fixdfti ; CHECK-NEXT: cmp x1, #1 ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload -; CHECK-NEXT: csinc x9, x1, xzr, lt ; CHECK-NEXT: csel x8, x0, xzr, lt -; CHECK-NEXT: cset w10, lt -; CHECK-NEXT: cmp x9, #0 -; CHECK-NEXT: cset w9, gt -; CHECK-NEXT: csel w9, w10, w9, eq +; CHECK-NEXT: csinc x9, x1, xzr, lt ; CHECK-NEXT: cmp x20, #1 -; CHECK-NEXT: csinc x11, x20, xzr, lt ; CHECK-NEXT: csel x10, x19, xzr, lt -; CHECK-NEXT: cset w12, lt +; CHECK-NEXT: csinc x11, x20, xzr, lt +; CHECK-NEXT: cmp x10, #0 +; CHECK-NEXT: cset w12, ne ; CHECK-NEXT: cmp x11, #0 ; CHECK-NEXT: cset w11, gt ; CHECK-NEXT: csel w11, w12, w11, eq -; CHECK-NEXT: cmp w11, #0 -; CHECK-NEXT: csel x10, x10, xzr, ne +; CHECK-NEXT: cmp x8, #0 +; CHECK-NEXT: cset w12, ne +; CHECK-NEXT: cmp x9, #0 +; CHECK-NEXT: cset w9, gt +; CHECK-NEXT: csel w9, w12, w9, eq ; CHECK-NEXT: cmp w9, #0 ; CHECK-NEXT: csel x8, x8, xzr, ne +; CHECK-NEXT: cmp w11, #0 +; CHECK-NEXT: csel x9, x10, xzr, ne ; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: fmov d0, x10 -; CHECK-NEXT: fmov d1, x8 +; CHECK-NEXT: fmov d0, x8 +; CHECK-NEXT: fmov d1, x9 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: add sp, sp, #48 ; CHECK-NEXT: ret @@ -509,35 +511,37 @@ ; CHECK-NEXT: .cfi_offset w30, -32 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill -; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0 +; CHECK-NEXT: mov s0, v0.s[1] ; CHECK-NEXT: bl __fixsfti ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: mov x19, x0 ; CHECK-NEXT: mov x20, x1 -; CHECK-NEXT: mov s0, v0.s[1] +; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0 ; CHECK-NEXT: bl __fixsfti ; CHECK-NEXT: cmp x1, #1 ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload -; CHECK-NEXT: csinc x9, x1, xzr, lt -; CHECK-NEXT: csel x8, x0, xzr, lt -; CHECK-NEXT: cset w10, lt -; CHECK-NEXT: cmp x9, #0 -; CHECK-NEXT: cset w9, gt -; CHECK-NEXT: csel w9, w10, w9, eq +; CHECK-NEXT: csinc x8, x1, xzr, lt +; CHECK-NEXT: csel x9, x0, xzr, lt ; CHECK-NEXT: cmp x20, #1 -; CHECK-NEXT: csinc x11, x20, xzr, lt ; CHECK-NEXT: csel x10, x19, xzr, lt -; CHECK-NEXT: cset w12, lt +; CHECK-NEXT: csinc x11, x20, xzr, lt +; CHECK-NEXT: cmp x10, #0 +; CHECK-NEXT: cset w12, ne ; CHECK-NEXT: cmp x11, #0 ; CHECK-NEXT: cset w11, gt ; CHECK-NEXT: csel w11, w12, w11, eq +; CHECK-NEXT: cmp x9, #0 +; CHECK-NEXT: cset w12, ne +; CHECK-NEXT: cmp x8, #0 +; CHECK-NEXT: cset w8, gt +; CHECK-NEXT: csel w8, w12, w8, eq +; CHECK-NEXT: cmp w8, #0 +; CHECK-NEXT: csel x8, x9, xzr, ne ; CHECK-NEXT: cmp w11, #0 -; CHECK-NEXT: csel x10, x10, xzr, ne -; CHECK-NEXT: cmp w9, #0 -; CHECK-NEXT: csel x8, x8, xzr, ne +; CHECK-NEXT: csel x9, x10, xzr, ne ; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: fmov d0, x10 -; CHECK-NEXT: fmov d1, x8 +; CHECK-NEXT: fmov d0, x8 +; CHECK-NEXT: fmov d1, x9 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: add sp, sp, #48 ; CHECK-NEXT: ret @@ -633,35 +637,37 @@ ; CHECK-NEXT: .cfi_offset w30, -32 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill -; CHECK-NEXT: // kill: def $h0 killed $h0 killed $q0 +; CHECK-NEXT: mov h0, v0.h[1] ; CHECK-NEXT: bl __fixhfti ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: mov x19, x0 ; CHECK-NEXT: mov x20, x1 -; CHECK-NEXT: mov h0, v0.h[1] +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $q0 ; CHECK-NEXT: bl __fixhfti ; CHECK-NEXT: cmp x1, #1 ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload -; CHECK-NEXT: csinc x9, x1, xzr, lt -; CHECK-NEXT: csel x8, x0, xzr, lt -; CHECK-NEXT: cset w10, lt -; CHECK-NEXT: cmp x9, #0 -; CHECK-NEXT: cset w9, gt -; CHECK-NEXT: csel w9, w10, w9, eq +; CHECK-NEXT: csinc x8, x1, xzr, lt +; CHECK-NEXT: csel x9, x0, xzr, lt ; CHECK-NEXT: cmp x20, #1 -; CHECK-NEXT: csinc x11, x20, xzr, lt ; CHECK-NEXT: csel x10, x19, xzr, lt -; CHECK-NEXT: cset w12, lt +; CHECK-NEXT: csinc x11, x20, xzr, lt +; CHECK-NEXT: cmp x10, #0 +; CHECK-NEXT: cset w12, ne ; CHECK-NEXT: cmp x11, #0 ; CHECK-NEXT: cset w11, gt ; CHECK-NEXT: csel w11, w12, w11, eq +; CHECK-NEXT: cmp x9, #0 +; CHECK-NEXT: cset w12, ne +; CHECK-NEXT: cmp x8, #0 +; CHECK-NEXT: cset w8, gt +; CHECK-NEXT: csel w8, w12, w8, eq +; CHECK-NEXT: cmp w8, #0 +; CHECK-NEXT: csel x8, x9, xzr, ne ; CHECK-NEXT: cmp w11, #0 -; CHECK-NEXT: csel x10, x10, xzr, ne -; CHECK-NEXT: cmp w9, #0 -; CHECK-NEXT: csel x8, x8, xzr, ne +; CHECK-NEXT: csel x9, x10, xzr, ne ; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: fmov d0, x10 -; CHECK-NEXT: fmov d1, x8 +; CHECK-NEXT: fmov d0, x8 +; CHECK-NEXT: fmov d1, x9 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: add sp, sp, #48 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll --- a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll +++ b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll @@ -23,6 +23,7 @@ ; CHECK-NEXT: fcvtzs w8, s0 ; CHECK-NEXT: cmp w8, #0 ; CHECK-NEXT: csel w8, w8, wzr, lt +; CHECK-NEXT: cmp w8, #0 ; CHECK-NEXT: csinv w8, w8, wzr, ge ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret @@ -200,6 +201,7 @@ ; CHECK-NEXT: fcvtzs w8, d0 ; CHECK-NEXT: cmp w8, #0 ; CHECK-NEXT: csel w8, w8, wzr, lt +; CHECK-NEXT: cmp w8, #0 ; CHECK-NEXT: csinv w8, w8, wzr, ge ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret @@ -380,6 +382,7 @@ ; CHECK-CVT-NEXT: fcvtzs w8, s0 ; CHECK-CVT-NEXT: cmp w8, #0 ; CHECK-CVT-NEXT: csel w8, w8, wzr, lt +; CHECK-CVT-NEXT: cmp w8, #0 ; CHECK-CVT-NEXT: csinv w8, w8, wzr, ge ; CHECK-CVT-NEXT: and w0, w8, #0x1 ; CHECK-CVT-NEXT: ret @@ -389,6 +392,7 @@ ; CHECK-FP16-NEXT: fcvtzs w8, h0 ; CHECK-FP16-NEXT: cmp w8, #0 ; CHECK-FP16-NEXT: csel w8, w8, wzr, lt +; CHECK-FP16-NEXT: cmp w8, #0 ; CHECK-FP16-NEXT: csinv w8, w8, wzr, ge ; CHECK-FP16-NEXT: and w0, w8, #0x1 ; CHECK-FP16-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll --- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll +++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll @@ -1306,9 +1306,11 @@ ; CHECK-NEXT: fcvtzs w8, d1 ; CHECK-NEXT: cmp w8, #0 ; CHECK-NEXT: csel w8, w8, wzr, lt +; CHECK-NEXT: cmp w8, #0 ; CHECK-NEXT: csinv w8, w8, wzr, ge ; CHECK-NEXT: cmp w9, #0 ; CHECK-NEXT: csel w9, w9, wzr, lt +; CHECK-NEXT: cmp w9, #0 ; CHECK-NEXT: csinv w9, w9, wzr, ge ; CHECK-NEXT: fmov s0, w9 ; CHECK-NEXT: mov v0.s[1], w8 @@ -2060,49 +2062,57 @@ ; CHECK-CVT-NEXT: fcvtl2 v1.4s, v0.8h ; CHECK-CVT-NEXT: fcvtl v0.4s, v0.4h ; CHECK-CVT-NEXT: mov s2, v1.s[1] -; CHECK-CVT-NEXT: mov s3, v1.s[2] ; CHECK-CVT-NEXT: fcvtzs w9, s1 -; CHECK-CVT-NEXT: mov s1, v1.s[3] ; CHECK-CVT-NEXT: fcvtzs w13, s0 ; CHECK-CVT-NEXT: fcvtzs w8, s2 -; CHECK-CVT-NEXT: mov s2, v0.s[1] -; CHECK-CVT-NEXT: fcvtzs w10, s3 -; CHECK-CVT-NEXT: fcvtzs w11, s1 -; CHECK-CVT-NEXT: mov s1, v0.s[2] -; CHECK-CVT-NEXT: mov s0, v0.s[3] +; CHECK-CVT-NEXT: mov s2, v1.s[2] +; CHECK-CVT-NEXT: mov s1, v1.s[3] ; CHECK-CVT-NEXT: cmp w8, #0 ; CHECK-CVT-NEXT: csel w8, w8, wzr, lt -; CHECK-CVT-NEXT: fcvtzs w12, s2 +; CHECK-CVT-NEXT: fcvtzs w10, s2 +; CHECK-CVT-NEXT: cmp w8, #0 +; CHECK-CVT-NEXT: mov s2, v0.s[1] ; CHECK-CVT-NEXT: csinv w8, w8, wzr, ge ; CHECK-CVT-NEXT: cmp w9, #0 ; CHECK-CVT-NEXT: csel w9, w9, wzr, lt +; CHECK-CVT-NEXT: fcvtzs w11, s1 +; CHECK-CVT-NEXT: cmp w9, #0 +; CHECK-CVT-NEXT: mov s1, v0.s[2] ; CHECK-CVT-NEXT: csinv w9, w9, wzr, ge ; CHECK-CVT-NEXT: cmp w10, #0 ; CHECK-CVT-NEXT: csel w10, w10, wzr, lt +; CHECK-CVT-NEXT: fcvtzs w12, s2 +; CHECK-CVT-NEXT: cmp w10, #0 +; CHECK-CVT-NEXT: mov s0, v0.s[3] ; CHECK-CVT-NEXT: csinv w10, w10, wzr, ge ; CHECK-CVT-NEXT: cmp w11, #0 ; CHECK-CVT-NEXT: csel w11, w11, wzr, lt ; CHECK-CVT-NEXT: fmov s2, w9 +; CHECK-CVT-NEXT: cmp w11, #0 ; CHECK-CVT-NEXT: csinv w11, w11, wzr, ge ; CHECK-CVT-NEXT: cmp w12, #0 ; CHECK-CVT-NEXT: csel w12, w12, wzr, lt -; CHECK-CVT-NEXT: fcvtzs w9, s1 +; CHECK-CVT-NEXT: cmp w12, #0 ; CHECK-CVT-NEXT: csinv w12, w12, wzr, ge ; CHECK-CVT-NEXT: cmp w13, #0 ; CHECK-CVT-NEXT: csel w13, w13, wzr, lt -; CHECK-CVT-NEXT: csinv w13, w13, wzr, ge -; CHECK-CVT-NEXT: cmp w9, #0 +; CHECK-CVT-NEXT: cmp w13, #0 +; CHECK-CVT-NEXT: csinv w9, w13, wzr, ge +; CHECK-CVT-NEXT: fcvtzs w13, s1 ; CHECK-CVT-NEXT: mov v2.s[1], w8 -; CHECK-CVT-NEXT: csel w8, w9, wzr, lt +; CHECK-CVT-NEXT: fmov s1, w9 +; CHECK-CVT-NEXT: cmp w13, #0 +; CHECK-CVT-NEXT: csel w8, w13, wzr, lt ; CHECK-CVT-NEXT: fcvtzs w9, s0 +; CHECK-CVT-NEXT: cmp w8, #0 +; CHECK-CVT-NEXT: mov v1.s[1], w12 ; CHECK-CVT-NEXT: csinv w8, w8, wzr, ge -; CHECK-CVT-NEXT: fmov s1, w13 ; CHECK-CVT-NEXT: cmp w9, #0 -; CHECK-CVT-NEXT: mov v1.s[1], w12 +; CHECK-CVT-NEXT: csel w9, w9, wzr, lt ; CHECK-CVT-NEXT: mov v2.s[2], w10 +; CHECK-CVT-NEXT: cmp w9, #0 ; CHECK-CVT-NEXT: mov v1.s[2], w8 -; CHECK-CVT-NEXT: csel w8, w9, wzr, lt -; CHECK-CVT-NEXT: csinv w8, w8, wzr, ge +; CHECK-CVT-NEXT: csinv w8, w9, wzr, ge ; CHECK-CVT-NEXT: mov v2.s[3], w11 ; CHECK-CVT-NEXT: mov v1.s[3], w8 ; CHECK-CVT-NEXT: uzp1 v0.8h, v1.8h, v2.8h