diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1153,12 +1153,14 @@ setOperationAction(ISD::MULHS, VT, Expand); setOperationAction(ISD::UMUL_LOHI, VT, Expand); setOperationAction(ISD::SMUL_LOHI, VT, Expand); + setOperationAction(ISD::SELECT_CC, VT, Expand); } // Illegal unpacked integer vector types. for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) { setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); + setOperationAction(ISD::SELECT_CC, VT, Expand); } for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) { @@ -1218,6 +1220,8 @@ setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); + + setOperationAction(ISD::SELECT_CC, VT, Expand); } for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) { @@ -15265,6 +15269,9 @@ if (N0.getOpcode() != ISD::SETCC) return SDValue(); + if (ResVT.isScalableVector()) + return SDValue(); + // Make sure the SETCC result is either i1 (initial DAG), or i32, the lowered // scalar SetCCResultType. We also don't expect vectors, because we assume // that selects fed by vector SETCCs are canonicalized to VSELECT. diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -1402,7 +1402,9 @@ def : SVE_3_Op_Pat(NAME # _D)>; def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _S)>; def : SVE_3_Op_Pat(NAME # _S)>; + def : SVE_3_Op_Pat(NAME # _D)>; def : SVE_3_Op_Pat(NAME # _D)>; def : SVE_3_Op_Pat(NAME # _D)>; diff --git a/llvm/test/CodeGen/AArch64/select-sve.ll b/llvm/test/CodeGen/AArch64/select-sve.ll --- a/llvm/test/CodeGen/AArch64/select-sve.ll +++ b/llvm/test/CodeGen/AArch64/select-sve.ll @@ -132,3 +132,448 @@ %res = select i1 %cond, %a, %b ret %res } + +; Check icmp+select + +define @icmp_select_nxv2f16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2f16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2f32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2f32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2f64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2f64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4f16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4f16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4f32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4f32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4f64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4f64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z2.d +; CHECK-NEXT: sel z1.d, p0, z1.d, z3.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8f16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8f16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.h, xzr, x8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8f32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8f32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z2.s +; CHECK-NEXT: sel z1.s, p0, z1.s, z3.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8f64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8f64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z4.d +; CHECK-NEXT: sel z1.d, p0, z1.d, z5.d +; CHECK-NEXT: sel z2.d, p0, z2.d, z6.d +; CHECK-NEXT: sel z3.d, p0, z3.d, z7.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16f16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16f16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.h, xzr, x8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z2.h +; CHECK-NEXT: sel z1.h, p0, z1.h, z3.h +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16f32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16f32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z4.s +; CHECK-NEXT: sel z1.s, p0, z1.s, z5.s +; CHECK-NEXT: sel z2.s, p0, z2.s, z6.s +; CHECK-NEXT: sel z3.s, p0, z3.s, z7.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16f64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16f64 +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: cmp x1, #0 // =0 +; CHECK-NEXT: ld1d { z24.d }, p0/z, [x0, #7, mul vl] +; CHECK-NEXT: ld1d { z25.d }, p0/z, [x0, #6, mul vl] +; CHECK-NEXT: ld1d { z26.d }, p0/z, [x0, #5, mul vl] +; CHECK-NEXT: ld1d { z27.d }, p0/z, [x0, #4, mul vl] +; CHECK-NEXT: ld1d { z28.d }, p0/z, [x0, #3, mul vl] +; CHECK-NEXT: ld1d { z29.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z30.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z31.d +; CHECK-NEXT: sel z1.d, p0, z1.d, z30.d +; CHECK-NEXT: sel z2.d, p0, z2.d, z29.d +; CHECK-NEXT: sel z3.d, p0, z3.d, z28.d +; CHECK-NEXT: sel z4.d, p0, z4.d, z27.d +; CHECK-NEXT: sel z5.d, p0, z5.d, z26.d +; CHECK-NEXT: sel z6.d, p0, z6.d, z25.d +; CHECK-NEXT: sel z7.d, p0, z7.d, z24.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2i8( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2i8 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2i16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2i16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2i32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2i32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2i64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2i64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4i8( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4i8 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4i16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4i16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4i32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4i32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4i64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4i64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z2.d +; CHECK-NEXT: sel z1.d, p0, z1.d, z3.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8i8( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8i8 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.h, xzr, x8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8i16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8i16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.h, xzr, x8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8i32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8i32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z2.s +; CHECK-NEXT: sel z1.s, p0, z1.s, z3.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8i64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8i64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z4.d +; CHECK-NEXT: sel z1.d, p0, z1.d, z5.d +; CHECK-NEXT: sel z2.d, p0, z2.d, z6.d +; CHECK-NEXT: sel z3.d, p0, z3.d, z7.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16i8( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16i8 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.b, xzr, x8 +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16i16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16i16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.h, xzr, x8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z2.h +; CHECK-NEXT: sel z1.h, p0, z1.h, z3.h +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16i32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16i32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z4.s +; CHECK-NEXT: sel z1.s, p0, z1.s, z5.s +; CHECK-NEXT: sel z2.s, p0, z2.s, z6.s +; CHECK-NEXT: sel z3.s, p0, z3.s, z7.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16i64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16i64 +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: cmp x1, #0 // =0 +; CHECK-NEXT: ld1d { z24.d }, p0/z, [x0, #7, mul vl] +; CHECK-NEXT: ld1d { z25.d }, p0/z, [x0, #6, mul vl] +; CHECK-NEXT: ld1d { z26.d }, p0/z, [x0, #5, mul vl] +; CHECK-NEXT: ld1d { z27.d }, p0/z, [x0, #4, mul vl] +; CHECK-NEXT: ld1d { z28.d }, p0/z, [x0, #3, mul vl] +; CHECK-NEXT: ld1d { z29.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z30.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z31.d +; CHECK-NEXT: sel z1.d, p0, z1.d, z30.d +; CHECK-NEXT: sel z2.d, p0, z2.d, z29.d +; CHECK-NEXT: sel z3.d, p0, z3.d, z28.d +; CHECK-NEXT: sel z4.d, p0, z4.d, z27.d +; CHECK-NEXT: sel z5.d, p0, z5.d, z26.d +; CHECK-NEXT: sel z6.d, p0, z6.d, z25.d +; CHECK-NEXT: sel z7.d, p0, z7.d, z24.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} +