diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1153,6 +1153,7 @@ setOperationAction(ISD::MULHS, VT, Expand); setOperationAction(ISD::UMUL_LOHI, VT, Expand); setOperationAction(ISD::SMUL_LOHI, VT, Expand); + setOperationAction(ISD::SELECT_CC, VT, Expand); } // Illegal unpacked integer vector types. @@ -1171,6 +1172,8 @@ setOperationAction(ISD::VECREDUCE_OR, VT, Custom); setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); + setOperationAction(ISD::SELECT_CC, VT, Expand); + // There are no legal MVT::nxv16f## based types. if (VT != MVT::nxv16i1) { setOperationAction(ISD::SINT_TO_FP, VT, Custom); @@ -1218,6 +1221,8 @@ setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); + + setOperationAction(ISD::SELECT_CC, VT, Expand); } for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) { @@ -15312,6 +15317,9 @@ if (N0.getOpcode() != ISD::SETCC) return SDValue(); + if (ResVT.isScalableVector()) + return SDValue(); + // Make sure the SETCC result is either i1 (initial DAG), or i32, the lowered // scalar SetCCResultType. We also don't expect vectors, because we assume // that selects fed by vector SETCCs are canonicalized to VSELECT. diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -1402,7 +1402,9 @@ def : SVE_3_Op_Pat(NAME # _D)>; def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _S)>; def : SVE_3_Op_Pat(NAME # _S)>; + def : SVE_3_Op_Pat(NAME # _D)>; def : SVE_3_Op_Pat(NAME # _D)>; def : SVE_3_Op_Pat(NAME # _D)>; diff --git a/llvm/test/CodeGen/AArch64/sve-select.ll b/llvm/test/CodeGen/AArch64/sve-select.ll --- a/llvm/test/CodeGen/AArch64/sve-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-select.ll @@ -216,3 +216,198 @@ %sel = select %p, %a, %dst ret %sel } + +; Check icmp+select + +define @icmp_select_nxv2f16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2f16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2f32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2f32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2f64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2f64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4f16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4f16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4f32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4f32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8f16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8f16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.h, xzr, x8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2i64( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2i64 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.d, xzr, x8 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv4i32( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4i32 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.s, xzr, x8 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv8i16( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8i16 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.h, xzr, x8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv16i8( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16i8 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p0.b, xzr, x8 +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} + +define @icmp_select_nxv2i1( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv2i1 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p2.d, xzr, x8 +; CHECK-NEXT: sel p0.b, p2, p0.b, p1.b +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} +define @icmp_select_nxv4i1( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv4i1 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p2.s, xzr, x8 +; CHECK-NEXT: sel p0.b, p2, p0.b, p1.b +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} +define @icmp_select_nxv8i1( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv8i1 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p2.h, xzr, x8 +; CHECK-NEXT: sel p0.b, p2, p0.b, p1.b +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +} +define @icmp_select_nxv16i1( %a, %b, i64 %x0) { +; CHECK-LABEL: icmp_select_nxv16i1 +; CHECK: // %bb.0: +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: cset w8, eq +; CHECK-NEXT: sbfx x8, x8, #0, #1 +; CHECK-NEXT: whilelo p2.b, xzr, x8 +; CHECK-NEXT: sel p0.b, p2, p0.b, p1.b +; CHECK-NEXT: ret + %mask = icmp eq i64 %x0, 0 + %sel = select i1 %mask, %a, %b + ret %sel +}