diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -137,6 +137,23 @@ } } +static inline MVT getPromotedVTForPredicate(MVT VT) { + assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) && + "Expected scalable predicate vector type!"); + switch (VT.getVectorMinNumElements()) { + default: + llvm_unreachable("unexpected element count for vector"); + case 2: + return MVT::nxv2i64; + case 4: + return MVT::nxv4i32; + case 8: + return MVT::nxv8i16; + case 16: + return MVT::nxv16i8; + } +} + /// Returns true if VT's elements occupy the lowest bit positions of its /// associated register class without any intervening space. /// @@ -973,76 +990,74 @@ // FIXME: Add custom lowering of MLOAD to handle different passthrus (not a // splat of 0 or undef) once vector selects supported in SVE codegen. See // D68877 for more details. - for (MVT VT : MVT::integer_scalable_vector_valuetypes()) { - if (isTypeLegal(VT)) { - setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); - setOperationAction(ISD::UINT_TO_FP, VT, Custom); - setOperationAction(ISD::SINT_TO_FP, VT, Custom); - setOperationAction(ISD::FP_TO_UINT, VT, Custom); - setOperationAction(ISD::FP_TO_SINT, VT, Custom); - setOperationAction(ISD::MUL, VT, Custom); - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - setOperationAction(ISD::SELECT, VT, Custom); - setOperationAction(ISD::SDIV, VT, Custom); - setOperationAction(ISD::UDIV, VT, Custom); - setOperationAction(ISD::SMIN, VT, Custom); - setOperationAction(ISD::UMIN, VT, Custom); - setOperationAction(ISD::SMAX, VT, Custom); - setOperationAction(ISD::UMAX, VT, Custom); - setOperationAction(ISD::SHL, VT, Custom); - setOperationAction(ISD::SRL, VT, Custom); - setOperationAction(ISD::SRA, VT, Custom); - if (VT.getScalarType() == MVT::i1) { - setOperationAction(ISD::SETCC, VT, Custom); - setOperationAction(ISD::TRUNCATE, VT, Custom); - setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); - } - } - } + for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) { + setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); + setOperationAction(ISD::UINT_TO_FP, VT, Custom); + setOperationAction(ISD::SINT_TO_FP, VT, Custom); + setOperationAction(ISD::FP_TO_UINT, VT, Custom); + setOperationAction(ISD::FP_TO_SINT, VT, Custom); + setOperationAction(ISD::MUL, VT, Custom); + setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); + setOperationAction(ISD::SELECT, VT, Custom); + setOperationAction(ISD::SDIV, VT, Custom); + setOperationAction(ISD::UDIV, VT, Custom); + setOperationAction(ISD::SMIN, VT, Custom); + setOperationAction(ISD::UMIN, VT, Custom); + setOperationAction(ISD::SMAX, VT, Custom); + setOperationAction(ISD::UMAX, VT, Custom); + setOperationAction(ISD::SHL, VT, Custom); + setOperationAction(ISD::SRL, VT, Custom); + setOperationAction(ISD::SRA, VT, Custom); + } + + // Illegal unpacked integer vector types. for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) { setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); } - setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom); - setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); + for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) { + setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); + setOperationAction(ISD::SELECT, VT, Custom); + setOperationAction(ISD::SETCC, VT, Custom); + setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); + setOperationAction(ISD::TRUNCATE, VT, Custom); - for (MVT VT : MVT::fp_scalable_vector_valuetypes()) { - if (isTypeLegal(VT)) { - setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - setOperationAction(ISD::SELECT, VT, Custom); - setOperationAction(ISD::FADD, VT, Custom); - setOperationAction(ISD::FDIV, VT, Custom); - setOperationAction(ISD::FMA, VT, Custom); - setOperationAction(ISD::FMUL, VT, Custom); - setOperationAction(ISD::FNEG, VT, Custom); - setOperationAction(ISD::FSUB, VT, Custom); - setOperationAction(ISD::FCEIL, VT, Custom); - setOperationAction(ISD::FFLOOR, VT, Custom); - setOperationAction(ISD::FNEARBYINT, VT, Custom); - setOperationAction(ISD::FRINT, VT, Custom); - setOperationAction(ISD::FROUND, VT, Custom); - setOperationAction(ISD::FROUNDEVEN, VT, Custom); - setOperationAction(ISD::FTRUNC, VT, Custom); - setOperationAction(ISD::FSQRT, VT, Custom); + // There are no legal MVT::nxv16f## based types. + if (VT != MVT::nxv16i1) { + setOperationAction(ISD::SINT_TO_FP, VT, Promote); + AddPromotedToType(ISD::SINT_TO_FP, VT, getPromotedVTForPredicate(VT)); + setOperationAction(ISD::UINT_TO_FP, VT, Promote); + AddPromotedToType(ISD::UINT_TO_FP, VT, getPromotedVTForPredicate(VT)); } } - setOperationAction(ISD::SINT_TO_FP, MVT::nxv2i1, Promote); - AddPromotedToType(ISD::SINT_TO_FP, MVT::nxv2i1, MVT::nxv2i64); - setOperationAction(ISD::SINT_TO_FP, MVT::nxv4i1, Promote); - AddPromotedToType(ISD::SINT_TO_FP, MVT::nxv4i1, MVT::nxv4i32); - setOperationAction(ISD::SINT_TO_FP, MVT::nxv8i1, Promote); - AddPromotedToType(ISD::SINT_TO_FP, MVT::nxv8i1, MVT::nxv8i16); + for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32, + MVT::nxv4f32, MVT::nxv2f64}) { + setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); + setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); + setOperationAction(ISD::SELECT, VT, Custom); + setOperationAction(ISD::FADD, VT, Custom); + setOperationAction(ISD::FDIV, VT, Custom); + setOperationAction(ISD::FMA, VT, Custom); + setOperationAction(ISD::FMUL, VT, Custom); + setOperationAction(ISD::FNEG, VT, Custom); + setOperationAction(ISD::FSUB, VT, Custom); + setOperationAction(ISD::FCEIL, VT, Custom); + setOperationAction(ISD::FFLOOR, VT, Custom); + setOperationAction(ISD::FNEARBYINT, VT, Custom); + setOperationAction(ISD::FRINT, VT, Custom); + setOperationAction(ISD::FROUND, VT, Custom); + setOperationAction(ISD::FROUNDEVEN, VT, Custom); + setOperationAction(ISD::FTRUNC, VT, Custom); + setOperationAction(ISD::FSQRT, VT, Custom); + } + + setOperationAction(ISD::SPLAT_VECTOR, MVT::nxv8bf16, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::nxv2i1, Promote); - AddPromotedToType(ISD::UINT_TO_FP, MVT::nxv2i1, MVT::nxv2i64); - setOperationAction(ISD::UINT_TO_FP, MVT::nxv4i1, Promote); - AddPromotedToType(ISD::UINT_TO_FP, MVT::nxv4i1, MVT::nxv4i32); - setOperationAction(ISD::UINT_TO_FP, MVT::nxv8i1, Promote); - AddPromotedToType(ISD::UINT_TO_FP, MVT::nxv8i1, MVT::nxv8i16); + setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom); + setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); // NOTE: Currently this has to happen after computeRegisterProperties rather // than the preferred option of combining it with the addRegisterClass call. @@ -3456,7 +3471,7 @@ Op.getOperand(2), Op.getOperand(3), Op.getOperand(1)); case Intrinsic::aarch64_sve_frintm: return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, dl, Op.getValueType(), - Op.getOperand(2), Op.getOperand(3), Op.getOperand(1)); + Op.getOperand(2), Op.getOperand(3), Op.getOperand(1)); case Intrinsic::aarch64_sve_frinti: return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, dl, Op.getValueType(), Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));