diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -62,6 +62,7 @@ SHL_PRED, SRL_PRED, SRA_PRED, + SETCC_PRED, // Arithmetic instructions which write flags. ADDS, diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -886,6 +886,8 @@ setOperationAction(ISD::SHL, VT, Custom); setOperationAction(ISD::SRL, VT, Custom); setOperationAction(ISD::SRA, VT, Custom); + if (VT.getScalarType() == MVT::i1) + setOperationAction(ISD::SETCC, VT, Custom); } } setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom); @@ -1294,6 +1296,7 @@ case AArch64ISD::SHL_PRED: return "AArch64ISD::SHL_PRED"; case AArch64ISD::SRL_PRED: return "AArch64ISD::SRL_PRED"; case AArch64ISD::SRA_PRED: return "AArch64ISD::SRA_PRED"; + case AArch64ISD::SETCC_PRED: return "AArch64ISD::SETCC_PRED"; case AArch64ISD::ADC: return "AArch64ISD::ADC"; case AArch64ISD::SBC: return "AArch64ISD::SBC"; case AArch64ISD::ADDS: return "AArch64ISD::ADDS"; @@ -7719,7 +7722,9 @@ VT.getVectorNumElements(), true); SDValue Mask = getPTrue(DAG, DL, PredTy, AArch64SVEPredPattern::all); - return DAG.getNode(NewOp, DL, VT, Mask, Op.getOperand(0), Op.getOperand(1)); + SmallVector Operands = {Mask}; + Operands.append(Op->op_begin(), Op->op_end()); + return DAG.getNode(NewOp, DL, VT, Operands); } static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits, @@ -8788,6 +8793,12 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const { + if (Op.getValueType().isScalableVector()) { + if (Op.getOperand(0).getValueType().isFloatingPoint()) + return Op; + return LowerToPredicatedOp(Op, DAG, AArch64ISD::SETCC_PRED); + } + ISD::CondCode CC = cast(Op.getOperand(2))->get(); SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); @@ -11291,8 +11302,7 @@ return DAG.getNode(ISD::BITCAST, dl, VT, EXT); } -static SDValue tryConvertSVEWideCompare(SDNode *N, unsigned ReplacementIID, - bool Invert, +static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { if (DCI.isBeforeLegalize()) @@ -11346,17 +11356,8 @@ } SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm); - SDValue ID = DAG.getTargetConstant(ReplacementIID, DL, MVT::i64); - SDValue Op0, Op1; - if (Invert) { - Op0 = Splat; - Op1 = N->getOperand(2); - } else { - Op0 = N->getOperand(2); - Op1 = Splat; - } - return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, - ID, Pred, Op0, Op1); + return DAG.getNode(AArch64ISD::SETCC_PRED, DL, VT, Pred, N->getOperand(2), + Splat, DAG.getCondCode(CC)); } return SDValue(); @@ -11530,6 +11531,42 @@ case Intrinsic::aarch64_sve_asr: return DAG.getNode(AArch64ISD::SRA_PRED, SDLoc(N), N->getValueType(0), N->getOperand(1), N->getOperand(2), N->getOperand(3)); + case Intrinsic::aarch64_sve_cmphs: + if (!N->getOperand(2).getValueType().isFloatingPoint()) + return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3), + DAG.getCondCode(ISD::SETUGE)); + break; + case Intrinsic::aarch64_sve_cmphi: + if (!N->getOperand(2).getValueType().isFloatingPoint()) + return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3), + DAG.getCondCode(ISD::SETUGT)); + break; + case Intrinsic::aarch64_sve_cmpge: + if (!N->getOperand(2).getValueType().isFloatingPoint()) + return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3), + DAG.getCondCode(ISD::SETGE)); + break; + case Intrinsic::aarch64_sve_cmpgt: + if (!N->getOperand(2).getValueType().isFloatingPoint()) + return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3), + DAG.getCondCode(ISD::SETGT)); + break; + case Intrinsic::aarch64_sve_cmpeq: + if (!N->getOperand(2).getValueType().isFloatingPoint()) + return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3), + DAG.getCondCode(ISD::SETEQ)); + break; + case Intrinsic::aarch64_sve_cmpne: + if (!N->getOperand(2).getValueType().isFloatingPoint()) + return DAG.getNode(AArch64ISD::SETCC_PRED, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3), + DAG.getCondCode(ISD::SETNE)); + break; case Intrinsic::aarch64_sve_fadda: return combineSVEReductionOrderedFP(N, AArch64ISD::FADDA_PRED, DAG); case Intrinsic::aarch64_sve_faddv: @@ -11546,35 +11583,25 @@ return DAG.getNode(ISD::VSELECT, SDLoc(N), N->getValueType(0), N->getOperand(1), N->getOperand(2), N->getOperand(3)); case Intrinsic::aarch64_sve_cmpeq_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmpeq, - false, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETEQ, DCI, DAG); case Intrinsic::aarch64_sve_cmpne_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmpne, - false, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETNE, DCI, DAG); case Intrinsic::aarch64_sve_cmpge_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmpge, - false, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETGE, DCI, DAG); case Intrinsic::aarch64_sve_cmpgt_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmpgt, - false, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETGT, DCI, DAG); case Intrinsic::aarch64_sve_cmplt_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmpgt, - true, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETLT, DCI, DAG); case Intrinsic::aarch64_sve_cmple_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmpge, - true, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETLE, DCI, DAG); case Intrinsic::aarch64_sve_cmphs_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmphs, - false, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETUGE, DCI, DAG); case Intrinsic::aarch64_sve_cmphi_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmphi, - false, DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETUGT, DCI, DAG); case Intrinsic::aarch64_sve_cmplo_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmphi, true, - DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETULT, DCI, DAG); case Intrinsic::aarch64_sve_cmpls_wide: - return tryConvertSVEWideCompare(N, Intrinsic::aarch64_sve_cmphs, true, - DCI, DAG); + return tryConvertSVEWideCompare(N, ISD::SETULE, DCI, DAG); case Intrinsic::aarch64_sve_ptest_any: return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2), AArch64CC::ANY_ACTIVE); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1005,12 +1005,12 @@ defm TRN1_PPP : sve_int_perm_bin_perm_pp<0b100, "trn1", AArch64trn1>; defm TRN2_PPP : sve_int_perm_bin_perm_pp<0b101, "trn2", AArch64trn2>; - defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs", int_aarch64_sve_cmphs, SETUGE>; - defm CMPHI_PPzZZ : sve_int_cmp_0<0b001, "cmphi", int_aarch64_sve_cmphi, SETUGT>; - defm CMPGE_PPzZZ : sve_int_cmp_0<0b100, "cmpge", int_aarch64_sve_cmpge, SETGE>; - defm CMPGT_PPzZZ : sve_int_cmp_0<0b101, "cmpgt", int_aarch64_sve_cmpgt, SETGT>; - defm CMPEQ_PPzZZ : sve_int_cmp_0<0b110, "cmpeq", int_aarch64_sve_cmpeq, SETEQ>; - defm CMPNE_PPzZZ : sve_int_cmp_0<0b111, "cmpne", int_aarch64_sve_cmpne, SETNE>; + defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs", SETUGE, SETULE>; + defm CMPHI_PPzZZ : sve_int_cmp_0<0b001, "cmphi", SETUGT, SETULT>; + defm CMPGE_PPzZZ : sve_int_cmp_0<0b100, "cmpge", SETGE, SETLE>; + defm CMPGT_PPzZZ : sve_int_cmp_0<0b101, "cmpgt", SETGT, SETLT>; + defm CMPEQ_PPzZZ : sve_int_cmp_0<0b110, "cmpeq", SETEQ, SETEQ>; + defm CMPNE_PPzZZ : sve_int_cmp_0<0b111, "cmpne", SETNE, SETNE>; defm CMPEQ_WIDE_PPzZZ : sve_int_cmp_0_wide<0b010, "cmpeq", int_aarch64_sve_cmpeq_wide>; defm CMPNE_WIDE_PPzZZ : sve_int_cmp_0_wide<0b011, "cmpne", int_aarch64_sve_cmpne_wide>; @@ -1023,16 +1023,16 @@ defm CMPLO_WIDE_PPzZZ : sve_int_cmp_1_wide<0b110, "cmplo", int_aarch64_sve_cmplo_wide>; defm CMPLS_WIDE_PPzZZ : sve_int_cmp_1_wide<0b111, "cmpls", int_aarch64_sve_cmpls_wide>; - defm CMPGE_PPzZI : sve_int_scmp_vi<0b000, "cmpge", SETGE, int_aarch64_sve_cmpge>; - defm CMPGT_PPzZI : sve_int_scmp_vi<0b001, "cmpgt", SETGT, int_aarch64_sve_cmpgt>; - defm CMPLT_PPzZI : sve_int_scmp_vi<0b010, "cmplt", SETLT, null_frag, int_aarch64_sve_cmpgt>; - defm CMPLE_PPzZI : sve_int_scmp_vi<0b011, "cmple", SETLE, null_frag, int_aarch64_sve_cmpge>; - defm CMPEQ_PPzZI : sve_int_scmp_vi<0b100, "cmpeq", SETEQ, int_aarch64_sve_cmpeq>; - defm CMPNE_PPzZI : sve_int_scmp_vi<0b101, "cmpne", SETNE, int_aarch64_sve_cmpne>; - defm CMPHS_PPzZI : sve_int_ucmp_vi<0b00, "cmphs", SETUGE, int_aarch64_sve_cmphs>; - defm CMPHI_PPzZI : sve_int_ucmp_vi<0b01, "cmphi", SETUGT, int_aarch64_sve_cmphi>; - defm CMPLO_PPzZI : sve_int_ucmp_vi<0b10, "cmplo", SETULT, null_frag, int_aarch64_sve_cmphi>; - defm CMPLS_PPzZI : sve_int_ucmp_vi<0b11, "cmpls", SETULE, null_frag, int_aarch64_sve_cmphs>; + defm CMPGE_PPzZI : sve_int_scmp_vi<0b000, "cmpge", SETGE, SETLE>; + defm CMPGT_PPzZI : sve_int_scmp_vi<0b001, "cmpgt", SETGT, SETLT>; + defm CMPLT_PPzZI : sve_int_scmp_vi<0b010, "cmplt", SETLT, SETGT>; + defm CMPLE_PPzZI : sve_int_scmp_vi<0b011, "cmple", SETLE, SETGE>; + defm CMPEQ_PPzZI : sve_int_scmp_vi<0b100, "cmpeq", SETEQ, SETEQ>; + defm CMPNE_PPzZI : sve_int_scmp_vi<0b101, "cmpne", SETNE, SETEQ>; + defm CMPHS_PPzZI : sve_int_ucmp_vi<0b00, "cmphs", SETUGE, SETULE>; + defm CMPHI_PPzZI : sve_int_ucmp_vi<0b01, "cmphi", SETUGT, SETULT>; + defm CMPLO_PPzZI : sve_int_ucmp_vi<0b10, "cmplo", SETULT, SETUGT>; + defm CMPLS_PPzZI : sve_int_ucmp_vi<0b11, "cmpls", SETULE, SETUGE>; defm FCMGE_PPzZZ : sve_fp_3op_p_pd_cc<0b000, "fcmge", int_aarch64_sve_fcmpge, setoge>; defm FCMGT_PPzZZ : sve_fp_3op_p_pd_cc<0b001, "fcmgt", int_aarch64_sve_fcmpgt, setogt>; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -10,6 +10,14 @@ // //===----------------------------------------------------------------------===// +def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ + SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, + SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, + SDTCisVT<4, OtherVT> +]>; + +def AArch64setcc_pred : SDNode<"AArch64ISD::SETCC_PRED", SDT_AArch64Setcc>; + def SVEPatternOperand : AsmOperandClass { let Name = "SVEPattern"; let ParserMethod = "tryParseSVEPattern"; @@ -4147,17 +4155,24 @@ let Defs = [NZCV]; } -multiclass sve_int_cmp_0 opc, string asm, SDPatternOperator op, - CondCode cc> { +multiclass SVE_SETCC_Pat { + def : Pat<(predvt (AArch64setcc_pred predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), + (cmp $Op1, $Op2, $Op3)>; + def : Pat<(predvt (AArch64setcc_pred predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), + (cmp $Op1, $Op3, $Op2)>; +} + +multiclass sve_int_cmp_0 opc, string asm, CondCode cc, CondCode invcc> { def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; - def : SVE_3_Op_Pat(NAME # _B)>; - def : SVE_3_Op_Pat(NAME # _H)>; - def : SVE_3_Op_Pat(NAME # _S)>; - def : SVE_3_Op_Pat(NAME # _D)>; + defm : SVE_SETCC_Pat(NAME # _B)>; + defm : SVE_SETCC_Pat(NAME # _H)>; + defm : SVE_SETCC_Pat(NAME # _S)>; + defm : SVE_SETCC_Pat(NAME # _D)>; } multiclass sve_int_cmp_0_wide opc, string asm, SDPatternOperator op> { @@ -4212,67 +4227,35 @@ let ElementSize = pprty.ElementSize; } -multiclass sve_int_scmp_vi opc, string asm, CondCode cc, - SDPatternOperator op = null_frag, - SDPatternOperator inv_op = null_frag> { +multiclass SVE_SETCC_Imm_Pat { + def : Pat<(predvt (AArch64setcc_pred (predvt PPR_3b:$Pg), + (intvt ZPR:$Zs1), + (intvt (AArch64dup (immtype:$imm))), + cc)), + (cmp $Pg, $Zs1, immtype:$imm)>; + def : Pat<(predvt (AArch64setcc_pred (predvt PPR_3b:$Pg), + (intvt (AArch64dup (immtype:$imm))), + (intvt ZPR:$Zs1), + commuted_cc)), + (cmp $Pg, $Zs1, immtype:$imm)>; +} + +multiclass sve_int_scmp_vi opc, string asm, CondCode cc, CondCode commuted_cc> { def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; - // IR version - def : Pat<(nxv16i1 (setcc (nxv16i8 ZPR:$Zs1), - (nxv16i8 (AArch64dup (simm5_32b:$imm))), - cc)), - (!cast(NAME # "_B") (PTRUE_B 31), ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv8i1 (setcc (nxv8i16 ZPR:$Zs1), - (nxv8i16 (AArch64dup (simm5_32b:$imm))), - cc)), - (!cast(NAME # "_H") (PTRUE_H 31), ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv4i1 (setcc (nxv4i32 ZPR:$Zs1), - (nxv4i32 (AArch64dup (simm5_32b:$imm))), - cc)), - (!cast(NAME # "_S") (PTRUE_S 31), ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv2i1 (setcc (nxv2i64 ZPR:$Zs1), - (nxv2i64 (AArch64dup (simm5_64b:$imm))), - cc)), - (!cast(NAME # "_D") (PTRUE_D 31), ZPR:$Zs1, simm5_64b:$imm)>; - - // Intrinsic version - def : Pat<(nxv16i1 (op (nxv16i1 PPR_3b:$Pg), - (nxv16i8 ZPR:$Zs1), - (nxv16i8 (AArch64dup (simm5_32b:$imm))))), - (!cast(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv8i1 (op (nxv8i1 PPR_3b:$Pg), - (nxv8i16 ZPR:$Zs1), - (nxv8i16 (AArch64dup (simm5_32b:$imm))))), - (!cast(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv4i1 (op (nxv4i1 PPR_3b:$Pg), - (nxv4i32 ZPR:$Zs1), - (nxv4i32 (AArch64dup (simm5_32b:$imm))))), - (!cast(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv2i1 (op (nxv2i1 PPR_3b:$Pg), - (nxv2i64 ZPR:$Zs1), - (nxv2i64 (AArch64dup (simm5_64b:$imm))))), - (!cast(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, simm5_64b:$imm)>; - - // Inverted intrinsic version - def : Pat<(nxv16i1 (inv_op (nxv16i1 PPR_3b:$Pg), - (nxv16i8 (AArch64dup (simm5_32b:$imm))), - (nxv16i8 ZPR:$Zs1))), - (!cast(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv8i1 (inv_op (nxv8i1 PPR_3b:$Pg), - (nxv8i16 (AArch64dup (simm5_32b:$imm))), - (nxv8i16 ZPR:$Zs1))), - (!cast(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv4i1 (inv_op (nxv4i1 PPR_3b:$Pg), - (nxv4i32 (AArch64dup (simm5_32b:$imm))), - (nxv4i32 ZPR:$Zs1))), - (!cast(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, simm5_32b:$imm)>; - def : Pat<(nxv2i1 (inv_op (nxv2i1 PPR_3b:$Pg), - (nxv2i64 (AArch64dup (simm5_64b:$imm))), - (nxv2i64 ZPR:$Zs1))), - (!cast(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, simm5_64b:$imm)>; + defm : SVE_SETCC_Imm_Pat(NAME # _B)>; + defm : SVE_SETCC_Imm_Pat(NAME # _H)>; + defm : SVE_SETCC_Imm_Pat(NAME # _S)>; + defm : SVE_SETCC_Imm_Pat(NAME # _D)>; } @@ -4304,66 +4287,20 @@ } multiclass sve_int_ucmp_vi opc, string asm, CondCode cc, - SDPatternOperator op = null_frag, - SDPatternOperator inv_op = null_frag> { + CondCode commuted_cc> { def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; - // IR version - def : Pat<(nxv16i1 (setcc (nxv16i8 ZPR:$Zs1), - (nxv16i8 (AArch64dup (imm0_127:$imm))), - cc)), - (!cast(NAME # "_B") (PTRUE_B 31), ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv8i1 (setcc (nxv8i16 ZPR:$Zs1), - (nxv8i16 (AArch64dup (imm0_127:$imm))), - cc)), - (!cast(NAME # "_H") (PTRUE_H 31), ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv4i1 (setcc (nxv4i32 ZPR:$Zs1), - (nxv4i32 (AArch64dup (imm0_127:$imm))), - cc)), - (!cast(NAME # "_S") (PTRUE_S 31), ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv2i1 (setcc (nxv2i64 ZPR:$Zs1), - (nxv2i64 (AArch64dup (imm0_127_64b:$imm))), - cc)), - (!cast(NAME # "_D") (PTRUE_D 31), ZPR:$Zs1, imm0_127_64b:$imm)>; - - // Intrinsic version - def : Pat<(nxv16i1 (op (nxv16i1 PPR_3b:$Pg), - (nxv16i8 ZPR:$Zs1), - (nxv16i8 (AArch64dup (imm0_127:$imm))))), - (!cast(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv8i1 (op (nxv8i1 PPR_3b:$Pg), - (nxv8i16 ZPR:$Zs1), - (nxv8i16 (AArch64dup (imm0_127:$imm))))), - (!cast(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv4i1 (op (nxv4i1 PPR_3b:$Pg), - (nxv4i32 ZPR:$Zs1), - (nxv4i32 (AArch64dup (imm0_127:$imm))))), - (!cast(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv2i1 (op (nxv2i1 PPR_3b:$Pg), - (nxv2i64 ZPR:$Zs1), - (nxv2i64 (AArch64dup (imm0_127_64b:$imm))))), - (!cast(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, imm0_127_64b:$imm)>; - - // Inverted intrinsic version - def : Pat<(nxv16i1 (inv_op (nxv16i1 PPR_3b:$Pg), - (nxv16i8 (AArch64dup (imm0_127:$imm))), - (nxv16i8 ZPR:$Zs1))), - (!cast(NAME # "_B") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv8i1 (inv_op (nxv8i1 PPR_3b:$Pg), - (nxv8i16 (AArch64dup (imm0_127:$imm))), - (nxv8i16 ZPR:$Zs1))), - (!cast(NAME # "_H") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv4i1 (inv_op (nxv4i1 PPR_3b:$Pg), - (nxv4i32 (AArch64dup (imm0_127:$imm))), - (nxv4i32 ZPR:$Zs1))), - (!cast(NAME # "_S") PPR_3b:$Pg, ZPR:$Zs1, imm0_127:$imm)>; - def : Pat<(nxv2i1 (inv_op (nxv2i1 PPR_3b:$Pg), - (nxv2i64 (AArch64dup (imm0_127_64b:$imm))), - (nxv2i64 ZPR:$Zs1))), - (!cast(NAME # "_D") PPR_3b:$Pg, ZPR:$Zs1, imm0_127_64b:$imm)>; + defm : SVE_SETCC_Imm_Pat(NAME # _B)>; + defm : SVE_SETCC_Imm_Pat(NAME # _H)>; + defm : SVE_SETCC_Imm_Pat(NAME # _S)>; + defm : SVE_SETCC_Imm_Pat(NAME # _D)>; } diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares.ll @@ -74,6 +74,42 @@ ret %out } +define @cmpeq_ir_b( %a, %b) { +; CHECK-LABEL: cmpeq_ir_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ret + %out = icmp eq %a, %b + ret %out +} + +define @cmpeq_ir_h( %a, %b) { +; CHECK-LABEL: cmpeq_ir_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ret + %out = icmp eq %a, %b + ret %out +} + +define @cmpeq_ir_s( %a, %b) { +; CHECK-LABEL: cmpeq_ir_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ret + %out = icmp eq %a, %b + ret %out +} + +define @cmpeq_ir_d( %a, %b) { +; CHECK-LABEL: cmpeq_ir_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ret + %out = icmp eq %a, %b + ret %out +} + ; ; CMPGE ; @@ -148,6 +184,78 @@ ret %out } +define @cmpge_ir_b( %a, %b) { +; CHECK-LABEL: cmpge_ir_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmpge p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ret + %out = icmp sge %a, %b + ret %out +} + +define @cmpge_ir_h( %a, %b) { +; CHECK-LABEL: cmpge_ir_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmpge p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ret + %out = icmp sge %a, %b + ret %out +} + +define @cmpge_ir_s( %a, %b) { +; CHECK-LABEL: cmpge_ir_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmpge p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ret + %out = icmp sge %a, %b + ret %out +} + +define @cmpge_ir_d( %a, %b) { +; CHECK-LABEL: cmpge_ir_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmpge p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ret + %out = icmp sge %a, %b + ret %out +} + +define @cmpge_ir_comm_b( %a, %b) { +; CHECK-LABEL: cmpge_ir_comm_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmpge p0.b, p0/z, z1.b, z0.b +; CHECK-NEXT: ret + %out = icmp sle %a, %b + ret %out +} + +define @cmpge_ir_comm_h( %a, %b) { +; CHECK-LABEL: cmpge_ir_comm_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmpge p0.h, p0/z, z1.h, z0.h +; CHECK-NEXT: ret + %out = icmp sle %a, %b + ret %out +} + +define @cmpge_ir_comm_s( %a, %b) { +; CHECK-LABEL: cmpge_ir_comm_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmpge p0.s, p0/z, z1.s, z0.s +; CHECK-NEXT: ret + %out = icmp sle %a, %b + ret %out +} + +define @cmpge_ir_comm_d( %a, %b) { +; CHECK-LABEL: cmpge_ir_comm_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmpge p0.d, p0/z, z1.d, z0.d +; CHECK-NEXT: ret + %out = icmp sle %a, %b + ret %out +} + ; ; CMPGT ; @@ -222,6 +330,78 @@ ret %out } +define @cmpgt_ir_b( %a, %b) { +; CHECK-LABEL: cmpgt_ir_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ret + %out = icmp sgt %a, %b + ret %out +} + +define @cmpgt_ir_h( %a, %b) { +; CHECK-LABEL: cmpgt_ir_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmpgt p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ret + %out = icmp sgt %a, %b + ret %out +} + +define @cmpgt_ir_s( %a, %b) { +; CHECK-LABEL: cmpgt_ir_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmpgt p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ret + %out = icmp sgt %a, %b + ret %out +} + +define @cmpgt_ir_d( %a, %b) { +; CHECK-LABEL: cmpgt_ir_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmpgt p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ret + %out = icmp sgt %a, %b + ret %out +} + +define @cmpgt_ir_comm_b( %a, %b) { +; CHECK-LABEL: cmpgt_ir_comm_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b +; CHECK-NEXT: ret + %out = icmp slt %a, %b + ret %out +} + +define @cmpgt_ir_comm_h( %a, %b) { +; CHECK-LABEL: cmpgt_ir_comm_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmpgt p0.h, p0/z, z1.h, z0.h +; CHECK-NEXT: ret + %out = icmp slt %a, %b + ret %out +} + +define @cmpgt_ir_comm_s( %a, %b) { +; CHECK-LABEL: cmpgt_ir_comm_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmpgt p0.s, p0/z, z1.s, z0.s +; CHECK-NEXT: ret + %out = icmp slt %a, %b + ret %out +} + +define @cmpgt_ir_comm_d( %a, %b) { +; CHECK-LABEL: cmpgt_ir_comm_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmpgt p0.d, p0/z, z1.d, z0.d +; CHECK-NEXT: ret + %out = icmp slt %a, %b + ret %out +} + ; ; CMPHI ; @@ -296,6 +476,78 @@ ret %out } +define @cmphi_ir_b( %a, %b) { +; CHECK-LABEL: cmphi_ir_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ret + %out = icmp ugt %a, %b + ret %out +} + +define @cmphi_ir_h( %a, %b) { +; CHECK-LABEL: cmphi_ir_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmphi p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ret + %out = icmp ugt %a, %b + ret %out +} + +define @cmphi_ir_s( %a, %b) { +; CHECK-LABEL: cmphi_ir_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmphi p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ret + %out = icmp ugt %a, %b + ret %out +} + +define @cmphi_ir_d( %a, %b) { +; CHECK-LABEL: cmphi_ir_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmphi p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ret + %out = icmp ugt %a, %b + ret %out +} + +define @cmphi_ir_comm_b( %a, %b) { +; CHECK-LABEL: cmphi_ir_comm_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b +; CHECK-NEXT: ret + %out = icmp ult %a, %b + ret %out +} + +define @cmphi_ir_comm_h( %a, %b) { +; CHECK-LABEL: cmphi_ir_comm_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmphi p0.h, p0/z, z1.h, z0.h +; CHECK-NEXT: ret + %out = icmp ult %a, %b + ret %out +} + +define @cmphi_ir_comm_s( %a, %b) { +; CHECK-LABEL: cmphi_ir_comm_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmphi p0.s, p0/z, z1.s, z0.s +; CHECK-NEXT: ret + %out = icmp ult %a, %b + ret %out +} + +define @cmphi_ir_comm_d( %a, %b) { +; CHECK-LABEL: cmphi_ir_comm_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmphi p0.d, p0/z, z1.d, z0.d +; CHECK-NEXT: ret + %out = icmp ult %a, %b + ret %out +} + ; ; CMPHS ; @@ -370,6 +622,78 @@ ret %out } +define @cmphs_ir_b( %a, %b) { +; CHECK-LABEL: cmphs_ir_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmphs p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ret + %out = icmp uge %a, %b + ret %out +} + +define @cmphs_ir_h( %a, %b) { +; CHECK-LABEL: cmphs_ir_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmphs p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ret + %out = icmp uge %a, %b + ret %out +} + +define @cmphs_ir_s( %a, %b) { +; CHECK-LABEL: cmphs_ir_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmphs p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ret + %out = icmp uge %a, %b + ret %out +} + +define @cmphs_ir_d( %a, %b) { +; CHECK-LABEL: cmphs_ir_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmphs p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ret + %out = icmp uge %a, %b + ret %out +} + +define @cmphs_ir_comm_b( %a, %b) { +; CHECK-LABEL: cmphs_ir_comm_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmphs p0.b, p0/z, z1.b, z0.b +; CHECK-NEXT: ret + %out = icmp ule %a, %b + ret %out +} + +define @cmphs_ir_comm_h( %a, %b) { +; CHECK-LABEL: cmphs_ir_comm_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmphs p0.h, p0/z, z1.h, z0.h +; CHECK-NEXT: ret + %out = icmp ule %a, %b + ret %out +} + +define @cmphs_ir_comm_s( %a, %b) { +; CHECK-LABEL: cmphs_ir_comm_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmphs p0.s, p0/z, z1.s, z0.s +; CHECK-NEXT: ret + %out = icmp ule %a, %b + ret %out +} + +define @cmphs_ir_comm_d( %a, %b) { +; CHECK-LABEL: cmphs_ir_comm_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmphs p0.d, p0/z, z1.d, z0.d +; CHECK-NEXT: ret + %out = icmp ule %a, %b + ret %out +} + ; ; CMPLE ; @@ -580,6 +904,42 @@ ret %out } +define @cmpne_ir_b( %a, %b) { +; CHECK-LABEL: cmpne_ir_b: +; CHECK: ptrue p0.b +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ret + %out = icmp ne %a, %b + ret %out +} + +define @cmpne_ir_h( %a, %b) { +; CHECK-LABEL: cmpne_ir_h: +; CHECK: ptrue p0.h +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ret + %out = icmp ne %a, %b + ret %out +} + +define @cmpne_ir_s( %a, %b) { +; CHECK-LABEL: cmpne_ir_s: +; CHECK: ptrue p0.s +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ret + %out = icmp ne %a, %b + ret %out +} + +define @cmpne_ir_d( %a, %b) { +; CHECK-LABEL: cmpne_ir_d: +; CHECK: ptrue p0.d +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ret + %out = icmp ne %a, %b + ret %out +} + declare @llvm.aarch64.sve.cmpeq.nxv16i8(, , ) declare @llvm.aarch64.sve.cmpeq.nxv8i16(, , ) declare @llvm.aarch64.sve.cmpeq.nxv4i32(, , )