diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -136,15 +136,15 @@ // // SVE CNT/INC/RDVL -def sve_rdvl_imm : ComplexPattern">; -def sve_cnth_imm : ComplexPattern">; -def sve_cntw_imm : ComplexPattern">; -def sve_cntd_imm : ComplexPattern">; +def sve_rdvl_imm : ComplexPattern">; +def sve_cnth_imm : ComplexPattern">; +def sve_cntw_imm : ComplexPattern">; +def sve_cntd_imm : ComplexPattern">; // SVE DEC -def sve_cnth_imm_neg : ComplexPattern">; -def sve_cntw_imm_neg : ComplexPattern">; -def sve_cntd_imm_neg : ComplexPattern">; +def sve_cnth_imm_neg : ComplexPattern">; +def sve_cntw_imm_neg : ComplexPattern">; +def sve_cntd_imm_neg : ComplexPattern">; def SDT_AArch64Reduce : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisVec<2>]>; def AArch64faddv_p : SDNode<"AArch64ISD::FADDV_PRED", SDT_AArch64Reduce>; @@ -602,13 +602,13 @@ def : Pat<(nxv8bf16 (AArch64dup (bf16 fpimm0))), (DUP_ZI_H 0, 0)>; // Duplicate Int immediate into all vector elements - def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_B $a, $b)>; - def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_H $a, $b)>; - def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_S $a, $b)>; - def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm64 i32:$a, i32:$b)))), (DUP_ZI_D $a, $b)>; // Duplicate immediate FP into all vector elements. @@ -2498,13 +2498,13 @@ // Splice with lane bigger or equal to 0 def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 (sve_ext_imm_0_15 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_15:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 (sve_ext_imm_0_7 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_7:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 (sve_ext_imm_0_3 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_3:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 (sve_ext_imm_0_1 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_1:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; } // End HasSVEorStreamingSVE diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -197,34 +197,42 @@ def SVEAddSubImm8Pat : ComplexPattern", []>; def SVEAddSubImm16Pat : ComplexPattern", []>; def SVEAddSubImm32Pat : ComplexPattern", []>; -def SVEAddSubImm64Pat : ComplexPattern", []>; +def SVEAddSubImm64Pat : ComplexPattern", []>; -def SVELogicalImm8Pat : ComplexPattern", []>; -def SVELogicalImm16Pat : ComplexPattern", []>; -def SVELogicalImm32Pat : ComplexPattern", []>; +def SVELogicalImm8Pat : ComplexPattern", []>; +def SVELogicalImm16Pat : ComplexPattern", []>; +def SVELogicalImm32Pat : ComplexPattern", []>; def SVELogicalImm64Pat : ComplexPattern", []>; -def SVELogicalImm8NotPat : ComplexPattern", []>; -def SVELogicalImm16NotPat : ComplexPattern", []>; -def SVELogicalImm32NotPat : ComplexPattern", []>; +def SVELogicalImm8NotPat : ComplexPattern", []>; +def SVELogicalImm16NotPat : ComplexPattern", []>; +def SVELogicalImm32NotPat : ComplexPattern", []>; def SVELogicalImm64NotPat : ComplexPattern", []>; -def SVE8BitLslImm : ComplexPattern; +def SVE8BitLslImm32 : ComplexPattern; +def SVE8BitLslImm64 : ComplexPattern; +class SVE8BitLslImm { + ComplexPattern Pat = !cond( + !eq(ty, i32): SVE8BitLslImm32, + !eq(ty, i64): SVE8BitLslImm64); +} def SVEArithUImm8Pat : ComplexPattern", []>; def SVEArithUImm16Pat : ComplexPattern", []>; def SVEArithUImm32Pat : ComplexPattern", []>; -def SVEArithUImm64Pat : ComplexPattern", []>; -def SVEArithSImmPat : ComplexPattern; +def SVEArithUImm64Pat : ComplexPattern", []>; + +def SVEArithSImmPat32 : ComplexPattern; +def SVEArithSImmPat64 : ComplexPattern; def SVEShiftImmL8 : ComplexPattern", []>; def SVEShiftImmL16 : ComplexPattern", []>; def SVEShiftImmL32 : ComplexPattern", []>; -def SVEShiftImmL64 : ComplexPattern", []>; +def SVEShiftImmL64 : ComplexPattern", []>; def SVEShiftImmR8 : ComplexPattern", []>; def SVEShiftImmR16 : ComplexPattern", []>; def SVEShiftImmR32 : ComplexPattern", []>; -def SVEShiftImmR64 : ComplexPattern", []>; +def SVEShiftImmR64 : ComplexPattern", []>; def SVEAllActive : ComplexPattern; @@ -260,14 +268,14 @@ } // This allows i32 immediate extraction from i64 based arithmetic. -def sve_cnt_mul_imm : ComplexPattern">; -def sve_cnt_shl_imm : ComplexPattern">; +def sve_cnt_mul_imm : ComplexPattern">; +def sve_cnt_shl_imm : ComplexPattern">; -def sve_ext_imm_0_1 : ComplexPattern">; -def sve_ext_imm_0_3 : ComplexPattern">; -def sve_ext_imm_0_7 : ComplexPattern">; -def sve_ext_imm_0_15 : ComplexPattern">; +def sve_ext_imm_0_1 : ComplexPattern">; +def sve_ext_imm_0_3 : ComplexPattern">; +def sve_ext_imm_0_7 : ComplexPattern">; +def sve_ext_imm_0_15 : ComplexPattern">; def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), (int_aarch64_sve_cntp node:$pred, node:$src2), [{ @@ -429,8 +437,8 @@ : Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; -def SVEDup0 : ComplexPattern; -def SVEDup0Undef : ComplexPattern; +def SVEDup0 : ComplexPattern; +def SVEDup0Undef : ComplexPattern; let AddedComplexity = 1 in { class SVE_3_Op_Pat_SelZero; def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; } multiclass sve_int_arith_imm1_unsigned opc, string asm, SDPatternOperator op> { @@ -4233,10 +4241,10 @@ def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; } //===----------------------------------------------------------------------===// @@ -4427,7 +4435,7 @@ (!cast(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; def : Pat<(intty (vselect predty:$Pg, - (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), + (intty (AArch64dup (scalarty (SVE8BitLslImm.Pat i32:$imm, i32:$shift)))), intty:$Zd)), (!cast(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>; } @@ -4465,7 +4473,7 @@ (!cast(NAME) PPRAny:$Ps1, 1, 0)>; def : Pat<(intty (vselect predty:$Pg, - (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), + (intty (AArch64dup (scalarty (SVE8BitLslImm.Pat i32:$imm, i32:$shift)))), (intty (AArch64dup (scalarty 0))))), (!cast(NAME) $Pg, i32:$imm, i32:$shift)>; }