diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -136,15 +136,15 @@ // // SVE CNT/INC/RDVL -def sve_rdvl_imm : ComplexPattern">; -def sve_cnth_imm : ComplexPattern">; -def sve_cntw_imm : ComplexPattern">; -def sve_cntd_imm : ComplexPattern">; +def sve_rdvl_imm : ComplexPattern">; +def sve_cnth_imm : ComplexPattern">; +def sve_cntw_imm : ComplexPattern">; +def sve_cntd_imm : ComplexPattern">; // SVE DEC -def sve_cnth_imm_neg : ComplexPattern">; -def sve_cntw_imm_neg : ComplexPattern">; -def sve_cntd_imm_neg : ComplexPattern">; +def sve_cnth_imm_neg : ComplexPattern">; +def sve_cntw_imm_neg : ComplexPattern">; +def sve_cntd_imm_neg : ComplexPattern">; def SDT_AArch64Reduce : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisVec<2>]>; def AArch64faddv_p : SDNode<"AArch64ISD::FADDV_PRED", SDT_AArch64Reduce>; @@ -624,13 +624,13 @@ def : Pat<(nxv8bf16 (AArch64dup (bf16 fpimm0))), (DUP_ZI_H 0, 0)>; // Duplicate Int immediate into all vector elements - def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_B $a, $b)>; - def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_H $a, $b)>; - def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_S $a, $b)>; - def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm64 i32:$a, i32:$b)))), (DUP_ZI_D $a, $b)>; // Duplicate immediate FP into all vector elements. @@ -2686,13 +2686,13 @@ // Splice with lane bigger or equal to 0 def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 (sve_ext_imm_0_255 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_255:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 (sve_ext_imm_0_127 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_127:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 (sve_ext_imm_0_63 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_63:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 (sve_ext_imm_0_31 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_31:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; } // End HasSVEorStreamingSVE diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -197,34 +197,42 @@ def SVEAddSubImm8Pat : ComplexPattern", []>; def SVEAddSubImm16Pat : ComplexPattern", []>; def SVEAddSubImm32Pat : ComplexPattern", []>; -def SVEAddSubImm64Pat : ComplexPattern", []>; +def SVEAddSubImm64Pat : ComplexPattern", []>; -def SVELogicalImm8Pat : ComplexPattern", []>; -def SVELogicalImm16Pat : ComplexPattern", []>; -def SVELogicalImm32Pat : ComplexPattern", []>; +def SVELogicalImm8Pat : ComplexPattern", []>; +def SVELogicalImm16Pat : ComplexPattern", []>; +def SVELogicalImm32Pat : ComplexPattern", []>; def SVELogicalImm64Pat : ComplexPattern", []>; -def SVELogicalImm8NotPat : ComplexPattern", []>; -def SVELogicalImm16NotPat : ComplexPattern", []>; -def SVELogicalImm32NotPat : ComplexPattern", []>; +def SVELogicalImm8NotPat : ComplexPattern", []>; +def SVELogicalImm16NotPat : ComplexPattern", []>; +def SVELogicalImm32NotPat : ComplexPattern", []>; def SVELogicalImm64NotPat : ComplexPattern", []>; -def SVE8BitLslImm : ComplexPattern; +def SVE8BitLslImm32 : ComplexPattern; +def SVE8BitLslImm64 : ComplexPattern; +class SVE8BitLslImm { + ComplexPattern Pat = !cond( + !eq(ty, i32): SVE8BitLslImm32, + !eq(ty, i64): SVE8BitLslImm64); +} def SVEArithUImm8Pat : ComplexPattern", []>; def SVEArithUImm16Pat : ComplexPattern", []>; def SVEArithUImm32Pat : ComplexPattern", []>; -def SVEArithUImm64Pat : ComplexPattern", []>; -def SVEArithSImmPat : ComplexPattern; +def SVEArithUImm64Pat : ComplexPattern", []>; + +def SVEArithSImmPat32 : ComplexPattern; +def SVEArithSImmPat64 : ComplexPattern; def SVEShiftImmL8 : ComplexPattern", []>; def SVEShiftImmL16 : ComplexPattern", []>; def SVEShiftImmL32 : ComplexPattern", []>; -def SVEShiftImmL64 : ComplexPattern", []>; +def SVEShiftImmL64 : ComplexPattern", []>; def SVEShiftImmR8 : ComplexPattern", []>; def SVEShiftImmR16 : ComplexPattern", []>; def SVEShiftImmR32 : ComplexPattern", []>; -def SVEShiftImmR64 : ComplexPattern", []>; +def SVEShiftImmR64 : ComplexPattern", []>; def SVEAllActive : ComplexPattern; @@ -260,14 +268,15 @@ } // This allows i32 immediate extraction from i64 based arithmetic. -def sve_cnt_mul_imm : ComplexPattern">; -def sve_cnt_shl_imm : ComplexPattern">; +def sve_cnt_mul_imm_i32 : ComplexPattern">; +def sve_cnt_mul_imm_i64 : ComplexPattern">; +def sve_cnt_shl_imm : ComplexPattern">; -def sve_ext_imm_0_31 : ComplexPattern">; -def sve_ext_imm_0_63 : ComplexPattern">; -def sve_ext_imm_0_127 : ComplexPattern">; -def sve_ext_imm_0_255 : ComplexPattern">; +def sve_ext_imm_0_31 : ComplexPattern">; +def sve_ext_imm_0_63 : ComplexPattern">; +def sve_ext_imm_0_127 : ComplexPattern">; +def sve_ext_imm_0_255 : ComplexPattern">; def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), (int_aarch64_sve_cntp node:$pred, node:$src2), [{ @@ -435,8 +444,8 @@ : Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; -def SVEDup0 : ComplexPattern; -def SVEDup0Undef : ComplexPattern; +def SVEDup0 : ComplexPattern; +def SVEDup0Undef : ComplexPattern; let AddedComplexity = 1 in { class SVE_3_Op_Pat_SelZero(NAME) GPR64:$Rd, 0b11111, 1), 2>; - def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))), + def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))), (!cast(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))), @@ -951,7 +960,7 @@ def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))), (!cast(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>; - def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm)))), + def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))), (!cast(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm))))), @@ -962,7 +971,7 @@ GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1), sub_32))>; - def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm i32:$imm)))), + def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))), (i32 (EXTRACT_SUBREG (!cast(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), sub_32))>; @@ -4324,10 +4333,10 @@ def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; } multiclass sve_int_arith_imm1_unsigned opc, string asm, SDPatternOperator op> { @@ -4348,10 +4357,10 @@ def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; - def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _B)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _H)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _S)>; + def : SVE_1_Op_Imm_Arith_All_Active(NAME # _D)>; } //===----------------------------------------------------------------------===// @@ -4542,7 +4551,7 @@ (!cast(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; def : Pat<(intty (vselect predty:$Pg, - (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), + (intty (AArch64dup (scalarty (SVE8BitLslImm.Pat i32:$imm, i32:$shift)))), intty:$Zd)), (!cast(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>; } @@ -4580,7 +4589,7 @@ (!cast(NAME) PPRAny:$Ps1, 1, 0)>; def : Pat<(intty (vselect predty:$Pg, - (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), + (intty (AArch64dup (scalarty (SVE8BitLslImm.Pat i32:$imm, i32:$shift)))), (intty (AArch64dup (scalarty 0))))), (!cast(NAME) $Pg, i32:$imm, i32:$shift)>; }