diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -794,7 +794,7 @@ [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; class AdvSIMD_3VectorArgIndexed_Intrinsic : Intrinsic<[llvm_anyvector_ty], @@ -802,7 +802,7 @@ LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; class AdvSIMD_Pred1VectorArg_Intrinsic : Intrinsic<[llvm_anyvector_ty], @@ -894,7 +894,7 @@ [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; class AdvSIMD_SVE_ShiftWide_Intrinsic : Intrinsic<[llvm_anyvector_ty], @@ -932,7 +932,7 @@ LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; class AdvSIMD_SVE_EXPA_Intrinsic : Intrinsic<[llvm_anyvector_ty], @@ -1012,7 +1012,7 @@ LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; class AdvSIMD_SVE_PTEST_Intrinsic : Intrinsic<[llvm_i1_ty], @@ -1039,7 +1039,7 @@ LLVMSubdivide2VectorType<0>, LLVMSubdivide2VectorType<0>, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; class SVE2_1VectorArg_Narrowing_Intrinsic : Intrinsic<[LLVMSubdivide2VectorType<0>], diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -647,6 +647,13 @@ let DecoderMethod = "DecodeVecShiftR32Imm"; let ParserMatchClass = Imm1_32Operand; } +def tvecshiftR64 : Operand, TImmLeaf 0) && (((uint32_t)Imm) < 65); +}]> { + let EncoderMethod = "getVecShiftR64OpValue"; + let DecoderMethod = "DecodeVecShiftR64Imm"; + let ParserMatchClass = Imm1_64Operand; +} def Imm0_1Operand : AsmImmRange<0, 1>; def Imm0_7Operand : AsmImmRange<0, 7>; @@ -683,7 +690,6 @@ let ParserMatchClass = Imm0_63Operand; } - // Crazy immediate formats used by 32-bit and 64-bit logical immediate // instructions for splatting repeating bit patterns across the immediate. def logical_imm32_XFORM : SDNodeXForm, ImmLeaf, TImmLeaf { let ParserMatchClass = Imm0_7Operand; @@ -1091,29 +1097,44 @@ let RenderMethod = "addVectorIndexOperands"; } -class AsmVectorIndexOpnd - : Operand, ImmLeaf { +class AsmVectorIndexOpnd + : Operand { let ParserMatchClass = mc; let PrintMethod = "printVectorIndex"; } +multiclass VectorIndex { + def "" : AsmVectorIndexOpnd, ImmLeaf; + def _timm : AsmVectorIndexOpnd, TImmLeaf; +} + def VectorIndex1Operand : AsmVectorIndex<1, 1>; def VectorIndexBOperand : AsmVectorIndex<0, 15>; def VectorIndexHOperand : AsmVectorIndex<0, 7>; def VectorIndexSOperand : AsmVectorIndex<0, 3>; def VectorIndexDOperand : AsmVectorIndex<0, 1>; -def VectorIndex1 : AsmVectorIndexOpnd; -def VectorIndexB : AsmVectorIndexOpnd; -def VectorIndexH : AsmVectorIndexOpnd; -def VectorIndexS : AsmVectorIndexOpnd; -def VectorIndexD : AsmVectorIndexOpnd; - -def VectorIndex132b : AsmVectorIndexOpnd; -def VectorIndexB32b : AsmVectorIndexOpnd; -def VectorIndexH32b : AsmVectorIndexOpnd; -def VectorIndexS32b : AsmVectorIndexOpnd; -def VectorIndexD32b : AsmVectorIndexOpnd; +defm VectorIndex1 : VectorIndex; +defm VectorIndexB : VectorIndex; +defm VectorIndexH : VectorIndex; +defm VectorIndexS : VectorIndex; +defm VectorIndexD : VectorIndex; + +defm VectorIndex132b : VectorIndex; +defm VectorIndexB32b : VectorIndex; +defm VectorIndexH32b : VectorIndex; +defm VectorIndexS32b : VectorIndex; +defm VectorIndexD32b : VectorIndex; def SVEVectorIndexExtDupBOperand : AsmVectorIndex<0, 63, "SVE">; def SVEVectorIndexExtDupHOperand : AsmVectorIndex<0, 31, "SVE">; @@ -1121,16 +1142,21 @@ def SVEVectorIndexExtDupDOperand : AsmVectorIndex<0, 7, "SVE">; def SVEVectorIndexExtDupQOperand : AsmVectorIndex<0, 3, "SVE">; -def sve_elm_idx_extdup_b - : AsmVectorIndexOpnd; -def sve_elm_idx_extdup_h - : AsmVectorIndexOpnd; -def sve_elm_idx_extdup_s - : AsmVectorIndexOpnd; -def sve_elm_idx_extdup_d - : AsmVectorIndexOpnd; -def sve_elm_idx_extdup_q - : AsmVectorIndexOpnd; +defm sve_elm_idx_extdup_b + : VectorIndex; +defm sve_elm_idx_extdup_h + : VectorIndex; +defm sve_elm_idx_extdup_s + : VectorIndex; +defm sve_elm_idx_extdup_d + : VectorIndex; +defm sve_elm_idx_extdup_q + : VectorIndex; // 8-bit immediate for AdvSIMD where 64-bit values of the form: // aaaaaaaa bbbbbbbb cccccccc dddddddd eeeeeeee ffffffff gggggggg hhhhhhhh diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -1646,12 +1646,12 @@ let Inst{19-16} = Zm; } - def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b:$idx))), - (!cast(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b:$idx)>; - def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b:$idx))), - (!cast(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b:$idx)>; - def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b:$idx))), - (!cast(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b:$idx)>; + def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), + (!cast(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; + def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), + (!cast(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; + def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), + (!cast(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; } @@ -1694,12 +1694,12 @@ let Inst{19-16} = Zm; } - def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b:$idx))), - (!cast(NAME # _H) $Op1, $Op2, VectorIndexH32b:$idx)>; - def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b:$idx))), - (!cast(NAME # _S) $Op1, $Op2, VectorIndexS32b:$idx)>; - def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b:$idx))), - (!cast(NAME # _D) $Op1, $Op2, VectorIndexD32b:$idx)>; + def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), + (!cast(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; + def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), + (!cast(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; + def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), + (!cast(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; } //===----------------------------------------------------------------------===// @@ -1785,10 +1785,10 @@ let Inst{19-16} = Zm; } - def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b:$idx), (i32 complexrotateop:$imm))), - (!cast(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b:$idx, complexrotateop:$imm)>; - def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b:$idx), (i32 complexrotateop:$imm))), - (!cast(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b:$idx, complexrotateop:$imm)>; + def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), + (!cast(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; + def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), + (!cast(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; } //===----------------------------------------------------------------------===// @@ -1949,7 +1949,7 @@ multiclass sve2_fp_mla_long_by_indexed_elem opc, string asm, SDPatternOperator op> { def NAME : sve2_fp_mla_long_by_indexed_elem; - def : SVE_4_Op_Imm_Pat(NAME)>; + def : SVE_4_Op_Imm_Pat(NAME)>; } //===----------------------------------------------------------------------===// @@ -2479,23 +2479,23 @@ multiclass sve_intx_dot_by_indexed_elem { - def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { + def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { bits<2> iop; bits<3> Zm; let Inst{20-19} = iop; let Inst{18-16} = Zm; } - def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { + def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { bits<1> iop; bits<4> Zm; let Inst{20} = iop; let Inst{19-16} = Zm; } - def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv16i8:$Op2, nxv16i8:$Op3, (i32 VectorIndexS32b:$idx))), - (!cast(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b:$idx)>; - def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv8i16:$Op2, nxv8i16:$Op3, (i32 VectorIndexD32b:$idx))), - (!cast(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b:$idx)>; + def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv16i8:$Op2, nxv16i8:$Op3, (i32 VectorIndexS32b_timm:$idx))), + (!cast(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; + def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv8i16:$Op2, nxv8i16:$Op3, (i32 VectorIndexD32b_timm:$idx))), + (!cast(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; } //===----------------------------------------------------------------------===// @@ -4406,10 +4406,10 @@ let Inst{9-8} = imm{4-3}; } - def : SVE_3_Op_Imm_Pat(NAME # _B)>; - def : SVE_3_Op_Imm_Pat(NAME # _H)>; - def : SVE_3_Op_Imm_Pat(NAME # _S)>; - def : SVE_3_Op_Imm_Pat(NAME # _D)>; + def : SVE_3_Op_Imm_Pat(NAME # _B)>; + def : SVE_3_Op_Imm_Pat(NAME # _H)>; + def : SVE_3_Op_Imm_Pat(NAME # _S)>; + def : SVE_3_Op_Imm_Pat(NAME # _D)>; } class sve_int_bin_pred_shift sz8_64, bit wide, bits<3> opc,