diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -965,6 +965,12 @@ LLVMPointerToElt<0>], [IntrReadMem, IntrArgMemOnly]>; + class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerToElt<0>], + [IntrInaccessibleMemOrArgMemOnly]>; + class AdvSIMD_1Vec_PredStore_Intrinsic : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, @@ -1397,6 +1403,15 @@ ], [IntrReadMem, IntrArgMemOnly]>; +class AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [ + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> + ], + [IntrInaccessibleMemOrArgMemOnly]>; + class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [ @@ -1406,6 +1421,15 @@ ], [IntrReadMem, IntrArgMemOnly]>; +class AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [ + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> + ], + [IntrInaccessibleMemOrArgMemOnly]>; + class AdvSIMD_GatherLoad_VS_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [ @@ -1415,6 +1439,15 @@ ], [IntrReadMem]>; +class AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + [ + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty, + llvm_i64_ty + ], + [IntrInaccessibleMemOrArgMemOnly]>; + class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic : DefaultAttrsIntrinsic<[], [ @@ -1506,8 +1539,8 @@ def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic; def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic; -def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic; -def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic; +def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; +def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic; def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic; @@ -1692,10 +1725,10 @@ // FFR manipulation // -def int_aarch64_sve_rdffr : GCCBuiltin<"__builtin_sve_svrdffr">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], []>; -def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>; -def int_aarch64_sve_setffr : GCCBuiltin<"__builtin_sve_svsetffr">, DefaultAttrsIntrinsic<[], []>; -def int_aarch64_sve_wrffr : GCCBuiltin<"__builtin_sve_svwrffr">, DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty]>; +def int_aarch64_sve_rdffr : GCCBuiltin<"__builtin_sve_svrdffr">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>; +def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>; +def int_aarch64_sve_setffr : GCCBuiltin<"__builtin_sve_svsetffr">, DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>; +def int_aarch64_sve_wrffr : GCCBuiltin<"__builtin_sve_svwrffr">, DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>; // // Saturating scalar arithmetic @@ -2046,24 +2079,24 @@ // // 64 bit unscaled offsets -def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; +def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; // 64 bit scaled offsets -def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; +def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; // 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits -def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; -def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; +def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; +def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; // 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits -def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; -def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; +def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; +def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; // // First-faulting gather loads: vector base + scalar offset // -def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; +def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic; //