diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -1265,11 +1265,21 @@ static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC, IntrinsicInst &II) { + if (auto FMLA_U = + instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, + Intrinsic::aarch64_sve_fmla_u>( + IC, II, true)) + return FMLA_U; if (auto FMLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, Intrinsic::aarch64_sve_fmla>(IC, II, true)) return FMLA; + if (auto MLA_U = + instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, + Intrinsic::aarch64_sve_mla_u>( + IC, II, true)) + return MLA_U; if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul, Intrinsic::aarch64_sve_mla>( IC, II, true)) @@ -1288,11 +1298,21 @@ static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC, IntrinsicInst &II) { + if (auto FMLS_U = + instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, + Intrinsic::aarch64_sve_fmls_u>( + IC, II, true)) + return FMLS_U; if (auto FMLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, Intrinsic::aarch64_sve_fmls>(IC, II, true)) return FMLS; + if (auto MLS_U = + instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, + Intrinsic::aarch64_sve_mls_u>( + IC, II, true)) + return MLS_U; if (auto MLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul, Intrinsic::aarch64_sve_mls>( IC, II, true)) @@ -1677,27 +1697,15 @@ case Intrinsic::aarch64_sve_fmul: return instCombineSVEVectorMul(IC, II); case Intrinsic::aarch64_sve_fadd: - case Intrinsic::aarch64_sve_add: - return instCombineSVEVectorAdd(IC, II); case Intrinsic::aarch64_sve_fadd_u: - return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, - Intrinsic::aarch64_sve_fmla_u>( - IC, II, true); + case Intrinsic::aarch64_sve_add: case Intrinsic::aarch64_sve_add_u: - return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, - Intrinsic::aarch64_sve_mla_u>( - IC, II, true); + return instCombineSVEVectorAdd(IC, II); case Intrinsic::aarch64_sve_fsub: - case Intrinsic::aarch64_sve_sub: - return instCombineSVEVectorSub(IC, II); case Intrinsic::aarch64_sve_fsub_u: - return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, - Intrinsic::aarch64_sve_fmls_u>( - IC, II, true); + case Intrinsic::aarch64_sve_sub: case Intrinsic::aarch64_sve_sub_u: - return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, - Intrinsic::aarch64_sve_mls_u>( - IC, II, true); + return instCombineSVEVectorSub(IC, II); case Intrinsic::aarch64_sve_tbl: return instCombineSVETBL(IC, II); case Intrinsic::aarch64_sve_uunpkhi: