diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -736,6 +736,22 @@ bool matchCombineFMinMaxNaN(MachineInstr &MI, unsigned &Info); + /// Transform G_FMUL(G_FMUL(x, c0) c1) to G_FMUL(x, c0 * c1). + /// Unsafe floating point math only. + bool matchFMulFMulWithConstantOps( + MachineInstr &MI, std::pair> &MatchInfo); + bool applyFMulFMulWithConstantOps( + MachineInstr &MI, std::pair> &MatchInfo); + + /// Transform G_FMUL(G_FADD(x, x) c) to G_FMUL(x, c * 2.0). + /// Unsafe floating point math only. + bool + matchFMulFAddWithSameReg(MachineInstr &MI, + std::pair> &MatchInfo); + bool + applyFMulFAddWithSameReg(MachineInstr &MI, + std::pair> &MatchInfo); + private: /// Given a non-indexed load or store instruction \p MI, find an offset that /// can be usefully and legally folded into it as a post-indexing operation. diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -898,6 +898,28 @@ [{ return Helper.matchCombineFMinMaxNaN(*${root}, ${info}); }]), (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, ${info}); }])>; +// Transform (fmul (fmul x, c0), c1) -> (fmul x, c0 * c1) +// Unsafe fp math only +def fmul_fmul_cst_matchinfo : GIDefMatchData<"std::pair>">; +def fmul_fmul_cst: GICombineRule< + (defs root:$root, fmul_fmul_cst_matchinfo:$matchinfo), + (match (wip_match_opcode G_FMUL):$root, + [{ return Helper.matchFMulFMulWithConstantOps(*${root}, + ${matchinfo}); }]), + (apply [{ return Helper.applyFMulFMulWithConstantOps(*${root}, + ${matchinfo}); }])>; + +// Transform (fmul (fadd x, x), c) -> (fmul x, c * 2.0) +// Unsafe fp math only +def fmul_fadd_cst_matchinfo : GIDefMatchData<"std::pair>">; +def fmul_fadd_cst: GICombineRule< + (defs root:$root, fmul_fadd_cst_matchinfo:$matchinfo), + (match (wip_match_opcode G_FMUL):$root, + [{ return Helper.matchFMulFAddWithSameReg(*${root}, + ${matchinfo}); }]), + (apply [{ return Helper.applyFMulFAddWithSameReg(*${root}, + ${matchinfo}); }])>; + // FIXME: These should use the custom predicate feature once it lands. def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero, undef_to_negative_one, @@ -917,7 +939,8 @@ def const_combines : GICombineGroup<[constant_fp_op, const_ptradd_to_i2p, overlapping_and, mulo_by_2, mulo_by_0, - addo_by_0, combine_minmax_nan]>; + addo_by_0, combine_minmax_nan, + fmul_fmul_cst, fmul_fadd_cst]>; def known_bits_simplifications : GICombineGroup<[ redundant_and, redundant_sext_inreg, redundant_or, urem_pow2_to_mask, diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -5633,6 +5633,88 @@ return MatchNaN(1) || MatchNaN(2); } +bool CombinerHelper::matchFMulFMulWithConstantOps( + MachineInstr &MI, std::pair> &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_FMUL && "Expected a G_FMUL"); + auto Options = MI.getMF()->getTarget().Options; + if (!(Options.UnsafeFPMath || MI.getFlag(MachineInstr::MIFlag::FmReassoc))) + return false; + + Register Dst = MI.getOperand(0).getReg(); + Register Src; + const ConstantFP *FP0{}; + const ConstantFP *FP1{}; + if (mi_match(Dst, MRI, + m_GFMul(m_GFMul(m_Reg(Src), m_GFCst(FP0)), m_GFCst(FP1)))) { + const fltSemantics &FPSemantic = getFltSemanticForLLT(MRI.getType(Dst)); + APFloat RHS = FP0->getValueAPF(); + APFloat LHS = FP1->getValueAPF(); + bool Unused; + RHS.convert(FPSemantic, APFloat::rmNearestTiesToEven, &Unused); + LHS.convert(FPSemantic, APFloat::rmNearestTiesToEven, &Unused); + MatchInfo = std::make_pair(Src, RHS * LHS); + return true; + } + return false; +} + +bool CombinerHelper::applyFMulFMulWithConstantOps( + MachineInstr &MI, std::pair> &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_FMUL && + MatchInfo.second.hasValue() && + "Expected a G_FMUL and an non-empty optional"); + Register Src = MatchInfo.first; + APFloat NewFP = MatchInfo.second.getValue(); + Register Dst = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(Dst); + + Builder.setInstrAndDebugLoc(MI); + auto MIBFPNewC = Builder.buildFConstant( + DstTy, *ConstantFP::get(MI.getMF()->getFunction().getContext(), NewFP)); + Builder.buildFMul({Dst}, Src, MIBFPNewC, MI.getFlags()); + MI.eraseFromParent(); + return true; +} + +bool CombinerHelper::matchFMulFAddWithSameReg( + MachineInstr &MI, std::pair> &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_FMUL && "Expected a G_FMUL"); + auto Options = MI.getMF()->getTarget().Options; + if (!Options.UnsafeFPMath && !MI.getFlag(MachineInstr::MIFlag::FmReassoc)) + return false; + + Register Dst = MI.getOperand(0).getReg(); + Register Src0; + Register Src1; + const ConstantFP *FP{}; + if (mi_match(Dst, MRI, + m_GFMul(m_GFAdd(m_Reg(Src0), m_Reg(Src1)), m_GFCst(FP))) && + Src0 == Src1) { + APFloat OldFP = FP->getValueAPF(); + APFloat NewFP = OldFP * APFloat(OldFP.getSemantics(), "2.0"); + MatchInfo = std::make_pair(Src0, NewFP); + return true; + } + return false; +} + +bool CombinerHelper::applyFMulFAddWithSameReg( + MachineInstr &MI, std::pair> &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_FMUL && + MatchInfo.second.hasValue() && "Expected a G_FMUL"); + Register Src = MatchInfo.first; + APFloat NewFP = MatchInfo.second.getValue(); + Register Dst = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(Dst); + + Builder.setInstrAndDebugLoc(MI); + auto MIBFPNewC = Builder.buildFConstant( + DstTy, *ConstantFP::get(MI.getMF()->getFunction().getContext(), NewFP)); + Builder.buildFMul({Dst}, Src, MIBFPNewC, MI.getFlags()); + MI.eraseFromParent(); + return true; +} + bool CombinerHelper::tryCombine(MachineInstr &MI) { if (tryCombineCopy(MI)) return true; diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-unsafe-fp-math.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-unsafe-fp-math.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-unsafe-fp-math.mir @@ -0,0 +1,66 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s --check-prefixes=SAFE +# RUN: llc -enable-unsafe-fp-math -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s --check-prefixes=UNSAFE + +--- +name: fmul_fmul_cst +alignment: 4 +tracksRegLiveness: true +frameInfo: + maxAlignment: 1 +machineFunctionInfo: {} +body: | + bb.0: + liveins: $x0 + ; SAFE-LABEL: name: fmul_fmul_cst + ; SAFE: liveins: $x0 + ; SAFE-NEXT: {{ $}} + ; SAFE-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; SAFE-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 2.000000e+00 + ; SAFE-NEXT: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[C]] + ; SAFE-NEXT: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[FMUL]], [[C]] + ; SAFE-NEXT: $x0 = COPY [[FMUL1]](s64) + ; UNSAFE-LABEL: name: fmul_fmul_cst + ; UNSAFE: liveins: $x0 + ; UNSAFE-NEXT: {{ $}} + ; UNSAFE-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; UNSAFE-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 4.000000e+00 + ; UNSAFE-NEXT: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[C]] + ; UNSAFE-NEXT: $x0 = COPY [[FMUL]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 2.0 + %2:_(s64) = G_FMUL %0, %1(s64) + %3:_(s64) = G_FMUL %2, %1(s64) + $x0 = COPY %3(s64) +... +--- +name: fmul_fadd_same_reg +alignment: 4 +tracksRegLiveness: true +frameInfo: + maxAlignment: 1 +machineFunctionInfo: {} +body: | + bb.0: + liveins: $x0 + ; SAFE-LABEL: name: fmul_fadd_same_reg + ; SAFE: liveins: $x0 + ; SAFE-NEXT: {{ $}} + ; SAFE-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; SAFE-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 4.000000e+00 + ; SAFE-NEXT: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[COPY]] + ; SAFE-NEXT: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[FADD]], [[C]] + ; SAFE-NEXT: $x0 = COPY [[FMUL]](s64) + ; UNSAFE-LABEL: name: fmul_fadd_same_reg + ; UNSAFE: liveins: $x0 + ; UNSAFE-NEXT: {{ $}} + ; UNSAFE-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; UNSAFE-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 8.000000e+00 + ; UNSAFE-NEXT: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[C]] + ; UNSAFE-NEXT: $x0 = COPY [[FMUL]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 4.0 + %2:_(s64) = G_FADD %0, %0 + %3:_(s64) = G_FMUL %2, %1(s64) + $x0 = COPY %3(s64) +...