diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -105,6 +105,18 @@ : InstrsToBuild(InstrsToBuild) {} }; +struct FMinMaxNanMatchInfo { + bool PropogateNaN; + unsigned NaNIdx; + unsigned getIdxToPropogate() const { + assert((NaNIdx == 1 || NaNIdx == 2) && "Unsupported NaNIdx!"); + return PropogateNaN ? NaNIdx : getOppositeIdx(); + } + +private: + unsigned getOppositeIdx() const { return NaNIdx == 1 ? 2 : 1; } +}; + class CombinerHelper { protected: MachineIRBuilder &Builder; @@ -734,6 +746,9 @@ /// Fold boolean selects to logical operations. bool matchSelectToLogical(MachineInstr &MI, BuildFnTy &MatchInfo); + bool matchCombineFMinMaxNaN(MachineInstr &MI, FMinMaxNanMatchInfo &Info); + void applyCombineFMinMaxNaN(MachineInstr &MI, FMinMaxNanMatchInfo &Info); + private: /// Given a non-indexed load or store instruction \p MI, find an offset that /// can be usefully and legally folded into it as a post-indexing operation. diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -891,6 +891,13 @@ *${root}, ${info}); }]), (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>; +def combine_minmax_nan_matchinfo: GIDefMatchData<"FMinMaxNanMatchInfo">; +def combine_minmax_nan: GICombineRule< + (defs root:$root, combine_minmax_nan_matchinfo:$info), + (match (wip_match_opcode G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root, + [{ return Helper.matchCombineFMinMaxNaN(*${root}, ${info}); }]), + (apply [{ Helper.applyCombineFMinMaxNaN(*${root}, ${info}); }])>; + // FIXME: These should use the custom predicate feature once it lands. def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero, undef_to_negative_one, @@ -910,7 +917,7 @@ def const_combines : GICombineGroup<[constant_fp_op, const_ptradd_to_i2p, overlapping_and, mulo_by_2, mulo_by_0, - addo_by_0]>; + addo_by_0, combine_minmax_nan]>; def known_bits_simplifications : GICombineGroup<[ redundant_and, redundant_sext_inreg, redundant_or, urem_pow2_to_mask, diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -5600,6 +5600,39 @@ return false; } +bool CombinerHelper::matchCombineFMinMaxNaN(MachineInstr &MI, + FMinMaxNanMatchInfo &Info) { + switch (MI.getOpcode()) { + default: + return false; + case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMAXNUM: + Info.PropogateNaN = false; + break; + case TargetOpcode::G_FMINIMUM: + case TargetOpcode::G_FMAXIMUM: + Info.PropogateNaN = true; + break; + } + + auto MatchNaN = [&](unsigned Idx) { + Register MaybeNaNReg = MI.getOperand(Idx).getReg(); + const ConstantFP *MaybeCst = getConstantFPVRegVal(MaybeNaNReg, MRI); + if (!MaybeCst || !MaybeCst->getValueAPF().isNaN()) + return false; + Info.NaNIdx = Idx; + return true; + }; + return MatchNaN(1) || MatchNaN(2); +} + +void CombinerHelper::applyCombineFMinMaxNaN(MachineInstr &MI, + FMinMaxNanMatchInfo &Info) { + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(Info.getIdxToPropogate()).getReg(); + MI.eraseFromParent(); + replaceRegWith(MRI, DstReg, SrcReg); +} bool CombinerHelper::tryCombine(MachineInstr &MI) { if (tryCombineCopy(MI)) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-fminimum-fmaximum.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-fminimum-fmaximum.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-fminimum-fmaximum.mir @@ -0,0 +1,159 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s + +--- +name: test_combine_nan_rhs_fminimum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fminimum_half + ; CHECK: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH7C01 + ; CHECK-NEXT: $h0 = COPY [[C]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMINIMUM %0, %1 + $h0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fminimum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fminimum_float + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + ; CHECK-NEXT: $w0 = COPY [[C]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMINIMUM %0, %1 + $w0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fminimum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fminimum_double + ; CHECK: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + ; CHECK-NEXT: $x0 = COPY [[C]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMINIMUM %0, %1 + $x0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fminimum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fminimum_half + ; CHECK: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH7C01 + ; CHECK-NEXT: $h0 = COPY [[C]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMINIMUM %1, %0 + $h0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fminimum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fminimum_float + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + ; CHECK-NEXT: $w0 = COPY [[C]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMINIMUM %1, %0 + $w0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fminimum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fminimum_double + ; CHECK: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + ; CHECK-NEXT: $x0 = COPY [[C]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMINIMUM %1, %0 + $x0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fmaximum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fmaximum_half + ; CHECK: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH7C01 + ; CHECK-NEXT: $h0 = COPY [[C]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMAXIMUM %0, %1 + $h0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fmaximum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fmaximum_float + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + ; CHECK-NEXT: $w0 = COPY [[C]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMAXIMUM %0, %1 + $w0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fmaximum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fmaximum_double + ; CHECK: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + ; CHECK-NEXT: $x0 = COPY [[C]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMAXIMUM %0, %1 + $x0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fmaximum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fmaximum_half + ; CHECK: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH7C01 + ; CHECK-NEXT: $h0 = COPY [[C]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMAXIMUM %1, %0 + $h0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fmaximum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fmaximum_float + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + ; CHECK-NEXT: $w0 = COPY [[C]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMAXIMUM %1, %0 + $w0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fmaximum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fmaximum_double + ; CHECK: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + ; CHECK-NEXT: $x0 = COPY [[C]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMAXIMUM %1, %0 + $x0 = COPY %2 +... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-fminnum-fmaxnum.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-fminnum-fmaxnum.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-fminnum-fmaxnum.mir @@ -0,0 +1,159 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s + +--- +name: test_combine_nan_rhs_fminnum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fminnum_half + ; CHECK: [[COPY:%[0-9]+]]:_(s16) = COPY $h0 + ; CHECK-NEXT: $h0 = COPY [[COPY]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMINNUM %0, %1 + $h0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fminnum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fminnum_float + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMINNUM %0, %1 + $w0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fminnum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fminnum_double + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY [[COPY]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMINNUM %0, %1 + $x0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fminnum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fminnum_half + ; CHECK: [[COPY:%[0-9]+]]:_(s16) = COPY $h0 + ; CHECK-NEXT: $h0 = COPY [[COPY]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMINNUM %1, %0 + $h0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fminnum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fminnum_float + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMINNUM %1, %0 + $w0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fminnum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fminnum_double + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY [[COPY]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMINNUM %1, %0 + $x0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fmaxnum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fmaxnum_half + ; CHECK: [[COPY:%[0-9]+]]:_(s16) = COPY $h0 + ; CHECK-NEXT: $h0 = COPY [[COPY]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMAXNUM %0, %1 + $h0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fmaxnum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fmaxnum_float + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMAXNUM %0, %1 + $w0 = COPY %2 +... +--- +name: test_combine_nan_rhs_fmaxnum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_rhs_fmaxnum_double + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY [[COPY]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMAXNUM %0, %1 + $x0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fmaxnum_half +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fmaxnum_half + ; CHECK: [[COPY:%[0-9]+]]:_(s16) = COPY $h0 + ; CHECK-NEXT: $h0 = COPY [[COPY]](s16) + %0:_(s16) = COPY $h0 + %1:_(s16) = G_FCONSTANT half 0xH7C01 + %2:_(s16) = G_FMAXNUM %1, %0 + $h0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fmaxnum_float +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fmaxnum_float + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) + %0:_(s32) = COPY $w0 + %1:_(s32) = G_FCONSTANT float 0x7FF8000000000000 + %2:_(s32) = G_FMAXNUM %1, %0 + $w0 = COPY %2 +... +--- +name: test_combine_nan_lhs_fmaxnum_double +body: | + bb.1: + liveins: + ; CHECK-LABEL: name: test_combine_nan_lhs_fmaxnum_double + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY [[COPY]](s64) + %0:_(s64) = COPY $x0 + %1:_(s64) = G_FCONSTANT double 0x7FF8000000000000 + %2:_(s64) = G_FMAXNUM %1, %0 + $x0 = COPY %2 +...