diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -312,6 +312,7 @@ /// This helper class is used to match scalar and vector constants that /// satisfy a specified predicate, and bind them to an APInt. +/// Undefs are not allowed in splat vector constants. template struct api_pred_ty : public Predicate { const APInt *&Res; @@ -335,6 +336,32 @@ } }; +/// This helper class is used to match scalar and vector constants that +/// satisfy a specified predicate, and bind them to an APFloat. +/// Undefs are not allowed in splat vector constants. +template struct apf_pred_ty : public Predicate { + const APFloat *&Res; + + apf_pred_ty(const APFloat *&R) : Res(R) {} + + template bool match(ITy *V) { + if (const auto *CI = dyn_cast(V)) + if (this->isValue(CI->getValue())) { + Res = &CI->getValue(); + return true; + } + if (V->getType()->isVectorTy()) + if (const auto *C = dyn_cast(V)) + if (auto *CI = dyn_cast_or_null(C->getSplatValue())) + if (this->isValue(CI->getValue())) { + Res = &CI->getValue(); + return true; + } + + return false; + } +}; + /////////////////////////////////////////////////////////////////////////////// // // Encapsulate constant value queries for use in templated predicate matchers. @@ -555,6 +582,15 @@ return cstfp_pred_ty(); } +struct is_nonnan { + bool isValue(const APFloat &C) { return !C.isNaN(); } +}; +/// Match a non-NaN FP constant. +/// For vectors, this includes constants with undefined elements. +inline cstfp_pred_ty m_NonNaN() { + return cstfp_pred_ty(); +} + struct is_inf { bool isValue(const APFloat &C) { return C.isInfinity(); } }; @@ -564,6 +600,25 @@ return cstfp_pred_ty(); } +struct is_noninf { + bool isValue(const APFloat &C) { return !C.isInfinity(); } +}; +/// Match a non-infinity FP constant, i.e. finite or NaN. +/// For vectors, this includes constants with undefined elements. +inline cstfp_pred_ty m_NonInf() { + return cstfp_pred_ty(); +} + +struct is_finite { + bool isValue(const APFloat &C) { return C.isFinite(); } +}; +/// Match a finite FP constant, i.e. not infinity or NaN. +/// For vectors, this includes constants with undefined elements. +inline cstfp_pred_ty m_Finite() { + return cstfp_pred_ty(); +} +inline apf_pred_ty m_Finite(const APFloat *&V) { return V; } + struct is_any_zero_fp { bool isValue(const APFloat &C) { return C.isZero(); } }; @@ -591,6 +646,15 @@ return cstfp_pred_ty(); } +struct is_non_zero_fp { + bool isValue(const APFloat &C) { return C.isNonZero(); } +}; +/// Match a floating-point non-zero. +/// For vectors, this includes constants with undefined elements. +inline cstfp_pred_ty m_NonZeroFP() { + return cstfp_pred_ty(); +} + /////////////////////////////////////////////////////////////////////////////// template struct bind_ty { diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -4070,9 +4070,7 @@ // change the sign of a zero operand. bool HasNoSignedZeros = Q.CxtI && isa(Q.CxtI) && Q.CxtI->hasNoSignedZeros(); - const APFloat *C; - if (HasNoSignedZeros || (match(T, m_APFloat(C)) && C->isNonZero()) || - (match(F, m_APFloat(C)) && C->isNonZero())) { + if (HasNoSignedZeros || match(T, m_NonZeroFP()) || match(F, m_NonZeroFP())) { // (T == F) ? T : F --> F // (F == T) ? T : F --> F if (Pred == FCmpInst::FCMP_OEQ) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -3485,9 +3485,9 @@ if (FPMathOp->hasNoInfs()) return true; - // Handle scalar constants. - if (auto *CFP = dyn_cast(V)) - return !CFP->isInfinity(); + // Handle scalar and fixed width vector constants. + if (isa(V)) + return match(V, m_NonInf()); if (Depth == MaxAnalysisRecursionDepth) return false; @@ -3517,25 +3517,6 @@ } } - // try to handle fixed width vector constants - auto *VFVTy = dyn_cast(V->getType()); - if (VFVTy && isa(V)) { - // For vectors, verify that each element is not infinity. - unsigned NumElts = VFVTy->getNumElements(); - for (unsigned i = 0; i != NumElts; ++i) { - Constant *Elt = cast(V)->getAggregateElement(i); - if (!Elt) - return false; - if (isa(Elt)) - continue; - auto *CElt = dyn_cast(Elt); - if (!CElt || CElt->isInfinity()) - return false; - } - // All elements were confirmed non-infinity or undefined. - return true; - } - // was not able to prove that V never contains infinity return false; } @@ -3549,9 +3530,9 @@ if (FPMathOp->hasNoNaNs()) return true; - // Handle scalar constants. - if (auto *CFP = dyn_cast(V)) - return !CFP->isNaN(); + // Handle scalar and fixed width vector constants. + if (isa(V)) + return match(V, m_NonNaN()); if (Depth == MaxAnalysisRecursionDepth) return false; @@ -3622,25 +3603,6 @@ } } - // Try to handle fixed width vector constants - auto *VFVTy = dyn_cast(V->getType()); - if (VFVTy && isa(V)) { - // For vectors, verify that each element is not NaN. - unsigned NumElts = VFVTy->getNumElements(); - for (unsigned i = 0; i != NumElts; ++i) { - Constant *Elt = cast(V)->getAggregateElement(i); - if (!Elt) - return false; - if (isa(Elt)) - continue; - auto *CElt = dyn_cast(Elt); - if (!CElt || CElt->isNaN()) - return false; - } - // All elements were confirmed not-NaN or undefined. - return true; - } - // Was not able to prove that V never contains NaN return false; } @@ -5326,7 +5288,7 @@ RHS = FalseVal; const APFloat *FC1; - if (CmpRHS != TrueVal || !match(CmpRHS, m_APFloat(FC1)) || !FC1->isFinite()) + if (CmpRHS != TrueVal || !match(CmpRHS, m_Finite(FC1))) return {SPF_UNKNOWN, SPNB_NA, false}; const APFloat *FC2; diff --git a/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll --- a/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll +++ b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll @@ -16,6 +16,21 @@ ret float %r } +; (X < C1) ? C1 : MIN(X, C2) +define <2 x float> @clamp_float_fast_ordered_strict_maxmin_vec(<2 x float> %x) { +; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_ordered_strict_maxmin_vec( +; CHECK-NEXT: [[CMP2:%.*]] = fcmp fast olt <2 x float> [[X:%.*]], +; CHECK-NEXT: [[MIN:%.*]] = select <2 x i1> [[CMP2]], <2 x float> [[X]], <2 x float> +; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x float> @llvm.maxnum.v2f32(<2 x float> [[MIN]], <2 x float> ) +; CHECK-NEXT: ret <2 x float> [[TMP1]] +; + %cmp2 = fcmp fast olt <2 x float> %x, + %min = select <2 x i1> %cmp2, <2 x float> %x, <2 x float> + %cmp1 = fcmp fast olt <2 x float> %x, + %r = select <2 x i1> %cmp1, <2 x float> , <2 x float> %min + ret <2 x float> %r +} + ; (X <= C1) ? C1 : MIN(X, C2) define float @clamp_float_fast_ordered_nonstrict_maxmin(float %x) { ; CHECK-LABEL: define {{[^@]+}}@clamp_float_fast_ordered_nonstrict_maxmin( diff --git a/llvm/test/Transforms/InstSimplify/fcmp-select.ll b/llvm/test/Transforms/InstSimplify/fcmp-select.ll --- a/llvm/test/Transforms/InstSimplify/fcmp-select.ll +++ b/llvm/test/Transforms/InstSimplify/fcmp-select.ll @@ -12,6 +12,17 @@ ret double %cond } +; X == <4.0, 2.0, undef> ? X : <4.0, 2.0, undef> --> <4.0, 2.0, undef> + +define <3 x double> @oeq_vec(<3 x double> %x) { +; CHECK-LABEL: @oeq_vec( +; CHECK-NEXT: ret <3 x double> +; + %cmp = fcmp oeq <3 x double> %x, + %cond = select <3 x i1> %cmp, <3 x double> %x, <3 x double> + ret <3 x double> %cond +} + ; X == 42.0 ? 42.0 : X --> X define float @oeq_swapped(float %x) { diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp --- a/llvm/unittests/IR/PatternMatch.cpp +++ b/llvm/unittests/IR/PatternMatch.cpp @@ -1093,39 +1093,77 @@ Constant *VectorZero = Constant::getNullValue(VectorTy); Constant *ScalarPosInf = ConstantFP::getInfinity(ScalarTy, false); Constant *ScalarNegInf = ConstantFP::getInfinity(ScalarTy, true); + Constant *ScalarNaN = ConstantFP::getNaN(ScalarTy, true); - SmallVector Elems; - Elems.push_back(ScalarUndef); - Elems.push_back(ScalarZero); - Elems.push_back(ScalarUndef); - Elems.push_back(ScalarZero); - Constant *VectorZeroUndef = ConstantVector::get(Elems); + Constant *VectorZeroUndef = + ConstantVector::get({ScalarUndef, ScalarZero, ScalarUndef, ScalarZero}); + + Constant *VectorInfUndef = ConstantVector::get( + {ScalarPosInf, ScalarNegInf, ScalarUndef, ScalarPosInf}); - SmallVector InfElems; - InfElems.push_back(ScalarPosInf); - InfElems.push_back(ScalarNegInf); - InfElems.push_back(ScalarUndef); - InfElems.push_back(ScalarPosInf); - Constant *VectorInfUndef = ConstantVector::get(InfElems); + Constant *VectorNaNUndef = + ConstantVector::get({ScalarUndef, ScalarNaN, ScalarNaN, ScalarNaN}); EXPECT_TRUE(match(ScalarUndef, m_Undef())); EXPECT_TRUE(match(VectorUndef, m_Undef())); EXPECT_FALSE(match(ScalarZero, m_Undef())); EXPECT_FALSE(match(VectorZero, m_Undef())); EXPECT_FALSE(match(VectorZeroUndef, m_Undef())); + EXPECT_FALSE(match(VectorInfUndef, m_Undef())); + EXPECT_FALSE(match(VectorNaNUndef, m_Undef())); EXPECT_FALSE(match(ScalarUndef, m_AnyZeroFP())); EXPECT_FALSE(match(VectorUndef, m_AnyZeroFP())); EXPECT_TRUE(match(ScalarZero, m_AnyZeroFP())); EXPECT_TRUE(match(VectorZero, m_AnyZeroFP())); EXPECT_TRUE(match(VectorZeroUndef, m_AnyZeroFP())); + EXPECT_FALSE(match(VectorInfUndef, m_AnyZeroFP())); + EXPECT_FALSE(match(VectorNaNUndef, m_AnyZeroFP())); + + EXPECT_FALSE(match(ScalarUndef, m_NaN())); + EXPECT_FALSE(match(VectorUndef, m_NaN())); + EXPECT_FALSE(match(VectorZeroUndef, m_NaN())); + EXPECT_FALSE(match(ScalarPosInf, m_NaN())); + EXPECT_FALSE(match(ScalarNegInf, m_NaN())); + EXPECT_TRUE(match(ScalarNaN, m_NaN())); + EXPECT_FALSE(match(VectorInfUndef, m_NaN())); + EXPECT_TRUE(match(VectorNaNUndef, m_NaN())); + + EXPECT_FALSE(match(ScalarUndef, m_NonNaN())); + EXPECT_FALSE(match(VectorUndef, m_NonNaN())); + EXPECT_TRUE(match(VectorZeroUndef, m_NonNaN())); + EXPECT_TRUE(match(ScalarPosInf, m_NonNaN())); + EXPECT_TRUE(match(ScalarNegInf, m_NonNaN())); + EXPECT_FALSE(match(ScalarNaN, m_NonNaN())); + EXPECT_TRUE(match(VectorInfUndef, m_NonNaN())); + EXPECT_FALSE(match(VectorNaNUndef, m_NonNaN())); EXPECT_FALSE(match(ScalarUndef, m_Inf())); EXPECT_FALSE(match(VectorUndef, m_Inf())); EXPECT_FALSE(match(VectorZeroUndef, m_Inf())); EXPECT_TRUE(match(ScalarPosInf, m_Inf())); EXPECT_TRUE(match(ScalarNegInf, m_Inf())); + EXPECT_FALSE(match(ScalarNaN, m_Inf())); EXPECT_TRUE(match(VectorInfUndef, m_Inf())); + EXPECT_FALSE(match(VectorNaNUndef, m_Inf())); + + EXPECT_FALSE(match(ScalarUndef, m_NonInf())); + EXPECT_FALSE(match(VectorUndef, m_NonInf())); + EXPECT_TRUE(match(VectorZeroUndef, m_NonInf())); + EXPECT_FALSE(match(ScalarPosInf, m_NonInf())); + EXPECT_FALSE(match(ScalarNegInf, m_NonInf())); + EXPECT_TRUE(match(ScalarNaN, m_NonInf())); + EXPECT_FALSE(match(VectorInfUndef, m_NonInf())); + EXPECT_TRUE(match(VectorNaNUndef, m_NonInf())); + + EXPECT_FALSE(match(ScalarUndef, m_Finite())); + EXPECT_FALSE(match(VectorUndef, m_Finite())); + EXPECT_TRUE(match(VectorZeroUndef, m_Finite())); + EXPECT_FALSE(match(ScalarPosInf, m_Finite())); + EXPECT_FALSE(match(ScalarNegInf, m_Finite())); + EXPECT_FALSE(match(ScalarNaN, m_Finite())); + EXPECT_FALSE(match(VectorInfUndef, m_Finite())); + EXPECT_FALSE(match(VectorNaNUndef, m_Finite())); const APFloat *C; // Regardless of whether undefs are allowed,