diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -1389,6 +1389,59 @@ return PerformLoadPRE(LI, ValuesPerBlock, UnavailableBlocks); } +static bool impliesEquivalanceIfTrue(CmpInst* Cmp) { + if (Cmp->getPredicate() == CmpInst::Predicate::ICMP_EQ) + return true; + + // Floating point comparisons can be equal, but not equivalent. Cases: + // NaNs for unordered operators + // +0.0 vs 0.0 for all operators + if (Cmp->getPredicate() == CmpInst::Predicate::FCMP_OEQ || + (Cmp->getPredicate() == CmpInst::Predicate::FCMP_UEQ && + Cmp->getFastMathFlags().noNaNs())) { + Value *LHS = Cmp->getOperand(0); + Value *RHS = Cmp->getOperand(1); + // If we can prove either side non-zero, then equality must imply + // equivalence. + // FIXME: We should do this optimization if 'no signed zeros' is + // applicable via an instruction-level fast-math-flag or some other + // indicator that relaxed FP semantics are being used. + if (isa(LHS) && !cast(LHS)->isZero()) + return true; + if (isa(RHS) && !cast(RHS)->isZero()) + return true;; + // TODO: Handle vector floating point constants + } + return false; +} + +static bool impliesEquivalanceIfFalse(CmpInst* Cmp) { + if (Cmp->getPredicate() == CmpInst::Predicate::ICMP_NE) + return true; + + // Floating point comparisons can be equal, but not equivelent. Cases: + // NaNs for unordered operators + // +0.0 vs 0.0 for all operators + if ((Cmp->getPredicate() == CmpInst::Predicate::FCMP_ONE && + Cmp->getFastMathFlags().noNaNs()) || + Cmp->getPredicate() == CmpInst::Predicate::FCMP_UNE) { + Value *LHS = Cmp->getOperand(0); + Value *RHS = Cmp->getOperand(1); + // If we can prove either side non-zero, then equality must imply + // equivalence. + // FIXME: We should do this optimization if 'no signed zeros' is + // applicable via an instruction-level fast-math-flag or some other + // indicator that relaxed FP semantics are being used. + if (isa(LHS) && !cast(LHS)->isZero()) + return true; + if (isa(RHS) && !cast(RHS)->isZero()) + return true;; + // TODO: Handle vector floating point constants + } + return false; +} + + static bool hasUsersIn(Value *V, BasicBlock *BB) { for (User *U : V->users()) if (isa(U) && @@ -1453,10 +1506,7 @@ // call void @llvm.assume(i1 %cmp) // ret float %load ; will change it to ret float %0 if (auto *CmpI = dyn_cast(V)) { - if (CmpI->getPredicate() == CmpInst::Predicate::ICMP_EQ || - CmpI->getPredicate() == CmpInst::Predicate::FCMP_OEQ || - (CmpI->getPredicate() == CmpInst::Predicate::FCMP_UEQ && - CmpI->getFastMathFlags().noNaNs())) { + if (impliesEquivalanceIfTrue(CmpI)) { Value *CmpLHS = CmpI->getOperand(0); Value *CmpRHS = CmpI->getOperand(1); // Heuristically pick the better replacement -- the choice of heuristic @@ -1483,12 +1533,6 @@ if (isa(CmpLHS) && isa(CmpRHS)) return Changed; - // +0.0 and -0.0 compare equal, but do not imply equivalence. Unless we - // can prove equivalence, bail. - if (CmpRHS->getType()->isFloatTy() && - (!isa(CmpRHS) || cast(CmpRHS)->isZero())) - return Changed; - LLVM_DEBUG(dbgs() << "Replacing dominated uses of " << *CmpLHS << " with " << *CmpRHS << " in block " @@ -1877,27 +1921,12 @@ Value *Op0 = Cmp->getOperand(0), *Op1 = Cmp->getOperand(1); // If "A == B" is known true, or "A != B" is known false, then replace - // A with B everywhere in the scope. - if ((isKnownTrue && Cmp->getPredicate() == CmpInst::ICMP_EQ) || - (isKnownFalse && Cmp->getPredicate() == CmpInst::ICMP_NE)) + // A with B everywhere in the scope. For floating point operations, we + // have to be careful since equality does not always imply equivalance. + if ((isKnownTrue && impliesEquivalanceIfTrue(Cmp)) || + (isKnownFalse && impliesEquivalanceIfFalse(Cmp))) Worklist.push_back(std::make_pair(Op0, Op1)); - // Handle the floating point versions of equality comparisons too. - if ((isKnownTrue && Cmp->getPredicate() == CmpInst::FCMP_OEQ) || - (isKnownFalse && Cmp->getPredicate() == CmpInst::FCMP_UNE)) { - - // Floating point -0.0 and 0.0 compare equal, so we can only - // propagate values if we know that we have a constant and that - // its value is non-zero. - - // FIXME: We should do this optimization if 'no signed zeros' is - // applicable via an instruction-level fast-math-flag or some other - // indicator that relaxed FP semantics are being used. - - if (isa(Op1) && !cast(Op1)->isZero()) - Worklist.push_back(std::make_pair(Op0, Op1)); - } - // If "A >= B" is known true, replace "A < B" with false everywhere. CmpInst::Predicate NotPred = Cmp->getInversePredicate(); Constant *NotVal = ConstantInt::get(Cmp->getType(), isKnownFalse); diff --git a/llvm/test/Transforms/GVN/edge.ll b/llvm/test/Transforms/GVN/edge.ll --- a/llvm/test/Transforms/GVN/edge.ll +++ b/llvm/test/Transforms/GVN/edge.ll @@ -93,6 +93,40 @@ ; CHECK: %div = fdiv double %x, 2.0 } +define double @fcmp_one_possibly_nan(double %x, double %y) { +entry: + %cmp = fcmp one double %y, 2.0 + br i1 %cmp, label %return, label %else + +else: + %div = fdiv double %x, %y + br label %return + +return: + %retval = phi double [ %div, %else ], [ %x, %entry ] + ret double %retval + +; CHECK-LABEL: define double @fcmp_one_possibly_nan( +; CHECK: %div = fdiv double %x, %y +} + +define double @fcmp_one_not_zero_or_nan(double %x, double %y) { +entry: + %cmp = fcmp nnan one double %y, 2.0 + br i1 %cmp, label %return, label %else + +else: + %div = fdiv double %x, %y + br label %return + +return: + %retval = phi double [ %div, %else ], [ %x, %entry ] + ret double %retval + +; CHECK-LABEL: define double @fcmp_one_not_zero_or_nan( +; CHECK: %div = fdiv double %x, 2.0 +} + ; PR22376 - We can't propagate zero constants because -0.0 ; compares equal to 0.0. If %y is -0.0 in this test case, ; we would produce the wrong sign on the infinity return value. @@ -168,3 +202,38 @@ ; CHECK-LABEL: define double @fcmp_une_maybe_zero( ; CHECK: %div = fdiv double %x, %z } + + +define double @fcmp_ueq_possibly_nan(double %x, double %y) { +entry: + %cmp = fcmp ueq double %y, 2.0 + br i1 %cmp, label %do_div, label %return + +do_div: + %div = fdiv double %x, %y + br label %return + +return: + %retval = phi double [ %div, %do_div ], [ %x, %entry ] + ret double %retval + +; CHECK-LABEL: define double @fcmp_ueq_possibly_nan( +; CHECK: %div = fdiv double %x, %y +} + +define double @fcmp_ueq_not_zero_or_nan(double %x, double %y) { +entry: + %cmp = fcmp nnan ueq double %y, 2.0 + br i1 %cmp, label %do_div, label %return + +do_div: + %div = fdiv double %x, %y + br label %return + +return: + %retval = phi double [ %div, %do_div ], [ %x, %entry ] + ret double %retval + +; CHECK-LABEL: define double @fcmp_ueq_not_zero_or_nan( +; CHECK: %div = fdiv double %x, 2.0 +}