diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -1656,10 +1656,12 @@ // If we have special knowledge that this addrec won't overflow, // we don't need to do any further analysis. - if (AR->hasNoUnsignedWrap()) - return getAddRecExpr( - getExtendAddRecStart(AR, Ty, this, Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + if (AR->hasNoUnsignedWrap()) { + Start = + getExtendAddRecStart(AR, Ty, this, Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); + } // Check whether the backedge-taken count is SCEVCouldNotCompute. // Note that this serves two purposes: It filters out loops that are @@ -1701,10 +1703,10 @@ // Cache knowledge of AR NUW, which is propagated to this AddRec. setNoWrapFlags(const_cast(AR), SCEV::FlagNUW); // Return the expression with the addrec on the outside. - return getAddRecExpr(getExtendAddRecStart( - AR, Ty, this, Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart(AR, Ty, this, + Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // Similar to above, only this time treat the step value as signed. // This covers loops that count down. @@ -1719,10 +1721,10 @@ // Negative step causes unsigned wrap, but it still can't self-wrap. setNoWrapFlags(const_cast(AR), SCEV::FlagNW); // Return the expression with the addrec on the outside. - return getAddRecExpr(getExtendAddRecStart( - AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart(AR, Ty, this, + Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } } @@ -1744,9 +1746,10 @@ // issue. It's not clear that the order of checks does matter, but // it's one of two issue possible causes for a change which was // reverted. Be conservative for the moment. - return getAddRecExpr( - getExtendAddRecStart(AR, Ty, this, Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + Start = + getExtendAddRecStart(AR, Ty, this, Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // For a negative step, we can extend the operands iff doing so only @@ -1761,10 +1764,10 @@ // still can't self-wrap. setNoWrapFlags(const_cast(AR), SCEV::FlagNW); // Return the expression with the addrec on the outside. - return getAddRecExpr(getExtendAddRecStart( - AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart(AR, Ty, this, + Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } } @@ -1788,9 +1791,10 @@ if (proveNoWrapByVaryingStart(Start, Step, L)) { setNoWrapFlags(const_cast(AR), SCEV::FlagNUW); - return getAddRecExpr( - getExtendAddRecStart(AR, Ty, this, Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + Start = + getExtendAddRecStart(AR, Ty, this, Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } @@ -1798,15 +1802,19 @@ { const SCEV *LHS; const SCEV *RHS; - if (matchURem(Op, LHS, RHS)) - return getURemExpr(getZeroExtendExpr(LHS, Ty, Depth + 1), - getZeroExtendExpr(RHS, Ty, Depth + 1)); + if (matchURem(Op, LHS, RHS)) { + const auto *LHSz = getZeroExtendExpr(LHS, Ty, Depth + 1); + const auto *RHSz = getZeroExtendExpr(RHS, Ty, Depth + 1); + return getURemExpr(LHSz, RHSz); + } } // zext(A / B) --> zext(A) / zext(B). - if (auto *Div = dyn_cast(Op)) - return getUDivExpr(getZeroExtendExpr(Div->getLHS(), Ty, Depth + 1), - getZeroExtendExpr(Div->getRHS(), Ty, Depth + 1)); + if (auto *Div = dyn_cast(Op)) { + const auto *LHSz = getZeroExtendExpr(Div->getLHS(), Ty, Depth + 1); + const auto *RHSz = getZeroExtendExpr(Div->getRHS(), Ty, Depth + 1); + return getUDivExpr(LHSz, RHSz); + } if (auto *SA = dyn_cast(Op)) { // zext((A + B + ...)) --> (zext(A) + zext(B) + ...) @@ -1871,11 +1879,10 @@ int NewTruncBits = getTypeSizeInBits(TruncRHS->getType()) - MulLHS->getAPInt().logBase2(); Type *NewTruncTy = IntegerType::get(getContext(), NewTruncBits); - return getMulExpr( - getZeroExtendExpr(MulLHS, Ty), - getZeroExtendExpr( - getTruncateExpr(TruncRHS->getOperand(), NewTruncTy), Ty), - SCEV::FlagNUW, Depth + 1); + const auto *LHSz = getZeroExtendExpr(MulLHS, Ty); + const auto *RHSz = getZeroExtendExpr( + getTruncateExpr(TruncRHS->getOperand(), NewTruncTy), Ty); + return getMulExpr(LHSz, RHSz, SCEV::FlagNUW, Depth + 1); } } @@ -1992,10 +1999,12 @@ // If we have special knowledge that this addrec won't overflow, // we don't need to do any further analysis. - if (AR->hasNoSignedWrap()) - return getAddRecExpr( - getExtendAddRecStart(AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, SCEV::FlagNSW); + if (AR->hasNoSignedWrap()) { + Start = + getExtendAddRecStart(AR, Ty, this, Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, SCEV::FlagNSW); + } // Check whether the backedge-taken count is SCEVCouldNotCompute. // Note that this serves two purposes: It filters out loops that are @@ -2038,10 +2047,10 @@ // Cache knowledge of AR NSW, which is propagated to this AddRec. setNoWrapFlags(const_cast(AR), SCEV::FlagNSW); // Return the expression with the addrec on the outside. - return getAddRecExpr(getExtendAddRecStart( - AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart(AR, Ty, this, + Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // Similar to above, only this time treat the step value as unsigned. // This covers loops that count up with an unsigned step. @@ -2063,10 +2072,10 @@ setNoWrapFlags(const_cast(AR), SCEV::FlagNW); // Return the expression with the addrec on the outside. - return getAddRecExpr(getExtendAddRecStart( - AR, Ty, this, Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart(AR, Ty, this, + Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } } @@ -2078,9 +2087,10 @@ // issue. It's not clear that the order of checks does matter, but // it's one of two issue possible causes for a change which was // reverted. Be conservative for the moment. - return getAddRecExpr( - getExtendAddRecStart(AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + Start = + getExtendAddRecStart(AR, Ty, this, Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step})) @@ -2102,9 +2112,10 @@ if (proveNoWrapByVaryingStart(Start, Step, L)) { setNoWrapFlags(const_cast(AR), SCEV::FlagNSW); - return getAddRecExpr( - getExtendAddRecStart(AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + Start = + getExtendAddRecStart(AR, Ty, this, Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } @@ -2306,9 +2317,9 @@ const SCEV *A = (this->*Extension)( (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0); - const SCEV *B = (this->*Operation)((this->*Extension)(LHS, WideTy, 0), - (this->*Extension)(RHS, WideTy, 0), - SCEV::FlagAnyWrap, 0); + const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0); + const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0); + const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0); return A == B; } @@ -3112,12 +3123,13 @@ // TODO: There are some cases where this transformation is not // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of // this transformation should be narrowed down. - if (Add->getNumOperands() == 2 && containsConstantInAddMulChain(Add)) - return getAddExpr(getMulExpr(LHSC, Add->getOperand(0), - SCEV::FlagAnyWrap, Depth + 1), - getMulExpr(LHSC, Add->getOperand(1), - SCEV::FlagAnyWrap, Depth + 1), - SCEV::FlagAnyWrap, Depth + 1); + if (Add->getNumOperands() == 2 && containsConstantInAddMulChain(Add)) { + const auto *LHS = getMulExpr(LHSC, Add->getOperand(0), + SCEV::FlagAnyWrap, Depth + 1); + const auto *RHS = getMulExpr(LHSC, Add->getOperand(1), + SCEV::FlagAnyWrap, Depth + 1); + return getAddExpr(LHS, RHS, SCEV::FlagAnyWrap, Depth + 1); + } if (Ops[0]->isAllOnesValue()) { // If we have a mul by -1 of an add, try distributing the -1 among the @@ -6111,8 +6123,10 @@ if (!isa(TrueVal) && !isa(FalseVal)) return None; - return createNodeForSelectViaUMinSeq( - SE, SE->getSCEV(Cond), SE->getSCEV(TrueVal), SE->getSCEV(FalseVal)); + const auto *SECond = SE->getSCEV(Cond); + const auto *SETrue = SE->getSCEV(TrueVal); + const auto *SEFalse = SE->getSCEV(FalseVal); + return createNodeForSelectViaUMinSeq(SE, SECond, SETrue, SEFalse); } const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq( @@ -7199,6 +7213,9 @@ else if (!isa(V)) return getUnknown(V); + const SCEV *LHS; + const SCEV *RHS; + Operator *U = cast(V); if (auto BO = MatchBinaryOp(U, DT)) { switch (BO->Opcode) { @@ -7264,8 +7281,9 @@ SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op); if (Flags != SCEV::FlagAnyWrap) { - MulOps.push_back( - getMulExpr(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags)); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + MulOps.push_back(getMulExpr(LHS, RHS, Flags)); break; } } @@ -7282,14 +7300,20 @@ return getMulExpr(MulOps); } case Instruction::UDiv: - return getUDivExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getUDivExpr(LHS, RHS); case Instruction::URem: - return getURemExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getURemExpr(LHS, RHS); case Instruction::Sub: { SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; if (BO->Op) Flags = getNoWrapFlagsFromUB(BO->Op); - return getMinusSCEV(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getMinusSCEV(LHS, RHS, Flags); } case Instruction::And: // For an expression like x&255 that merely masks off the high bits, @@ -7342,8 +7366,11 @@ } } // Binary `and` is a bit-wise `umin`. - if (BO->LHS->getType()->isIntegerTy(1)) - return getUMinExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); + if (BO->LHS->getType()->isIntegerTy(1)) { + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getUMinExpr(LHS, RHS); + } break; case Instruction::Or: @@ -7364,8 +7391,11 @@ } } // Binary `or` is a bit-wise `umax`. - if (BO->LHS->getType()->isIntegerTy(1)) - return getUMaxExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); + if (BO->LHS->getType()->isIntegerTy(1)) { + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getUMaxExpr(LHS, RHS); + } break; case Instruction::Xor: @@ -7542,16 +7572,16 @@ case Instruction::SDiv: // If both operands are non-negative, this is just an udiv. - if (isKnownNonNegative(getSCEV(U->getOperand(0))) && - isKnownNonNegative(getSCEV(U->getOperand(1)))) - return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1))); + if (isKnownNonNegative(LHS = getSCEV(U->getOperand(0))) && + isKnownNonNegative(RHS = getSCEV(U->getOperand(1)))) + return getUDivExpr(LHS, RHS); break; case Instruction::SRem: // If both operands are non-negative, this is just an urem. - if (isKnownNonNegative(getSCEV(U->getOperand(0))) && - isKnownNonNegative(getSCEV(U->getOperand(1)))) - return getURemExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1))); + if (isKnownNonNegative(LHS = getSCEV(U->getOperand(0))) && + isKnownNonNegative(RHS = getSCEV(U->getOperand(1)))) + return getURemExpr(LHS, RHS); break; case Instruction::GetElementPtr: @@ -7576,17 +7606,21 @@ getSCEV(II->getArgOperand(0)), /*IsNSW=*/cast(II->getArgOperand(1))->isOne()); case Intrinsic::umax: - return getUMaxExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getUMaxExpr(LHS, RHS); case Intrinsic::umin: - return getUMinExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getUMinExpr(LHS, RHS); case Intrinsic::smax: - return getSMaxExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getSMaxExpr(LHS, RHS); case Intrinsic::smin: - return getSMinExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getSMinExpr(LHS, RHS); case Intrinsic::usub_sat: { const SCEV *X = getSCEV(II->getArgOperand(0)); const SCEV *Y = getSCEV(II->getArgOperand(1)); @@ -10628,17 +10662,27 @@ return false; if (Pred == CmpInst::ICMP_NE) { - if (CheckRanges(getSignedRange(LHS), getSignedRange(RHS)) || - CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS))) + auto SL = getSignedRange(LHS); + auto SR = getSignedRange(RHS); + if (CheckRanges(SL, SR)) + return true; + auto UL = getUnsignedRange(LHS); + auto UR = getUnsignedRange(RHS); + if (CheckRanges(UL, UR)) return true; auto *Diff = getMinusSCEV(LHS, RHS); return !isa(Diff) && isKnownNonZero(Diff); } - if (CmpInst::isSigned(Pred)) - return CheckRanges(getSignedRange(LHS), getSignedRange(RHS)); + if (CmpInst::isSigned(Pred)) { + auto SL = getSignedRange(LHS); + auto SR = getSignedRange(RHS); + return CheckRanges(SL, SR); + } - return CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS)); + auto UL = getUnsignedRange(LHS); + auto UR = getUnsignedRange(RHS); + return CheckRanges(UL, UR); } bool ScalarEvolution::isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred, @@ -14511,8 +14555,9 @@ if (auto *Cmp = dyn_cast(Cond)) { auto Predicate = EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate(); - CollectCondition(Predicate, getSCEV(Cmp->getOperand(0)), - getSCEV(Cmp->getOperand(1)), RewriteMap); + const auto *LHS = getSCEV(Cmp->getOperand(0)); + const auto *RHS = getSCEV(Cmp->getOperand(1)); + CollectCondition(Predicate, LHS, RHS, RewriteMap); continue; } @@ -14533,8 +14578,9 @@ auto *Cmp = dyn_cast(AssumeI->getOperand(0)); if (!Cmp || !DT.dominates(AssumeI, L->getHeader())) continue; - CollectCondition(Cmp->getPredicate(), getSCEV(Cmp->getOperand(0)), - getSCEV(Cmp->getOperand(1)), RewriteMap); + const auto *LHS = getSCEV(Cmp->getOperand(0)); + const auto *RHS = getSCEV(Cmp->getOperand(1)); + CollectCondition(Cmp->getPredicate(), LHS, RHS, RewriteMap); } if (RewriteMap.empty()) diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp --- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -159,11 +159,12 @@ D = ConstantInt::get(UseInst->getContext(), APInt::getOneBitSet(BitWidth, D->getZExtValue())); } - FoldedExpr = SE->getUDivExpr(SE->getSCEV(IVSrc), SE->getSCEV(D)); + const auto *LHS = SE->getSCEV(IVSrc); + const auto *RHS = SE->getSCEV(D); + FoldedExpr = SE->getUDivExpr(LHS, RHS); // We might have 'exact' flag set at this point which will no longer be // correct after we make the replacement. - if (UseInst->isExact() && - SE->getSCEV(IVSrc) != SE->getMulExpr(FoldedExpr, SE->getSCEV(D))) + if (UseInst->isExact() && LHS != SE->getMulExpr(FoldedExpr, RHS)) MustDropExactFlag = true; } // We have something that might fold it's operand. Compare SCEVs.