diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -575,7 +575,7 @@ /// Set up the values of the IVs correctly when exiting the vector loop. void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II, - Value *CountRoundDown, Value *EndValue, + Value *VectorTripCount, Value *EndValue, BasicBlock *MiddleBlock); /// Introduce a conditional branch (on true, condition to be set later) at the @@ -3407,10 +3407,10 @@ Type *StepType = II.getStep()->getType(); Instruction::CastOps CastOp = CastInst::getCastOpcode(VectorTripCount, true, StepType, true); - Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd"); + Value *VTC = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.vtc"); Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint()); - EndValue = emitTransformedIndex(B, CRD, II.getStartValue(), Step, II); + EndValue = emitTransformedIndex(B, VTC, II.getStartValue(), Step, II); EndValue->setName("ind.end"); // Compute the end value for the additional bypass (if applicable). @@ -3420,10 +3420,10 @@ StepType, true); Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint()); - CRD = - B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd"); + VTC = + B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.vtc"); EndValueFromAdditionalBypass = - emitTransformedIndex(B, CRD, II.getStartValue(), Step, II); + emitTransformedIndex(B, VTC, II.getStartValue(), Step, II); EndValueFromAdditionalBypass->setName("ind.end"); } } @@ -3569,7 +3569,7 @@ // value for the IV when arriving directly from the middle block. void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II, - Value *CountRoundDown, Value *EndValue, + Value *VectorTripCount, Value *EndValue, BasicBlock *MiddleBlock) { // There are two kinds of external IV usages - those that use the value // computed in the last iteration (the PHI) and those that use the penultimate @@ -3593,34 +3593,34 @@ // An external user of the penultimate value need to see EndValue - Step. // The simplest way to get this is to recompute it from the constituent SCEVs, - // that is Start + (Step * (CRD - 1)). + // that is Start + (Step * (VTC - 1)). for (User *U : OrigPhi->users()) { auto *UI = cast(U); - if (!OrigLoop->contains(UI)) { - assert(isa(UI) && "Expected LCSSA form"); + if (OrigLoop->contains(UI)) + continue; - IRBuilder<> B(MiddleBlock->getTerminator()); + assert(isa(UI) && "Expected LCSSA form"); - // Fast-math-flags propagate from the original induction instruction. - if (II.getInductionBinOp() && isa(II.getInductionBinOp())) - B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags()); + IRBuilder<> B(MiddleBlock->getTerminator()); - Value *CountMinusOne = B.CreateSub( - CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1)); - Value *CMO = - !II.getStep()->getType()->isIntegerTy() - ? B.CreateCast(Instruction::SIToFP, CountMinusOne, - II.getStep()->getType()) - : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType()); - CMO->setName("cast.cmo"); - - Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(), - LoopVectorBody->getTerminator()); - Value *Escape = - emitTransformedIndex(B, CMO, II.getStartValue(), Step, II); - Escape->setName("ind.escape"); - MissingVals[UI] = Escape; - } + // Fast-math-flags propagate from the original induction instruction. + if (II.getInductionBinOp() && isa(II.getInductionBinOp())) + B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags()); + + Value *CountMinusOne = B.CreateSub( + VectorTripCount, ConstantInt::get(VectorTripCount->getType(), 1)); + Value *CMO = + !II.getStep()->getType()->isIntegerTy() + ? B.CreateCast(Instruction::SIToFP, CountMinusOne, + II.getStep()->getType()) + : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType()); + CMO->setName("cast.cmo"); + + Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(), + LoopVectorBody->getTerminator()); + Value *Escape = emitTransformedIndex(B, CMO, II.getStartValue(), Step, II); + Escape->setName("ind.escape"); + MissingVals[UI] = Escape; } for (auto &I : MissingVals) { @@ -8087,8 +8087,7 @@ emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, false); // Generate the induction variable. - Value *CountRoundDown = getOrCreateVectorTripCount(Lp); - EPI.VectorTripCount = CountRoundDown; + EPI.VectorTripCount = getOrCreateVectorTripCount(Lp); createHeaderBranch(Lp); // Skip induction resume value creation here because they will be created in