diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -535,6 +535,11 @@ // generated by fixReduction. PHINode *getReductionResumeValue(const RecurrenceDescriptor &RdxDesc); + PHINode *createInductionResumeValue( + PHINode *OrigPhi, const InductionDescriptor &ID, + ArrayRef BypassBlocks, + std::pair AdditionalBypass = {nullptr, nullptr}); + protected: friend class LoopVectorizationPlanner; @@ -3113,61 +3118,48 @@ DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock); } -void InnerLoopVectorizer::createInductionResumeValues( +PHINode *InnerLoopVectorizer::createInductionResumeValue( + PHINode *OrigPhi, const InductionDescriptor &II, + ArrayRef BypassBlocks, std::pair AdditionalBypass) { - assert(((AdditionalBypass.first && AdditionalBypass.second) || - (!AdditionalBypass.first && !AdditionalBypass.second)) && - "Inconsistent information about additional bypass."); - Value *VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader); assert(VectorTripCount && "Expected valid arguments"); - // We are going to resume the execution of the scalar loop. - // Go over all of the induction variables that we found and fix the - // PHIs that are left in the scalar version of the loop. - // The starting values of PHI nodes depend on the counter of the last - // iteration in the vectorized loop. - // If we come from a bypass edge then we need to start from the original - // start value. - Instruction *OldInduction = Legal->getPrimaryInduction(); - for (const auto &InductionEntry : Legal->getInductionVars()) { - PHINode *OrigPhi = InductionEntry.first; - InductionDescriptor II = InductionEntry.second; - - Value *&EndValue = IVEndValues[OrigPhi]; - Value *EndValueFromAdditionalBypass = AdditionalBypass.second; - if (OrigPhi == OldInduction) { - // We know what the end value is. - EndValue = VectorTripCount; - } else { - IRBuilder<> B(LoopVectorPreHeader->getTerminator()); - - // Fast-math-flags propagate from the original induction instruction. - if (II.getInductionBinOp() && isa(II.getInductionBinOp())) - B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags()); - Type *StepType = II.getStep()->getType(); - Instruction::CastOps CastOp = - CastInst::getCastOpcode(VectorTripCount, true, StepType, true); - Value *VTC = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.vtc"); + Instruction *OldInduction = Legal->getPrimaryInduction(); + Value *&EndValue = IVEndValues[OrigPhi]; + Value *EndValueFromAdditionalBypass = AdditionalBypass.second; + if (OrigPhi == OldInduction) { + // We know what the end value is. + EndValue = VectorTripCount; + } else { + IRBuilder<> B(LoopVectorPreHeader->getTerminator()); + + // Fast-math-flags propagate from the original induction instruction. + if (II.getInductionBinOp() && isa(II.getInductionBinOp())) + B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags()); + + Type *StepType = II.getStep()->getType(); + Instruction::CastOps CastOp = + CastInst::getCastOpcode(VectorTripCount, true, StepType, true); + Value *VTC = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.vtc"); + Value *Step = + CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint()); + EndValue = emitTransformedIndex(B, VTC, II.getStartValue(), Step, II); + EndValue->setName("ind.end"); + + // Compute the end value for the additional bypass (if applicable). + if (AdditionalBypass.first) { + B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt())); + CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true, StepType, + true); Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint()); - EndValue = emitTransformedIndex(B, VTC, II.getStartValue(), Step, II); - EndValue->setName("ind.end"); - - // Compute the end value for the additional bypass (if applicable). - if (AdditionalBypass.first) { - B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt())); - CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true, - StepType, true); - Value *Step = - CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint()); - VTC = - B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.vtc"); - EndValueFromAdditionalBypass = - emitTransformedIndex(B, VTC, II.getStartValue(), Step, II); - EndValueFromAdditionalBypass->setName("ind.end"); - } + VTC = B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.vtc"); + EndValueFromAdditionalBypass = + emitTransformedIndex(B, VTC, II.getStartValue(), Step, II); + EndValueFromAdditionalBypass->setName("ind.end"); } + } // Create phi nodes to merge from the backedge-taken check block. PHINode *BCResumeVal = @@ -3183,15 +3175,36 @@ // Fix the scalar body counter (PHI node). // The old induction's phi node in the scalar body needs the truncated // value. - for (BasicBlock *BB : LoopBypassBlocks) - BCResumeVal->addIncoming(II.getStartValue(), BB); + for (BasicBlock *BB : BypassBlocks) + BCResumeVal->addIncoming(II.getStartValue(), BB); if (AdditionalBypass.first) BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first, EndValueFromAdditionalBypass); + return BCResumeVal; +} - OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal); - } +DenseMap +InnerLoopVectorizer::createInductionResumeValues( + std::pair AdditionalBypass) { + assert(((AdditionalBypass.first && AdditionalBypass.second) || + (!AdditionalBypass.first && !AdditionalBypass.second)) && + "Inconsistent information about additional bypass."); + DenseMap ResumeValues; + // We are going to resume the execution of the scalar loop. + // Go over all of the induction variables that we found and fix the + // PHIs that are left in the scalar version of the loop. + // The starting values of PHI nodes depend on the counter of the last + // iteration in the vectorized loop. + // If we come from a bypass edge then we need to start from the original + // start value. + for (const auto &InductionEntry : Legal->getInductionVars()) { + PHINode *OrigPhi = InductionEntry.first; + const InductionDescriptor &II = InductionEntry.second; + PHINode *BCResumeVal = createInductionResumeValue( + OrigPhi, II, LoopBypassBlocks, AdditionalBypass); + OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal); + } } BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(MDNode *OrigLoopID) {