diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -753,25 +753,26 @@ } }; - - -bool unrollingLogic(bool FullOrPartial,Loop *L, - const TargetTransformInfo &TTI, DominatorTree &DT, - ScalarEvolution &SE, const SmallPtrSetImpl &EphValues, - OptimizationRemarkEmitter *ORE, bool MaxOrZero, unsigned LoopSize, - unsigned &TripMultiple, unsigned &TripCount, unsigned &MaxTripCount, UnrollCostEstimator UCE, - TargetTransformInfo::UnrollingPreferences &UP, TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound) - { - - bool UserUnrollCount = UnrollCount.getNumOccurrences() > 0; - - bool PragmaFullUnroll = hasUnrollFullPragma(L); - unsigned PragmaCount = unrollCountPragmaValue(L); - bool PragmaEnableUnroll = hasUnrollEnablePragma(L); - bool ExplicitUnroll = PragmaCount > 0 || PragmaFullUnroll || - PragmaEnableUnroll || UserUnrollCount; - - if (PragmaFullUnroll) +bool unrollingLogic(bool FullOrPartial, Loop *L, const TargetTransformInfo &TTI, + DominatorTree &DT, ScalarEvolution &SE, + const SmallPtrSetImpl &EphValues, + OptimizationRemarkEmitter *ORE, bool MaxOrZero, + unsigned LoopSize, unsigned &TripMultiple, + unsigned &TripCount, unsigned &MaxTripCount, + UnrollCostEstimator UCE, + TargetTransformInfo::UnrollingPreferences &UP, + TargetTransformInfo::PeelingPreferences &PP, + bool &UseUpperBound) { + + bool UserUnrollCount = UnrollCount.getNumOccurrences() > 0; + + bool PragmaFullUnroll = hasUnrollFullPragma(L); + unsigned PragmaCount = unrollCountPragmaValue(L); + bool PragmaEnableUnroll = hasUnrollEnablePragma(L); + bool ExplicitUnroll = PragmaCount > 0 || PragmaFullUnroll || + PragmaEnableUnroll || UserUnrollCount; + + if (PragmaFullUnroll) ORE->emit([&]() { return OptimizationRemarkMissed( DEBUG_TYPE, "CantFullUnrollAsDirectedRuntimeTripCount", @@ -781,10 +782,10 @@ "because loop has a runtime trip count."; }); - if (FullOrPartial){ + if (FullOrPartial) { - // Using unroll pragma - if (UnrollCount.getNumOccurrences() > 0){ + // Using unroll pragma + if (UnrollCount.getNumOccurrences() > 0) { UP.Count = UnrollCount; UP.AllowExpensiveTripCount = true; UP.Force = true; @@ -808,8 +809,6 @@ return false; } - - if (ExplicitUnroll && TripCount != 0) { // If the loop has an unrolling pragma, we want to be more aggressive with // unrolling limits. Set thresholds to at least the PragmaUnrollThreshold @@ -828,7 +827,7 @@ // compute the former when the latter is zero. unsigned ExactTripCount = TripCount; assert((ExactTripCount == 0 || FullUnrollMaxTripCount == 0) && - "ExtractTripCount and UnrollByMaxCount cannot both be non zero."); + "ExtractTripCount and UnrollByMaxCount cannot both be non zero."); unsigned FullUnrollTripCount = ExactTripCount ? ExactTripCount : FullUnrollMaxTripCount; @@ -860,76 +859,73 @@ } } } - } - else{ + } else { if (TripCount) { - UP.Partial |= ExplicitUnroll; - if (!UP.Partial) { - LLVM_DEBUG(dbgs() << " will not try to unroll partially because " - << "-unroll-allow-partial not given\n"); - UP.Count = 0; - return false; - } - if (UP.Count == 0) - UP.Count = TripCount; - if (UP.PartialThreshold != NoThreshold) { - // Reduce unroll count to be modulo of TripCount for partial unrolling. - if (UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold) - UP.Count = - (std::max(UP.PartialThreshold, UP.BEInsns + 1) - UP.BEInsns) / - (LoopSize - UP.BEInsns); - if (UP.Count > UP.MaxCount) - UP.Count = UP.MaxCount; - while (UP.Count != 0 && TripCount % UP.Count != 0) - UP.Count--; - if (UP.AllowRemainder && UP.Count <= 1) { - // If there is no Count that is modulo of TripCount, set Count to - // largest power-of-two factor that satisfies the threshold limit. - // As we'll create fixup loop, do the type of unrolling only if - // remainder loop is allowed. - UP.Count = UP.DefaultUnrollRuntimeCount; - while (UP.Count != 0 && - UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold) - UP.Count >>= 1; - } - if (UP.Count < 2) { - if (PragmaEnableUnroll) - ORE->emit([&]() { - return OptimizationRemarkMissed(DEBUG_TYPE, - "UnrollAsDirectedTooLarge", - L->getStartLoc(), L->getHeader()) - << "Unable to unroll loop as directed by unroll(enable) " - "pragma " - "because unrolled size is too large."; - }); + UP.Partial |= ExplicitUnroll; + if (!UP.Partial) { + LLVM_DEBUG(dbgs() << " will not try to unroll partially because " + << "-unroll-allow-partial not given\n"); UP.Count = 0; + return false; } - } else { - UP.Count = TripCount; - } - if (UP.Count > UP.MaxCount) - UP.Count = UP.MaxCount; - if ((PragmaFullUnroll || PragmaEnableUnroll) && TripCount && - UP.Count != TripCount) - ORE->emit([&]() { - return OptimizationRemarkMissed(DEBUG_TYPE, - "FullUnrollAsDirectedTooLarge", - L->getStartLoc(), L->getHeader()) - << "Unable to fully unroll loop as directed by unroll pragma " - "because " - "unrolled size is too large."; - }); - LLVM_DEBUG(dbgs() << " partially unrolling with count: " << UP.Count - << "\n"); - return ExplicitUnroll; - + if (UP.Count == 0) + UP.Count = TripCount; + if (UP.PartialThreshold != NoThreshold) { + // Reduce unroll count to be modulo of TripCount for partial unrolling. + if (UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold) + UP.Count = + (std::max(UP.PartialThreshold, UP.BEInsns + 1) - UP.BEInsns) / + (LoopSize - UP.BEInsns); + if (UP.Count > UP.MaxCount) + UP.Count = UP.MaxCount; + while (UP.Count != 0 && TripCount % UP.Count != 0) + UP.Count--; + if (UP.AllowRemainder && UP.Count <= 1) { + // If there is no Count that is modulo of TripCount, set Count to + // largest power-of-two factor that satisfies the threshold limit. + // As we'll create fixup loop, do the type of unrolling only if + // remainder loop is allowed. + UP.Count = UP.DefaultUnrollRuntimeCount; + while (UP.Count != 0 && + UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold) + UP.Count >>= 1; + } + if (UP.Count < 2) { + if (PragmaEnableUnroll) + ORE->emit([&]() { + return OptimizationRemarkMissed(DEBUG_TYPE, + "UnrollAsDirectedTooLarge", + L->getStartLoc(), L->getHeader()) + << "Unable to unroll loop as directed by unroll(enable) " + "pragma " + "because unrolled size is too large."; + }); + UP.Count = 0; + } + } else { + UP.Count = TripCount; + } + if (UP.Count > UP.MaxCount) + UP.Count = UP.MaxCount; + if ((PragmaFullUnroll || PragmaEnableUnroll) && TripCount && + UP.Count != TripCount) + ORE->emit([&]() { + return OptimizationRemarkMissed(DEBUG_TYPE, + "FullUnrollAsDirectedTooLarge", + L->getStartLoc(), L->getHeader()) + << "Unable to fully unroll loop as directed by unroll pragma " + "because " + "unrolled size is too large."; + }); + LLVM_DEBUG(dbgs() << " partially unrolling with count: " << UP.Count + << "\n"); + return ExplicitUnroll; } } - UP.Runtime |= PragmaEnableUnroll || PragmaCount > 0 || UserUnrollCount; + UP.Runtime |= PragmaEnableUnroll || PragmaCount > 0 || UserUnrollCount; - return ExplicitUnroll; - - } + return ExplicitUnroll; +} // Returns true if unroll count was set explicitly. // Calculates unroll count and writes it to UP.Count. // Unless IgnoreUser is true, will also use metadata and command-line options @@ -947,8 +943,6 @@ TargetTransformInfo::UnrollingPreferences &UP, TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound) { - - UnrollCostEstimator UCE(*L, LoopSize); // Use an explicit peel count that has been specified for testing. In this @@ -966,10 +960,10 @@ bool tryFullUnroll = true; bool ExplicitUnroll = false; - - - ExplicitUnroll = unrollingLogic(tryFullUnroll, L, TTI, DT, SE, EphValues, ORE, MaxOrZero, LoopSize, TripMultiple, TripCount, MaxTripCount, UCE, UP, PP, UseUpperBound); + ExplicitUnroll = unrollingLogic(tryFullUnroll, L, TTI, DT, SE, EphValues, ORE, + MaxOrZero, LoopSize, TripMultiple, TripCount, + MaxTripCount, UCE, UP, PP, UseUpperBound); // 4th priority is loop peeling. computePeelCount(L, LoopSize, PP, TripCount, SE, UP.Threshold); @@ -980,14 +974,13 @@ } tryFullUnroll = false; - ExplicitUnroll = unrollingLogic(tryFullUnroll, L, TTI, DT, SE, EphValues, ORE, MaxOrZero, LoopSize, TripMultiple, TripCount, MaxTripCount, UCE, UP, PP, UseUpperBound); - - + ExplicitUnroll = unrollingLogic(tryFullUnroll, L, TTI, DT, SE, EphValues, ORE, + MaxOrZero, LoopSize, TripMultiple, TripCount, + MaxTripCount, UCE, UP, PP, UseUpperBound); assert(TripCount == 0 && "All cases when TripCount is constant should be covered here."); - // 6th priority is runtime unrolling. // Don't unroll a runtime trip count loop when it is disabled. if (hasRuntimeUnrollDisablePragma(L)) {