Please use GitHub pull requests for new patches. Avoid migrating existing patches. Phabricator shutdown timeline

Differential D57598 Diff 192599 llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp

Changeset View

Standalone View

llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 1,377 Lines • ▼ Show 20 Lines	static bool isExplicitVecOuterLoop(Loop *OuterLp,

Function *Fn = OuterLp->getHeader()->getParent();		Function *Fn = OuterLp->getHeader()->getParent();
if (!Hints.allowVectorization(Fn, OuterLp,		if (!Hints.allowVectorization(Fn, OuterLp,
true /VectorizeOnlyWhenForced/)) {		true /VectorizeOnlyWhenForced/)) {
LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");		LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
return false;		return false;
}		}

if (!Hints.getWidth()) {
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: No user vector width.\n");
Hints.emitRemarkWithHints();
return false;
}

if (Hints.getInterleave() > 1) {		if (Hints.getInterleave() > 1) {
// TODO: Interleave support is future work.		// TODO: Interleave support is future work.
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "		LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n");		"outer loops.\n");
Hints.emitRemarkWithHints();		Hints.emitRemarkWithHints();
return false;		return false;
}		}

▲ Show 20 Lines • Show All 4,676 Lines • ▼ Show 20 Lines	void LoopVectorizationCostModel::collectValuesToIgnore() {
// detection.		// detection.
for (auto &Induction : *Legal->getInductionVars()) {		for (auto &Induction : *Legal->getInductionVars()) {
InductionDescriptor &IndDes = Induction.second;		InductionDescriptor &IndDes = Induction.second;
const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();		const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
VecValuesToIgnore.insert(Casts.begin(), Casts.end());		VecValuesToIgnore.insert(Casts.begin(), Casts.end());
}		}
}		}

		// TODO: we could return a pair of values that specify the max VF and
		// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
		// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
		// doesn't have a cost model that can choose which plan to execute if
		// more than one is generated.
		unsigned determineVPlanVF(const unsigned WidestVectorRegBits,
		LoopVectorizationCostModel &CM) {
		unsigned WidestType;
		std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
		return WidestVectorRegBits / WidestType;
		}

VectorizationFactor		VectorizationFactor
LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize,		LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize,
unsigned UserVF) {		unsigned UserVF) {
		unsigned VF = UserVF;
// Outer loop handling: They may require CFG and instruction level		// Outer loop handling: They may require CFG and instruction level
// transformations before even evaluating whether vectorization is profitable.		// transformations before even evaluating whether vectorization is profitable.
// Since we cannot modify the incoming IR, we need to build VPlan upfront in		// Since we cannot modify the incoming IR, we need to build VPlan upfront in
// the vectorization pipeline.		// the vectorization pipeline.
if (!OrigLoop->empty()) {		if (!OrigLoop->empty()) {
// TODO: If UserVF is not provided, we set UserVF to 4 for stress testing.		// If the user doesn't provide a vectorization factor, determine a
// This won't be necessary when UserVF is not required in the VPlan-native		// reasonable one.
// path.		if (!UserVF) {
if (VPlanBuildStressTest && !UserVF)		// We set VF to 4 for stress testing.
UserVF = 4;		if (VPlanBuildStressTest)
		VF = 4;
		else
		VF = determineVPlanVF(TTI->getRegisterBitWidth(true /* Vector*/), CM);
		}

assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");		assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
assert(UserVF && "Expected UserVF for outer loop vectorization.");		assert(isPowerOf2_32(VF) && "VF needs to be a power of two");
assert(isPowerOf2_32(UserVF) && "VF needs to be a power of two");		LLVM_DEBUG(dbgs() << "LV: Using " << (UserVF ? "user VF " : "computed VF ")
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");		<< VF << " to build VPlans.\n");
buildVPlans(UserVF, UserVF);		buildVPlans(VF, VF);

// For VPlan build stress testing, we bail out after VPlan construction.		// For VPlan build stress testing, we bail out after VPlan construction.
if (VPlanBuildStressTest)		if (VPlanBuildStressTest)
return VectorizationFactor::Disabled();		return VectorizationFactor::Disabled();

return {UserVF, 0};		return {VF, 0};
}		}

LLVM_DEBUG(		LLVM_DEBUG(
dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "		dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
"VPlan-native path.\n");		"VPlan-native path.\n");
return VectorizationFactor::Disabled();		return VectorizationFactor::Disabled();
}		}

▲ Show 20 Lines • Show All 1,006 Lines • ▼ Show 20 Lines	static bool processLoopInVPlanNativePath(
LoopVectorizationCostModel CM(L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,		LoopVectorizationCostModel CM(L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,
&Hints, IAI);		&Hints, IAI);
// Use the planner for outer loop vectorization.		// Use the planner for outer loop vectorization.
// TODO: CM is not used at this point inside the planner. Turn CM into an		// TODO: CM is not used at this point inside the planner. Turn CM into an
// optional argument if we don't need it in the future.		// optional argument if we don't need it in the future.
LoopVectorizationPlanner LVP(L, LI, TLI, TTI, LVL, CM);		LoopVectorizationPlanner LVP(L, LI, TLI, TTI, LVL, CM);

// Get user vectorization factor.		// Get user vectorization factor.
unsigned UserVF = Hints.getWidth();		const unsigned UserVF = Hints.getWidth();

// Check the function attributes to find out if this function should be		// Check the function attributes to find out if this function should be
// optimized for size.		// optimized for size.
bool OptForSize =		bool OptForSize =
Hints.getForce() != LoopVectorizeHints::FK_Enabled && F->optForSize();		Hints.getForce() != LoopVectorizeHints::FK_Enabled && F->optForSize();

// Plan how to best vectorize, return the best VF and its cost.		// Plan how to best vectorize, return the best VF and its cost.
VectorizationFactor VF = LVP.planInVPlanNativePath(OptForSize, UserVF);		const VectorizationFactor VF = LVP.planInVPlanNativePath(OptForSize, UserVF);

// If we are stress testing VPlan builds, do not attempt to generate vector		// If we are stress testing VPlan builds, do not attempt to generate vector
// code. Masked vector code generation support will follow soon.		// code. Masked vector code generation support will follow soon.
if (VPlanBuildStressTest \|\| EnableVPlanPredication)		// Also, do not attempt to vectorize if no vector code will be produced.
		if (VPlanBuildStressTest \|\| EnableVPlanPredication \|\|
		VectorizationFactor::Disabled() == VF)
return false;		return false;

LVP.setBestPlan(VF.Width, 1);		LVP.setBestPlan(VF.Width, 1);

InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, UserVF, 1, LVL,		InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, 1, LVL,
&CM);		&CM);
LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""		LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
<< L->getHeader()->getParent()->getName() << "\"\n");		<< L->getHeader()->getParent()->getName() << "\"\n");
LVP.executePlan(LB, DT);		LVP.executePlan(LB, DT);

// Mark the loop as already vectorized to avoid vectorizing again.		// Mark the loop as already vectorized to avoid vectorizing again.
Hints.setAlreadyVectorized();		Hints.setAlreadyVectorized();

▲ Show 20 Lines • Show All 433 Lines • Show Last 20 Lines