Differential D134745 Diff 463299 llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

Changeset View

Standalone View

llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

Show First 20 Lines • Show All 94 Lines • ▼ Show 20 Lines
}		}

LoopVectorizeHints::LoopVectorizeHints(const Loop *L,		LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
bool InterleaveOnlyWhenForced,		bool InterleaveOnlyWhenForced,
OptimizationRemarkEmitter &ORE,		OptimizationRemarkEmitter &ORE,
const TargetTransformInfo *TTI)		const TargetTransformInfo *TTI)
: Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH),		: Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH),
Interleave("interleave.count", InterleaveOnlyWhenForced, HK_INTERLEAVE),		Interleave("interleave.count", InterleaveOnlyWhenForced, HK_INTERLEAVE),
Force("vectorize.enable", FK_Undefined, HK_FORCE),		VectorizationForce("vectorize.enable", FK_Undefined, HK_FORCE),
		InterleaveForce("interleave.enable", FK_Undefined, HK_FORCE),
IsVectorized("isvectorized", 0, HK_ISVECTORIZED),		IsVectorized("isvectorized", 0, HK_ISVECTORIZED),
Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE),		Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE),
Scalable("vectorize.scalable.enable", SK_Unspecified, HK_SCALABLE),		Scalable("vectorize.scalable.enable", SK_Unspecified, HK_SCALABLE),
TheLoop(L), ORE(ORE) {		TheLoop(L), ORE(ORE) {
// Populate values with existing loop metadata.		// Populate values with existing loop metadata.
getHintsFromMetadata();		getHintsFromMetadata();

// force-vector-interleave overrides DisableInterleaving.		// force-vector-interleave overrides DisableInterleaving.
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines	void LoopVectorizeHints::setAlreadyVectorized() {
TheLoop->setLoopID(NewLoopID);		TheLoop->setLoopID(NewLoopID);

// Update internal cache.		// Update internal cache.
IsVectorized.Value = 1;		IsVectorized.Value = 1;
}		}

bool LoopVectorizeHints::allowVectorization(		bool LoopVectorizeHints::allowVectorization(
Function F, Loop L, bool VectorizeOnlyWhenForced) const {		Function F, Loop L, bool VectorizeOnlyWhenForced) const {
if (getForce() == LoopVectorizeHints::FK_Disabled) {		if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled &&
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n");		getInterleaveForce() == LoopVectorizeHints::FK_Disabled) {
		LLVM_DEBUG(
		dbgs()
		<< "LV: Not vectorizing: #pragma vectorize and interleave disable.\n");
emitRemarkWithHints();		emitRemarkWithHints();
return false;		return false;
}		}

if (VectorizeOnlyWhenForced && getForce() != LoopVectorizeHints::FK_Enabled) {		if (VectorizeOnlyWhenForced &&
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n");		getVectorizationForce() != LoopVectorizeHints::FK_Enabled &&
		getInterleaveForce() != LoopVectorizeHints::FK_Enabled) {
		LLVM_DEBUG(
		dbgs()
		<< "LV: Not vectorizing: No #pragma vectorize or interleave enable.\n");
emitRemarkWithHints();		emitRemarkWithHints();
return false;		return false;
}		}

if (getIsVectorized() == 1) {		if (getIsVectorized() == 1) {
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n");		LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n");
// FIXME: Add interleave.disable metadata. This will allow
// vectorize.disable to be used without disabling the pass and errors
// to differentiate between disabled vectorization and a width of 1.
ORE.emit([&]() {		ORE.emit([&]() {
return OptimizationRemarkAnalysis(vectorizeAnalysisPassName(),		return OptimizationRemarkAnalysis(vectorizeAnalysisPassName(),
"AllDisabled", L->getStartLoc(),		"AllDisabled", L->getStartLoc(),
L->getHeader())		L->getHeader())
<< "loop not vectorized: vectorization and interleaving are "		<< "loop not vectorized: vectorization and interleaving are "
"explicitly disabled, or the loop has already been "		"explicitly disabled, or the loop has already been "
"vectorized";		"vectorized";
});		});
return false;		return false;
}		}

return true;		return true;
}		}

void LoopVectorizeHints::emitRemarkWithHints() const {		void LoopVectorizeHints::emitRemarkWithHints() const {
using namespace ore;		using namespace ore;

ORE.emit([&]() {		ORE.emit([&]() {
if (Force.Value == LoopVectorizeHints::FK_Disabled)		if (VectorizationForce.Value == LoopVectorizeHints::FK_Disabled &&
		InterleaveForce.Value == LoopVectorizeHints::FK_Disabled)
return OptimizationRemarkMissed(LV_NAME, "MissedExplicitlyDisabled",		return OptimizationRemarkMissed(LV_NAME, "MissedExplicitlyDisabled",
TheLoop->getStartLoc(),		TheLoop->getStartLoc(),
TheLoop->getHeader())		TheLoop->getHeader())
<< "loop not vectorized: vectorization is explicitly disabled";		<< "loop not vectorized: vectorization and interleave is "
		"explicitly disabled";
else {		else {
OptimizationRemarkMissed R(LV_NAME, "MissedDetails",		OptimizationRemarkMissed R(LV_NAME, "MissedDetails",
TheLoop->getStartLoc(), TheLoop->getHeader());		TheLoop->getStartLoc(), TheLoop->getHeader());
R << "loop not vectorized";		R << "loop not vectorized";
if (Force.Value == LoopVectorizeHints::FK_Enabled) {		if (VectorizationForce.Value == LoopVectorizeHints::FK_Enabled) {
R << " (Force=" << NV("Force", true);		R << " (VectorizationForce=" << NV("VectorizationForce", true);
if (Width.Value != 0)		if (Width.Value != 0)
R << ", Vector Width=" << NV("VectorWidth", getWidth());		R << ", Vector Width=" << NV("VectorWidth", getWidth());
		R << ")";
		}
		if (InterleaveForce.Value == LoopVectorizeHints::FK_Enabled) {
		R << " (InterleaveForce=" << NV("InterleaveForce", true);
if (getInterleave() != 0)		if (getInterleave() != 0)
R << ", Interleave Count=" << NV("InterleaveCount", getInterleave());		R << ", Interleave Count=" << NV("InterleaveCount", getInterleave());
R << ")";		R << ")";
}		}
return R;		return R;
}		}
});		});
}		}

const char *LoopVectorizeHints::vectorizeAnalysisPassName() const {		const char *LoopVectorizeHints::vectorizeAnalysisPassName() const {
if (getWidth() == ElementCount::getFixed(1))		if (getWidth() == ElementCount::getFixed(1))
return LV_NAME;		return LV_NAME;
if (getForce() == LoopVectorizeHints::FK_Disabled)		if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled)
return LV_NAME;		return LV_NAME;
if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth().isZero())		if (getVectorizationForce() == LoopVectorizeHints::FK_Undefined &&
		getWidth().isZero())
return LV_NAME;		return LV_NAME;
return OptimizationRemarkAnalysis::AlwaysPrint;		return OptimizationRemarkAnalysis::AlwaysPrint;
}		}

bool LoopVectorizeHints::allowReordering() const {		bool LoopVectorizeHints::allowReordering() const {
// Allow the vectorizer to change the order of operations if enabling		// Allow the vectorizer to change the order of operations if enabling
// loop hints are provided		// loop hints are provided
ElementCount EC = getWidth();		ElementCount EC = getWidth();
return HintsAllowReordering &&		return HintsAllowReordering &&
(getForce() == LoopVectorizeHints::FK_Enabled \|\|		(getVectorizationForce() == LoopVectorizeHints::FK_Enabled \|\|
EC.getKnownMinValue() > 1);		EC.getKnownMinValue() > 1);
}		}

void LoopVectorizeHints::getHintsFromMetadata() {		void LoopVectorizeHints::getHintsFromMetadata() {
MDNode *LoopID = TheLoop->getLoopID();		MDNode *LoopID = TheLoop->getLoopID();
if (!LoopID)		if (!LoopID)
return;		return;

Show All 33 Lines	if (!Name.startswith(Prefix()))
return;		return;
Name = Name.substr(Prefix().size(), StringRef::npos);		Name = Name.substr(Prefix().size(), StringRef::npos);

const ConstantInt *C = mdconst::dyn_extract<ConstantInt>(Arg);		const ConstantInt *C = mdconst::dyn_extract<ConstantInt>(Arg);
if (!C)		if (!C)
return;		return;
unsigned Val = C->getZExtValue();		unsigned Val = C->getZExtValue();

Hint *Hints[] = {&Width, &Interleave, &Force,		Hint *Hints[] = {&Width, &Interleave, &VectorizationForce,
&IsVectorized, &Predicate, &Scalable};		&InterleaveForce, &IsVectorized, &Predicate,
		&Scalable};
for (auto *H : Hints) {		for (auto *H : Hints) {
if (Name == H->Name) {		if (Name == H->Name) {
if (H->validate(Val))		if (H->validate(Val))
H->Value = Val;		H->Value = Val;
else		else
LLVM_DEBUG(dbgs() << "LV: ignoring invalid hint '" << Name << "'\n");		LLVM_DEBUG(dbgs() << "LV: ignoring invalid hint '" << Name << "'\n");
break;		break;
}		}
▲ Show 20 Lines • Show All 1,032 Lines • ▼ Show 20 Lines	bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {

LLVM_DEBUG(dbgs() << "LV: We can vectorize this loop"		LLVM_DEBUG(dbgs() << "LV: We can vectorize this loop"
<< (LAI->getRuntimePointerChecking()->Need		<< (LAI->getRuntimePointerChecking()->Need
? " (with a runtime bound check)"		? " (with a runtime bound check)"
: "")		: "")
<< "!\n");		<< "!\n");

unsigned SCEVThreshold = VectorizeSCEVCheckThreshold;		unsigned SCEVThreshold = VectorizeSCEVCheckThreshold;
if (Hints->getForce() == LoopVectorizeHints::FK_Enabled)		if (Hints->getVectorizationForce() == LoopVectorizeHints::FK_Enabled)
SCEVThreshold = PragmaVectorizeSCEVCheckThreshold;		SCEVThreshold = PragmaVectorizeSCEVCheckThreshold;

if (PSE.getPredicate().getComplexity() > SCEVThreshold) {		if (PSE.getPredicate().getComplexity() > SCEVThreshold) {
reportVectorizationFailure("Too many SCEV checks needed",		reportVectorizationFailure("Too many SCEV checks needed",
"Too many SCEV assumptions need to be made and checked at runtime",		"Too many SCEV assumptions need to be made and checked at runtime",
"TooManySCEVRunTimeChecks", ORE, TheLoop);		"TooManySCEVRunTimeChecks", ORE, TheLoop);
if (DoExtraAnalysis)		if (DoExtraAnalysis)
Result = false;		Result = false;
▲ Show 20 Lines • Show All 64 Lines • Show Last 20 Lines