Differential D117952 Diff 402225 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Changeset View

Standalone View

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 2,356 Lines • ▼ Show 20 Lines	bool isPartOfBundle() const {
return NextInBundle != nullptr \|\| FirstInBundle != this;		return NextInBundle != nullptr \|\| FirstInBundle != this;
}		}

/// Returns true if it is ready for scheduling, i.e. it has no more		/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.		/// unscheduled depending instructions/bundles.
bool isReady() const {		bool isReady() const {
assert(isSchedulingEntity() &&		assert(isSchedulingEntity() &&
"can't consider non-scheduling entity for ready list");		"can't consider non-scheduling entity for ready list");
		assert(hasValidDependencies() &&
		"can't schedule without dependencies");
return UnscheduledDepsInBundle == 0 && !IsScheduled;		return UnscheduledDepsInBundle == 0 && !IsScheduled;
}		}

/// Modifies the number of unscheduled dependencies, also updating it for		/// Modifies the number of unscheduled dependencies, also updating it for
/// the whole bundle.		/// the whole bundle.
int incrementUnscheduledDeps(int Incr) {		int incrementUnscheduledDeps(int Incr) {
UnscheduledDeps += Incr;		UnscheduledDeps += Incr;
return FirstInBundle->UnscheduledDepsInBundle += Incr;		return FirstInBundle->UnscheduledDepsInBundle += Incr;
▲ Show 20 Lines • Show All 4,968 Lines • ▼ Show 20 Lines	BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value > VL, BoUpSLP SLP,
LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n");		LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n");

auto TryScheduleBundleImpl = [this, OldScheduleEnd, SLP](ScheduleData *Bundle) {		auto TryScheduleBundleImpl = [this, OldScheduleEnd, SLP](ScheduleData *Bundle) {
// The scheduling region got new instructions at the lower end (or it is a		// The scheduling region got new instructions at the lower end (or it is a
// new region for the first bundle). This makes it necessary to		// new region for the first bundle). This makes it necessary to
// recalculate all dependencies.		// recalculate all dependencies.
// It is seldom that this needs to be done a second time after adding the		// It is seldom that this needs to be done a second time after adding the
// initial bundle to the region.		// initial bundle to the region.
bool ReSchedule = false;
if (ScheduleEnd != OldScheduleEnd) {		if (ScheduleEnd != OldScheduleEnd) {
for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode())		for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode())
doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); });		doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); });
ReSchedule = true;
}
if (ReSchedule) {
resetSchedule();		resetSchedule();
initialFillReadyList(ReadyInsts);
		// NOTE: At this point, we've cleared dependencies and schedule information
		// we can not restart scheduling without computing dependencies again. During
		// prescheduling, we only build dependencies for things which are transitive
		// users of a proposed node in the tree. This means we leave large sections
		// of the overall window without valid dependencies until final scheduling.
}		}
if (Bundle) {		if (!Bundle)
		return;

LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle		LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle
<< " in block " << BB->getName() << "\n");		<< " in block " << BB->getName() << "\n");
calculateDependencies(Bundle, /InsertInReadyList=/true, SLP);		calculateDependencies(Bundle, /InsertInReadyList=/true, SLP);
}

// Now try to schedule the new bundle or (if no bundle) just calculate		// Now try to schedule the new bundle. As soon as the bundle is "ready" it
// dependencies. As soon as the bundle is "ready" it means that there are no		// means that there are no cyclic dependencies and we can schedule it. Note
// cyclic dependencies and we can schedule it. Note that's important that we		// that it's important that we don't "schedule" the bundle yet (see
// don't "schedule" the bundle yet (see cancelScheduling).		// cancelScheduling).
while (((!Bundle && ReSchedule) \|\| (Bundle && !Bundle->isReady())) &&		while (!Bundle->isReady() && !ReadyInsts.empty()) {
!ReadyInsts.empty()) {
ScheduleData *Picked = ReadyInsts.pop_back_val();		ScheduleData *Picked = ReadyInsts.pop_back_val();
if (Picked->isSchedulingEntity() && Picked->isReady())		if (Picked->isSchedulingEntity() && Picked->isReady())
schedule(Picked, ReadyInsts);		schedule(Picked, ReadyInsts);
}		}
};		};

// Make sure that the scheduling region contains all		// Make sure that the scheduling region contains all
// instructions of the bundle.		// instructions of the bundle.
▲ Show 20 Lines • Show All 2,953 Lines • Show Last 20 Lines