Page MenuHomePhabricator
Differential D134745 Diff 463299 clang/lib/CodeGen/CGLoopInfo.cpp

Changeset View

Standalone View

View Options

clang/lib/CodeGen/CGLoopInfo.cpp

Show First 20 Lines • Show All 207 Lines • ▼ Show 20 Lines
MDNode * MDNode *
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
ArrayRef<Metadata *> LoopProperties, ArrayRef<Metadata *> LoopProperties,
bool &HasUserTransforms) { bool &HasUserTransforms) {
LLVMContext &Ctx = Header->getContext(); LLVMContext &Ctx = Header->getContext();
Optional<bool> Enabled; Optional<bool> Enabled;
if (Attrs.VectorizeEnable == LoopAttributes::Disable) if (Attrs.VectorizeEnable == LoopAttributes::Disable &&
Attrs.InterleaveEnable == LoopAttributes::Disable)
Enabled = false; Enabled = false;
else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified || Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
Attrs.InterleaveEnable != LoopAttributes::Unspecified ||
Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 || Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
Attrs.VectorizeScalable != LoopAttributes::Unspecified) Attrs.VectorizeScalable != LoopAttributes::Unspecified)
Enabled = true; Enabled = true;
if (Enabled != true) { if (Enabled != true) {
SmallVector<Metadata *, 4> NewLoopProperties; SmallVector<Metadata *, 4> NewLoopProperties;
if (Enabled == false) { if (Enabled == false) {
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
NewLoopProperties.push_back( NewLoopProperties.push_back(
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.interleave.enable"),
ConstantAsMetadata::get(ConstantInt::get(
llvm::Type::getInt1Ty(Ctx), 0))}));
NewLoopProperties.push_back(
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
ConstantAsMetadata::get(ConstantInt::get( ConstantAsMetadata::get(ConstantInt::get(
llvm::Type::getInt1Ty(Ctx), 0))})); llvm::Type::getInt1Ty(Ctx), 0))}));
LoopProperties = NewLoopProperties; LoopProperties = NewLoopProperties;
} }
return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms); return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
} }
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 LinesLoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
if (Attrs.InterleaveCount > 0) { if (Attrs.InterleaveCount > 0) {
Metadata *Vals[] = { Metadata *Vals[] = {
MDString::get(Ctx, "llvm.loop.interleave.count"), MDString::get(Ctx, "llvm.loop.interleave.count"),
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
Attrs.InterleaveCount))}; Attrs.InterleaveCount))};
Args.push_back(MDNode::get(Ctx, Vals)); Args.push_back(MDNode::get(Ctx, Vals));
} }
// Setting interleave.enable
if (Attrs.InterleaveEnable != LoopAttributes::Unspecified ||
Attrs.InterleaveCount > 0) {
bool IsInterleaveEnabled =
(Attrs.InterleaveEnable == LoopAttributes::Enable) ||
Attrs.InterleaveCount > 1;
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.enable"),
ConstantAsMetadata::get(ConstantInt::get(
llvm::Type::getInt1Ty(Ctx), IsInterleaveEnabled))};
Args.push_back(MDNode::get(Ctx, Vals));
}
// vectorize.enable is set if: // vectorize.enable is set if:
// 1) loop hint vectorize.enable is set, or // 1) loop hint vectorize.enable is set, or
// 2) it is implied when vectorize.predicate is set, or // 2) it is implied when vectorize.predicate is set, or
// 3) it is implied when vectorize.width is set to a value > 1 // 3) it is implied when vectorize.width is set to a value > 1
// 4) it is implied when vectorize.scalable.enable is true // 4) it is implied when vectorize.scalable.enable is true
// 5) it is implied when vectorize.width is unset (0) and the user // 5) it is implied when vectorize.width is unset (0) and the user
// explicitly requested fixed-width vectorization, i.e. // explicitly requested fixed-width vectorization, i.e.
// vectorize.scalable.enable is false. // vectorize.scalable.enable is false.
▲ Show 20 Lines • Show All 141 Lines • ▼ Show 20 LinesMDNode *LoopInfo::createMetadata(
return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms); return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
} }
LoopAttributes::LoopAttributes(bool IsParallel) LoopAttributes::LoopAttributes(bool IsParallel)
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
UnrollEnable(LoopAttributes::Unspecified), UnrollEnable(LoopAttributes::Unspecified),
UnrollAndJamEnable(LoopAttributes::Unspecified), UnrollAndJamEnable(LoopAttributes::Unspecified),
VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0), VectorizeScalable(LoopAttributes::Unspecified),
InterleaveEnable(LoopAttributes::Unspecified), InterleaveCount(0),
UnrollCount(0), UnrollAndJamCount(0), UnrollCount(0), UnrollAndJamCount(0),
DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
PipelineInitiationInterval(0), MustProgress(false) {} PipelineInitiationInterval(0), MustProgress(false) {}
void LoopAttributes::clear() { void LoopAttributes::clear() {
IsParallel = false; IsParallel = false;
VectorizeWidth = 0; VectorizeWidth = 0;
VectorizeScalable = LoopAttributes::Unspecified; VectorizeScalable = LoopAttributes::Unspecified;
InterleaveCount = 0; InterleaveCount = 0;
UnrollCount = 0; UnrollCount = 0;
UnrollAndJamCount = 0; UnrollAndJamCount = 0;
VectorizeEnable = LoopAttributes::Unspecified; VectorizeEnable = LoopAttributes::Unspecified;
InterleaveEnable = LoopAttributes::Unspecified;
UnrollEnable = LoopAttributes::Unspecified; UnrollEnable = LoopAttributes::Unspecified;
UnrollAndJamEnable = LoopAttributes::Unspecified; UnrollAndJamEnable = LoopAttributes::Unspecified;
VectorizePredicateEnable = LoopAttributes::Unspecified; VectorizePredicateEnable = LoopAttributes::Unspecified;
DistributeEnable = LoopAttributes::Unspecified; DistributeEnable = LoopAttributes::Unspecified;
PipelineDisabled = false; PipelineDisabled = false;
PipelineInitiationInterval = 0; PipelineInitiationInterval = 0;
MustProgress = false; MustProgress = false;
} }
Show All 12 LinesLoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
Attrs.VectorizeScalable == LoopAttributes::Unspecified && Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
Attrs.PipelineInitiationInterval == 0 && Attrs.PipelineInitiationInterval == 0 &&
Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified && Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified && Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
Attrs.InterleaveEnable == LoopAttributes::Unspecified &&
Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.UnrollEnable == LoopAttributes::Unspecified &&
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc && Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
!EndLoc && !Attrs.MustProgress) !EndLoc && !Attrs.MustProgress)
return; return;
TempLoopID = MDNode::getTemporary(Header->getContext(), None); TempLoopID = MDNode::getTemporary(Header->getContext(), None);
} }
Show All 17 Lines if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
LoopAttributes BeforeJam, AfterJam; LoopAttributes BeforeJam, AfterJam;
BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel; BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
BeforeJam.VectorizeWidth = Attrs.VectorizeWidth; BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
BeforeJam.VectorizeScalable = Attrs.VectorizeScalable; BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
BeforeJam.InterleaveCount = Attrs.InterleaveCount; BeforeJam.InterleaveCount = Attrs.InterleaveCount;
BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
BeforeJam.InterleaveEnable = Attrs.InterleaveEnable;
BeforeJam.DistributeEnable = Attrs.DistributeEnable; BeforeJam.DistributeEnable = Attrs.DistributeEnable;
BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
switch (Attrs.UnrollEnable) { switch (Attrs.UnrollEnable) {
case LoopAttributes::Unspecified: case LoopAttributes::Unspecified:
case LoopAttributes::Disable: case LoopAttributes::Disable:
BeforeJam.UnrollEnable = Attrs.UnrollEnable; BeforeJam.UnrollEnable = Attrs.UnrollEnable;
AfterJam.UnrollEnable = Attrs.UnrollEnable; AfterJam.UnrollEnable = Attrs.UnrollEnable;
Show All 23 Lines if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
// consider the first inner loop. // consider the first inner loop.
if (!Parent->UnrollAndJamInnerFollowup) { if (!Parent->UnrollAndJamInnerFollowup) {
// Splitting the attributes into a BeforeJam and an AfterJam part will // Splitting the attributes into a BeforeJam and an AfterJam part will
// stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam) // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
// to be forwarded to the AfterJam part. We detect the situation here and // to be forwarded to the AfterJam part. We detect the situation here and
// add it manually. // add it manually.
SmallVector<Metadata *, 1> BeforeLoopProperties; SmallVector<Metadata *, 1> BeforeLoopProperties;
if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
BeforeJam.InterleaveEnable != LoopAttributes::Unspecified ||
BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified || BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 || BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
BeforeJam.VectorizeScalable == LoopAttributes::Enable) BeforeJam.VectorizeScalable == LoopAttributes::Enable)
BeforeLoopProperties.push_back( BeforeLoopProperties.push_back(
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
bool InnerFollowupHasTransform = false; bool InnerFollowupHasTransform = false;
MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties, MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines if (OpenCLHint) {
Option = LH->getOption(); Option = LH->getOption();
State = LH->getState(); State = LH->getState();
} }
switch (State) { switch (State) {
case LoopHintAttr::Disable: case LoopHintAttr::Disable:
switch (Option) { switch (Option) {
case LoopHintAttr::Vectorize: case LoopHintAttr::Vectorize:
// Disable vectorization by specifying a width of 1. // Disable vectorization with {vectorization.enable, false} and
// vectorization width of 1.
setVectorizeEnable(false);
setVectorizeWidth(1); setVectorizeWidth(1);
setVectorizeScalable(LoopAttributes::Unspecified); setVectorizeScalable(LoopAttributes::Unspecified);
break; break;
case LoopHintAttr::Interleave: case LoopHintAttr::Interleave:
// Disable interleaving by speciyfing a count of 1. // Disable interleave with {interleave.enable, false} and interleave
// count of 1.
setInterleaveEnable(false);
setInterleaveCount(1); setInterleaveCount(1);
mcberg2021Unsubmitted
Done
ReplyInline Actions

Can you update the comments on lines: 665 and 671 as they both need update.

mcberg2021: Can you update the comments on lines: 665 and 671 as they both need update.
break; break;
case LoopHintAttr::Unroll: case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Disable); setUnrollState(LoopAttributes::Disable);
break; break;
case LoopHintAttr::UnrollAndJam: case LoopHintAttr::UnrollAndJam:
setUnrollAndJamState(LoopAttributes::Disable); setUnrollAndJamState(LoopAttributes::Disable);
break; break;
case LoopHintAttr::VectorizePredicate: case LoopHintAttr::VectorizePredicate:
Show All 12 Lines case LoopHintAttr::Disable:
case LoopHintAttr::PipelineInitiationInterval: case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot be disabled."); llvm_unreachable("Options cannot be disabled.");
break; break;
} }
break; break;
case LoopHintAttr::Enable: case LoopHintAttr::Enable:
switch (Option) { switch (Option) {
case LoopHintAttr::Vectorize: case LoopHintAttr::Vectorize:
case LoopHintAttr::Interleave:
setVectorizeEnable(true); setVectorizeEnable(true);
break; break;
case LoopHintAttr::Interleave:
setInterleaveEnable(true);
break;
case LoopHintAttr::Unroll: case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Enable); setUnrollState(LoopAttributes::Enable);
break; break;
case LoopHintAttr::UnrollAndJam: case LoopHintAttr::UnrollAndJam:
setUnrollAndJamState(LoopAttributes::Enable); setUnrollAndJamState(LoopAttributes::Enable);
break; break;
case LoopHintAttr::VectorizePredicate: case LoopHintAttr::VectorizePredicate:
setVectorizePredicateState(LoopAttributes::Enable); setVectorizePredicateState(LoopAttributes::Enable);
Show All 9 Lines case LoopHintAttr::Enable:
case LoopHintAttr::PipelineInitiationInterval: case LoopHintAttr::PipelineInitiationInterval:
llvm_unreachable("Options cannot enabled."); llvm_unreachable("Options cannot enabled.");
break; break;
} }
break; break;
case LoopHintAttr::AssumeSafety: case LoopHintAttr::AssumeSafety:
switch (Option) { switch (Option) {
case LoopHintAttr::Vectorize: case LoopHintAttr::Vectorize:
case LoopHintAttr::Interleave:
// Apply "llvm.mem.parallel_loop_access" metadata to load/stores. // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
setParallel(true); setParallel(true);
setVectorizeEnable(true); setVectorizeEnable(true);
break; break;
case LoopHintAttr::Interleave:
// Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
setParallel(true);
setInterleaveEnable(true);
break;
case LoopHintAttr::Unroll: case LoopHintAttr::Unroll:
case LoopHintAttr::UnrollAndJam: case LoopHintAttr::UnrollAndJam:
case LoopHintAttr::VectorizePredicate: case LoopHintAttr::VectorizePredicate:
case LoopHintAttr::UnrollCount: case LoopHintAttr::UnrollCount:
case LoopHintAttr::UnrollAndJamCount: case LoopHintAttr::UnrollAndJamCount:
case LoopHintAttr::VectorizeWidth: case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount: case LoopHintAttr::InterleaveCount:
case LoopHintAttr::Distribute: case LoopHintAttr::Distribute:
▲ Show 20 Lines • Show All 124 Lines • Show Last 20 Lines