diff --git a/clang/lib/CodeGen/CGLoopInfo.h b/clang/lib/CodeGen/CGLoopInfo.h --- a/clang/lib/CodeGen/CGLoopInfo.h +++ b/clang/lib/CodeGen/CGLoopInfo.h @@ -61,6 +61,9 @@ // Value for llvm.loop.vectorize.scalable.enable LVEnableState VectorizeScalable; + /// Value for llvm.loop.interleave.enable metadata. + LVEnableState InterleaveEnable; + /// Value for llvm.loop.interleave.count metadata. unsigned InterleaveCount; @@ -265,6 +268,11 @@ StagedAttrs.VectorizeScalable = State; } + void setInterleaveEnable(bool Enable = true) { + StagedAttrs.InterleaveEnable = + Enable ? LoopAttributes::Enable : LoopAttributes::Disable; + } + /// Set the interleave count for the next loop pushed. void setInterleaveCount(unsigned C) { StagedAttrs.InterleaveCount = C; } diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp --- a/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/clang/lib/CodeGen/CGLoopInfo.cpp @@ -213,10 +213,12 @@ LLVMContext &Ctx = Header->getContext(); Optional Enabled; - if (Attrs.VectorizeEnable == LoopAttributes::Disable) + if (Attrs.VectorizeEnable == LoopAttributes::Disable && + Attrs.InterleaveEnable == LoopAttributes::Disable) Enabled = false; else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified || + Attrs.InterleaveEnable != LoopAttributes::Unspecified || Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 || Attrs.VectorizeScalable != LoopAttributes::Unspecified) Enabled = true; @@ -225,6 +227,10 @@ SmallVector NewLoopProperties; if (Enabled == false) { NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.interleave.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 0))})); NewLoopProperties.push_back( MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantAsMetadata::get(ConstantInt::get( @@ -292,6 +298,18 @@ Args.push_back(MDNode::get(Ctx, Vals)); } + // Setting interleave.enable + if (Attrs.InterleaveEnable != LoopAttributes::Unspecified || + Attrs.InterleaveCount > 0) { + bool IsInterleaveEnabled = + (Attrs.InterleaveEnable == LoopAttributes::Enable) || + Attrs.InterleaveCount > 1; + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), IsInterleaveEnabled))}; + Args.push_back(MDNode::get(Ctx, Vals)); + } + // vectorize.enable is set if: // 1) loop hint vectorize.enable is set, or // 2) it is implied when vectorize.predicate is set, or @@ -449,7 +467,8 @@ UnrollEnable(LoopAttributes::Unspecified), UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), - VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0), + VectorizeScalable(LoopAttributes::Unspecified), + InterleaveEnable(LoopAttributes::Unspecified), InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0), DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), PipelineInitiationInterval(0), MustProgress(false) {} @@ -462,6 +481,7 @@ UnrollCount = 0; UnrollAndJamCount = 0; VectorizeEnable = LoopAttributes::Unspecified; + InterleaveEnable = LoopAttributes::Unspecified; UnrollEnable = LoopAttributes::Unspecified; UnrollAndJamEnable = LoopAttributes::Unspecified; VectorizePredicateEnable = LoopAttributes::Unspecified; @@ -490,6 +510,7 @@ Attrs.PipelineInitiationInterval == 0 && Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified && Attrs.VectorizeEnable == LoopAttributes::Unspecified && + Attrs.InterleaveEnable == LoopAttributes::Unspecified && Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc && @@ -523,6 +544,7 @@ BeforeJam.VectorizeScalable = Attrs.VectorizeScalable; BeforeJam.InterleaveCount = Attrs.InterleaveCount; BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; + BeforeJam.InterleaveEnable = Attrs.InterleaveEnable; BeforeJam.DistributeEnable = Attrs.DistributeEnable; BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; @@ -562,6 +584,7 @@ // add it manually. SmallVector BeforeLoopProperties; if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || + BeforeJam.InterleaveEnable != LoopAttributes::Unspecified || BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified || BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 || BeforeJam.VectorizeScalable == LoopAttributes::Enable) @@ -639,12 +662,16 @@ case LoopHintAttr::Disable: switch (Option) { case LoopHintAttr::Vectorize: - // Disable vectorization by specifying a width of 1. + // Disable vectorization with {vectorization.enable, false} and + // vectorization width of 1. + setVectorizeEnable(false); setVectorizeWidth(1); setVectorizeScalable(LoopAttributes::Unspecified); break; case LoopHintAttr::Interleave: - // Disable interleaving by speciyfing a count of 1. + // Disable interleave with {interleave.enable, false} and interleave + // count of 1. + setInterleaveEnable(false); setInterleaveCount(1); break; case LoopHintAttr::Unroll: @@ -674,9 +701,11 @@ case LoopHintAttr::Enable: switch (Option) { case LoopHintAttr::Vectorize: - case LoopHintAttr::Interleave: setVectorizeEnable(true); break; + case LoopHintAttr::Interleave: + setInterleaveEnable(true); + break; case LoopHintAttr::Unroll: setUnrollState(LoopAttributes::Enable); break; @@ -702,11 +731,15 @@ case LoopHintAttr::AssumeSafety: switch (Option) { case LoopHintAttr::Vectorize: - case LoopHintAttr::Interleave: // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. setParallel(true); setVectorizeEnable(true); break; + case LoopHintAttr::Interleave: + // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. + setParallel(true); + setInterleaveEnable(true); + break; case LoopHintAttr::Unroll: case LoopHintAttr::UnrollAndJam: case LoopHintAttr::VectorizePredicate: diff --git a/clang/test/CodeGenCXX/pragma-loop-predicate.cpp b/clang/test/CodeGenCXX/pragma-loop-predicate.cpp --- a/clang/test/CodeGenCXX/pragma-loop-predicate.cpp +++ b/clang/test/CodeGenCXX/pragma-loop-predicate.cpp @@ -113,10 +113,11 @@ // CHECK-NEXT: ![[LOOP3]] = distinct !{![[LOOP3]], [[MP]], [[GEN6]], [[GEN3]]} -// CHECK-NEXT: ![[LOOP4]] = distinct !{![[LOOP4]], [[MP]], [[GEN10:![0-9]+]]} +// CHECK-NEXT: ![[LOOP4]] = distinct !{![[LOOP4]], [[MP]], [[GEN10:![0-9]+]], [[VECTORIZE_DISABLE:.*]]} // CHECK-NEXT: [[GEN10]] = !{!"llvm.loop.vectorize.width", i32 1} +// CHECK-NEXT: [[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false} -// CHECK-NEXT: ![[LOOP5]] = distinct !{![[LOOP5]], [[MP]], [[GEN6]], [[GEN10]]} +// CHECK-NEXT: ![[LOOP5]] = distinct !{![[LOOP5]], [[MP]], [[GEN6]], [[GEN10]], [[VECTORIZE_DISABLE]]} // CHECK-NEXT: ![[LOOP6]] = distinct !{![[LOOP6]], [[MP]], [[GEN8]], [[GEN10]], [[GEN11:![0-9]+]]} // CHECK-NEXT: [[GEN11]] = !{!"llvm.loop.vectorize.scalable.enable", i1 false} diff --git a/clang/test/CodeGenCXX/pragma-loop-safety.cpp b/clang/test/CodeGenCXX/pragma-loop-safety.cpp --- a/clang/test/CodeGenCXX/pragma-loop-safety.cpp +++ b/clang/test/CodeGenCXX/pragma-loop-safety.cpp @@ -47,12 +47,15 @@ } // CHECK: ![[ACCESS_GROUP_2]] = distinct !{} -// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], [[MP:![0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]]} +// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], [[MP:![0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTERLEAVE_DISABLE:.*]], ![[VECTORIZE_ENABLE:[0-9]+]]} // CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]} // CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"} // CHECK: ![[INTERLEAVE_1]] = !{!"llvm.loop.interleave.count", i32 1} -// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.enable", i1 false} +// CHECK: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true} // CHECK: ![[ACCESS_GROUP_8]] = distinct !{} -// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], [[MP]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]]} +// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], [[MP]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_DISABLE:[0-9]+]]} // CHECK: ![[PARALLEL_ACCESSES_11]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_8]]} // CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1} +// CHECK: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.interleave.enable", i1 true} +// CHECK: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false} diff --git a/clang/test/CodeGenCXX/pragma-loop.cpp b/clang/test/CodeGenCXX/pragma-loop.cpp --- a/clang/test/CodeGenCXX/pragma-loop.cpp +++ b/clang/test/CodeGenCXX/pragma-loop.cpp @@ -56,7 +56,7 @@ // Verify disable pragma clang loop directive generates correct metadata void disable_test(int *List, int Length) { -#pragma clang loop vectorize(disable) unroll(disable) distribute(disable) +#pragma clang loop vectorize(disable) interleave(disable) unroll(disable) distribute(disable) for (int i = 0; i < Length; i++) { // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_5:.*]] List[i] = i * 2; @@ -206,26 +206,28 @@ // CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], [[MP:![0-9]+]], ![[UNROLL_FULL:.*]]} // CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"} -// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], [[MP]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[FIXED_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]} +// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], [[MP]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[FIXED_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_ENABLE:.*]]} // CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"} // CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false} // CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8} // CHECK: ![[FIXED_VEC]] = !{!"llvm.loop.vectorize.scalable.enable", i1 false} // CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4} +// CHECK: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.interleave.enable", i1 true} // CHECK: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true} -// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], [[MP]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_3:.*]]} +// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], [[MP]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[FOLLOWUP_VECTOR_3:.*]]} // CHECK: ![[FOLLOWUP_VECTOR_3]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_3:.*]]} // CHECK: ![[AFTER_VECTOR_3]] = distinct !{![[AFTER_VECTOR_3]], [[MP]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]} // CHECK: ![[ISVECTORIZED]] = !{!"llvm.loop.isvectorized"} // CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8} -// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[VECTORIZE_ENABLE]]} +// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_ENABLE]]} // CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2} // CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2} -// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]]} -// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1} +// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[INTERLEAVE_DISABLE:.*]], ![[VECTORIZE_DISABLE:.*]]} +// CHECK: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.enable", i1 false} +// CHECK: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false} // CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], [[MP]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_6:.*]]} // CHECK: ![[FOLLOWUP_VECTOR_6]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_6:.*]]} @@ -253,7 +255,7 @@ // CHECK: ![[AFTER_VECTOR_12]] = distinct !{![[AFTER_VECTOR_12]], ![[ISVECTORIZED:.*]], ![[UNROLL_24:.*]]} // CHECK: ![[UNROLL_24]] = !{!"llvm.loop.unroll.count", i32 24} -// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_13:.*]]} +// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_13:.*]]} // CHECK: ![[INTERLEAVE_16]] = !{!"llvm.loop.interleave.count", i32 16} // CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]} // CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]} @@ -262,12 +264,13 @@ // CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], [[MP]], ![[WIDTH_10:.*]], ![[FIXED_VEC]], ![[VECTORIZE_ENABLE]]} // CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10} -// CHECK: ![[LOOP_15]] = distinct !{![[LOOP_15]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]} +// CHECK: ![[LOOP_15]] = distinct !{![[LOOP_15]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]} // CHECK: ![[WIDTH_16]] = !{!"llvm.loop.vectorize.width", i32 16} -// CHECK: ![[LOOP_16]] = distinct !{![[LOOP_16]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16]], ![[SCALABLE_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]} +// CHECK: ![[LOOP_16]] = distinct !{![[LOOP_16]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16]], ![[SCALABLE_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]} // CHECK: ![[SCALABLE_VEC]] = !{!"llvm.loop.vectorize.scalable.enable", i1 true} -// CHECK: ![[LOOP_17]] = distinct !{![[LOOP_17]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]} -// CHECK: ![[LOOP_18]] = distinct !{![[LOOP_18]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]} -// CHECK: ![[LOOP_19]] = distinct !{![[LOOP_19]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]} +// CHECK: ![[LOOP_17]] = distinct !{![[LOOP_17]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]} +// CHECK: ![[LOOP_18]] = distinct !{![[LOOP_18]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]} +// CHECK: ![[LOOP_19]] = distinct !{![[LOOP_19]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]} +// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1} diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -87,7 +87,10 @@ Hint Interleave; /// Vectorization forced - Hint Force; + Hint VectorizationForce; + + /// Interleave forced + Hint InterleaveForce; /// Already Vectorized Hint IsVectorized; @@ -151,11 +154,17 @@ } unsigned getIsVectorized() const { return IsVectorized.Value; } unsigned getPredicate() const { return Predicate.Value; } - enum ForceKind getForce() const { - if ((ForceKind)Force.Value == FK_Undefined && + enum ForceKind getVectorizationForce() const { + if ((ForceKind)VectorizationForce.Value == FK_Undefined && + hasDisableAllTransformsHint(TheLoop)) + return FK_Disabled; + return (ForceKind)VectorizationForce.Value; + } + enum ForceKind getInterleaveForce() const { + if ((ForceKind)InterleaveForce.Value == FK_Undefined && hasDisableAllTransformsHint(TheLoop)) return FK_Disabled; - return (ForceKind)Force.Value; + return (ForceKind)InterleaveForce.Value; } /// \return true if scalable vectorization has been explicitly disabled. @@ -180,7 +189,8 @@ // IEEE 754 FP ops properly, or bad single-to-double promotions. // Otherwise, a sequence of vectorized loops, even without reduction, // could lead to different end results on the destination vectors. - return getForce() != LoopVectorizeHints::FK_Enabled && PotentiallyUnsafe; + return getVectorizationForce() != LoopVectorizeHints::FK_Enabled && + PotentiallyUnsafe; } void setPotentiallyUnsafe() { PotentiallyUnsafe = true; } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -100,7 +100,8 @@ const TargetTransformInfo *TTI) : Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH), Interleave("interleave.count", InterleaveOnlyWhenForced, HK_INTERLEAVE), - Force("vectorize.enable", FK_Undefined, HK_FORCE), + VectorizationForce("vectorize.enable", FK_Undefined, HK_FORCE), + InterleaveForce("interleave.enable", FK_Undefined, HK_FORCE), IsVectorized("isvectorized", 0, HK_ISVECTORIZED), Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE), Scalable("vectorize.scalable.enable", SK_Unspecified, HK_SCALABLE), @@ -171,23 +172,27 @@ bool LoopVectorizeHints::allowVectorization( Function *F, Loop *L, bool VectorizeOnlyWhenForced) const { - if (getForce() == LoopVectorizeHints::FK_Disabled) { - LLVM_DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n"); + if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled && + getInterleaveForce() == LoopVectorizeHints::FK_Disabled) { + LLVM_DEBUG( + dbgs() + << "LV: Not vectorizing: #pragma vectorize and interleave disable.\n"); emitRemarkWithHints(); return false; } - if (VectorizeOnlyWhenForced && getForce() != LoopVectorizeHints::FK_Enabled) { - LLVM_DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n"); + if (VectorizeOnlyWhenForced && + getVectorizationForce() != LoopVectorizeHints::FK_Enabled && + getInterleaveForce() != LoopVectorizeHints::FK_Enabled) { + LLVM_DEBUG( + dbgs() + << "LV: Not vectorizing: No #pragma vectorize or interleave enable.\n"); emitRemarkWithHints(); return false; } if (getIsVectorized() == 1) { LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n"); - // FIXME: Add interleave.disable metadata. This will allow - // vectorize.disable to be used without disabling the pass and errors - // to differentiate between disabled vectorization and a width of 1. ORE.emit([&]() { return OptimizationRemarkAnalysis(vectorizeAnalysisPassName(), "AllDisabled", L->getStartLoc(), @@ -206,19 +211,25 @@ using namespace ore; ORE.emit([&]() { - if (Force.Value == LoopVectorizeHints::FK_Disabled) + if (VectorizationForce.Value == LoopVectorizeHints::FK_Disabled && + InterleaveForce.Value == LoopVectorizeHints::FK_Disabled) return OptimizationRemarkMissed(LV_NAME, "MissedExplicitlyDisabled", TheLoop->getStartLoc(), TheLoop->getHeader()) - << "loop not vectorized: vectorization is explicitly disabled"; + << "loop not vectorized: vectorization and interleave is " + "explicitly disabled"; else { OptimizationRemarkMissed R(LV_NAME, "MissedDetails", TheLoop->getStartLoc(), TheLoop->getHeader()); R << "loop not vectorized"; - if (Force.Value == LoopVectorizeHints::FK_Enabled) { - R << " (Force=" << NV("Force", true); + if (VectorizationForce.Value == LoopVectorizeHints::FK_Enabled) { + R << " (VectorizationForce=" << NV("VectorizationForce", true); if (Width.Value != 0) R << ", Vector Width=" << NV("VectorWidth", getWidth()); + R << ")"; + } + if (InterleaveForce.Value == LoopVectorizeHints::FK_Enabled) { + R << " (InterleaveForce=" << NV("InterleaveForce", true); if (getInterleave() != 0) R << ", Interleave Count=" << NV("InterleaveCount", getInterleave()); R << ")"; @@ -231,9 +242,10 @@ const char *LoopVectorizeHints::vectorizeAnalysisPassName() const { if (getWidth() == ElementCount::getFixed(1)) return LV_NAME; - if (getForce() == LoopVectorizeHints::FK_Disabled) + if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled) return LV_NAME; - if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth().isZero()) + if (getVectorizationForce() == LoopVectorizeHints::FK_Undefined && + getWidth().isZero()) return LV_NAME; return OptimizationRemarkAnalysis::AlwaysPrint; } @@ -243,7 +255,7 @@ // loop hints are provided ElementCount EC = getWidth(); return HintsAllowReordering && - (getForce() == LoopVectorizeHints::FK_Enabled || + (getVectorizationForce() == LoopVectorizeHints::FK_Enabled || EC.getKnownMinValue() > 1); } @@ -293,8 +305,9 @@ return; unsigned Val = C->getZExtValue(); - Hint *Hints[] = {&Width, &Interleave, &Force, - &IsVectorized, &Predicate, &Scalable}; + Hint *Hints[] = {&Width, &Interleave, &VectorizationForce, + &InterleaveForce, &IsVectorized, &Predicate, + &Scalable}; for (auto *H : Hints) { if (Name == H->Name) { if (H->validate(Val)) @@ -1343,7 +1356,7 @@ << "!\n"); unsigned SCEVThreshold = VectorizeSCEVCheckThreshold; - if (Hints->getForce() == LoopVectorizeHints::FK_Enabled) + if (Hints->getVectorizationForce() == LoopVectorizeHints::FK_Enabled) SCEVThreshold = PragmaVectorizeSCEVCheckThreshold; if (PSE.getPredicate().getComplexity() > SCEVThreshold) { diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2129,7 +2129,7 @@ // Only outer loops with an explicit vectorization hint are supported. // Unannotated outer loops are ignored. - if (Hints.getForce() == LoopVectorizeHints::FK_Undefined) + if (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Undefined) return false; Function *Fn = OuterLp->getHeader()->getParent(); @@ -2139,7 +2139,7 @@ return false; } - if (Hints.getInterleave() > 1) { + if (Hints.getInterleaveForce()) { // TODO: Interleave support is future work. LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for " "outer loops.\n"); @@ -3010,11 +3010,10 @@ return nullptr; assert(!(SCEVCheckBlock->getParent()->hasOptSize() || - (OptForSizeBasedOnProfile && - Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && + (OptForSizeBasedOnProfile && Cost->Hints->getVectorizationForce() != + LoopVectorizeHints::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"); - // Update dominator only if this is first RT check. if (LoopBypassBlocks.empty()) { DT->changeImmediateDominator(Bypass, SCEVCheckBlock); @@ -3045,7 +3044,8 @@ return nullptr; if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) { - assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && + assert(Cost->Hints->getVectorizationForce() == + LoopVectorizeHints::FK_Enabled && "Cannot emit memory checks when optimizing for size, unless forced " "to vectorize."); ORE->emit([&]() { @@ -5327,7 +5327,8 @@ ExpectedCost); VectorizationFactor ChosenFactor = ScalarCost; - bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled; + bool ForceVectorization = + Hints->getVectorizationForce() == LoopVectorizeHints::FK_Enabled; if (ForceVectorization && VFCandidates.size() > 1) { // Ignore scalar width, because the user explicitly wants vectorization. // Initialize cost to max so that VF = 2 is, at least, chosen during cost @@ -9827,9 +9828,10 @@ // of strides in LoopAccessInfo::analyzeLoop() and vectorize without // versioning when the vectorization is forced, unlike hasOptSize. So revert // back to the old way and vectorize with versioning when forced. See D81345.) - if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI, - PGSOQueryType::IRPass) && - Hints.getForce() != LoopVectorizeHints::FK_Enabled)) + if (F->hasOptSize() || + (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI, + PGSOQueryType::IRPass) && + Hints.getInterleaveForce() != LoopVectorizeHints::FK_Enabled)) return CM_ScalarEpilogueNotAllowedOptSize; // 2) If set, obey the directives @@ -10153,15 +10155,21 @@ LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI); LLVM_DEBUG( - dbgs() << "LV: Loop hints:" - << " force=" - << (Hints.getForce() == LoopVectorizeHints::FK_Disabled - ? "disabled" - : (Hints.getForce() == LoopVectorizeHints::FK_Enabled - ? "enabled" - : "?")) - << " width=" << Hints.getWidth() - << " interleave=" << Hints.getInterleave() << "\n"); + dbgs() + << "LV: Loop hints:" + << " force_vectorization=" + << (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Disabled + ? "disabled" + : (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled + ? "enabled" + : "?")) + << " width=" << Hints.getWidth() << " force_interleave=" + << (Hints.getInterleaveForce() == LoopVectorizeHints::FK_Disabled + ? "disabled" + : (Hints.getInterleaveForce() == LoopVectorizeHints::FK_Enabled + ? "enabled" + : "?")) + << " interleave=" << Hints.getInterleave() << "\n"); // Function containing loop Function *F = L->getHeader()->getParent(); @@ -10225,7 +10233,7 @@ LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. " << "This loop is worth vectorizing only if no scalar " << "iteration overheads are incurred."); - if (Hints.getForce() == LoopVectorizeHints::FK_Enabled) + if (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled) LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n"); else { if (*ExpectedTC > TTI->getMinTripCountTailFoldingThreshold()) { @@ -10324,7 +10332,7 @@ // Check if it is profitable to vectorize with runtime checks. bool ForceVectorization = - Hints.getForce() == LoopVectorizeHints::FK_Enabled; + Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled; if (!ForceVectorization && !areRuntimeChecksProfitable(Checks, VF, CM.getVScaleForTuning(), L, *PSE.getSE())) { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll @@ -140,5 +140,6 @@ } -!1 = distinct !{!1, !2} +!1 = distinct !{!1, !2, !3} !2 = !{!"llvm.loop.vectorize.enable", i1 true} +!3 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll --- a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll @@ -1,7 +1,7 @@ ; REQUIRES: asserts ; RUN: opt < %s -loop-vectorize -mtriple riscv64 -riscv-v-vector-bits-min=128 -mattr="+v" -debug-only=loop-vectorize -S 2>&1 | FileCheck %s -; CHECK: LV: Loop hints: force=enabled +; CHECK: LV: Loop hints: force_vectorization=enabled width=vscale x 0 force_interleave=? interleave=0 ; CHECK: LV: Scalar loop costs: 7. ; ChosenFactor.Cost is 8, but the real cost will be divided by the width, which is 4. ; CHECK: LV: Vector loop of width 2 costs: 4. diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -12,7 +12,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) { ; CHECK-LABEL: 'vector_reverse_i64' -; CHECK-NEXT: LV: Loop hints: force=enabled width=vscale x 4 interleave=0 +; CHECK-NEXT: LV: Loop hints: force_vectorization=enabled width=vscale x 4 force_interleave=? interleave=0 ; CHECK-NEXT: LV: Found a loop: for.body ; CHECK-NEXT: LV: Found an induction variable. ; CHECK-NEXT: LV: Found an induction variable. @@ -142,7 +142,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) { ; CHECK-LABEL: 'vector_reverse_f32' -; CHECK-NEXT: LV: Loop hints: force=enabled width=vscale x 4 interleave=0 +; CHECK-NEXT: LV: Loop hints: force_vectorization=enabled width=vscale x 4 force_interleave=? interleave=0 ; CHECK-NEXT: LV: Found a loop: for.body ; CHECK-NEXT: LV: Found an induction variable. ; CHECK-NEXT: LV: Found an induction variable. diff --git a/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll --- a/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll @@ -1957,7 +1957,8 @@ ret i32 %1 } -!0 = !{!0, !1} +!0 = !{!0, !1, !4} !1 = !{!"llvm.loop.vectorize.enable", i1 1} -!2 = !{!2, !3} +!2 = !{!2, !3, !4} !3 = !{!"llvm.loop.vectorize.enable", i1 0} +!4 = !{!"llvm.loop.interleave.enable", i1 0} diff --git a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll --- a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll @@ -109,5 +109,6 @@ ret void } -!1 = distinct !{!1, !2} +!1 = distinct !{!1, !2, !3} !2 = !{!"llvm.loop.vectorize.enable", i1 true} +!3 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll --- a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll @@ -1,8 +1,8 @@ ; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -debug-only=loop-vectorize -stats -S 2>&1 | FileCheck %s ; REQUIRES: asserts -; CHECK: LV: Loop hints: force=enabled -; CHECK: LV: Loop hints: force=? +; CHECK: LV: Loop hints: force_vectorization=enabled width=0 force_interleave=disabled interleave=0 +; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0 ; No more loops in the module ; CHECK-NOT: LV: Loop hints: force= ; CHECK: 2 loop-vectorize - Number of loops analyzed for vectorization @@ -48,9 +48,10 @@ ret void } -!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}} +!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}, !4} !2 = !{!"llvm.loop.vectorize.enable", i1 true} !11 = distinct !{} +!4 = !{!"llvm.loop.interleave.enable", i1 false} ; ; This method will not be vectorized, as scalar cost is lower than any of vector costs. diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll --- a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll @@ -28,7 +28,7 @@ ; A[i] = A[B[i]]; ; } ; CHECK: remark: source.cpp:19:5: loop not vectorized: cannot identify array bounds -; CHECK: remark: source.cpp:18:8: loop not vectorized (Force=true) +; CHECK: remark: source.cpp:18:8: loop not vectorized (VectorizationForce=true) ; CHECK: warning: source.cpp:18:8: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering ; int foo(); @@ -85,8 +85,8 @@ ; YAML-NEXT: Function: _Z17test_array_boundsPiS_i ; YAML-NEXT: Args: ; YAML-NEXT: - String: loop not vectorized -; YAML-NEXT: - String: ' (Force=' -; YAML-NEXT: - Force: 'true' +; YAML-NEXT: - String: ' (VectorizationForce=' +; YAML-NEXT: - VectorizationForce: 'true' ; YAML-NEXT: - String: ')' ; YAML-NEXT: ... ; YAML-NEXT: --- !Failure diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll --- a/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll +++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll @@ -19,10 +19,10 @@ ; Case 1: Annotated outer loop WITH vector width information must be collected. ; CHECK-LABEL: vector_width -; CHECK: LV: Loop hints: force=enabled width=4 interleave=0 +; CHECK: LV: Loop hints: force_vectorization=enabled width=4 force_interleave=disabled interleave=0 ; CHECK: LV: We can vectorize this outer loop! ; CHECK: LV: Using user VF 4 to build VPlans. -; CHECK-NOT: LV: Loop hints: force=? +; CHECK-NOT: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0 ; CHECK-NOT: LV: Found a loop: inner.body target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -71,7 +71,7 @@ ; Case 2: Annotated outer loop WITHOUT vector width information must be collected. ; CHECK-LABEL: case2 -; CHECK: LV: Loop hints: force=enabled width=0 interleave=0 +; CHECK: LV: Loop hints: force_vectorization=enabled width=0 force_interleave=disabled interleave=0 ; CHECK: LV: We can vectorize this outer loop! ; CHECK: LV: Using VF 1 to build VPlans. @@ -122,7 +122,7 @@ ; CHECK-LABEL: case3 ; CHECK-NOT: LV: Loop hints: force=enabled ; CHECK-NOT: LV: We can vectorize this outer loop! -; CHECK: LV: Loop hints: force=? +; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0 ; CHECK: LV: Found a loop: inner.body define void @case3(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr { @@ -172,7 +172,7 @@ ; CHECK-LABEL: case4 ; CHECK-NOT: LV: Loop hints: force=enabled ; CHECK-NOT: LV: We can vectorize this outer loop! -; CHECK: LV: Loop hints: force=? +; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0 ; CHECK: LV: Found a loop: inner.body define void @case4(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr { @@ -226,11 +226,13 @@ !4 = !{!"omnipotent char", !5, i64 0} !5 = !{!"Simple C/C++ TBAA"} ; Case 1 -!6 = distinct !{!6, !7, !8} +!6 = distinct !{!6, !7, !8, !12} !7 = !{!"llvm.loop.vectorize.width", i32 4} !8 = !{!"llvm.loop.vectorize.enable", i1 true} ; Case 2 -!9 = distinct !{!9, !8} +!9 = distinct !{!9, !8, !12} ; Case 3 !10 = !{!"llvm.loop.interleave.count", i32 2} -!11 = distinct !{!11, !7, !10, !8} +!11 = distinct !{!11, !7, !10, !8, !13} +!12 = !{!"llvm.loop.interleave.enable", i1 false} +!13 = !{!"llvm.loop.interleave.enable", i1 true} diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll --- a/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll +++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll @@ -168,6 +168,7 @@ !3 = !{!"int", !4, i64 0} !4 = !{!"omnipotent char", !5, i64 0} !5 = !{!"Simple C/C++ TBAA"} -!6 = distinct !{!6, !7, !8} +!6 = distinct !{!6, !7, !8, !9} !7 = !{!"llvm.loop.vectorize.width", i32 8} !8 = !{!"llvm.loop.vectorize.enable", i1 true} +!9 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll --- a/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll +++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll @@ -131,6 +131,7 @@ !3 = !{!"int", !4, i64 0} !4 = !{!"omnipotent char", !5, i64 0} !5 = !{!"Simple C/C++ TBAA"} -!6 = distinct !{!6, !7, !8} +!6 = distinct !{!6, !7, !8, !9} !7 = !{!"llvm.loop.vectorize.width", i32 8} !8 = !{!"llvm.loop.vectorize.enable", i1 true} +!9 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/no_switch.ll b/llvm/test/Transforms/LoopVectorize/no_switch.ll --- a/llvm/test/Transforms/LoopVectorize/no_switch.ll +++ b/llvm/test/Transforms/LoopVectorize/no_switch.ll @@ -9,7 +9,7 @@ ; NOANALYSIS: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering ; MOREINFO: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement -; MOREINFO: remark: source.cpp:4:5: loop not vectorized (Force=true, Vector Width=4) +; MOREINFO: remark: source.cpp:4:5: loop not vectorized (VectorizationForce=true, Vector Width=4) ; MOREINFO: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering ; CHECK: _Z11test_switchPii diff --git a/llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll b/llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll --- a/llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll @@ -3,7 +3,7 @@ ; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -pass-remarks-missed='loop-vectorize' -S 2>&1 | FileCheck %s -check-prefix=MOREINFO ; This test is a copy of no_switch.ll, with the "llvm.loop.vectorize.enable" metadata set to false. -; It tests that vectorization is explicitly disabled and no warnings are emitted. +; It tests that vectorization and interleave is explicitly disabled and no warnings are emitted. ; CHECK-NOT: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement ; CHECK-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering @@ -11,7 +11,7 @@ ; NOANALYSIS-NOT: remark: {{.*}} ; NOANALYSIS-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering -; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization is explicitly disabled +; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization and interleave is explicitly disabled ; MOREINFO-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering ; CHECK: _Z11test_switchPii @@ -80,7 +80,7 @@ !9 = !{!"clang version 3.5.0"} !10 = !DILocation(line: 3, column: 8, scope: !11) !11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4) -!12 = !{!12, !13, !13} +!12 = !{!12, !13, !25} !13 = !{!"llvm.loop.vectorize.enable", i1 false} !14 = !DILocation(line: 4, column: 5, scope: !15) !15 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !11) @@ -93,3 +93,4 @@ !22 = !DILocation(line: 7, column: 5, scope: !21) !23 = !DILocation(line: 9, column: 7, scope: !21) !24 = !DILocation(line: 14, column: 1, scope: !4) +!25 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/nounroll.ll b/llvm/test/Transforms/LoopVectorize/nounroll.ll --- a/llvm/test/Transforms/LoopVectorize/nounroll.ll +++ b/llvm/test/Transforms/LoopVectorize/nounroll.ll @@ -4,7 +4,7 @@ target datalayout = "e-m:e-i64:64-n32:64-S128-v256:256:256-v512:512:512" ; CHECK: LV: Checking a loop in 'f1' -; CHECK: LV: Loop hints: force=? width=0 interleave=1 +; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=1 define dso_local void @f1(i32 signext %n, i32* %A) { entry: %cmp1 = icmp sgt i32 %n, 0 @@ -31,7 +31,7 @@ } ; CHECK: LV: Checking a loop in 'f2' -; CHECK: LV: Loop hints: force=? width=0 interleave=4 +; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=4 define dso_local void @f2(i32 signext %n, i32* %A) { entry: %cmp1 = icmp sgt i32 %n, 0 @@ -58,7 +58,7 @@ } ; CHECK: LV: Checking a loop in 'f3' -; CHECK: LV: Loop hints: force=? width=0 interleave=1 +; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=1 define dso_local void @f3(i32 signext %n, i32* %A) { entry: %cmp1 = icmp sgt i32 %n, 0 diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll --- a/llvm/test/Transforms/LoopVectorize/optsize.ll +++ b/llvm/test/Transforms/LoopVectorize/optsize.ll @@ -377,5 +377,6 @@ !12 = !{i32 999000, i64 100, i32 1} !13 = !{i32 999999, i64 1, i32 2} !14 = !{!"function_entry_count", i64 0} -!15 = distinct !{!15, !16} -!16 = !{!"llvm.loop.vectorize.enable", i1 true} +!15 = distinct !{!15, !16, !17} +!16 = !{!"llvm.loop.interleave.enable", i1 true} +!17 = !{!"llvm.loop.vectorize.enable", i1 true} diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll --- a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll +++ b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll @@ -118,6 +118,7 @@ uselistorder label %loop.2.header, { 1, 0 } } -!0 = distinct !{!0, !1, !2} +!0 = distinct !{!0, !1, !2, !3} !1 = !{!"llvm.loop.vectorize.width", i32 4} !2 = !{!"llvm.loop.vectorize.enable", i1 true} +!3 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll --- a/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll +++ b/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll @@ -77,6 +77,7 @@ ret void } -!1 = distinct !{!1, !2, !3} +!1 = distinct !{!1, !2, !3, !4} !2 = !{!"llvm.loop.vectorize.width", i32 4} !3 = !{!"llvm.loop.vectorize.enable", i1 true} +!4 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll --- a/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll +++ b/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll @@ -104,6 +104,7 @@ ret void } -!1 = distinct !{!1, !2, !3} +!1 = distinct !{!1, !2, !3, !4} !2 = !{!"llvm.loop.vectorize.width", i32 4} !3 = !{!"llvm.loop.vectorize.enable", i1 true} +!4 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll b/llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll --- a/llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll +++ b/llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll @@ -2,7 +2,7 @@ ; REQUIRES: asserts ; CHECK: LV: Checking a loop in 'foo' -; CHECK: LV: Loop hints: force=enabled +; CHECK: LV: Loop hints: force_vectorization=enabled width=4 force_interleave=? interleave=2 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" diff --git a/llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll b/llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll @@ -43,5 +43,6 @@ ret void } -!0 = distinct !{!0, !1} +!0 = distinct !{!0, !1, !2} !1 = !{!"llvm.loop.vectorize.enable", i1 true} +!2 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -69,6 +69,7 @@ ret void } -!1 = distinct !{!1, !2, !3} +!1 = distinct !{!1, !2, !3, !4} !2 = !{!"llvm.loop.vectorize.width", i32 4} !3 = !{!"llvm.loop.vectorize.enable", i1 true} +!4 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll b/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll @@ -77,5 +77,6 @@ ret void } -!0 = distinct !{!0, !1} +!0 = distinct !{!0, !1, !2} !1 = !{!"llvm.loop.vectorize.enable", i1 true} +!2 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll @@ -68,5 +68,6 @@ ret void } -!0 = distinct !{!0, !1} +!0 = distinct !{!0, !1, !2} !1 = !{!"llvm.loop.vectorize.enable", i1 true} +!2 = !{!"llvm.loop.interleave.enable", i1 false} diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll @@ -196,5 +196,6 @@ exit: ret void } -!0 = distinct !{!0, !1} +!0 = distinct !{!0, !1, !2} !1 = !{!"llvm.loop.vectorize.enable", i1 true} +!2 = !{!"llvm.loop.interleave.enable", i1 false}