diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -115,6 +115,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" @@ -502,6 +503,9 @@ /// Fix the non-induction PHIs in the OrigPHIsToFix vector. void fixNonInductionPHIs(void); + /// Update profile info of the vector and prolog/epilog loops. + void fixProfileInfo(); + protected: friend class LoopVectorizationPlanner; @@ -3460,6 +3464,8 @@ // Remove redundant induction instructions. cse(LoopVectorBody); + + fixProfileInfo(); } void InnerLoopVectorizer::fixCrossIterationPHIs() { @@ -3969,6 +3975,81 @@ } } +// Update profile info since expected TC of vectorized loop is less by VFxUF +// than original TC. At the same time original loop becomes prolog/epilog loop +// and has less than VFxUF iterations. +// Note: There are cases like foldTailByMasking() and requiresScalarEpiloque() +// for which computations below don't give precise result but that should be OK +// since profile is not precise anyway. +void InnerLoopVectorizer::fixProfileInfo() { + uint64_t OrigTakenWeight = 0; + uint64_t OrigExitWeight = 0; + Loop *OrigLoop = LI->getLoopFor(LoopScalarBody); + auto *OrigBackBranchI = OrigLoop->getLoopLatch()->getTerminator(); + + if (!OrigBackBranchI->extractProfMetadata(OrigTakenWeight, + OrigExitWeight)) + return; + + Loop *VecLoop = LI->getLoopFor(LoopVectorBody); + MDBuilder MDB(OrigBackBranchI->getContext()); + auto *VectorBackBranchI = VecLoop->getLoopLatch()->getTerminator(); + bool IsTrueBackEdgeOrigLoop = + OrigLoop->contains(*succ_begin(OrigLoop->getLoopLatch())); + bool IsTrueBackEdgeVecLoop = + VecLoop->contains(*succ_begin(VecLoop->getLoopLatch())); + + if (!IsTrueBackEdgeOrigLoop) + std::swap(OrigTakenWeight, OrigExitWeight); + + if (OrigExitWeight == 0) + return; + + // Calculate number of iterations in original scalar loop which is equal to + // (number of times header block was executed) / + // (number of times header was reached from pre-header) == + // (OrigTakenWeight + OrigExitWeight) / OrigExitWeight == + // (OrigTakenWeight / OrigExitWeight + 1) + // Note: Uses of OrigIterCount below should not be simplified as it will + // produce a different value. In other words: (A mod N) * B != (A*B) mod N + const uint64_t OrigAverageTripCount = OrigTakenWeight / OrigExitWeight + 1; + // Calculate number of iterations in vector loop. + uint64_t VecAverageTripCount = OrigAverageTripCount / (VF * UF); + // Calculate number of iterations for prolog/epilog loop. + uint64_t PEIterCount = OrigAverageTripCount % (VF * UF); + + // Calculate taken and fall through counts for vector loop. + uint64_t VecTakenCount = 0; + uint64_t VecFallThrough = 0; + if (VecAverageTripCount > 0) { + VecTakenCount = (VecAverageTripCount - 1) * OrigExitWeight; + VecFallThrough = OrigExitWeight; + } + + // Now calculate counters for prolog/epilog loop. + uint64_t PETakenCount = 0; + uint64_t PEFallThroughCount = 0; + if (PEIterCount > 0) { + PETakenCount = (PEIterCount - 1) * OrigExitWeight; + PEFallThroughCount = OrigExitWeight; + } + + // Make a swap if back edge is taken when condition "false". + if (!IsTrueBackEdgeVecLoop) + std::swap(VecTakenCount, VecFallThrough); + // Set new profile metadata. + VectorBackBranchI->setMetadata( + LLVMContext::MD_prof, + MDB.createBranchWeights(VecTakenCount, VecFallThrough)); + // Make a swap if back edge is taken when condition "false". + if (!IsTrueBackEdgeOrigLoop) + std::swap(PETakenCount, PEFallThroughCount); + // Set new profile metadata. + OrigBackBranchI->setMetadata( + LLVMContext::MD_prof, + MDB.createBranchWeights(PETakenCount, PEFallThroughCount)); +} + void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, unsigned UF, unsigned VF) { PHINode *P = cast(PN); diff --git a/llvm/test/Transforms/LoopVectorize/check-prof-info.ll b/llvm/test/Transforms/LoopVectorize/check-prof-info.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/check-prof-info.ll @@ -0,0 +1,97 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -passes="print,loop-vectorize" -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s +; RUN: opt -passes="print,loop-vectorize" -force-vector-width=4 -force-vector-interleave=4 -S < %s | FileCheck %s -check-prefix=CHECK-MASKED + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@a = dso_local global [1024 x i32] zeroinitializer, align 16 +@b = dso_local global [1024 x i32] zeroinitializer, align 16 + +; Check correctness of profile info for vectorization without epilog. +; Function Attrs: nofree norecurse nounwind uwtable +define dso_local void @_Z3foov() local_unnamed_addr #0 { +; CHECK-LABEL: @_Z3foov( +; CHECK: [[VECTOR_BODY:vector\.body]]: +; CHECK: br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_255:\!.*]], +; CHECK: [[FOR_BODY:for\.body]]: +; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP0_0:\!.*]], +; CHECK-MASKED: [[VECTOR_BODY:vector\.body]]: +; CHECK-MASKED: br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_63:\!.*]], +; CHECK-MASKED: [[FOR_BODY:for\.body]]: +; CHECK-MASKED: br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP0_0:\!.*]], +; +entry: + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret void + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @b, i64 0, i64 %indvars.iv + %0 = load i32, i32* %arrayidx, align 4, !tbaa !2 + %1 = trunc i64 %indvars.iv to i32 + %mul = mul nsw i32 %0, %1 + %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %indvars.iv + %2 = load i32, i32* %arrayidx2, align 4, !tbaa !2 + %add = add nsw i32 %2, %mul + store i32 %add, i32* %arrayidx2, align 4, !tbaa !2 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1024 + br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !6 +} + +; Check correctness of profile info for vectorization with epilog. +; Function Attrs: nofree norecurse nounwind uwtable +define dso_local void @_Z3foo2v() local_unnamed_addr #0 { +; CHECK-LABEL: @_Z3foo2v( +; CHECK: [[VECTOR_BODY:vector\.body]]: +; CHECK: br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_255:\!.*]], +; CHECK: [[FOR_BODY:for\.body]]: +; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP1_2:\!.*]], +; CHECK-MASKED: [[VECTOR_BODY:vector\.body]]: +; CHECK-MASKED: br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_63:\!.*]], +; CHECK-MASKED: [[FOR_BODY:for\.body]]: +; CHECK-MASKED: br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP1_2:\!.*]], +; +entry: + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret void + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @b, i64 0, i64 %indvars.iv + %0 = load i32, i32* %arrayidx, align 4, !tbaa !2 + %1 = trunc i64 %indvars.iv to i32 + %mul = mul nsw i32 %0, %1 + %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %indvars.iv + %2 = load i32, i32* %arrayidx2, align 4, !tbaa !2 + %add = add nsw i32 %2, %mul + store i32 %add, i32* %arrayidx2, align 4, !tbaa !2 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1027 + br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !7 +} + +attributes #0 = { "use-soft-float"="false" } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +; CHECK: [[LP1_255]] = !{!"branch_weights", i32 1, i32 255} +; CHECK: [[LP0_0]] = !{!"branch_weights", i32 0, i32 0} +; CHECK-MASKED: [[LP1_63]] = !{!"branch_weights", i32 1, i32 63} +; CHECK-MASKED: [[LP0_0]] = !{!"branch_weights", i32 0, i32 0} +; CHECK: [[LP1_2]] = !{!"branch_weights", i32 1, i32 2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project c292b5b5e059e6ce3e6449e6827ef7e1037c21c4)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"int", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C++ TBAA"} +!6 = !{!"branch_weights", i32 1, i32 1023} +!7 = !{!"branch_weights", i32 1, i32 1026} diff --git a/llvm/test/Transforms/LoopVectorize/hot_short_tc_loop.ll b/llvm/test/Transforms/LoopVectorize/hot_short_tc_loop.ll --- a/llvm/test/Transforms/LoopVectorize/hot_short_tc_loop.ll +++ b/llvm/test/Transforms/LoopVectorize/hot_short_tc_loop.ll @@ -13,12 +13,16 @@ ; Function Attrs: uwtable define dso_local void @_Z3fooi(i32 %M) local_unnamed_addr #0 !prof !11 { +; CHECK-LABEL: @_Z3fooi( +; CHECK: [[VECTOR_BODY:vector\.body]]: ; CHECK: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP15:%.*]] ; CHECK: [[TMP18:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD]], [[VEC_IND6:%.*]] ; CHECK: [[WIDE_LOAD10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP23:%.*]] ; CHECK: [[TMP26:%.*]] = add nsw <4 x i32> [[WIDE_LOAD10]], [[TMP18]] ; CHECK: store <4 x i32> [[TMP26]], <4 x i32>* [[TMP28:%.*]] -; +; CHECK: br i1 [[TMP30:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP12:\!.*]], +; CHECK: [[SCAL_BODY:for\.body4\.us]]: +; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]], label %[[SCAL_BODY]], !prof [[LP15:\!.*]], entry: %a = alloca [5 x i32], align 16 %b = alloca [5 x i32], align 16 @@ -74,6 +78,7 @@ ; Function Attrs: uwtable define dso_local void @_Z3fooi2() local_unnamed_addr #0 !prof !11 { +; CHECK-LABEL: @_Z3fooi2( ; CHECK: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP15:%.*]] ; CHECK: [[TMP18:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD]], [[VEC_IND6:%.*]] ; CHECK: [[WIDE_LOAD10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP23:%.*]] @@ -118,11 +123,14 @@ ; Function Attrs: uwtable define dso_local void @_Z3fooi3(i32 %M) local_unnamed_addr #0 !prof !11 { +; CHECK-LABEL: @_Z3fooi3( +; CHECK: [[SCAL_BODY:for\.body4\.us]]: ; CHECK: [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_US:%.*]] ; CHECK: [[MUL_US:%.*]] = mul nsw i32 [[TMP2]], [[TMP3:%.*]] ; CHECK: [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX6_US:%.*]] ; CHECK: [[ADD_US:%.*]] = add nsw i32 [[TMP4]], [[MUL_US]] ; CHECK: store i32 [[ADD_US]], i32* [[ARRAYIDX6_US]] +; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]], label %[[SCAL_BODY]], !prof [[LP26:\!.*]] ; entry: %a = alloca [5 x i32], align 16 @@ -188,6 +196,10 @@ !llvm.module.flags = !{!0} !llvm.ident = !{!1} +; CHECK: [[LP12]] = !{!"branch_weights", i32 999, i32 0} +; CHECK: [[LP15]] = !{!"branch_weights", i32 999, i32 999} +; CHECK: [[LP26]] = !{!"branch_weights", i32 9, i32 45} + !0 = !{i32 1, !"wchar_size", i32 4} !1 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project f379dd57b978c4e1483d721f422c79e3c0c5ccdc)"} !2 = !{!3, !3, i64 0} diff --git a/llvm/test/Transforms/LoopVectorize/tripcount.ll b/llvm/test/Transforms/LoopVectorize/tripcount.ll --- a/llvm/test/Transforms/LoopVectorize/tripcount.ll +++ b/llvm/test/Transforms/LoopVectorize/tripcount.ll @@ -61,8 +61,10 @@ ; but has a high trip count per invocation. Vectorize it. ; CHECK-LABEL: @foo_low_trip_count3( -; CHECK: vector.body: - +; CHECK: [[VECTOR_BODY:vector\.body]]: +; CHECK: br i1 [[TMP9:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP3:\!.*]], +; CHECK: [[FOR_BODY:for\.body]]: +; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP6:\!.*]], entry: br i1 %cond, label %for.preheader, label %for.end, !prof !2 @@ -205,6 +207,9 @@ ret i32 0 } +; CHECK: [[LP3]] = !{!"branch_weights", i32 10, i32 2490} +; CHECK: [[LP6]] = !{!"branch_weights", i32 10, i32 0} + !0 = !{!"function_entry_count", i64 100} !1 = !{!"branch_weights", i32 100, i32 0} !2 = !{!"branch_weights", i32 10, i32 90}