Index: include/llvm/ProfileData/InstrProf.h =================================================================== --- include/llvm/ProfileData/InstrProf.h +++ include/llvm/ProfileData/InstrProf.h @@ -269,6 +269,7 @@ /// Get the value profile data for value site \p SiteIdx from \p InstrProfR /// and annotate the instruction \p Inst with the value profile meta data. /// Annotate up to \p MaxMDCount (default 3) number of records per value site. +/// 0 value of \p MaxMDCount indicates to annotate all the available records. void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, Index: lib/ProfileData/InstrProf.cpp =================================================================== --- lib/ProfileData/InstrProf.cpp +++ lib/ProfileData/InstrProf.cpp @@ -788,12 +788,13 @@ // Value Profile Data uint32_t MDCount = MaxMDCount; + bool CheckMaxCount = MDCount; for (auto &VD : VDs) { Vals.push_back(MDHelper.createConstant( ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value))); Vals.push_back(MDHelper.createConstant( ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count))); - if (--MDCount == 0) + if (CheckMaxCount && --MDCount == 0) break; } Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -247,6 +247,7 @@ GlobalVariable *FuncNameVar = nullptr; uint64_t FuncHash = 0; PGOUseFunc *UseFunc = nullptr; + std::vector Candidates; MemIntrinsicVisitor(Function &Func) : F(Func) {} @@ -255,6 +256,7 @@ Mode = VM_counting; visit(Func); } + void instrumentMemIntrinsics(Function &Func, unsigned TotalNC, GlobalVariable *FNV, uint64_t FHash) { Mode = VM_instrument; @@ -264,6 +266,13 @@ visit(Func); } + std::vector findMemIntrinsics(Function &Func) { + Candidates.clear(); + Mode = VM_annotate; + visit(Func); + return Candidates; + } + // Visit the IR stream and annotate all mem intrinsic call instructions. void instrumentOneMemIntrinsic(MemIntrinsic &MI); // Visit \p MI instruction and perform tasks according to visit mode. @@ -429,6 +438,7 @@ NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); ValueSites[IPVK_IndirectCallTarget] = findIndirectCallSites(Func); + ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func); FuncName = getPGOFuncName(F); computeCFGHash(); @@ -1150,7 +1160,8 @@ instrumentOneMemIntrinsic(MI); return; case VM_annotate: - break; + Candidates.push_back(&MI); + return; } llvm_unreachable("Unknown visiting mode"); } @@ -1188,7 +1199,7 @@ << NumValueSites << "\n"); annotateValueSite(*M, *I, ProfileRecord, static_cast(Kind), ValueSiteIndex, - MaxNumAnnotations); + Kind == IPVK_IndirectCallTarget ? MaxNumAnnotations : 0); ValueSiteIndex++; } } Index: test/Transforms/PGOProfile/Inputs/memop_size.proftext =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/Inputs/memop_size.proftext @@ -0,0 +1,27 @@ +# IR level Instrumentation Flag +:ir +foo +# Func Hash: +53929068288 +# Num Counters: +3 +# Counter Values: +556 +20 +1 +# Num Value Kinds: +1 +# ValueKind = IPVK_MemOPSize: +1 +# NumValueSites: +1 +9 +1:99 +2:88 +3:77 +9:72 +4:66 +5:55 +6:44 +7:33 +8:22 Index: test/Transforms/PGOProfile/memop_size.ll =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/memop_size.ll @@ -0,0 +1,56 @@ +; RUN: llvm-profdata merge %S/Inputs/memop_size.proftext -o %t.profdata +; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=MEMOP_ANNOTATION +; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=MEMOP_ANNOTATION + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @foo(i8* %dst, i8* %src, i32* %a, i32 %n) { +entry: + br label %for.cond + +for.cond: + %i.0 = phi i32 [ 0, %entry ], [ %inc5, %for.inc4 ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end6 + +for.body: + br label %for.cond1 + +for.cond1: + %j.0 = phi i32 [ 0, %for.body ], [ %inc, %for.inc ] + %idx.ext = sext i32 %i.0 to i64 + %add.ptr = getelementptr inbounds i32, i32* %a, i64 %idx.ext + %0 = load i32, i32* %add.ptr, align 4 + %cmp2 = icmp slt i32 %j.0, %0 + br i1 %cmp2, label %for.body3, label %for.end + +for.body3: + %add = add nsw i32 %i.0, 1 + %conv = sext i32 %add to i64 + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i32 1, i1 false) +; MEMOP_ANNOTATION: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i32 1, i1 false) +; MEMOP_ANNOTATION-SAME: !prof ![[MEMOP_VALUESITE:[0-9]+]] +; MEMOP_ANNOTATION: ![[MEMOP_VALUESITE]] = !{!"VP", i32 1, i64 556, i64 1, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22} + br label %for.inc + +for.inc: + %inc = add nsw i32 %j.0, 1 + br label %for.cond1 + +for.end: + br label %for.inc4 + +for.inc4: + %inc5 = add nsw i32 %i.0, 1 + br label %for.cond + +for.end6: + ret void +} + +declare void @llvm.lifetime.start(i64, i8* nocapture) + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) + +declare void @llvm.lifetime.end(i64, i8* nocapture)