Index: docs/CommandGuide/llvm-profdata.rst =================================================================== --- docs/CommandGuide/llvm-profdata.rst +++ docs/CommandGuide/llvm-profdata.rst @@ -196,6 +196,10 @@ Specify that the input profile is a sample-based profile. +.. option:: -memop-sizes + + Show the profiled sizes of the memory intrinsic calls for shown functions. + EXIT STATUS ----------- Index: include/llvm/ProfileData/InstrProf.h =================================================================== --- include/llvm/ProfileData/InstrProf.h +++ include/llvm/ProfileData/InstrProf.h @@ -79,6 +79,11 @@ return INSTR_PROF_VALUE_PROF_FUNC_STR; } +/// Return the name profile runtime entry point to do value range profiling. +inline StringRef getInstrProfValueRangeProfFuncName() { + return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR; +} + /// Return the name of the section containing function coverage mapping /// data. inline StringRef getInstrProfCoverageSectionName(bool AddSegment) { @@ -646,11 +651,14 @@ private: std::vector IndirectCallSites; + std::vector MemOPSizes; const std::vector & getValueSitesForKind(uint32_t ValueKind) const { switch (ValueKind) { case IPVK_IndirectCallTarget: return IndirectCallSites; + case IPVK_MemOPSize: + return MemOPSizes; default: llvm_unreachable("Unknown value kind!"); } Index: include/llvm/ProfileData/InstrProfData.inc =================================================================== --- include/llvm/ProfileData/InstrProfData.inc +++ include/llvm/ProfileData/InstrProfData.inc @@ -153,7 +153,17 @@ VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ INSTR_PROF_COMMA VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA +#ifndef VALUE_RANGE_PROF VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) +#else /* VALUE_RANGE_PROF */ +VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) +#endif /*VALUE_RANGE_PROF */ #undef VALUE_PROF_FUNC_PARAM #undef INSTR_PROF_COMMA /* VALUE_PROF_FUNC_PARAM end */ @@ -174,13 +184,15 @@ * name hash and the function address. */ VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +/* For memory intrinsic functions size profiling. */ +VALUE_PROF_KIND(IPVK_MemOPSize, 1) /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) -VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize) #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -649,6 +661,9 @@ #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target #define INSTR_PROF_VALUE_PROF_FUNC_STR \ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) +#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range +#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) /* InstrProfile per-function control data alignment. */ #define INSTR_PROF_DATA_ALIGNMENT 8 Index: include/llvm/Transforms/InstrProfiling.h =================================================================== --- include/llvm/Transforms/InstrProfiling.h +++ include/llvm/Transforms/InstrProfiling.h @@ -59,6 +59,14 @@ GlobalVariable *NamesVar; size_t NamesSize; + // The start value of precise value profile range for memory intrinsic sizes. + const int64_t DefaultMemOPSizeRangeStart = 0; + int64_t MemOPSizeRangeStart; + // The end value of precise value profile range for memory intrinsic sizes. + const int64_t DefaultMemOPSizeRangeLast = 8; + int64_t MemOPSizeRangeLast; + int64_t MemOPSizeLargeVal; + bool isMachO() const; /// Get the section name for the counter variables. Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -182,7 +182,7 @@ CHECK_LINE_END(Line); std::pair VD = Line->rsplit(':'); uint64_t TakenCount, Value; - if (VK == IPVK_IndirectCallTarget) { + if (ValueKind == IPVK_IndirectCallTarget) { Symtab->addFuncName(VD.first); Value = IndexedInstrProf::ComputeHash(VD.first); } else { @@ -192,7 +192,7 @@ CurrentValues.push_back({Value, TakenCount}); Line++; } - Record.addValueData(VK, S, CurrentValues.data(), NumValueData, nullptr); + Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData, nullptr); } } return success(); Index: lib/Transforms/Instrumentation/InstrProfiling.cpp =================================================================== --- lib/Transforms/Instrumentation/InstrProfiling.cpp +++ lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -77,6 +77,17 @@ // is usually smaller than 2. cl::init(1.0)); +cl::opt MemOPSizeRange( + "memop-size-range", + cl::desc("Set the range of size in memory intrinsic calls to be profiled " + "precisely, in a format of :"), + cl::init("")); +cl::opt MemOPSizeLarge( + "memop-size-large", + cl::desc("Set large value thresthold in memory intrinsic size profiling. " + "Value of 0 disables the large value profiling."), + cl::init(8192)); + class InstrProfilingLegacyPass : public ModulePass { InstrProfiling InstrProf; @@ -166,6 +177,25 @@ ProfileDataMap.clear(); UsedVars.clear(); + // Parse the value profile options. + MemOPSizeRangeStart = DefaultMemOPSizeRangeStart; + MemOPSizeRangeLast = DefaultMemOPSizeRangeLast; + if (!MemOPSizeRange.empty()) { + auto Pos = MemOPSizeRange.find(":"); + if (Pos != std::string::npos) { + if (Pos > 0) + MemOPSizeRangeStart = std::stoi(MemOPSizeRange.substr(0, Pos)); + if (Pos < MemOPSizeRange.size() - 1) + MemOPSizeRangeLast = std::stoi(MemOPSizeRange.substr(Pos + 1)); + } else + MemOPSizeRangeLast = std::stoi(MemOPSizeRange); + } + assert(MemOPSizeRangeLast >= MemOPSizeRangeStart); + + MemOPSizeLargeVal = MemOPSizeLarge; + if (MemOPSizeLargeVal == 0) + MemOPSizeLargeVal = INT64_MIN; + // We did not know how many value sites there would be inside // the instrumented function. This is counting the number of instrumented // target value sites to enter it as field in the profile data variable. @@ -217,17 +247,34 @@ } static Constant *getOrInsertValueProfilingCall(Module &M, - const TargetLibraryInfo &TLI) { + const TargetLibraryInfo &TLI, + bool IsRange = false) { LLVMContext &Ctx = M.getContext(); auto *ReturnTy = Type::getVoidTy(M.getContext()); - Type *ParamTypes[] = { + + Constant *Res; + if (!IsRange) { + Type *ParamTypes[] = { #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType #include "llvm/ProfileData/InstrProfData.inc" - }; - auto *ValueProfilingCallTy = - FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false); - Constant *Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(), - ValueProfilingCallTy); + }; + auto *ValueProfilingCallTy = + FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false); + Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(), + ValueProfilingCallTy); + } else { + Type *RangeParamTypes[] = { +#define VALUE_RANGE_PROF 1 +#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType +#include "llvm/ProfileData/InstrProfData.inc" +#undef VALUE_RANGE_PROF + }; + auto *ValueRangeProfilingCallTy = + FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false); + Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(), + ValueRangeProfilingCallTy); + } + if (Function *FunRes = dyn_cast(Res)) { if (auto AK = TLI.getExtAttrForI32Param(false)) FunRes->addAttribute(3, AK); @@ -261,11 +308,24 @@ Index += It->second.NumValueSites[Kind]; IRBuilder<> Builder(Ind); - Value *Args[3] = {Ind->getTargetValue(), - Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), - Builder.getInt32(Index)}; - CallInst *Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), - Args); + bool IsRange = (Ind->getValueKind()->getZExtValue() == + llvm::InstrProfValueKind::IPVK_MemOPSize); + CallInst *Call = nullptr; + if (!IsRange) { + Value *Args[3] = {Ind->getTargetValue(), + Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), + Builder.getInt32(Index)}; + Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args); + } else { + Value *Args[6] = {Ind->getTargetValue(), + Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), + Builder.getInt32(Index), + Builder.getInt64(MemOPSizeRangeStart), + Builder.getInt64(MemOPSizeRangeLast), + Builder.getInt64(MemOPSizeLargeVal)}; + Call = + Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args); + } if (auto AK = TLI->getExtAttrForI32Param(false)) Call->addAttribute(3, AK); Ind->replaceAllUsesWith(Call); @@ -477,8 +537,14 @@ size_t TotalNS = 0; for (auto &PD : ProfileDataMap) { - for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - TotalNS += PD.second.NumValueSites[Kind]; + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { + auto I = PD.second.NumValueSites[Kind]; + TotalNS += I; + // For value range profiles, such as IPVK_MemOPSize, we need preallocate + // more value sites. + if (Kind == IPVK_MemOPSize) + TotalNS += I * (MemOPSizeRangeLast + 1); + } } if (!TotalNS) Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -89,6 +89,7 @@ STATISTIC(NumOfPGOInstrument, "Number of edges instrumented."); STATISTIC(NumOfPGOSelectInsts, "Number of select instruction instrumented."); +STATISTIC(NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented."); STATISTIC(NumOfPGOEdge, "Number of edges."); STATISTIC(NumOfPGOBB, "Number of basic-blocks."); STATISTIC(NumOfPGOSplit, "Number of critical edge splits."); @@ -153,6 +154,10 @@ "the name of the function " "whose CFG will be displayed.")); +// Command line option to enable/disable memop intrinsic calls.. +static cl::opt PGOInstrMemOP("pgo-instr-memop", cl::init(true), + cl::Hidden); + // Command line option to turn on CFG dot dump after profile annotation. extern cl::opt PGOViewCounts; @@ -163,7 +168,7 @@ /// select instructions. In \c VM_instrument mode, it inserts code to count /// the number times TrueValue of select is taken. In \c VM_annotate mode, /// it reads the profile data and annotate the select instruction with metadata. -enum VisitMode { VM_counting, VM_instrument, VM_annotate }; +enum VisitMode { VM_counting, VM_instrument, VM_annotate, VM_optimize }; class PGOUseFunc; /// Instruction Visitor class to visit select instructions. @@ -212,6 +217,42 @@ unsigned getNumOfSelectInsts() const { return NSIs; } }; +/// Instruction Visitor class to visit memory intrinsic calls. +struct MemIntrinsicVisitor : public InstVisitor { + Function &F; + unsigned NMemIs = 0; // Number of memIntrinsics instrumented. + VisitMode Mode = VM_counting; // Visiting mode. + unsigned CurCtrId = 0; // Current counter index. + unsigned TotalNumCtrs = 0; // Total number of counters + GlobalVariable *FuncNameVar = nullptr; + uint64_t FuncHash = 0; + PGOUseFunc *UseFunc = nullptr; + + MemIntrinsicVisitor(Function &Func) : F(Func) {} + + void countMemIntrinsics(Function &Func) { + NMemIs = 0; + Mode = VM_counting; + visit(Func); + } + void instrumentMemIntrinsics(Function &Func, unsigned TotalNC, + GlobalVariable *FNV, uint64_t FHash) { + Mode = VM_instrument; + TotalNumCtrs = TotalNC; + FuncHash = FHash; + FuncNameVar = FNV; + visit(Func); + } + + // Visit the IR stream and annotate all mem intrinsic call instructions. + void optimizeMemIntrinsics(Function &Func, PGOUseFunc *UF); + void instrumentOneMemIntrinsic(MemIntrinsic &MI); + void optimizeOneMemIntrinsic(MemIntrinsic &MI); + // Visit \p MI instruction and perform tasks according to visit mode. + void visitMemIntrinsic(MemIntrinsic &SI); + unsigned getNumOfMemIntrinsics() const { return NMemIs; } +}; + class PGOInstrumentationGenLegacyPass : public ModulePass { public: static char ID; @@ -331,6 +372,7 @@ public: std::vector IndirectCallSites; SelectInstVisitor SIVisitor; + MemIntrinsicVisitor MIVisitor; std::string FuncName; GlobalVariable *FuncNameVar; // CFG hash value for this function. @@ -360,12 +402,14 @@ std::unordered_multimap &ComdatMembers, bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr) - : F(Func), ComdatMembers(ComdatMembers), SIVisitor(Func), FunctionHash(0), - MST(F, BPI, BFI) { + : F(Func), ComdatMembers(ComdatMembers), SIVisitor(Func), + MIVisitor(Func), FunctionHash(0), MST(F, BPI, BFI) { // This should be done before CFG hash computation. SIVisitor.countSelects(Func); + MIVisitor.countMemIntrinsics(Func); NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); + NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); IndirectCallSites = findIndirectCallSites(Func); FuncName = getPGOFuncName(F); @@ -582,6 +626,11 @@ Builder.getInt32(NumIndirectCallSites++)}); } NumOfPGOICall += NumIndirectCallSites; + + // Now instrument memop instrinsic calls: + FuncInfo.MIVisitor.instrumentMemIntrinsics(F, NumCounters, + FuncInfo.FuncNameVar, + FuncInfo.FunctionHash); } // This class represents a CFG edge in profile use compilation. @@ -955,7 +1004,7 @@ Weights.push_back(scaleBranchCount(ECI, Scale)); DEBUG(dbgs() << "Weight is: "; - for (const auto &W : Weights) { dbgs() << W << " "; } + for (const auto &W : Weights) { dbgs() << W << " "; } dbgs() << "\n";); TI->setMetadata(llvm::LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); } @@ -1043,11 +1092,50 @@ case VM_annotate: annotateOneSelectInst(SI); return; + default: + break; } llvm_unreachable("Unknown visiting mode"); } +void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) { + Module *M = F.getParent(); + IRBuilder<> Builder(&MI); + Type *Int64Ty = Builder.getInt64Ty(); + Type *I8PtrTy = Builder.getInt8PtrTy(); + Value *Length = MI.getLength(); + assert(!dyn_cast(Length)); + Builder.CreateCall( + Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), + {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + Builder.getInt64(FuncHash), Builder.CreatePtrToInt(Length, Int64Ty), + Builder.getInt32(llvm::InstrProfValueKind::IPVK_MemOPSize), + Builder.getInt32(CurCtrId)}); + ++CurCtrId; +} + +void MemIntrinsicVisitor::visitMemIntrinsic(MemIntrinsic &MI) { + if (!PGOInstrMemOP) + return; + Value *Length = MI.getLength(); + // Not instrument constant length calls. + if (dyn_cast(Length)) + return; + + NMemIs++; + switch (Mode) { + case VM_counting: + return; + case VM_instrument: + instrumentOneMemIntrinsic(MI); + return; + default: + break; + } + llvm_unreachable("Unknown visiting mode"); +} + // Traverse all the indirect callsites and annotate the instructions. void PGOUseFunc::annotateIndirectCallSites() { if (DisableValueProfiling) @@ -1210,6 +1298,7 @@ Func.populateCounters(); Func.setBranchWeights(); Func.annotateIndirectCallSites(); + PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr(); if (FreqAttr == PGOUseFunc::FFA_Cold) ColdFunctions.push_back(&F); Index: test/Instrumentation/InstrProfiling/PR23499.ll =================================================================== --- test/Instrumentation/InstrProfiling/PR23499.ll +++ test/Instrumentation/InstrProfiling/PR23499.ll @@ -15,13 +15,13 @@ ; CHECK: @__profn__Z3barIvEvv = private constant [11 x i8] c"_Z3barIvEvv", align 1 ; CHECK: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat($__profv__Z3barIvEvv), align 8 -; CHECK: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [1 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [1 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat($__profv__Z3barIvEvv), align 8 +; CHECK: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat($__profv__Z3barIvEvv), align 8 ; CHECK: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names" ; COFF: @__profn__Z3barIvEvv = private constant [11 x i8] c"_Z3barIvEvv", align 1 ; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat, align 8 -; COFF: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [1 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [1 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat($__profc__Z3barIvEvv), align 8 +; COFF: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat($__profc__Z3barIvEvv), align 8 declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #1 Index: test/Instrumentation/InstrProfiling/icall.ll =================================================================== --- test/Instrumentation/InstrProfiling/icall.ll +++ test/Instrumentation/InstrProfiling/icall.ll @@ -37,9 +37,9 @@ ; DYN-NOT: @__profvp_foo ; DYN-NOT: @__llvm_prf_vnodes -; STATIC: call void @__llvm_profile_instrument_target(i64 %3, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [1 x i16] }* @__profd_foo to i8*), i32 0) -; STATIC-EXT: call void @__llvm_profile_instrument_target(i64 %3, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [1 x i16] }* @__profd_foo to i8*), i32 zeroext 0) -; STATIC-SEXT: call void @__llvm_profile_instrument_target(i64 %3, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [1 x i16] }* @__profd_foo to i8*), i32 signext 0) +; STATIC: call void @__llvm_profile_instrument_target(i64 %3, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0) +; STATIC-EXT: call void @__llvm_profile_instrument_target(i64 %3, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 zeroext 0) +; STATIC-SEXT: call void @__llvm_profile_instrument_target(i64 %3, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 signext 0) ; STATIC: declare void @__llvm_profile_instrument_target(i64, i8*, i32) ; STATIC-EXT: declare void @__llvm_profile_instrument_target(i64, i8*, i32 zeroext) Index: test/Transforms/PGOProfile/comdat_internal.ll =================================================================== --- test/Transforms/PGOProfile/comdat_internal.ll +++ test/Transforms/PGOProfile/comdat_internal.ll @@ -14,9 +14,9 @@ ; CHECK: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat ; CHECK: @__profn__stdin__foo = private constant [11 x i8] c":foo" ; CHECK: @__profc__stdin__foo.[[FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8 -; CHECK: @__profd__stdin__foo.[[FOO_HASH]] = private global { i64, i64, i64*, i8*, i8*, i32, [1 x i16] } { i64 -5640069336071256030, i64 [[FOO_HASH]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__stdin__foo.[[FOO_HASH]], i32 0, i32 0), i8* null +; CHECK: @__profd__stdin__foo.[[FOO_HASH]] = private global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 -5640069336071256030, i64 [[FOO_HASH]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__stdin__foo.[[FOO_HASH]], i32 0, i32 0), i8* null ; CHECK-NOT: bitcast (i32 ()* @foo to i8*) -; CHECK-SAME: , i8* null, i32 1, [1 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8 +; CHECK-SAME: , i8* null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8 ; CHECK: @__llvm_prf_nm ; CHECK: @llvm.used Index: test/Transforms/PGOProfile/memcpy.ll =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/memcpy.ll @@ -0,0 +1,35 @@ +; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s +; RUN: opt <%s -passes=pgo-instr-gen,instrprof -S | FileCheck %s +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @foo(i8* %dst, i8* %src, i32* %a, i32 %n) { +entry: + br label %for.cond + +for.cond: + %i.0 = phi i32 [ 0, %entry ], [ %add, %for.cond1 ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.cond1, label %for.end6 + +for.cond1: + %j.0 = phi i32 [ %inc, %for.body3 ], [ 0, %for.cond ] + %idx.ext = sext i32 %i.0 to i64 + %add.ptr = getelementptr inbounds i32, i32* %a, i64 %idx.ext + %0 = load i32, i32* %add.ptr, align 4 + %cmp2 = icmp slt i32 %j.0, %0 + %add = add nsw i32 %i.0, 1 + br i1 %cmp2, label %for.body3, label %for.cond + +for.body3: + %conv = sext i32 %add to i64 +; CHECK: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192) + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i32 1, i1 false) + %inc = add nsw i32 %j.0, 1 + br label %for.cond1 + +for.end6: + ret void +} + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) Index: test/tools/llvm-profdata/memop-size-prof.proftext =================================================================== --- /dev/null +++ test/tools/llvm-profdata/memop-size-prof.proftext @@ -0,0 +1,76 @@ +# RUN: llvm-profdata show -memop-sizes -all-functions %s | FileCheck %s --check-prefix=MEMOP --check-prefix=MEMOP_SUM +# RUN: llvm-profdata show -memop-sizes -counts -text -all-functions %s | FileCheck %s --check-prefix=MEMOP_TEXT +# RUN: llvm-profdata merge -o %t.profdata %s +# RUN: llvm-profdata show -memop-sizes -all-functions %t.profdata | FileCheck %s --check-prefix=MEMOP --check-prefix=MEMOP_SUM + +# IR level Instrumentation Flag +:ir +foo +# Func Hash: +35277121310 +# Num Counters: +3 +# Counter Values: +20 +556 +1 +# Num Value Kinds: +1 +# ValueKind = IPVK_MemOPSize: +1 +# NumValueSites: +1 +9 +1:99 +2:88 +3:77 +9:72 +4:66 +5:55 +6:44 +7:33 +8:22 + +#MEMOP: Memory Intrinsics Size: 1 +#MEMOP-NEXT: Memory Instrinsic Size Results: +#MEMOP-NEXT: [ 0, 1, 99 ] +#MEMOP-NEXT: [ 0, 2, 88 ] +#MEMOP-NEXT: [ 0, 3, 77 ] +#MEMOP-NEXT: [ 0, 9, 72 ] +#MEMOP-NEXT: [ 0, 4, 66 ] +#MEMOP-NEXT: [ 0, 5, 55 ] +#MEMOP-NEXT: [ 0, 6, 44 ] +#MEMOP-NEXT: [ 0, 7, 33 ] +#MEMOP-NEXT: [ 0, 8, 22 ] + +#MEMOP_SUM: Total Number of Memory Intrinsic Calls Profiled: 1 +#MEMOP_SUM: Total Number of Memory Intrinsic Calls With Values : 1 +#MEMOP_SUM: Total Number of Memory Intrinsic Calls Profiled Values : 9 +#MEMOP_SUM: MemOPSize Value histogram: +#MEMOP_SUM: Size, SiteCount +#MEMOP_SUM: 1, 0 +#MEMOP_SUM: 2, 0 +#MEMOP_SUM: 3, 0 +#MEMOP_SUM: 4, 0 +#MEMOP_SUM: 5, 0 +#MEMOP_SUM: 6, 0 +#MEMOP_SUM: 7, 0 +#MEMOP_SUM: 8, 0 +#MEMOP_SUM: 9, 1 + +#MEMOP_TEXT: # Num Value Kinds: +#MEMOP_TEXT: 1 +#MEMOP_TEXT: # ValueKind = IPVK_MemOPSize: +#MEMOP_TEXT: 1 +#MEMOP_TEXT: # NumValueSites: +#MEMOP_TEXT: 1 +#MEMOP_TEXT: 9 +#MEMOP_TEXT: 1:99 +#MEMOP_TEXT: 2:88 +#MEMOP_TEXT: 3:77 +#MEMOP_TEXT: 9:72 +#MEMOP_TEXT: 4:66 +#MEMOP_TEXT: 5:55 +#MEMOP_TEXT: 6:44 +#MEMOP_TEXT: 7:33 +#MEMOP_TEXT: 8:22 Index: tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- tools/llvm-profdata/llvm-profdata.cpp +++ tools/llvm-profdata/llvm-profdata.cpp @@ -447,7 +447,7 @@ } static int showInstrProfile(const std::string &Filename, bool ShowCounts, - bool ShowIndirectCallTargets, + bool ShowIndirectCallTargets, bool ShowMemOPSizes, bool ShowDetailedSummary, std::vector DetailedSummaryCutoffs, bool ShowAllFunctions, @@ -469,6 +469,10 @@ uint64_t TotalNumValueSitesWithValueProfile = 0; uint64_t TotalNumValues = 0; std::vector ICHistogram; + uint64_t MemOPSizeTotalNumValueSites = 0; + uint64_t MemOPSizeTotalNumValueSitesWithValueProfile = 0; + uint64_t MemOPSizeTotalNumValues = 0; + std::vector MemOPSizeHistogram; for (const auto &Func : *Reader) { bool Show = ShowAllFunctions || (!ShowFunction.empty() && @@ -502,6 +506,10 @@ OS << " Indirect Call Site Count: " << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; + uint32_t MemOPSize = Func.getNumValueSites(IPVK_MemOPSize); + if (ShowMemOPSizes && MemOPSize > 0) + OS << " Memory Intrinsics Size: " << MemOPSize << "\n"; + if (ShowCounts) { OS << " Block counts: ["; size_t Start = (IsIRInstr ? 0 : 1); @@ -534,6 +542,27 @@ } } } + if (ShowMemOPSizes && MemOPSize > 0) { + uint32_t NS = Func.getNumValueSites(IPVK_MemOPSize); + OS << " Memory Instrinsic Size Results:\n"; + MemOPSizeTotalNumValueSites += NS; + for (size_t I = 0; I < NS; ++I) { + uint32_t NV = Func.getNumValueDataForSite(IPVK_MemOPSize, I); + std::unique_ptr VD = + Func.getValueForSite(IPVK_MemOPSize, I); + MemOPSizeTotalNumValues += NV; + if (NV) { + MemOPSizeTotalNumValueSitesWithValueProfile++; + if (NV > MemOPSizeHistogram.size()) + MemOPSizeHistogram.resize(NV, 0); + MemOPSizeHistogram[NV - 1]++; + } + for (uint32_t V = 0; V < NV; V++) { + OS << "\t[ " << I << ", "; + OS << VD[V].Value << ", " << VD[V].Count << " ]\n"; + } + } + } } } if (Reader->hasError()) @@ -559,6 +588,19 @@ OS << "\t" << I + 1 << ", " << ICHistogram[I] << "\n"; } } + if (ShownFunctions && ShowMemOPSizes) { + OS << "Total Number of Memory Intrinsic Calls Profiled: " + << MemOPSizeTotalNumValueSites << "\n"; + OS << "Total Number of Memory Intrinsic Calls With Values : " + << MemOPSizeTotalNumValueSitesWithValueProfile << "\n"; + OS << "Total Number of Memory Intrinsic Calls Profiled Values : " + << MemOPSizeTotalNumValues << "\n"; + + OS << "MemOPSize Value histogram:\n\tSize, SiteCount\n"; + for (unsigned I = 0; I < MemOPSizeHistogram.size(); I++) { + OS << "\t" << I + 1 << ", " << MemOPSizeHistogram[I] << "\n"; + } + } if (ShowDetailedSummary) { OS << "Detailed summary:\n"; @@ -608,6 +650,10 @@ cl::opt ShowIndirectCallTargets( "ic-targets", cl::init(false), cl::desc("Show indirect call site target values for shown functions")); + cl::opt ShowMemOPSizes( + "memop-sizes", cl::init(false), + cl::desc("Show the profiled sizes of the memory intrinsic calls " + "for shown functions")); cl::opt ShowDetailedSummary("detailed-summary", cl::init(false), cl::desc("Show detailed profile summary")); cl::list DetailedSummaryCutoffs( @@ -646,8 +692,9 @@ DetailedSummaryCutoffs.end()); if (ProfileKind == instr) return showInstrProfile(Filename, ShowCounts, ShowIndirectCallTargets, - ShowDetailedSummary, DetailedSummaryCutoffs, - ShowAllFunctions, ShowFunction, TextFormat, OS); + ShowMemOPSizes, ShowDetailedSummary, + DetailedSummaryCutoffs, ShowAllFunctions, + ShowFunction, TextFormat, OS); else return showSampleProfile(Filename, ShowCounts, ShowAllFunctions, ShowFunction, OS);