diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -94,10 +94,17 @@ llvm::PassPluginLibraryInfo get##Ext##PluginInfo(); #include "llvm/Support/Extension.def" +namespace llvm { +extern cl::opt DebugInfoCorrelate; +} + namespace { // Default filename used for profile generation. -static constexpr StringLiteral DefaultProfileGenName = "default_%m.profraw"; +Twine getDefaultProfileGenName() { + const Twine Extension = DebugInfoCorrelate ? "proflite" : "profraw"; + return "default_%m." + Extension; +} class EmitAssemblyHelper { DiagnosticsEngine &Diags; @@ -886,7 +893,7 @@ if (!CodeGenOpts.InstrProfileOutput.empty()) PMBuilder.PGOInstrGen = CodeGenOpts.InstrProfileOutput; else - PMBuilder.PGOInstrGen = std::string(DefaultProfileGenName); + PMBuilder.PGOInstrGen = getDefaultProfileGenName().str(); } if (CodeGenOpts.hasProfileIRUse()) { PMBuilder.PGOInstrUse = CodeGenOpts.ProfileInstrumentUsePath; @@ -1229,7 +1236,7 @@ if (CodeGenOpts.hasProfileIRInstr()) // -fprofile-generate. PGOOpt = PGOOptions(CodeGenOpts.InstrProfileOutput.empty() - ? std::string(DefaultProfileGenName) + ? getDefaultProfileGenName().str() : CodeGenOpts.InstrProfileOutput, "", "", PGOOptions::IRInstr, PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling); @@ -1267,13 +1274,13 @@ "Cannot run CSProfileGen pass with ProfileGen or SampleUse " " pass"); PGOOpt->CSProfileGenFile = CodeGenOpts.InstrProfileOutput.empty() - ? std::string(DefaultProfileGenName) + ? getDefaultProfileGenName().str() : CodeGenOpts.InstrProfileOutput; PGOOpt->CSAction = PGOOptions::CSIRInstr; } else PGOOpt = PGOOptions("", CodeGenOpts.InstrProfileOutput.empty() - ? std::string(DefaultProfileGenName) + ? getDefaultProfileGenName().str() : CodeGenOpts.InstrProfileOutput, "", PGOOptions::NoAction, PGOOptions::CSIRInstr, CodeGenOpts.DebugInfoForProfiling); diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -653,15 +653,17 @@ /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 - * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton + * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. * 1 in bit 57 indicates there are context-sensitive records in the profile. + * The 59th bit indicates whether to use debug info to correlate profiles. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) +#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -38,7 +38,7 @@ } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) { - return __llvm_profile_raw_version; + return INSTR_PROF_RAW_VERSION_VAR; } COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -95,6 +95,11 @@ COMPILER_RT_VISIBILITY int __llvm_profile_merge_from_buffer(const char *ProfileData, uint64_t ProfileSize) { + int DebugInfoCorrelate = + (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL; + if (DebugInfoCorrelate) + return 1; + __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; uint64_t *SrcCountersStart; diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -259,6 +259,8 @@ const uint64_t *CountersBegin, const uint64_t *CountersEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { + int DebugInfoCorrelate = + (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL; /* Calculate size of sections. */ const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); @@ -268,7 +270,7 @@ /* Create the header. */ __llvm_profile_header Header; - if (!DataSize) + if (!DataSize && (!DebugInfoCorrelate || !CountersSize)) return 0; /* Determine how much padding is needed before/after the counters and after @@ -294,6 +296,15 @@ if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) return -1; + if (DebugInfoCorrelate) { + ProfDataIOVec IOVecData[] = { + {CountersBegin, sizeof(uint64_t), CountersSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, + }; + return Writer->Write(Writer, IOVecData, + sizeof(IOVecData) / sizeof(*IOVecData)); + } + /* Write the binary id lengths and data. */ if (__llvm_write_binary_ids(Writer) == -1) return -1; diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -1149,7 +1149,8 @@ // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime // aware this is an ir_level profile so it can set the version flag. GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS, - bool InstrEntryBBEnabled); + bool InstrEntryBBEnabled, + bool DebugInfoCorrelate); // Create the variable for the profile file name. void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput); diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -653,15 +653,17 @@ /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 - * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton + * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. * 1 in bit 57 indicates there are context-sensitive records in the profile. + * The 59th bit indicates whether to use debug info to correlate profiles. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) +#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp --- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp @@ -842,13 +842,17 @@ for (const Metadata *Annotation : Annotations->operands()) { const MDNode *MD = cast(Annotation); const MDString *Name = cast(MD->getOperand(0)); - - // Currently, only MDString is supported with btf_decl_tag attribute. - const MDString *Value = cast(MD->getOperand(1)); + const auto &Value = MD->getOperand(1); DIE &AnnotationDie = createAndAddDIE(dwarf::DW_TAG_LLVM_annotation, Buffer); addString(AnnotationDie, dwarf::DW_AT_name, Name->getString()); - addString(AnnotationDie, dwarf::DW_AT_const_value, Value->getString()); + if (const auto *Data = dyn_cast(Value)) + addString(AnnotationDie, dwarf::DW_AT_const_value, Data->getString()); + else if (const auto *Data = dyn_cast(Value)) + addConstantValue(AnnotationDie, Data->getValue()->getUniqueInteger(), + /*Unsigned=*/true); + else + assert(false && "Unsupported annotation value type"); } } diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1175,7 +1175,8 @@ // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime // aware this is an ir_level profile so it can set the version flag. GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS, - bool InstrEntryBBEnabled) { + bool InstrEntryBBEnabled, + bool DebugInfoCorrelate) { const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); Type *IntTy64 = Type::getInt64Ty(M.getContext()); uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); @@ -1183,6 +1184,8 @@ ProfileVersion |= VARIANT_MASK_CSIR_PROF; if (InstrEntryBBEnabled) ProfileVersion |= VARIANT_MASK_INSTR_ENTRY; + if (DebugInfoCorrelate) + ProfileVersion |= VARIANT_MASK_DBG_CORRELATE; auto IRLevelVersionVariable = new GlobalVariable( M, IntTy64, true, GlobalValue::WeakAnyLinkage, Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName); diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -32,6 +32,7 @@ #include using namespace llvm; +extern cl::opt DebugInfoCorrelate; // A struct to define how the data stream should be patched. For Indexed // profiling, only uint64_t data type is needed. diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -26,6 +26,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DIBuilder.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" @@ -57,6 +58,13 @@ #define DEBUG_TYPE "instrprof" +namespace llvm { +cl::opt + DebugInfoCorrelate("debug-info-correlate", cl::ZeroOrMore, + cl::desc("Use debug info to correlate profiles."), + cl::init(false)); +} // namespace llvm + namespace { cl::opt DoHashBasedCounterSplit( @@ -641,6 +649,12 @@ } void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { + // TODO: Value profiling heavily depends on the data section which is omitted + // in lightweight mode. We need to move the value profile pointer to the + // Counter struct to get this working. + assert( + !DebugInfoCorrelate && + "Value profiling is not yet supported with lightweight instrumentation"); GlobalVariable *Name = Ind->getName(); auto It = ProfileDataMap.find(Name); assert(It != ProfileDataMap.end() && It->second.DataVar && @@ -916,6 +930,36 @@ MaybeSetComdat(CounterPtr); CounterPtr->setLinkage(Linkage); PD.RegionCounters = CounterPtr; + if (DebugInfoCorrelate) { + if (auto *SP = Fn->getSubprogram()) { + DIBuilder DB(*M, true, SP->getUnit()); + Metadata *FunctionNameAnnotation[] = { + MDString::get(Ctx, "Function Name"), + MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)), + }; + Metadata *CFGHashAnnotation[] = { + MDString::get(Ctx, "CFG Hash"), + ConstantAsMetadata::get(Inc->getHash()), + }; + Metadata *NumCountersAnnotation[] = { + MDString::get(Ctx, "Num Counters"), + ConstantAsMetadata::get(Inc->getNumCounters()), + }; + auto Annotations = DB.getOrCreateArray({ + MDNode::get(Ctx, FunctionNameAnnotation), + MDNode::get(Ctx, CFGHashAnnotation), + MDNode::get(Ctx, NumCountersAnnotation), + }); + auto *DICounter = DB.createGlobalVariableExpression( + SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(), + /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"), + CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr, + /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0, + Annotations); + CounterPtr->addDebugInfo(DICounter); + DB.finalize(); + } + } auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); // Allocate statically the array of pointers to value profile nodes for @@ -939,6 +983,9 @@ ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); } + if (DebugInfoCorrelate) + return PD.RegionCounters; + // Create data variable. auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); auto *Int16Ty = Type::getInt16Ty(Ctx); diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -291,6 +291,8 @@ // Command line option to specify the name of the function for CFG dump // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; + +extern cl::opt DebugInfoCorrelate; } // namespace llvm static cl::opt @@ -467,8 +469,9 @@ createProfileFileNameVar(M, InstrProfileOutput); // The variable in a comdat may be discarded by LTO. Ensure the // declaration will be retained. - appendToCompilerUsed( - M, createIRLevelProfileFlagVar(M, /*IsCS=*/true, PGOInstrumentEntry)); + appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true, + PGOInstrumentEntry, + DebugInfoCorrelate)); return false; } std::string InstrProfileOutput; @@ -1616,7 +1619,8 @@ // For the context-sensitve instrumentation, we should have a separated pass // (before LTO/ThinLTO linking) to create these variables. if (!IsCS) - createIRLevelProfileFlagVar(M, /*IsCS=*/false, PGOInstrumentEntry); + createIRLevelProfileFlagVar(M, /*IsCS=*/false, PGOInstrumentEntry, + DebugInfoCorrelate); std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers); @@ -1638,8 +1642,9 @@ createProfileFileNameVar(M, CSInstrName); // The variable in a comdat may be discarded by LTO. Ensure the declaration // will be retained. - appendToCompilerUsed( - M, createIRLevelProfileFlagVar(M, /*IsCS=*/true, PGOInstrumentEntry)); + appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true, + PGOInstrumentEntry, + DebugInfoCorrelate)); return PreservedAnalyses::all(); } diff --git a/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/debug-info-correlate.ll @@ -0,0 +1,74 @@ +; RUN: opt < %s -instrprof -debug-info-correlate -S > %t.ll +; RUN: llc < %t.ll --filetype=asm > %t.s +; RUN: llc < %t.ll --filetype=obj > %t.o +; RUN: FileCheck < %t.ll %s +; RUN: llvm-dwarfdump %t.o | FileCheck %s --implicit-check-not "{{DW_TAG|NULL}}" --check-prefix CHECK-DWARF +; RUN: FileCheck < %t.s %s --check-prefix CHECK-ASM + +target triple = "aarch64-unknown-linux-gnu" + +@__profn_foo = private constant [3 x i8] c"foo" +; CHECK: @__profc_foo = +; CHECK-SAME: !dbg ![[EXPR:[0-9]+]] + +; CHECK: ![[EXPR]] = !DIGlobalVariableExpression(var: ![[GLOBAL:[0-9]+]] +; CHECK: ![[GLOBAL]] = {{.*}} !DIGlobalVariable(name: "__profc_foo" +; CHECK-SAME: scope: ![[SCOPE:[0-9]+]] +; CHECK-SAME: annotations: ![[ANNOTATIONS:[0-9]+]] +; CHECK: ![[SCOPE]] = {{.*}} !DISubprogram(name: "foo" +; CHECK: ![[ANNOTATIONS]] = !{![[NAME:[0-9]+]], ![[HASH:[0-9]+]], ![[COUNTERS:[0-9]+]]} +; CHECK: ![[NAME]] = !{!"Function Name", !"foo"} +; CHECK: ![[HASH]] = !{!"CFG Hash", i64 12345678} +; CHECK: ![[COUNTERS]] = !{!"Num Counters", i32 2} + +define void @_Z3foov() !dbg !12 { + call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12345678, i32 2, i32 0) + ret void +} + +declare void @llvm.instrprof.increment(i8*, i64, i32, i32) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9, !10} +!llvm.ident = !{!11} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "debug-info-correlate.cpp", directory: "") +!2 = !{i32 7, !"Dwarf Version", i32 4} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !{i32 1, !"wchar_size", i32 4} +!5 = !{i32 1, !"branch-target-enforcement", i32 0} +!6 = !{i32 1, !"sign-return-address", i32 0} +!7 = !{i32 1, !"sign-return-address-all", i32 0} +!8 = !{i32 1, !"sign-return-address-with-bkey", i32 0} +!9 = !{i32 7, !"uwtable", i32 1} +!10 = !{i32 7, !"frame-pointer", i32 1} +!11 = !{!"clang version 14.0.0"} +!12 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !13, file: !13, line: 1, type: !14, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !16) +!13 = !DIFile(filename: "debug-info-correlate.cpp", directory: "") +!14 = !DISubroutineType(types: !15) +!15 = !{null} +!16 = !{} + +; CHECK-DWARF: DW_TAG_compile_unit +; CHECK-DWARF: DW_TAG_subprogram +; CHECK-DWARF: DW_AT_name ("foo") +; CHECK-DWARF: DW_TAG_variable +; CHECK-DWARF: DW_AT_name ("__profc_foo") +; CHECK-DWARF: DW_AT_type ({{.*}} "Profile Data Type") +; CHECK-DWARF: DW_TAG_LLVM_annotation +; CHECK-DWARF: DW_AT_name ("Function Name") +; CHECK-DWARF: DW_AT_const_value ("foo") +; CHECK-DWARF: DW_TAG_LLVM_annotation +; CHECK-DWARF: DW_AT_name ("CFG Hash") +; CHECK-DWARF: DW_AT_const_value (12345678) +; CHECK-DWARF: DW_TAG_LLVM_annotation +; CHECK-DWARF: DW_AT_name ("Num Counters") +; CHECK-DWARF: DW_AT_const_value (2) +; CHECK-DWARF: NULL +; CHECK-DWARF: NULL +; CHECK-DWARF: DW_TAG_unspecified_type +; CHECK-DWARF: NULL + +; CHECK-ASM-NOT: .section __llvm_prf_data +; CHECK-ASM-NOT: .section __llvm_prf_names