diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -15,6 +15,8 @@ #ifndef LLVM_CODEGEN_ASMPRINTER_H #define LLVM_CODEGEN_ASMPRINTER_H +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/BinaryFormat/Dwarf.h" @@ -189,6 +191,9 @@ /// Output stream for the stack usage file (i.e., .su file). std::unique_ptr StackUsageStream; + /// List of symbols to be inserted into PC sections. + DenseMap> PCSectionsSymbols; + static char ID; protected: @@ -413,6 +418,12 @@ void emitRemarksSection(remarks::RemarkStreamer &RS); + /// Emits a label as reference for PC sections. + void emitPCSectionsLabel(const MachineFunction &MF, const MDNode &MD); + + /// Emits the PC sections collected from instructions. + void emitPCSections(const MachineFunction &MF); + /// Get the CFISection type for a function. CFISection getFunctionCFISectionType(const Function &F) const; diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -366,6 +366,8 @@ MCSection *getPseudoProbeDescSection(StringRef FuncName) const; + MCSection *getPCSection(StringRef Name, const MCSection *TextSec) const; + // ELF specific sections. MCSection *getDataRelROSection() const { return DataRelROSection; } const MCSection *getMergeableConst4Section() const { diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1454,9 +1454,87 @@ *StackUsageStream << "static\n"; } -static bool needFuncLabelsForEHOrDebugInfo(const MachineFunction &MF) { +void AsmPrinter::emitPCSectionsLabel(const MachineFunction &MF, + const MDNode &MD) { + MCSymbol *S = MF.getContext().createTempSymbol("pcsection"); + OutStreamer->emitLabel(S); + PCSectionsSymbols[&MD].emplace_back(S); +} + +void AsmPrinter::emitPCSections(const MachineFunction &MF) { + const Function &F = MF.getFunction(); + if (PCSectionsSymbols.empty() && !F.hasMetadata(LLVMContext::MD_pcsections)) + return; + + const CodeModel::Model CM = MF.getTarget().getCodeModel(); + const unsigned RelativeRelocSize = + (CM == CodeModel::Medium || CM == CodeModel::Large) ? getPointerSize() + : 4; + + // Switch to PCSection, short-circuiting the common case where the current + // section is still valid (assume most MD_pcsections contain just 1 section). + auto SwitchSection = [&, Prev = StringRef()](const StringRef &Sec) mutable { + if (Sec == Prev) + return; + MCSection *S = getObjFileLowering().getPCSection(Sec, MF.getSection()); + assert(S && "PC section is not initialized"); + OutStreamer->switchSection(S); + Prev = Sec; + }; + // Emit symbols into sections and data as specified in the pcsections MDNode. + auto EmitForMD = [&](const MDNode &MD, ArrayRef Syms, + bool Deltas) { + // Expect the first operand to be a section name. After that, a tuple of + // constants may appear, which will simply be emitted into the current + // section (the user of MD_pcsections decides the format of encoded data). + assert(isa(MD.getOperand(0)) && "first operand not a string"); + for (const MDOperand &MDO : MD.operands()) { + if (auto *S = dyn_cast(MDO)) { + SwitchSection(S->getString()); + const MCSymbol *Prev = Syms.front(); + for (const MCSymbol *Sym : Syms) { + if (Sym == Prev || !Deltas) { + // Use the entry itself as the base of the relative offset. + MCSymbol *Base = MF.getContext().createTempSymbol("pcsection_base"); + OutStreamer->emitLabel(Base); + // Emit relative relocation `addr - base`, which avoids a dynamic + // relocation in the final binary. User will get the address with + // `base + addr`. + emitLabelDifference(Sym, Base, RelativeRelocSize); + } else { + emitLabelDifference(Sym, Prev, 4); + } + Prev = Sym; + } + } else { + assert(isa(MDO) && "expecting either string or tuple"); + const auto *AuxMDs = cast(MDO); + for (const MDOperand &AuxMDO : AuxMDs->operands()) { + assert(isa(AuxMDO) && "expecting a constant"); + const auto *C = cast(AuxMDO); + emitGlobalConstant(F.getParent()->getDataLayout(), C->getValue()); + } + } + } + }; + + OutStreamer->pushSection(); + // Emit PCs for function start and function size. + if (const MDNode *MD = F.getMetadata(LLVMContext::MD_pcsections)) + EmitForMD(*MD, {getFunctionBegin(), getFunctionEnd()}, true); + // Emit PCs for instructions collected. + for (const auto &MS : PCSectionsSymbols) + EmitForMD(*MS.first, MS.second, false); + OutStreamer->popSection(); + PCSectionsSymbols.clear(); +} + +/// Returns true if function begin and end labels should be emitted. +static bool needFuncLabels(const MachineFunction &MF) { MachineModuleInfo &MMI = MF.getMMI(); - if (!MF.getLandingPads().empty() || MF.hasEHFunclets() || MMI.hasDebugInfo()) + if (!MF.getLandingPads().empty() || MF.hasEHFunclets() || + MMI.hasDebugInfo() || + MF.getFunction().hasMetadata(LLVMContext::MD_pcsections)) return true; // We might emit an EH table that uses function begin and end labels even if @@ -1514,6 +1592,9 @@ if (MCSymbol *S = MI.getPreInstrSymbol()) OutStreamer->emitLabel(S); + if (MDNode *MD = MI.getPCSections()) + emitPCSectionsLabel(*MF, *MD); + for (const HandlerInfo &HI : Handlers) { NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, HI.TimerGroupDescription, TimePassesIsEnabled); @@ -1703,7 +1784,7 @@ // are automatically sized. bool EmitFunctionSize = MAI->hasDotTypeDotSizeDirective() && !TT.isWasm(); - if (needFuncLabelsForEHOrDebugInfo(*MF) || EmitFunctionSize) { + if (needFuncLabels(*MF) || EmitFunctionSize) { // Create a symbol for the end of function. CurrentFnEnd = createTempSymbol("func_end"); OutStreamer->emitLabel(CurrentFnEnd); @@ -1746,6 +1827,9 @@ if (MF->hasBBLabels() && HasAnyRealCode) emitBBAddrMapSection(*MF); + // Emit sections containing instruction and function PCs. + emitPCSections(*MF); + // Emit section containing stack size metadata. emitStackSizeSection(*MF); @@ -2265,7 +2349,7 @@ if (F.hasFnAttribute("patchable-function-entry") || F.hasFnAttribute("function-instrument") || F.hasFnAttribute("xray-instruction-threshold") || - needFuncLabelsForEHOrDebugInfo(MF) || NeedsLocalForSize || + needFuncLabels(MF) || NeedsLocalForSize || MF.getTarget().Options.EmitStackSizeSection || MF.hasBBLabels()) { CurrentFnBegin = createTempSymbol("func_begin"); if (NeedsLocalForSize) diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -1198,3 +1198,25 @@ } return PseudoProbeDescSection; } + +MCSection *MCObjectFileInfo::getPCSection(StringRef Name, + const MCSection *TextSec) const { + if (Ctx->getObjectFileType() != MCContext::IsELF) + return nullptr; + + // SHF_WRITE for relocations, and let user post-process data in-place. + unsigned Flags = ELF::SHF_WRITE | ELF::SHF_ALLOC | ELF::SHF_LINK_ORDER; + + if (!TextSec) + TextSec = getTextSection(); + + StringRef GroupName; + const auto &ElfSec = static_cast(*TextSec); + if (const MCSymbol *Group = ElfSec.getGroup()) { + GroupName = Group->getName(); + Flags |= ELF::SHF_GROUP; + } + return Ctx->getELFSection(Name, ELF::SHT_PROGBITS, Flags, 0, GroupName, true, + ElfSec.getUniqueID(), + cast(TextSec->getBeginSymbol())); +} diff --git a/llvm/test/CodeGen/AArch64/pcsections.ll b/llvm/test/CodeGen/AArch64/pcsections.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/pcsections.ll @@ -0,0 +1,117 @@ +; RUN: llc -O0 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-UNOPT,DEFCM +; RUN: llc -O1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-OPT,DEFCM +; RUN: llc -O2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-OPT,DEFCM +; RUN: llc -O3 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-OPT,DEFCM +; RUN: llc -O1 -code-model=large < %s | FileCheck %s --check-prefixes=CHECK,CHECK-OPT,LARGE + +target triple = "aarch64-unknown-linux-gnu" + +@foo = dso_local global i64 0, align 8 +@bar = dso_local global i64 0, align 8 + +define i64 @multiple() !pcsections !0 { +; CHECK-LABEL: multiple: +; CHECK: .Lfunc_begin0: +; CHECK: // %bb.0: // %entry +; CHECK: .Lpcsection0: +; CHECK-NEXT: ldr +; CHECK-NEXT: ret +; CHECK: .section section_no_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base0: +; DEFCM-NEXT: .word .Lfunc_begin0-.Lpcsection_base0 +; LARGE-NEXT: .xword .Lfunc_begin0-.Lpcsection_base0 +; CHECK-NEXT: .word .Lfunc_end0-.Lfunc_begin0 +; CHECK-NEXT: .section section_aux_42,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base1: +; DEFCM-NEXT: .word .Lpcsection0-.Lpcsection_base1 +; LARGE-NEXT: .xword .Lpcsection0-.Lpcsection_base1 +; CHECK-NEXT: .word 42 +; CHECK-NEXT: .section section_aux_21264,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base2: +; DEFCM-NEXT: .word .Lpcsection0-.Lpcsection_base2 +; LARGE-NEXT: .xword .Lpcsection0-.Lpcsection_base2 +; CHECK-NEXT: .word 21264 +; CHECK-NEXT: .text +entry: + %0 = load i64, i64* @bar, align 8, !pcsections !1 + ret i64 %0 +} + +define i64 @test_simple_atomic() { +; CHECK-LABEL: test_simple_atomic: +; CHECK: .Lpcsection1: +; CHECK-NEXT: ldr +; CHECK-NOT: .Lpcsection2 +; CHECK: ldr +; CHECK: add +; CHECK-NEXT: ret +; CHECK: .section section_no_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base3: +; DEFCM-NEXT: .word .Lpcsection1-.Lpcsection_base3 +; LARGE-NEXT: .xword .Lpcsection1-.Lpcsection_base3 +; CHECK-NEXT: .text +entry: + %0 = load atomic i64, i64* @foo monotonic, align 8, !pcsections !0 + %1 = load i64, i64* @bar, align 8 + %add = add nsw i64 %1, %0 + ret i64 %add +} + +define i64 @test_complex_atomic() { +; CHECK-LABEL: test_complex_atomic: +; --- +; CHECK-OPT: .Lpcsection2: +; CHECK-OPT-NEXT: ldxr +; CHECK-OPT: .Lpcsection3: +; CHECK-OPT-NEXT: add +; CHECK-OPT: .Lpcsection4: +; CHECK-OPT-NEXT: stxr +; CHECK-OPT: .Lpcsection5: +; CHECK-OPT-NEXT: cbnz +; --- +; CHECK-UNOPT: .Lpcsection2: +; CHECK-UNOPT-NEXT: ldr +; CHECK-UNOPT: .Lpcsection4: +; CHECK-UNOPT-NEXT: add +; CHECK-UNOPT: .Lpcsection5: +; CHECK-UNOPT-NEXT: ldaxr +; CHECK-UNOPT: .Lpcsection6: +; CHECK-UNOPT-NEXT: cmp +; CHECK-UNOPT: .Lpcsection8: +; CHECK-UNOPT-NEXT: stlxr +; CHECK-UNOPT: .Lpcsection9: +; CHECK-UNOPT-NEXT: cbnz +; CHECK-UNOPT: .Lpcsection12: +; CHECK-UNOPT-NEXT: b +; --- +; CHECK-NOT: .Lpcsection +; CHECK: ldr +; CHECK: ret +; CHECK: .section section_no_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base4: +; DEFCM-NEXT: .word .Lpcsection2-.Lpcsection_base4 +; LARGE-NEXT: .xword .Lpcsection2-.Lpcsection_base4 +; CHECK-NEXT: .Lpcsection_base5: +; DEFCM-NEXT: .word .Lpcsection3-.Lpcsection_base5 +; LARGE-NEXT: .xword .Lpcsection3-.Lpcsection_base5 +; CHECK-NEXT: .Lpcsection_base6: +; DEFCM-NEXT: .word .Lpcsection4-.Lpcsection_base6 +; LARGE-NEXT: .xword .Lpcsection4-.Lpcsection_base6 +; CHECK-NEXT: .Lpcsection_base7: +; DEFCM-NEXT: .word .Lpcsection5-.Lpcsection_base7 +; LARGE-NEXT: .xword .Lpcsection5-.Lpcsection_base7 +; CHECK-UNOPT: .word .Lpcsection12-.Lpcsection_base14 +; CHECK-NEXT: .text +entry: + %0 = atomicrmw add i64* @foo, i64 1 monotonic, align 8, !pcsections !0 + %1 = load i64, i64* @bar, align 8 + %inc = add nsw i64 %1, 1 + store i64 %inc, i64* @bar, align 8 + %add = add nsw i64 %1, %0 + ret i64 %add +} + +!0 = !{!"section_no_aux"} +!1 = !{!"section_aux_42", !2, !"section_aux_21264", !3} +!2 = !{i32 42} +!3 = !{i32 21264} diff --git a/llvm/test/CodeGen/X86/pcsections.ll b/llvm/test/CodeGen/X86/pcsections.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/pcsections.ll @@ -0,0 +1,125 @@ +; RUN: llc -O0 < %s | FileCheck %s --check-prefixes=CHECK,DEFCM +; RUN: llc -O1 < %s | FileCheck %s --check-prefixes=CHECK,DEFCM +; RUN: llc -O2 < %s | FileCheck %s --check-prefixes=CHECK,DEFCM +; RUN: llc -O3 < %s | FileCheck %s --check-prefixes=CHECK,DEFCM +; RUN: llc -O1 -code-model=large < %s | FileCheck %s --check-prefixes=CHECK,LARGE + +target triple = "x86_64-unknown-linux-gnu" + +@foo = dso_local global i64 0, align 8 +@bar = dso_local global i64 0, align 8 + +define void @empty_no_aux() !pcsections !0 { +; CHECK-LABEL: empty_no_aux: +; CHECK-NEXT: .Lfunc_begin0 +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: retq +; CHECK-NEXT: .Lfunc_end0: +; CHECK: .section section_no_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base0: +; DEFCM-NEXT: .long .Lfunc_begin0-.Lpcsection_base0 +; LARGE-NEXT: .quad .Lfunc_begin0-.Lpcsection_base0 +; CHECK-NEXT: .long .Lfunc_end0-.Lfunc_begin0 +; CHECK-NEXT: .text +entry: + ret void +} + +define void @empty_aux() !pcsections !1 { +; CHECK-LABEL: empty_aux: +; CHECK-NEXT: .Lfunc_begin1 +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: retq +; CHECK-NEXT: .Lfunc_end1: +; CHECK: .section section_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base1: +; DEFCM-NEXT: .long .Lfunc_begin1-.Lpcsection_base1 +; LARGE-NEXT: .quad .Lfunc_begin1-.Lpcsection_base1 +; CHECK-NEXT: .long .Lfunc_end1-.Lfunc_begin1 +; CHECK-NEXT: .long 10 +; CHECK-NEXT: .long 20 +; CHECK-NEXT: .long 30 +; CHECK-NEXT: .text +entry: + ret void +} + +define i64 @multiple() !pcsections !0 { +; CHECK-LABEL: multiple: +; CHECK-NEXT: .Lfunc_begin2 +; CHECK: # %bb.0: # %entry +; CHECK: .Lpcsection0: +; CHECK-NEXT: movq +; CHECK-NEXT: retq +; CHECK-NEXT: .Lfunc_end2: +; CHECK: .section section_no_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base2: +; DEFCM-NEXT: .long .Lfunc_begin2-.Lpcsection_base2 +; LARGE-NEXT: .quad .Lfunc_begin2-.Lpcsection_base2 +; CHECK-NEXT: .long .Lfunc_end2-.Lfunc_begin2 +; CHECK-NEXT: .section section_aux_42,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base3: +; DEFCM-NEXT: .long .Lpcsection0-.Lpcsection_base3 +; LARGE-NEXT: .quad .Lpcsection0-.Lpcsection_base3 +; CHECK-NEXT: .long 42 +; CHECK-NEXT: .section section_aux_21264,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base4: +; DEFCM-NEXT: .long .Lpcsection0-.Lpcsection_base4 +; LARGE-NEXT: .quad .Lpcsection0-.Lpcsection_base4 +; CHECK-NEXT: .long 21264 +; CHECK-NEXT: .text +entry: + %0 = load i64, i64* @bar, align 8, !pcsections !2 + ret i64 %0 +} + +define i64 @test_simple_atomic() { +; CHECK-LABEL: test_simple_atomic: +; CHECK: .Lpcsection1: +; CHECK-NEXT: movq +; CHECK-NOT: .Lpcsection +; CHECK: addq +; CHECK-NEXT: retq +; CHECK-NEXT: .Lfunc_end3: +; CHECK: .section section_no_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base5: +; DEFCM-NEXT: .long .Lpcsection1-.Lpcsection_base5 +; LARGE-NEXT: .quad .Lpcsection1-.Lpcsection_base5 +; CHECK-NEXT: .text +entry: + %0 = load atomic i64, i64* @foo monotonic, align 8, !pcsections !0 + %1 = load i64, i64* @bar, align 8 + %add = add nsw i64 %1, %0 + ret i64 %add +} + +define i64 @test_complex_atomic() { +; CHECK-LABEL: test_complex_atomic: +; CHECK: movl $1 +; CHECK-NEXT: .Lpcsection2: +; CHECK-NEXT: lock xaddq +; CHECK-NOT: .Lpcsection +; CHECK: movq +; CHECK: addq +; CHECK: retq +; CHECK-NEXT: .Lfunc_end4: +; CHECK: .section section_no_aux,"awo",@progbits,.text +; CHECK-NEXT: .Lpcsection_base6: +; DEFCM-NEXT: .long .Lpcsection2-.Lpcsection_base6 +; LARGE-NEXT: .quad .Lpcsection2-.Lpcsection_base6 +; CHECK-NEXT: .text +entry: + %0 = atomicrmw add i64* @foo, i64 1 monotonic, align 8, !pcsections !0 + %1 = load i64, i64* @bar, align 8 + %inc = add nsw i64 %1, 1 + store i64 %inc, i64* @bar, align 8 + %add = add nsw i64 %1, %0 + ret i64 %add +} + +!0 = !{!"section_no_aux"} +!1 = !{!"section_aux", !3} +!2 = !{!"section_aux_42", !4, !"section_aux_21264", !5} +!3 = !{i32 10, i32 20, i32 30} +!4 = !{i32 42} +!5 = !{i32 21264}