diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h --- a/bolt/include/bolt/Core/BinaryBasicBlock.h +++ b/bolt/include/bolt/Core/BinaryBasicBlock.h @@ -103,7 +103,7 @@ /// After output/codegen, map output offsets of instructions in this basic /// block to instruction offsets in the original function. Note that the /// output basic block could be different from the input basic block. - /// We only map instruction of interest, such as calls, and sdt markers. + /// We only map instruction of interest, such as calls and markers. /// /// We store the offset array in a basic block to facilitate BAT tables /// generation. Otherwise, the mapping could be done at function level. diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -673,9 +673,6 @@ /// List of functions that always trap. std::vector TrappedFunctions; - /// Map SDT locations to SDT markers info - std::unordered_map SDTMarkers; - /// Map linux kernel program locations/instructions to their pointers in /// special linux kernel sections std::unordered_map> LKMarkers; diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -1702,6 +1702,8 @@ /// Indicate that another function body was merged with this function. void setHasFunctionsFoldedInto() { HasFunctionsFoldedInto = true; } + void setHasSDTMarker(bool V) { HasSDTMarker = V; } + BinaryFunction &setPersonalityFunction(uint64_t Addr) { assert(!PersonalityFunction && "can't set personality function twice"); PersonalityFunction = BC.getOrCreateGlobalSymbol(Addr, "FUNCat"); diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -509,18 +509,6 @@ return OS; } -struct SDTMarkerInfo { - uint64_t PC; - uint64_t Base; - uint64_t Semaphore; - StringRef Provider; - StringRef Name; - StringRef Args; - - /// The offset of PC within the note section - unsigned PCOffset; -}; - /// Linux Kernel special sections point to a specific instruction in many cases. /// Unlike SDTMarkerInfo, these markers can come from different sections. struct LKInstructionMarkerInfo { diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -197,9 +197,6 @@ /// Update debug and other auxiliary information in the file. void updateMetadata(); - /// Update SDTMarkers' locations for the output binary. - void updateSDTMarkers(); - /// Update LKMarkers' locations for the output binary. void updateLKMarkers(); @@ -397,16 +394,10 @@ /// of appending contents to it. bool willOverwriteSection(StringRef SectionName); - /// Parse .note.stapsdt section - void parseSDTNotes(); - /// Parse .pseudo_probe_desc section and .pseudo_probe section /// Setup Pseudo probe decoder void parsePseudoProbe(); - /// Print all SDT markers - void printSDTMarkers(); - public: /// Standard ELF sections we overwrite. static constexpr const char *SectionsToOverwrite[] = { @@ -585,10 +576,6 @@ /// .note.gnu.build-id section. ErrorOr BuildIDSection{std::errc::bad_address}; - /// .note.stapsdt section. - /// Contains information about statically defined tracing points - ErrorOr SDTSection{std::errc::bad_address}; - /// .pseudo_probe_desc section. /// Contains information about pseudo probe description, like its related /// function diff --git a/bolt/include/bolt/Rewrite/Rewriters.h b/bolt/include/bolt/Rewrite/Rewriters.h --- a/bolt/include/bolt/Rewrite/Rewriters.h +++ b/bolt/include/bolt/Rewrite/Rewriters.h @@ -16,7 +16,9 @@ class RewriterBase; -/// The list of rewriter build functions (e.g. createNewRewriter()). +// The list of rewriter build functions. + +std::unique_ptr createSDTRewriter(); } // namespace bolt } // namespace llvm diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -1994,19 +1994,20 @@ updateOffset(LastInstrOffset); } - const uint64_t InstrInputAddr = I->first + Address; - bool IsSDTMarker = - MIB->isNoop(Instr) && BC.SDTMarkers.count(InstrInputAddr); - bool IsLKMarker = BC.LKMarkers.count(InstrInputAddr); + bool IsLKMarker = BC.LKMarkers.count(I->first + Address); // Mark all nops with Offset for profile tracking purposes. if (MIB->isNoop(Instr) || IsLKMarker) { - if (!MIB->getOffset(Instr)) + // If "Offset" annotation is not present, set it and mark the nop for + // deletion. + if (!MIB->getOffset(Instr)) { MIB->setOffset(Instr, static_cast(Offset), AllocatorId); - if (IsSDTMarker || IsLKMarker) - HasSDTMarker = true; - else // Annotate ordinary nops, so we can safely delete them if required. - MIB->addAnnotation(Instr, "NOP", static_cast(1), AllocatorId); + if (!IsLKMarker) + MIB->addAnnotation(Instr, "NOP", static_cast(1), + AllocatorId); + } + if (IsLKMarker) + HasSDTMarker = true; } if (!InsertBB) { diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt --- a/bolt/lib/Rewrite/CMakeLists.txt +++ b/bolt/lib/Rewrite/CMakeLists.txt @@ -17,6 +17,7 @@ MachORewriteInstance.cpp RewriteInstance.cpp RewriteManager.cpp + SDTRewriter.cpp DISABLE_LLVM_LINK_LLVM_DYLIB diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -193,10 +193,6 @@ cl::desc("print loop related information"), cl::Hidden, cl::cat(BoltCategory)); -static cl::opt PrintSDTMarkers("print-sdt", - cl::desc("print all SDT markers"), - cl::Hidden, cl::cat(BoltCategory)); - enum PrintPseudoProbesOptions { PPP_None = 0, PPP_Probes_Section_Decode = 0x1, @@ -387,6 +383,8 @@ RewriteInstance::~RewriteInstance() {} void RewriteInstance::initializeRewriters() { + RewriteManager::getManager()->registerRewriter(createSDTRewriter()); + RewriteManager::getManager()->initRewriters(*BC); } @@ -637,51 +635,6 @@ return Error::success(); } -void RewriteInstance::parseSDTNotes() { - if (!SDTSection) - return; - - StringRef Buf = SDTSection->getContents(); - DataExtractor DE = DataExtractor(Buf, BC->AsmInfo->isLittleEndian(), - BC->AsmInfo->getCodePointerSize()); - uint64_t Offset = 0; - - while (DE.isValidOffset(Offset)) { - uint32_t NameSz = DE.getU32(&Offset); - DE.getU32(&Offset); // skip over DescSz - uint32_t Type = DE.getU32(&Offset); - Offset = alignTo(Offset, 4); - - if (Type != 3) - errs() << "BOLT-WARNING: SDT note type \"" << Type - << "\" is not expected\n"; - - if (NameSz == 0) - errs() << "BOLT-WARNING: SDT note has empty name\n"; - - StringRef Name = DE.getCStr(&Offset); - - if (!Name.equals("stapsdt")) - errs() << "BOLT-WARNING: SDT note name \"" << Name - << "\" is not expected\n"; - - // Parse description - SDTMarkerInfo Marker; - Marker.PCOffset = Offset; - Marker.PC = DE.getU64(&Offset); - Marker.Base = DE.getU64(&Offset); - Marker.Semaphore = DE.getU64(&Offset); - Marker.Provider = DE.getCStr(&Offset); - Marker.Name = DE.getCStr(&Offset); - Marker.Args = DE.getCStr(&Offset); - Offset = alignTo(Offset, 4); - BC->SDTMarkers[Marker.PC] = Marker; - } - - if (opts::PrintSDTMarkers) - printSDTMarkers(); -} - void RewriteInstance::parsePseudoProbe() { if (!PseudoProbeDescSection && !PseudoProbeSection) { // pesudo probe is not added to binary. It is normal and no warning needed. @@ -731,19 +684,6 @@ } } -void RewriteInstance::printSDTMarkers() { - outs() << "BOLT-INFO: Number of SDT markers is " << BC->SDTMarkers.size() - << "\n"; - for (auto It : BC->SDTMarkers) { - SDTMarkerInfo &Marker = It.second; - outs() << "BOLT-INFO: PC: " << utohexstr(Marker.PC) - << ", Base: " << utohexstr(Marker.Base) - << ", Semaphore: " << utohexstr(Marker.Semaphore) - << ", Provider: " << Marker.Provider << ", Name: " << Marker.Name - << ", Args: " << Marker.Args << "\n"; - } -} - void RewriteInstance::parseBuildID() { if (!BuildIDSection) return; @@ -1856,7 +1796,6 @@ LSDASection = BC->getUniqueSectionByName(".gcc_except_table"); EHFrameSection = BC->getUniqueSectionByName(".eh_frame"); BuildIDSection = BC->getUniqueSectionByName(".note.gnu.build-id"); - SDTSection = BC->getUniqueSectionByName(".note.stapsdt"); PseudoProbeDescSection = BC->getUniqueSectionByName(".pseudo_probe_desc"); PseudoProbeSection = BC->getUniqueSectionByName(".pseudo_probe"); @@ -1913,8 +1852,6 @@ if (std::optional FileBuildID = getPrintableBuildID()) BC->setFileBuildID(*FileBuildID); - parseSDTNotes(); - // Read .dynamic/PT_DYNAMIC. return readELFDynamic(); } @@ -3608,7 +3545,6 @@ RewriteManager::getManager()->runFinalizersAfterEmit(); // TODO: use RewriteManager for updates. - updateSDTMarkers(); updateLKMarkers(); parsePseudoProbe(); updatePseudoProbes(); @@ -3889,29 +3825,6 @@ } } -void RewriteInstance::updateSDTMarkers() { - NamedRegionTimer T("updateSDTMarkers", "update SDT markers", TimerGroupName, - TimerGroupDesc, opts::TimeRewrite); - - if (!SDTSection) - return; - SDTSection->registerPatcher(std::make_unique()); - - SimpleBinaryPatcher *SDTNotePatcher = - static_cast(SDTSection->getPatcher()); - for (auto &SDTInfoKV : BC->SDTMarkers) { - const uint64_t OriginalAddress = SDTInfoKV.first; - SDTMarkerInfo &SDTInfo = SDTInfoKV.second; - const BinaryFunction *F = - BC->getBinaryFunctionContainingAddress(OriginalAddress); - if (!F) - continue; - const uint64_t NewAddress = - F->translateInputToOutputAddress(OriginalAddress); - SDTNotePatcher->addLE64Patch(SDTInfo.PCOffset, NewAddress); - } -} - void RewriteInstance::updateLKMarkers() { if (BC->LKMarkers.size() == 0) return; diff --git a/bolt/lib/Rewrite/SDTRewriter.cpp b/bolt/lib/Rewrite/SDTRewriter.cpp new file mode 100644 --- /dev/null +++ b/bolt/lib/Rewrite/SDTRewriter.cpp @@ -0,0 +1,177 @@ +//===- bolt/Rewrite/SDTRewriter.cpp ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implement the support for System Tap Statically-Defined Trace points stored +// in .note.stapsdt section. +// +//===----------------------------------------------------------------------===// + +#include "bolt/Core/BinaryFunction.h" +#include "bolt/Core/DebugData.h" +#include "bolt/Rewrite/RewriteManager.h" +#include "bolt/Rewrite/Rewriters.h" +#include "bolt/Utils/CommandLineOpts.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Timer.h" + +using namespace llvm; +using namespace bolt; + +namespace opts { +static cl::opt PrintSDTMarkers("print-sdt", + cl::desc("print all SDT markers"), + cl::Hidden, cl::cat(BoltCategory)); +} + +namespace { +class SDTRewriter final : public RewriterBase { + ErrorOr SDTSection{std::errc::bad_address}; + + struct SDTMarkerInfo { + uint64_t PC; + uint64_t Base; + uint64_t Semaphore; + StringRef Provider; + StringRef Name; + StringRef Args; + + /// The offset of PC within the note section + unsigned PCOffset; + }; + + /// Map SDT locations to SDT markers info + using SDTMarkersListType = std::unordered_map; + SDTMarkersListType SDTMarkers; + + /// Read section to populate SDTMarkers. + void readSection(); + + void printSDTMarkers() const; + +public: + SDTRewriter() : RewriterBase("sdt-rewriter") {} + + Error preCFGInitializer() override; + + Error postEmitFinalizer() override; +}; + +void SDTRewriter::readSection() { + SDTSection = BC->getUniqueSectionByName(".note.stapsdt"); + if (!SDTSection) + return; + + StringRef Buf = SDTSection->getContents(); + DataExtractor DE = DataExtractor(Buf, BC->AsmInfo->isLittleEndian(), + BC->AsmInfo->getCodePointerSize()); + uint64_t Offset = 0; + + while (DE.isValidOffset(Offset)) { + uint32_t NameSz = DE.getU32(&Offset); + DE.getU32(&Offset); // skip over DescSz + uint32_t Type = DE.getU32(&Offset); + Offset = alignTo(Offset, 4); + + if (Type != 3) + errs() << "BOLT-WARNING: SDT note type \"" << Type + << "\" is not expected\n"; + + if (NameSz == 0) + errs() << "BOLT-WARNING: SDT note has empty name\n"; + + StringRef Name = DE.getCStr(&Offset); + + if (!Name.equals("stapsdt")) + errs() << "BOLT-WARNING: SDT note name \"" << Name + << "\" is not expected\n"; + + // Parse description + SDTMarkerInfo Marker; + Marker.PCOffset = Offset; + Marker.PC = DE.getU64(&Offset); + Marker.Base = DE.getU64(&Offset); + Marker.Semaphore = DE.getU64(&Offset); + Marker.Provider = DE.getCStr(&Offset); + Marker.Name = DE.getCStr(&Offset); + Marker.Args = DE.getCStr(&Offset); + Offset = alignTo(Offset, 4); + SDTMarkers[Marker.PC] = Marker; + } + + if (opts::PrintSDTMarkers) + printSDTMarkers(); +} + +Error SDTRewriter::preCFGInitializer() { + // Populate SDTMarkers. + readSection(); + + // Mark nop instructions referenced by SDT and the containing function. + for (const uint64_t PC : llvm::make_first_range(SDTMarkers)) { + BinaryFunction *BF = BC->getBinaryFunctionContainingAddress(PC); + + if (!BF || !BC->shouldEmit(*BF)) + continue; + + const uint64_t Offset = PC - BF->getAddress(); + MCInst *Inst = BF->getInstructionAtOffset(Offset); + if (!Inst) + return createStringError(errc::executable_format_error, + "no instruction matches SDT offset"); + + if (!BC->MIB->isNoop(*Inst)) + return createStringError(std::make_error_code(std::errc::not_supported), + "nop instruction expected at SDT offset"); + + BC->MIB->setOffset(*Inst, static_cast(Offset)); + + BF->setHasSDTMarker(true); + } + + return Error::success(); +} + +Error SDTRewriter::postEmitFinalizer() { + if (!SDTSection) + return Error::success(); + + SDTSection->registerPatcher(std::make_unique()); + + SimpleBinaryPatcher *SDTNotePatcher = + static_cast(SDTSection->getPatcher()); + for (auto &SDTInfoKV : SDTMarkers) { + const uint64_t OriginalAddress = SDTInfoKV.first; + const SDTMarkerInfo &SDTInfo = SDTInfoKV.second; + const BinaryFunction *F = + BC->getBinaryFunctionContainingAddress(OriginalAddress); + if (!F) + continue; + const uint64_t NewAddress = + F->translateInputToOutputAddress(OriginalAddress); + SDTNotePatcher->addLE64Patch(SDTInfo.PCOffset, NewAddress); + } + + return Error::success(); +} + +void SDTRewriter::printSDTMarkers() const { + outs() << "BOLT-INFO: Number of SDT markers is " << SDTMarkers.size() << "\n"; + for (const SDTMarkerInfo &Marker : llvm::make_second_range(SDTMarkers)) { + outs() << "BOLT-INFO: PC: " << utohexstr(Marker.PC) + << ", Base: " << utohexstr(Marker.Base) + << ", Semaphore: " << utohexstr(Marker.Semaphore) + << ", Provider: " << Marker.Provider << ", Name: " << Marker.Name + << ", Args: " << Marker.Args << "\n"; + } +} +} // namespace + +std::unique_ptr llvm::bolt::createSDTRewriter() { + return std::make_unique(); +}