diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -672,10 +672,6 @@ /// List of functions that always trap. std::vector TrappedFunctions; - /// Map linux kernel program locations/instructions to their pointers in - /// special linux kernel sections - std::unordered_map> LKMarkers; - /// List of external addresses in the code that are not a function start /// and are referenced from BinaryFunction. std::list> InterproceduralReferences; diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -604,10 +604,6 @@ Islands->CodeOffsets.emplace(Offset); } - /// Register secondary entry point at a given \p Offset into the function. - /// Return global symbol for use by extern function references. - MCSymbol *addEntryPointAtOffset(uint64_t Offset); - /// Register an internal offset in a function referenced from outside. void registerReferencedOffset(uint64_t Offset) { ExternallyReferencedOffsets.emplace(Offset); @@ -1445,6 +1441,10 @@ /// symbol associated with the entry. MCSymbol *addEntryPoint(const BinaryBasicBlock &BB); + /// Register secondary entry point at a given \p Offset into the function. + /// Return global symbol for use by extern function references. + MCSymbol *addEntryPointAtOffset(uint64_t Offset); + /// Mark all blocks that are unreachable from a root (entry point /// or landing pad) as invalid. void markUnreachableBlocks(); diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -509,15 +509,6 @@ return OS; } -/// Linux Kernel special sections point to a specific instruction in many cases. -/// Unlike SDTMarkerInfo, these markers can come from different sections. -struct LKInstructionMarkerInfo { - uint64_t SectionOffset; - int32_t PCRelativeOffset; - bool IsPCRelative; - StringRef SectionName; -}; - } // namespace bolt } // namespace llvm diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -114,30 +114,6 @@ /// Process input relocations. void processRelocations(); - /// Insert an LKMarker for a given code pointer \p PC from a non-code section - /// \p SectionName. - void insertLKMarker(uint64_t PC, uint64_t SectionOffset, - int32_t PCRelativeOffset, bool IsPCRelative, - StringRef SectionName); - - /// Process linux kernel special sections and their relocations. - void processLKSections(); - - /// Process special linux kernel section, __ex_table. - void processLKExTable(); - - /// Process special linux kernel section, .pci_fixup. - void processLKPCIFixup(); - - /// Process __ksymtab and __ksymtab_gpl. - void processLKKSymtab(bool IsGPL = false); - - /// Process special linux kernel section, __bug_table. - void processLKBugTable(); - - /// Process special linux kernel section, .smp_locks. - void processLKSMPLocks(); - /// Read relocations from a given section. void readDynamicRelocations(const object::SectionRef &Section, bool IsJmpRel); @@ -198,9 +174,6 @@ /// Update debug and other auxiliary information in the file. void updateMetadata(); - /// Update LKMarkers' locations for the output binary. - void updateLKMarkers(); - /// Return the list of code sections in the output order. std::vector getCodeSections(); diff --git a/bolt/include/bolt/Rewrite/Rewriters.h b/bolt/include/bolt/Rewrite/Rewriters.h --- a/bolt/include/bolt/Rewrite/Rewriters.h +++ b/bolt/include/bolt/Rewrite/Rewriters.h @@ -19,6 +19,8 @@ // The list of rewriter build functions. +std::unique_ptr createLinuxKernelRewriter(BinaryContext &); + std::unique_ptr createPseudoProbeRewriter(BinaryContext &); std::unique_ptr createSDTRewriter(BinaryContext &); diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -1994,20 +1994,13 @@ updateOffset(LastInstrOffset); } - bool IsLKMarker = BC.LKMarkers.count(I->first + Address); // Mark all nops with Offset for profile tracking purposes. - if (MIB->isNoop(Instr) || IsLKMarker) { + if (MIB->isNoop(Instr) && !MIB->getOffset(Instr)) { // If "Offset" annotation is not present, set it and mark the nop for // deletion. - if (!MIB->getOffset(Instr)) { - MIB->setOffset(Instr, static_cast(Offset), AllocatorId); - // Annotate ordinary nops, so we can safely delete them if required. - if (!IsLKMarker) - MIB->addAnnotation(Instr, "NOP", static_cast(1), - AllocatorId); - } - if (IsLKMarker) - HasSDTMarker = true; + MIB->setOffset(Instr, static_cast(Offset), AllocatorId); + // Annotate ordinary nops, so we can safely delete them if required. + MIB->addAnnotation(Instr, "NOP", static_cast(1), AllocatorId); } if (!InsertBB) { diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt --- a/bolt/lib/Rewrite/CMakeLists.txt +++ b/bolt/lib/Rewrite/CMakeLists.txt @@ -14,6 +14,7 @@ DWARFRewriter.cpp ExecutableFileMemoryManager.cpp JITLinkLinker.cpp + LinuxKernelRewriter.cpp MachORewriteInstance.cpp PseudoProbeRewriter.cpp RewriteInstance.cpp diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp new file mode 100644 --- /dev/null +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -0,0 +1,369 @@ +//===- bolt/Rewrite/LinuxKernelRewriter.cpp -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Support for updating Linux Kernel metadata. +// +//===----------------------------------------------------------------------===// + +#include "bolt/Core/BinaryFunction.h" +#include "bolt/Rewrite/RewriteManager.h" +#include "bolt/Rewrite/Rewriters.h" +#include "bolt/Utils/CommandLineOpts.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Errc.h" + +using namespace llvm; +using namespace bolt; + +namespace { +class LinuxKernelRewriter final : public RewriterBase { + + /// Linux Kernel special sections point to a specific instruction in many + /// cases. Unlike SDTMarkerInfo, these markers can come from different + /// sections. + struct LKInstructionMarkerInfo { + uint64_t SectionOffset; + int32_t PCRelativeOffset; + bool IsPCRelative; + StringRef SectionName; + }; + + /// Map linux kernel program locations/instructions to their pointers in + /// special linux kernel sections + std::unordered_map> LKMarkers; + + /// Insert an LKMarker for a given code pointer \p PC from a non-code section + /// \p SectionName. + void insertLKMarker(uint64_t PC, uint64_t SectionOffset, + int32_t PCRelativeOffset, bool IsPCRelative, + StringRef SectionName); + + /// Process linux kernel special sections and their relocations. + void processLKSections(); + + /// Process special linux kernel section, __ex_table. + void processLKExTable(); + + /// Process special linux kernel section, .pci_fixup. + void processLKPCIFixup(); + + /// Process __ksymtab and __ksymtab_gpl. + void processLKKSymtab(bool IsGPL = false); + + /// Process special linux kernel section, __bug_table. + void processLKBugTable(); + + /// Process special linux kernel section, .smp_locks. + void processLKSMPLocks(); + + /// Update LKMarkers' locations for the output binary. + void updateLKMarkers(); + + /// Mark instructions referenced by kernel metadata. + Error markInstructions(); + +public: + LinuxKernelRewriter(BinaryContext &BC) + : RewriterBase("linux-kernel-rewriter", BC) {} + + Error preCFGInitializer() override { + if (opts::LinuxKernelMode) { + processLKSections(); + if (Error E = markInstructions()) + return E; + } + + return Error::success(); + } + + Error postEmitFinalizer() override { + updateLKMarkers(); + return Error::success(); + } +}; + +Error LinuxKernelRewriter::markInstructions() { + for (const uint64_t PC : llvm::make_first_range(LKMarkers)) { + BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(PC); + + if (!BF || !BC.shouldEmit(*BF)) + continue; + + const uint64_t Offset = PC - BF->getAddress(); + MCInst *Inst = BF->getInstructionAtOffset(Offset); + if (!Inst) + return createStringError(errc::executable_format_error, + "no instruction matches kernel marker offset"); + + BC.MIB->setOffset(*Inst, static_cast(Offset)); + + BF->setHasSDTMarker(true); + } + + return Error::success(); +} + +void LinuxKernelRewriter::insertLKMarker(uint64_t PC, uint64_t SectionOffset, + int32_t PCRelativeOffset, + bool IsPCRelative, + StringRef SectionName) { + LKMarkers[PC].emplace_back(LKInstructionMarkerInfo{ + SectionOffset, PCRelativeOffset, IsPCRelative, SectionName}); +} + +void LinuxKernelRewriter::processLKSections() { + assert(opts::LinuxKernelMode && + "process Linux Kernel special sections and their relocations only in " + "linux kernel mode.\n"); + + processLKExTable(); + processLKPCIFixup(); + processLKKSymtab(); + processLKKSymtab(true); + processLKBugTable(); + processLKSMPLocks(); +} + +/// Process __ex_table section of Linux Kernel. +/// This section contains information regarding kernel level exception +/// handling (https://www.kernel.org/doc/html/latest/x86/exception-tables.html). +/// More documentation is in arch/x86/include/asm/extable.h. +/// +/// The section is the list of the following structures: +/// +/// struct exception_table_entry { +/// int insn; +/// int fixup; +/// int handler; +/// }; +/// +void LinuxKernelRewriter::processLKExTable() { + ErrorOr SectionOrError = + BC.getUniqueSectionByName("__ex_table"); + if (!SectionOrError) + return; + + const uint64_t SectionSize = SectionOrError->getSize(); + const uint64_t SectionAddress = SectionOrError->getAddress(); + assert((SectionSize % 12) == 0 && + "The size of the __ex_table section should be a multiple of 12"); + for (uint64_t I = 0; I < SectionSize; I += 4) { + const uint64_t EntryAddress = SectionAddress + I; + ErrorOr Offset = BC.getSignedValueAtAddress(EntryAddress, 4); + assert(Offset && "failed reading PC-relative offset for __ex_table"); + int32_t SignedOffset = *Offset; + const uint64_t RefAddress = EntryAddress + SignedOffset; + + BinaryFunction *ContainingBF = + BC.getBinaryFunctionContainingAddress(RefAddress); + if (!ContainingBF) + continue; + + MCSymbol *ReferencedSymbol = ContainingBF->getSymbol(); + const uint64_t FunctionOffset = RefAddress - ContainingBF->getAddress(); + switch (I % 12) { + default: + llvm_unreachable("bad alignment of __ex_table"); + break; + case 0: + // insn + insertLKMarker(RefAddress, I, SignedOffset, true, "__ex_table"); + break; + case 4: + // fixup + if (FunctionOffset) + ReferencedSymbol = ContainingBF->addEntryPointAtOffset(FunctionOffset); + BC.addRelocation(EntryAddress, ReferencedSymbol, Relocation::getPC32(), 0, + *Offset); + break; + case 8: + // handler + assert(!FunctionOffset && + "__ex_table handler entry should point to function start"); + BC.addRelocation(EntryAddress, ReferencedSymbol, Relocation::getPC32(), 0, + *Offset); + break; + } + } +} + +/// Process .pci_fixup section of Linux Kernel. +/// This section contains a list of entries for different PCI devices and their +/// corresponding hook handler (code pointer where the fixup +/// code resides, usually on x86_64 it is an entry PC relative 32 bit offset). +/// Documentation is in include/linux/pci.h. +void LinuxKernelRewriter::processLKPCIFixup() { + ErrorOr SectionOrError = + BC.getUniqueSectionByName(".pci_fixup"); + assert(SectionOrError && + ".pci_fixup section not found in Linux Kernel binary"); + const uint64_t SectionSize = SectionOrError->getSize(); + const uint64_t SectionAddress = SectionOrError->getAddress(); + assert((SectionSize % 16) == 0 && ".pci_fixup size is not a multiple of 16"); + + for (uint64_t I = 12; I + 4 <= SectionSize; I += 16) { + const uint64_t PC = SectionAddress + I; + ErrorOr Offset = BC.getSignedValueAtAddress(PC, 4); + assert(Offset && "cannot read value from .pci_fixup"); + const int32_t SignedOffset = *Offset; + const uint64_t HookupAddress = PC + SignedOffset; + BinaryFunction *HookupFunction = + BC.getBinaryFunctionAtAddress(HookupAddress); + assert(HookupFunction && "expected function for entry in .pci_fixup"); + BC.addRelocation(PC, HookupFunction->getSymbol(), Relocation::getPC32(), 0, + *Offset); + } +} + +/// Process __ksymtab[_gpl] sections of Linux Kernel. +/// This section lists all the vmlinux symbols that kernel modules can access. +/// +/// All the entries are 4 bytes each and hence we can read them by one by one +/// and ignore the ones that are not pointing to the .text section. All pointers +/// are PC relative offsets. Always, points to the beginning of the function. +void LinuxKernelRewriter::processLKKSymtab(bool IsGPL) { + StringRef SectionName = "__ksymtab"; + if (IsGPL) + SectionName = "__ksymtab_gpl"; + ErrorOr SectionOrError = + BC.getUniqueSectionByName(SectionName); + assert(SectionOrError && + "__ksymtab[_gpl] section not found in Linux Kernel binary"); + const uint64_t SectionSize = SectionOrError->getSize(); + const uint64_t SectionAddress = SectionOrError->getAddress(); + assert((SectionSize % 4) == 0 && + "The size of the __ksymtab[_gpl] section should be a multiple of 4"); + + for (uint64_t I = 0; I < SectionSize; I += 4) { + const uint64_t EntryAddress = SectionAddress + I; + ErrorOr Offset = BC.getSignedValueAtAddress(EntryAddress, 4); + assert(Offset && "Reading valid PC-relative offset for a ksymtab entry"); + const int32_t SignedOffset = *Offset; + const uint64_t RefAddress = EntryAddress + SignedOffset; + BinaryFunction *BF = BC.getBinaryFunctionAtAddress(RefAddress); + if (!BF) + continue; + + BC.addRelocation(EntryAddress, BF->getSymbol(), Relocation::getPC32(), 0, + *Offset); + } +} + +/// Process __bug_table section. +/// This section contains information useful for kernel debugging. +/// Each entry in the section is a struct bug_entry that contains a pointer to +/// the ud2 instruction corresponding to the bug, corresponding file name (both +/// pointers use PC relative offset addressing), line number, and flags. +/// The definition of the struct bug_entry can be found in +/// `include/asm-generic/bug.h` +void LinuxKernelRewriter::processLKBugTable() { + ErrorOr SectionOrError = + BC.getUniqueSectionByName("__bug_table"); + if (!SectionOrError) + return; + + const uint64_t SectionSize = SectionOrError->getSize(); + const uint64_t SectionAddress = SectionOrError->getAddress(); + assert((SectionSize % 12) == 0 && + "The size of the __bug_table section should be a multiple of 12"); + for (uint64_t I = 0; I < SectionSize; I += 12) { + const uint64_t EntryAddress = SectionAddress + I; + ErrorOr Offset = BC.getSignedValueAtAddress(EntryAddress, 4); + assert(Offset && + "Reading valid PC-relative offset for a __bug_table entry"); + const int32_t SignedOffset = *Offset; + const uint64_t RefAddress = EntryAddress + SignedOffset; + assert(BC.getBinaryFunctionContainingAddress(RefAddress) && + "__bug_table entries should point to a function"); + + insertLKMarker(RefAddress, I, SignedOffset, true, "__bug_table"); + } +} + +/// .smp_locks section contains PC-relative references to instructions with LOCK +/// prefix. The prefix can be converted to NOP at boot time on non-SMP systems. +void LinuxKernelRewriter::processLKSMPLocks() { + ErrorOr SectionOrError = + BC.getUniqueSectionByName(".smp_locks"); + if (!SectionOrError) + return; + + uint64_t SectionSize = SectionOrError->getSize(); + const uint64_t SectionAddress = SectionOrError->getAddress(); + assert((SectionSize % 4) == 0 && + "The size of the .smp_locks section should be a multiple of 4"); + + for (uint64_t I = 0; I < SectionSize; I += 4) { + const uint64_t EntryAddress = SectionAddress + I; + ErrorOr Offset = BC.getSignedValueAtAddress(EntryAddress, 4); + assert(Offset && "Reading valid PC-relative offset for a .smp_locks entry"); + int32_t SignedOffset = *Offset; + uint64_t RefAddress = EntryAddress + SignedOffset; + + BinaryFunction *ContainingBF = + BC.getBinaryFunctionContainingAddress(RefAddress); + if (!ContainingBF) + continue; + + insertLKMarker(RefAddress, I, SignedOffset, true, ".smp_locks"); + } +} + +void LinuxKernelRewriter::updateLKMarkers() { + if (LKMarkers.size() == 0) + return; + + std::unordered_map PatchCounts; + for (std::pair> + &LKMarkerInfoKV : LKMarkers) { + const uint64_t OriginalAddress = LKMarkerInfoKV.first; + const BinaryFunction *BF = + BC.getBinaryFunctionContainingAddress(OriginalAddress, false, true); + if (!BF) + continue; + + uint64_t NewAddress = BF->translateInputToOutputAddress(OriginalAddress); + if (NewAddress == 0) + continue; + + // Apply base address. + if (OriginalAddress >= 0xffffffff00000000 && NewAddress < 0xffffffff) + NewAddress = NewAddress + 0xffffffff00000000; + + if (OriginalAddress == NewAddress) + continue; + + for (LKInstructionMarkerInfo &LKMarkerInfo : LKMarkerInfoKV.second) { + StringRef SectionName = LKMarkerInfo.SectionName; + SimpleBinaryPatcher *LKPatcher; + ErrorOr BSec = BC.getUniqueSectionByName(SectionName); + assert(BSec && "missing section info for kernel section"); + if (!BSec->getPatcher()) + BSec->registerPatcher(std::make_unique()); + LKPatcher = static_cast(BSec->getPatcher()); + PatchCounts[std::string(SectionName)]++; + if (LKMarkerInfo.IsPCRelative) + LKPatcher->addLE32Patch(LKMarkerInfo.SectionOffset, + NewAddress - OriginalAddress + + LKMarkerInfo.PCRelativeOffset); + else + LKPatcher->addLE64Patch(LKMarkerInfo.SectionOffset, NewAddress); + } + } + outs() << "BOLT-INFO: patching linux kernel sections. Total patches per " + "section are as follows:\n"; + for (const std::pair &KV : PatchCounts) + outs() << " Section: " << KV.first << ", patch-counts: " << KV.second + << '\n'; +} +} // namespace + +std::unique_ptr +llvm::bolt::createLinuxKernelRewriter(BinaryContext &BC) { + return std::make_unique(BC); +} diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -1266,13 +1266,11 @@ } } - if (opts::LinuxKernelMode) { - // Read all special linux kernel sections and their relocations - processLKSections(); - } else { + if (!opts::LinuxKernelMode) { // Read all relocations now that we have binary functions mapped. processRelocations(); } + registerFragments(); } @@ -2134,211 +2132,6 @@ << " relocations\n"; } -void RewriteInstance::insertLKMarker(uint64_t PC, uint64_t SectionOffset, - int32_t PCRelativeOffset, - bool IsPCRelative, StringRef SectionName) { - BC->LKMarkers[PC].emplace_back(LKInstructionMarkerInfo{ - SectionOffset, PCRelativeOffset, IsPCRelative, SectionName}); -} - -void RewriteInstance::processLKSections() { - assert(opts::LinuxKernelMode && - "process Linux Kernel special sections and their relocations only in " - "linux kernel mode.\n"); - - processLKExTable(); - processLKPCIFixup(); - processLKKSymtab(); - processLKKSymtab(true); - processLKBugTable(); - processLKSMPLocks(); -} - -/// Process __ex_table section of Linux Kernel. -/// This section contains information regarding kernel level exception -/// handling (https://www.kernel.org/doc/html/latest/x86/exception-tables.html). -/// More documentation is in arch/x86/include/asm/extable.h. -/// -/// The section is the list of the following structures: -/// -/// struct exception_table_entry { -/// int insn; -/// int fixup; -/// int handler; -/// }; -/// -void RewriteInstance::processLKExTable() { - ErrorOr SectionOrError = - BC->getUniqueSectionByName("__ex_table"); - if (!SectionOrError) - return; - - const uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 12) == 0 && - "The size of the __ex_table section should be a multiple of 12"); - for (uint64_t I = 0; I < SectionSize; I += 4) { - const uint64_t EntryAddress = SectionAddress + I; - ErrorOr Offset = BC->getSignedValueAtAddress(EntryAddress, 4); - assert(Offset && "failed reading PC-relative offset for __ex_table"); - int32_t SignedOffset = *Offset; - const uint64_t RefAddress = EntryAddress + SignedOffset; - - BinaryFunction *ContainingBF = - BC->getBinaryFunctionContainingAddress(RefAddress); - if (!ContainingBF) - continue; - - MCSymbol *ReferencedSymbol = ContainingBF->getSymbol(); - const uint64_t FunctionOffset = RefAddress - ContainingBF->getAddress(); - switch (I % 12) { - default: - llvm_unreachable("bad alignment of __ex_table"); - break; - case 0: - // insn - insertLKMarker(RefAddress, I, SignedOffset, true, "__ex_table"); - break; - case 4: - // fixup - if (FunctionOffset) - ReferencedSymbol = ContainingBF->addEntryPointAtOffset(FunctionOffset); - BC->addRelocation(EntryAddress, ReferencedSymbol, Relocation::getPC32(), - 0, *Offset); - break; - case 8: - // handler - assert(!FunctionOffset && - "__ex_table handler entry should point to function start"); - BC->addRelocation(EntryAddress, ReferencedSymbol, Relocation::getPC32(), - 0, *Offset); - break; - } - } -} - -/// Process .pci_fixup section of Linux Kernel. -/// This section contains a list of entries for different PCI devices and their -/// corresponding hook handler (code pointer where the fixup -/// code resides, usually on x86_64 it is an entry PC relative 32 bit offset). -/// Documentation is in include/linux/pci.h. -void RewriteInstance::processLKPCIFixup() { - ErrorOr SectionOrError = - BC->getUniqueSectionByName(".pci_fixup"); - assert(SectionOrError && - ".pci_fixup section not found in Linux Kernel binary"); - const uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 16) == 0 && ".pci_fixup size is not a multiple of 16"); - - for (uint64_t I = 12; I + 4 <= SectionSize; I += 16) { - const uint64_t PC = SectionAddress + I; - ErrorOr Offset = BC->getSignedValueAtAddress(PC, 4); - assert(Offset && "cannot read value from .pci_fixup"); - const int32_t SignedOffset = *Offset; - const uint64_t HookupAddress = PC + SignedOffset; - BinaryFunction *HookupFunction = - BC->getBinaryFunctionAtAddress(HookupAddress); - assert(HookupFunction && "expected function for entry in .pci_fixup"); - BC->addRelocation(PC, HookupFunction->getSymbol(), Relocation::getPC32(), 0, - *Offset); - } -} - -/// Process __ksymtab[_gpl] sections of Linux Kernel. -/// This section lists all the vmlinux symbols that kernel modules can access. -/// -/// All the entries are 4 bytes each and hence we can read them by one by one -/// and ignore the ones that are not pointing to the .text section. All pointers -/// are PC relative offsets. Always, points to the beginning of the function. -void RewriteInstance::processLKKSymtab(bool IsGPL) { - StringRef SectionName = "__ksymtab"; - if (IsGPL) - SectionName = "__ksymtab_gpl"; - ErrorOr SectionOrError = - BC->getUniqueSectionByName(SectionName); - assert(SectionOrError && - "__ksymtab[_gpl] section not found in Linux Kernel binary"); - const uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 4) == 0 && - "The size of the __ksymtab[_gpl] section should be a multiple of 4"); - - for (uint64_t I = 0; I < SectionSize; I += 4) { - const uint64_t EntryAddress = SectionAddress + I; - ErrorOr Offset = BC->getSignedValueAtAddress(EntryAddress, 4); - assert(Offset && "Reading valid PC-relative offset for a ksymtab entry"); - const int32_t SignedOffset = *Offset; - const uint64_t RefAddress = EntryAddress + SignedOffset; - BinaryFunction *BF = BC->getBinaryFunctionAtAddress(RefAddress); - if (!BF) - continue; - - BC->addRelocation(EntryAddress, BF->getSymbol(), Relocation::getPC32(), 0, - *Offset); - } -} - -/// Process __bug_table section. -/// This section contains information useful for kernel debugging. -/// Each entry in the section is a struct bug_entry that contains a pointer to -/// the ud2 instruction corresponding to the bug, corresponding file name (both -/// pointers use PC relative offset addressing), line number, and flags. -/// The definition of the struct bug_entry can be found in -/// `include/asm-generic/bug.h` -void RewriteInstance::processLKBugTable() { - ErrorOr SectionOrError = - BC->getUniqueSectionByName("__bug_table"); - if (!SectionOrError) - return; - - const uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 12) == 0 && - "The size of the __bug_table section should be a multiple of 12"); - for (uint64_t I = 0; I < SectionSize; I += 12) { - const uint64_t EntryAddress = SectionAddress + I; - ErrorOr Offset = BC->getSignedValueAtAddress(EntryAddress, 4); - assert(Offset && - "Reading valid PC-relative offset for a __bug_table entry"); - const int32_t SignedOffset = *Offset; - const uint64_t RefAddress = EntryAddress + SignedOffset; - assert(BC->getBinaryFunctionContainingAddress(RefAddress) && - "__bug_table entries should point to a function"); - - insertLKMarker(RefAddress, I, SignedOffset, true, "__bug_table"); - } -} - -/// .smp_locks section contains PC-relative references to instructions with LOCK -/// prefix. The prefix can be converted to NOP at boot time on non-SMP systems. -void RewriteInstance::processLKSMPLocks() { - ErrorOr SectionOrError = - BC->getUniqueSectionByName(".smp_locks"); - if (!SectionOrError) - return; - - uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 4) == 0 && - "The size of the .smp_locks section should be a multiple of 4"); - - for (uint64_t I = 0; I < SectionSize; I += 4) { - const uint64_t EntryAddress = SectionAddress + I; - ErrorOr Offset = BC->getSignedValueAtAddress(EntryAddress, 4); - assert(Offset && "Reading valid PC-relative offset for a .smp_locks entry"); - int32_t SignedOffset = *Offset; - uint64_t RefAddress = EntryAddress + SignedOffset; - - BinaryFunction *ContainingBF = - BC->getBinaryFunctionContainingAddress(RefAddress); - if (!ContainingBF) - continue; - - insertLKMarker(RefAddress, I, SignedOffset, true, ".smp_locks"); - } -} - void RewriteInstance::readDynamicRelocations(const SectionRef &Section, bool IsJmpRel) { assert(BinarySection(*BC, Section).isAllocatable() && "allocatable expected"); @@ -3104,6 +2897,9 @@ } void RewriteInstance::initializeRewriters() { + if (opts::LinuxKernelMode) + RewriteManager.registerRewriter(createLinuxKernelRewriter(*BC)); + RewriteManager.registerRewriter(createPseudoProbeRewriter(*BC)); RewriteManager.registerRewriter(createSDTRewriter(*BC)); @@ -3471,9 +3267,6 @@ void RewriteInstance::updateMetadata() { RewriteManager.runFinalizersAfterEmit(); - // TODO: use RewriteManager for updates. - updateLKMarkers(); - if (opts::UpdateDebugSections) { NamedRegionTimer T("updateDebugInfo", "update debug info", TimerGroupName, TimerGroupDesc, opts::TimeRewrite); @@ -3484,57 +3277,6 @@ addBoltInfoSection(); } -void RewriteInstance::updateLKMarkers() { - if (BC->LKMarkers.size() == 0) - return; - - NamedRegionTimer T("updateLKMarkers", "update LK markers", TimerGroupName, - TimerGroupDesc, opts::TimeRewrite); - - std::unordered_map PatchCounts; - for (std::pair> - &LKMarkerInfoKV : BC->LKMarkers) { - const uint64_t OriginalAddress = LKMarkerInfoKV.first; - const BinaryFunction *BF = - BC->getBinaryFunctionContainingAddress(OriginalAddress, false, true); - if (!BF) - continue; - - uint64_t NewAddress = BF->translateInputToOutputAddress(OriginalAddress); - if (NewAddress == 0) - continue; - - // Apply base address. - if (OriginalAddress >= 0xffffffff00000000 && NewAddress < 0xffffffff) - NewAddress = NewAddress + 0xffffffff00000000; - - if (OriginalAddress == NewAddress) - continue; - - for (LKInstructionMarkerInfo &LKMarkerInfo : LKMarkerInfoKV.second) { - StringRef SectionName = LKMarkerInfo.SectionName; - SimpleBinaryPatcher *LKPatcher; - ErrorOr BSec = BC->getUniqueSectionByName(SectionName); - assert(BSec && "missing section info for kernel section"); - if (!BSec->getPatcher()) - BSec->registerPatcher(std::make_unique()); - LKPatcher = static_cast(BSec->getPatcher()); - PatchCounts[std::string(SectionName)]++; - if (LKMarkerInfo.IsPCRelative) - LKPatcher->addLE32Patch(LKMarkerInfo.SectionOffset, - NewAddress - OriginalAddress + - LKMarkerInfo.PCRelativeOffset); - else - LKPatcher->addLE64Patch(LKMarkerInfo.SectionOffset, NewAddress); - } - } - outs() << "BOLT-INFO: patching linux kernel sections. Total patches per " - "section are as follows:\n"; - for (const std::pair &KV : PatchCounts) - outs() << " Section: " << KV.first << ", patch-counts: " << KV.second - << '\n'; -} - void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) { BC->deregisterUnusedSections();