diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -526,6 +526,10 @@ /// accordingly. void adjustCodePadding(); + /// Verify unclaimed Data to Code relocations, creating secondary entry points + /// for targets. + void postProcessDataRelocations(); + /// Regular page size. unsigned RegularPageSize{0x1000}; static constexpr unsigned RegularPageSizeX86 = 0x1000; diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -991,6 +991,74 @@ } } +void BinaryContext::postProcessDataRelocations() { + if (!HasRelocations) + return; + + std::unordered_set Functions; + std::unordered_set EntryPoints; + + for (BinarySection &Section : sections()) { + const uint64_t SourceSectionStart = Section.getAddress(); + for (const Relocation &Rel : Section.relocations()) { + LLVM_DEBUG({ + dbgs() << "BOLT-DEBUG: post-processing relocation "; + Rel.print(dbgs()); + dbgs() << '\n'; + }); + // PC-relative relocations have separate checking mechanism (strict mode) + if (Relocation::isPCRelative(Rel.Type)) + continue; + + // We expect relocations against function local labels, which have + // Addend == 0. + if (Rel.Addend) + continue; + + const MCSymbol *ReferencedSymbol = Rel.Symbol; + // Check if the relocation value is a function entry point + if (getFunctionForSymbol(ReferencedSymbol)) + continue; + + // Check if the relocation value is a recognized jump table target + const uint64_t RelValue = Rel.Value; + const uint64_t RelSourceAddress = SourceSectionStart + Rel.Offset; + BinaryFunction *BF = getBinaryFunctionContainingAddress( + RelValue, /* CheckPastEnd */ false, /* UseMaxSize */ false); + + // If there's no function at RelValue address, no need to register the + // target symbol as an entry point - i.e. checking past function end. + if (!BF) + continue; + + // Skip non-simple functions + if (!BF->isSimple()) + continue; + + const JumpTable *JT = BF->getJumpTableContainingAddress(RelSourceAddress); + if (JT && JT->Labels.find(RelValue) != JT->Labels.end()) + continue; + + // Otherwise register the relocation target as a secondary entry point + const uint64_t Offset = RelValue - BF->getAddress(); + const MCSymbol *Label = BF->addEntryPointAtOffset(Offset); + + Functions.insert(BF); + EntryPoints.insert(Label); + LLVM_DEBUG({ + dbgs() << "BOLT-DEBUG: registered label " << Label->getName() + << " as a secondary entry point in function " << *BF << '\n'; + }); + BF->setSimple(false); + } + } + if (opts::Verbosity >= 1) { + outs() << "BOLT-INFO: created " << EntryPoints.size() << " secondary entry " + << "points in " << Functions.size() << " functions for unclaimed " + << "data to code relocations.\n"; + } +} + MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, uint64_t Address, uint64_t Size, uint16_t Alignment, diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -2868,6 +2868,7 @@ BC->clearJumpTableTempData(); BC->adjustCodePadding(); + BC->postProcessDataRelocations(); for (auto &BFI : BC->getBinaryFunctions()) { BinaryFunction &Function = BFI.second;