diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -480,6 +480,10 @@ uint64_t Address, JumpTable::JumpTableType Type); + bool validJumpTableEntry(uint64_t EntryAddress, BinaryFunction *BF, + BinaryFunction *&TargetBF, + BinaryFunction *ValidExternalTargetBF); + /// Analyze a possible jump table of type \p Type at a given \p Address. /// \p BF is a function referencing the jump table. /// Return true if the jump table was detected at \p Address, and false @@ -585,6 +589,9 @@ /// Indicates if relocations are available for usage. bool HasRelocations{false}; + /// Indicates if the binary is stripped + bool IsStripped{false}; + /// Is the binary always loaded at a fixed address. Shared objects and /// position-independent executables (PIEs) are examples of binaries that /// will have HasFixedLoadAddress set to false. diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -2277,7 +2277,11 @@ } /// Process LSDA information for the function. - void parseLSDA(ArrayRef LSDAData, uint64_t LSDAAddress); + /// Run in two phases: + /// (a) Validate landing pad targets --> CheckOnly + /// (b) Skip validation, process LSDA only --> !CheckOnly + void parseLSDA(ArrayRef LSDAData, uint64_t LSDAAddress, + bool CheckOnly); /// Update exception handling ranges for the function. void updateEHRanges(); diff --git a/bolt/include/bolt/Core/Exceptions.h b/bolt/include/bolt/Core/Exceptions.h --- a/bolt/include/bolt/Core/Exceptions.h +++ b/bolt/include/bolt/Core/Exceptions.h @@ -38,6 +38,7 @@ public: explicit CFIReaderWriter(const DWARFDebugFrame &EHFrame); + void extractLSDAAddress(BinaryFunction &Function) const; bool fillCFIInfoFor(BinaryFunction &Function) const; /// Generate .eh_frame_hdr from old and new .eh_frame sections. diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -46,7 +46,6 @@ #define DEBUG_TYPE "bolt" namespace opts { - cl::opt NoHugePages("no-huge-pages", cl::desc("use regular size pages for code alignment"), cl::Hidden, cl::cat(BoltCategory)); @@ -419,8 +418,15 @@ } } + // IMPORTANT: + // creating jump table without BranchType == POSSIBLE_PIC_JUMP_TABLE + // is dangerous --> False Positive. + // --> Should only be done in BF.processIndirectBranch() + // --> Temporarily keep this code to avoid failing tests + // Original purpose: // With relocations, catch jump table references outside of the basic block // containing the indirect jump. + // --> There are binaries where all references are fake jump table. if (HasRelocations) { const MemoryContentsType MemType = analyzeMemoryAt(Address, BF); if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE && IsPCRel) { @@ -490,6 +496,65 @@ return false; } +bool BinaryContext::validJumpTableEntry(uint64_t EntryAddress, + BinaryFunction *BF, + BinaryFunction *&TargetBF, + BinaryFunction *ValidExternalTargetBF) { + TargetBF = getBinaryFunctionContainingAddress(EntryAddress); + // Property 1: Entry must target a valid function body + if (!TargetBF) + return false; + + // Property 2: Entry cannot target invalid instruction bounds + if (TargetBF->getState() == BinaryFunction::State::Disassembled && + !TargetBF->getInstructionAtOffset(EntryAddress - + TargetBF->getAddress())) { + LLVM_DEBUG(dbgs() << "FAIL: no instruction at this offset\n"); + return false; + } + + // Property 3: Entry cannot target more than 1 other-fragment + if (IsStripped && TargetBF != BF) { + // First time, the different fragment target can be any fragment + if (ValidExternalTargetBF == nullptr) { + ValidExternalTargetBF = TargetBF; + TargetBF->IsFragment = true; + return registerFragment(*TargetBF, *BF); + } + // Second time, the different fragment target must be similar + return TargetBF == ValidExternalTargetBF; + } + + // Property 4: EntryAddress cannot target a callable function --> Pending! + // (a) Previously: not allow jump table targeting current function entry + // --> Due to the assumption that function entry is always callable target + // --> The assumption is not realistic, temporarily relax this property + // IMPORTANT: + // (b) In future: + // --> Implement callable function identification + // --> EntryAddress cannot point to function entry + // --> EntryAddress can point to a sibling fragment entry + if (TargetBF == BF) + return EntryAddress != BF->getAddress(); + + // Property 5: Non-overlapping jump tables + // This property was already enforced in calculating UpperBound + // Relax policy for stripped binaries by accept all entries at this point + // --> No information about parent-fragment relation + if (IsStripped) { + TargetBF->IsFragment = true; + return registerFragment(*TargetBF, *BF); + } + + // Property 6: Perform further check for non-stripped binaries + // BF and TargetBF have parent-fragment relation + // (a) TargetBF is parent fragment + if (BF->isFragment()) + return registerFragment(*BF, *TargetBF); + // (b) BF is parent fragment + return TargetBF->isFragment() && registerFragment(*TargetBF, *BF); +} + bool BinaryContext::analyzeJumpTable( const uint64_t Address, const JumpTable::JumpTableType Type, BinaryFunction &BF, const uint64_t NextJTAddress, @@ -499,29 +564,13 @@ // Number of targets other than __builtin_unreachable. uint64_t NumRealEntries = 0; + BinaryFunction *ValidExternalTargetBF = nullptr; auto addEntryAddress = [&](uint64_t EntryAddress) { if (EntriesAsAddress) EntriesAsAddress->emplace_back(EntryAddress); }; - auto doesBelongToFunction = [&](const uint64_t Addr, - BinaryFunction *TargetBF) -> bool { - if (BF.containsAddress(Addr)) - return true; - // Nothing to do if we failed to identify the containing function. - if (!TargetBF) - return false; - // Case 1: check if BF is a fragment and TargetBF is its parent. - if (BF.isFragment()) { - // Parent function may or may not be already registered. - // Set parent link based on function name matching heuristic. - return registerFragment(BF, *TargetBF); - } - // Case 2: check if TargetBF is a fragment and BF is its parent. - return TargetBF->isFragment() && registerFragment(*TargetBF, BF); - }; - ErrorOr Section = getSectionForAddress(Address); if (!Section) return false; @@ -574,38 +623,31 @@ continue; } - // Function or one of its fragments. BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value); - // We assume that a jump table cannot have function start as an entry. - if (!doesBelongToFunction(Value, TargetBF) || Value == BF.getAddress()) { + // Verify if Value is a valid jump table entry + if (!validJumpTableEntry(Value, &BF, TargetBF, ValidExternalTargetBF)) { LLVM_DEBUG({ - if (!BF.containsAddress(Value)) { - dbgs() << "FAIL: function doesn't contain this address\n"; - if (TargetBF) { - dbgs() << " ! function containing this address: " - << TargetBF->getPrintName() << '\n'; - if (TargetBF->isFragment()) - dbgs() << " ! is a fragment\n"; - for (BinaryFunction *TargetParent : TargetBF->ParentFragments) - dbgs() << " ! its parent is " - << (TargetParent ? TargetParent->getPrintName() : "(none)") - << '\n'; - } - } if (Value == BF.getAddress()) dbgs() << "FAIL: jump table cannot have function start as an entry\n"; + else if (TargetBF == nullptr) + dbgs() << "FAIL: jump table targets are not contained in a valid " + << "function\n"; + else { + dbgs() << "FAIL: jump table are invalid:\n"; + dbgs() << " ! current function: " << BF.getPrintName() << "\n"; + dbgs() << " ! target function: " << TargetBF->getPrintName() << "\n"; + if (TargetBF->isFragment()) + dbgs() << " !! is a fragment\n"; + for (BinaryFunction *TargetParent : TargetBF->ParentFragments) + dbgs() << " !! its parent is " + << (TargetParent ? TargetParent->getPrintName() : "(none)") + << '\n'; + } }); break; } - // Check there's an instruction at this offset. - if (TargetBF->getState() == BinaryFunction::State::Disassembled && - !TargetBF->getInstructionAtOffset(Value - TargetBF->getAddress())) { - LLVM_DEBUG(dbgs() << "FAIL: no instruction at this offset\n"); - break; - } - ++NumRealEntries; if (TargetBF != &BF) @@ -1119,14 +1161,19 @@ bool BinaryContext::registerFragment(BinaryFunction &TargetFunction, BinaryFunction &Function) const { - if (!isPotentialFragmentByName(TargetFunction, Function)) + // Only use symbol name matching for non-stripped binaries + if (!IsStripped && !isPotentialFragmentByName(TargetFunction, Function)) return false; assert(TargetFunction.isFragment() && "TargetFunction must be a fragment"); if (TargetFunction.isParentFragment(&Function)) return true; TargetFunction.addParentFragment(Function); Function.addFragment(TargetFunction); - if (!HasRelocations) { + // When strip -s, target binaries don't have .rela.text + // Without .rela.text, BOLT cannot reliably move code + // --> BOLT could misidentify some other accesses to the code + // --> We relax this conservative approach for stripped binaries + if (!IsStripped && !HasRelocations) { TargetFunction.setSimple(false); Function.setSimple(false); } diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -790,6 +790,17 @@ } } + // IMPORTANT: analyzeIndirectBranch does not analyze PIC jump table pattern + // that cross fragments + // --> failed to recognize BranchType + // --> miscount jump table parents + // Solution: + // 1. Lift all instructions in every instruction to MCInst + // 2. Build map : (Branch -> Dest) from all direct branches + // 3. Identify indirect BranchType with respect to CFG + // 4. Create jump tables + // 5. Update map with new indirect branches, including jump tables + // 6. If some unclaimed PC-relative relocations in data, repeat from 3->5 IndirectBranchType BranchType = BC.MIB->analyzeIndirectBranch( Instruction, Begin, Instructions.end(), PtrSize, MemLocInstr, BaseRegNum, IndexRegNum, DispValue, DispExpr, PCRelBaseInstr); @@ -1602,10 +1613,11 @@ if (!getSecondaryEntryPointSymbol(Label)) continue; - // In non-relocation mode there's potentially an external undetectable - // reference to the entry point and hence we cannot move this entry - // point. Optimizing without moving could be difficult. - if (!BC.HasRelocations) + // When strip -s, target binaries don't have .rela.text + // Without .rela.text, BOLT cannot reliably move code + // --> Due to potentially external undetectable to the code + // --> We relax this conservative approach for stripped binaries + if (!BC.IsStripped && !BC.HasRelocations) setSimple(false); const uint32_t Offset = KV.first; @@ -1671,6 +1683,11 @@ TargetBF->getAddress()); JT.Entries.push_back(Label); } + // If jump table entry is fragment entry, the label already exists + else { + MCSymbol *Label = TargetBF->getLabels().at(0); + JT.Entries.push_back(Label); + } } } @@ -1883,8 +1900,12 @@ if (!EHInfo || !EHInfo->first) continue; - BinaryBasicBlock *LPBlock = getBasicBlockForLabel(EHInfo->first); - if (!BBLandingPads.count(LPBlock)) { + // For cross-fragment LPs, LPBlock = nullptr if LPFunc not yet buildCFG + // Solution: after buildCFG for all functions, rerun recomputeLandingPads + const MCSymbol *LPLabel = EHInfo->first; + BinaryFunction *LPFunc = BC.getFunctionForSymbol(LPLabel); + BinaryBasicBlock *LPBlock = LPFunc->getBasicBlockForLabel(LPLabel); + if (LPBlock != nullptr && !BBLandingPads.count(LPBlock)) { BBLandingPads.insert(LPBlock); BB->LandingPads.emplace_back(LPBlock); LPBlock->Throwers.emplace_back(BB); diff --git a/bolt/lib/Core/Exceptions.cpp b/bolt/lib/Core/Exceptions.cpp --- a/bolt/lib/Core/Exceptions.cpp +++ b/bolt/lib/Core/Exceptions.cpp @@ -99,7 +99,7 @@ // // Note: some functions have LSDA entries with 0 call site entries. void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, - uint64_t LSDASectionAddress) { + uint64_t LSDASectionAddress, bool CheckOnly) { assert(CurrentState == State::Disassembled && "unexpected function state"); if (!getLSDAAddress()) @@ -128,7 +128,7 @@ TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding); } - if (opts::PrintExceptions) { + if (opts::PrintExceptions && !CheckOnly) { outs() << "[LSDA at 0x" << Twine::utohexstr(getLSDAAddress()) << " for function " << *this << "]:\n"; outs() << "LPStart Encoding = 0x" << Twine::utohexstr(LPStartEncoding) @@ -158,7 +158,7 @@ uint64_t CallSitePtr = CallSiteTableStart; uint64_t ActionTableStart = CallSiteTableEnd; - if (opts::PrintExceptions) { + if (opts::PrintExceptions && !CheckOnly) { outs() << "CallSite Encoding = " << (unsigned)CallSiteEncoding << '\n'; outs() << "CallSite table length = " << CallSiteTableLength << '\n'; outs() << '\n'; @@ -177,56 +177,90 @@ uint64_t LPOffset = LPStart + LandingPad; uint64_t LPAddress = Address + LPOffset; + BinaryFunction *Fragment = BC.getBinaryFunctionContainingAddress(LPAddress); + // --------------------- Validate split landing pad --------------------- // Verify if landing pad code is located outside current function - // Support landing pad to builtin_unreachable - if (LPAddress < Address || LPAddress > Address + getSize()) { - BinaryFunction *Fragment = - BC.getBinaryFunctionContainingAddress(LPAddress); + // Otherwise, establish parent-fragment relation + // Skip case where landing pad targets builtin_unreachable + if ((LPAddress < Address || LPAddress > Address + getSize()) && CheckOnly) { + // Assume landing pad not target another fragment's builtin_unreachable + // If this assumption is violated, must run a global check first assert(Fragment != nullptr && "BOLT-ERROR: cannot find landing pad fragment"); + + // Update IsFragment: + // In stripped mode, always trust LSDA, consider the function that + // contains LP as a fragment + // In non-stripped mode, use pattern matching (adjustFunctionBoundaries) + if (BC.IsStripped) + Fragment->IsFragment = true; + + // Update parent-fragment relation, add Fragment as secondary entry of + // the current function, not an independent function BC.addInterproceduralReference(this, Fragment->getAddress()); BC.processInterproceduralReferences(); - auto isFragmentOf = [](BinaryFunction *Fragment, - BinaryFunction *Parent) -> bool { - return (Fragment->isFragment() && Fragment->isParentFragment(Parent)); - }; - assert((isFragmentOf(this, Fragment) || isFragmentOf(Fragment, this)) && - "BOLT-ERROR: cannot have landing pads in different " - "functions"); + + // In stripped mode, parent-fragment is always established --> skip check + // In non-stripped mode, parent-fragment depends on symbol name --> check + if (!BC.IsStripped) { + auto isFragmentOf = [](BinaryFunction *Fragment, + BinaryFunction *Parent) -> bool { + return (Fragment->isFragment() && Fragment->isParentFragment(Parent)); + }; + assert((isFragmentOf(this, Fragment) || isFragmentOf(Fragment, this)) && + "BOLT-ERROR: cannot have landing pads in different functions"); + } + // Mark that this fragment reaches LP in another fragment of same function setHasIndirectTargetToSplitFragment(true); BC.addFragmentsToSkip(this); - return; } - if (opts::PrintExceptions) { - outs() << "Call Site: [0x" << Twine::utohexstr(RangeBase + Start) - << ", 0x" << Twine::utohexstr(RangeBase + Start + Length) - << "); landing pad: 0x" << Twine::utohexstr(LPOffset) - << "; action entry: 0x" << Twine::utohexstr(ActionEntry) << "\n"; - outs() << " current offset is " << (CallSitePtr - CallSiteTableStart) - << '\n'; - } + if (CheckOnly) + continue; + // ---------------------------------------------------------------------- // Create a handler entry if necessary. - MCSymbol *LPSymbol = nullptr; - if (LPOffset) { - if (!getInstructionAtOffset(LPOffset)) { + MCSymbol *LPLabel = nullptr; + + // Special case, consider builtin_unreachable as part of this function + if (LPAddress == Address + getSize()) + Fragment = this; + + // Assumption: landing pad cannot target current fragment entry + // Note: landing pad can target other fragment entry -> split landing pad + if (LPAddress != Address) { + uint64_t FragmentOffset = LPAddress - Fragment->getAddress(); + if (!Fragment->getInstructionAtOffset(FragmentOffset)) { if (opts::Verbosity >= 1) errs() << "BOLT-WARNING: landing pad " << Twine::utohexstr(LPOffset) - << " not pointing to an instruction in function " << *this + << " not pointing to an instruction in function " << *Fragment << " - ignoring.\n"; } else { - auto Label = Labels.find(LPOffset); - if (Label != Labels.end()) { - LPSymbol = Label->second; - } else { - LPSymbol = BC.Ctx->createNamedTempSymbol("LP"); - Labels[LPOffset] = LPSymbol; - } + // Treat split landing pad as the fragment's secondary fragment + auto Label = Fragment->Labels.find(FragmentOffset); + LPLabel = (Label != Fragment->Labels.end()) + ? Label->second + : ((Fragment != this) + ? Fragment->addEntryPointAtOffset(FragmentOffset) + : BC.Ctx->createNamedTempSymbol("LP")); + // Support recomputeLandingPad to identify split landing pad + BC.setSymbolToFunctionMap(LPLabel, Fragment); + Labels[LPOffset] = LPLabel; } } + if (opts::PrintExceptions) { + outs() << "Call Site: [0x" << Twine::utohexstr(RangeBase + Start) + << ", 0x" << Twine::utohexstr(RangeBase + Start + Length) + << "); landing pad: 0x" << Twine::utohexstr(LPOffset) + << "; action entry: 0x" << Twine::utohexstr(ActionEntry) << "\n"; + outs() << " current offset is " << (CallSitePtr - CallSiteTableStart) + << '\n'; + if (LPLabel != nullptr) + outs() << " landing pad label: " << LPLabel->getName() << "\n"; + } + // Mark all call instructions in the range. auto II = Instructions.find(Start); auto IE = Instructions.end(); @@ -240,7 +274,7 @@ // Add extra operands to a call instruction making it an invoke from // now on. BC.MIB->addEHInfo(Instruction, - MCPlus::MCLandingPad(LPSymbol, ActionEntry)); + MCPlus::MCLandingPad(LPLabel, ActionEntry)); } ++II; } while (II != IE && II->first < Start + Length); @@ -506,16 +540,26 @@ } } -bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const { +void CFIReaderWriter::extractLSDAAddress(BinaryFunction &Function) const { uint64_t Address = Function.getAddress(); auto I = FDEs.find(Address); // Ignore zero-length FDE ranges. if (I == FDEs.end() || !I->second->getAddressRange()) - return true; + return; const FDE &CurFDE = *I->second; Optional LSDA = CurFDE.getLSDAAddress(); Function.setLSDAAddress(LSDA ? *LSDA : 0); +} + +bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const { + uint64_t Address = Function.getAddress(); + auto I = FDEs.find(Address); + // Ignore zero-length FDE ranges. + if (I == FDEs.end() || !I->second->getAddressRange()) + return true; + + const FDE &CurFDE = *I->second; uint64_t Offset = Function.getFirstInstructionOffset(); uint64_t CodeAlignment = CurFDE.getLinkedCIE()->getCodeAlignmentFactor(); diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp --- a/bolt/lib/Passes/SplitFunctions.cpp +++ b/bolt/lib/Passes/SplitFunctions.cpp @@ -333,8 +333,11 @@ if (!EHInfo || !EHInfo->first) continue; + // In case of split landing pad, LPFunc != BF const MCSymbol *LPLabel = EHInfo->first; - BinaryBasicBlock *LPBlock = BF.getBasicBlockForLabel(LPLabel); + BinaryFunction *LPFunc = + BF.getBinaryContext().getFunctionForSymbol(LPLabel); + BinaryBasicBlock *LPBlock = LPFunc->getBasicBlockForLabel(LPLabel); if (BB->isCold() == LPBlock->isCold()) continue; @@ -346,13 +349,14 @@ // Create a trampoline basic block in the same fragment as the thrower. // Note: there's no need to insert the jump instruction, it will be // added by fixBranches(). - BinaryBasicBlock *TrampolineBB = BF.addBasicBlock(); + BinaryBasicBlock *TrampolineBB = LPFunc->addBasicBlock(); TrampolineBB->setIsCold(BB->isCold()); TrampolineBB->setExecutionCount(LPBlock->getExecutionCount()); TrampolineBB->addSuccessor(LPBlock, TrampolineBB->getExecutionCount()); TrampolineBB->setCFIState(LPBlock->getCFIState()); TrampolineLabel = TrampolineBB->getLabel(); LPTrampolines.insert(std::make_pair(LPLabel, TrampolineLabel)); + BF.getBinaryContext().setSymbolToFunctionMap(TrampolineLabel, LPFunc); } // Substitute the landing pad with the trampoline. diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -92,12 +92,6 @@ "output binary via bolt info section"), cl::cat(BoltCategory)); -cl::opt -AllowStripped("allow-stripped", - cl::desc("allow processing of stripped binaries"), - cl::Hidden, - cl::cat(BoltCategory)); - cl::opt DumpDotAll( "dump-dot-all", cl::desc("dump function CFGs to graphviz format after each stage;" @@ -1557,6 +1551,7 @@ TimerGroupName, TimerGroupDesc, opts::TimeRewrite); bool HasTextRelocations = false; + bool HasSymbolTable = false; bool HasDebugInfo = false; // Process special sections. @@ -1588,6 +1583,7 @@ } HasTextRelocations = (bool)BC->getUniqueSectionByName(".rela.text"); + HasSymbolTable = (bool)BC->getUniqueSectionByName(".symtab"); LSDASection = BC->getUniqueSectionByName(".gcc_except_table"); EHFrameSection = BC->getUniqueSectionByName(".eh_frame"); GOTPLTSection = BC->getUniqueSectionByName(".got.plt"); @@ -1624,6 +1620,8 @@ BC->HasRelocations = HasTextRelocations && (opts::RelocationMode != cl::BOU_FALSE); + BC->IsStripped = !HasSymbolTable; + // Force non-relocation mode for heatmap generation if (opts::HeatmapMode) BC->HasRelocations = false; @@ -2800,13 +2798,11 @@ if (Error E = ProfileReader->preprocessProfile(*BC.get())) report_error("cannot pre-process profile", std::move(E)); - if (!BC->hasSymbolsWithFileName() && ProfileReader->hasLocalsWithFileName() && - !opts::AllowStripped) { + if (!BC->hasSymbolsWithFileName() && ProfileReader->hasLocalsWithFileName()) { errs() << "BOLT-ERROR: input binary does not have local file symbols " "but profile data includes function names with embedded file " "names. It appears that the input binary was stripped while a " - "profiled binary was not. If you know what you are doing and " - "wish to proceed, use -allow-stripped option.\n"; + "profiled binary was not.\n"; exit(1); } } @@ -2894,6 +2890,16 @@ } BC->processInterproceduralReferences(); + // Using LSDA to establish parent-fragment relation + for (auto &BFI : BC->getBinaryFunctions()) { + BinaryFunction &Function = BFI.second; + if (!Function.isSimple()) + continue; + CFIRdWrt->extractLSDAAddress(Function); + if (Function.getLSDAAddress() != 0) + Function.parseLSDA(getLSDAData(), getLSDAAddress(), true); + } + BC->populateJumpTables(); for (auto &BFI : BC->getBinaryFunctions()) { @@ -2915,9 +2921,10 @@ if (!shouldDisassemble(Function)) continue; - if (!Function.isSimple()) { - assert((!BC->HasRelocations || Function.getSize() == 0 || - Function.hasIndirectTargetToSplitFragment()) && + // If non-simple due to split jump table or split landing pad, process + // Every other non-simple cases, ignore + if (!Function.isSimple() && !Function.hasIndirectTargetToSplitFragment()) { + assert((!BC->HasRelocations || Function.getSize() == 0) && "unexpected non-simple function in relocation mode"); continue; } @@ -2935,9 +2942,8 @@ } // Parse LSDA. - if (Function.getLSDAAddress() != 0 && - !BC->getFragmentsToSkip().count(&Function)) - Function.parseLSDA(getLSDAData(), getLSDAAddress()); + if (Function.getLSDAAddress() != 0) + Function.parseLSDA(getLSDAData(), getLSDAAddress(), false); } } @@ -2971,6 +2977,14 @@ /*ForceSequential*/ opts::SequentialDisassembly || opts::PrintAll); BC->postProcessSymbolTable(); + + // buildCFG already executes recomputeLandingPads + // However, with split landing pad, target function might not have CFG yet + // --> Safely ignore split landing pad during buildCFG + // --> Rerun recomputeLandingPads after all functions have CFG + for (BinaryFunction *Function : BC->getAllBinaryFunctions()) + if (Function->getState() == BinaryFunction::State::CFG) + Function->recomputeLandingPads(); } void RewriteInstance::postProcessFunctions() { @@ -4465,6 +4479,14 @@ std::vector Symbols; auto getNewSectionIndex = [&](uint32_t OldIndex) { + // For dynamic symbol table, the section index could be wrong on the input, + // and its value is ignored by the runtime if it's different from + // SHN_UNDEF and SHN_ABS. + // However, we still need to update dynamic symbol table, so return a + // section index, even though the index is broken. + if (IsDynSym && OldIndex >= NewSectionIndex.size()) + return OldIndex; + assert(OldIndex < NewSectionIndex.size() && "section index out of bounds"); const uint32_t NewIndex = NewSectionIndex[OldIndex]; diff --git a/bolt/test/X86/false-jump-table.s b/bolt/test/X86/false-jump-table.s --- a/bolt/test/X86/false-jump-table.s +++ b/bolt/test/X86/false-jump-table.s @@ -1,5 +1,10 @@ -# Check that jump table detection does not fail on a false -# reference to a jump table. +# There are two conditions of a valid jump table +# (a) MemType is POSSIBLE_JUMP_TABLE +# (b) BranchType is POSSIBLE_JUMP_TABLE +# Only then, in principle, a jump table will be created/accessed. + +# This test checks if a jump table is created based solely on MemType. +# This test may need revision in future. # REQUIRES: system-linux diff --git a/bolt/test/X86/jump-table-move-pic.s b/bolt/test/X86/jump-table-move-pic.s new file mode 100644 --- /dev/null +++ b/bolt/test/X86/jump-table-move-pic.s @@ -0,0 +1,58 @@ +# REQUIRES: system-linux + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.out --lite=0 -v=1 --jump-tables=move -print-cfg 2>&1 | FileCheck %s -check-prefix=CHECK-NOSTRIP + + +# CHECK-NOSTRIP-NOT: unclaimed PC-relative relocations left in data +# CHECK-NOSTRIP: BOLT-INFO: marking main.cold.1 as a fragment of main +# CHECK-NOSTRIP: Ignoring main.cold.1 +# CHECK-NOSTRIP: Ignoring main +# CHECK-NOSTRIP: Binary Function "main.cold.1" after building cfg +# CHECK-NOSTRIP: PIC Jump table JUMP_TABLE for function main.cold.1 at {{.*}} with a total count of 0 +# CHECK-NOSTRIP-NEXT: 0x0000 : [[ENTRY:.+]] +# CHECK-NOSTRIP-NEXT: 0x0004 : [[ENTRY]] + + +# RUN: cp %t.exe %t.tmp.exe +# RUN: llvm-strip -s %t.tmp.exe +# RUN: llvm-bolt %t.tmp.exe -o %t.out --lite=0 -v=1 --jump-tables=move -print-cfg 2>&1 | FileCheck %s -check-prefix=CHECK-STRIP + +# CHECK-STRIP-NOT: unclaimed PC-relative relocations left in data +# CHECK-STRIP: BOLT-INFO: marking [[MAIN:.+]] as a fragment of [[MAIN_COLD:.+]] +# CHECK-STRIP: Ignoring [[MAIN_COLD]] +# CHECK-STRIP: Ignoring [[MAIN]] +# CHECK-STRIP: Binary Function "[[MAIN_COLD]]" after building cfg +# CHECK-STRIP: PIC Jump table JUMP_TABLE/[[MAIN_COLD]]{{.*}} for function [[MAIN_COLD]] at {{.*}} with a total count of 0 +# CHECK-STRIP-NEXT: 0x0000 : [[ENTRY:.+]] +# CHECK-STRIP-NEXT: 0x0004 : [[ENTRY]] + + .text + .globl main + .type main, %function + .p2align 2 +main: + .cfi_startproc + cmpl $0x67, %edi + jne main.cold.1 +LBB0: + retq + .cfi_endproc + + .globl main.cold.1 + .type main.cold.1, %function + .p2align 2 +main.cold.1: + .cfi_startproc + leaq JUMP_TABLE(%rip), %rax + movslq (%rax,%rdx,4), %rcx + addq %rax, %rcx + jmpq *%rcx + .cfi_endproc + + .rodata + .globl JUMP_TABLE +JUMP_TABLE: + .long LBB0-JUMP_TABLE + .long LBB0-JUMP_TABLE diff --git a/bolt/test/X86/split-func-jump-table-fragment-noparent.s b/bolt/test/X86/split-func-jump-table-fragment-noparent.s --- a/bolt/test/X86/split-func-jump-table-fragment-noparent.s +++ b/bolt/test/X86/split-func-jump-table-fragment-noparent.s @@ -1,22 +1,52 @@ # This reproduces a bug with jump table identification where jump table has -# entries pointing to code in function and its cold fragment. -# The fragment is only reachable through jump table. +# entries pointing to code in function and its cold fragment. The fragment is +# only reachable through jump table. +# This test verifies support for both stripped and non-stripped binaries. # REQUIRES: system-linux + # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o -# RUN: llvm-strip --strip-unneeded %t.o # RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -# RUN: llvm-bolt %t.exe -o %t.out --lite=0 -v=1 2>&1 | FileCheck %s +# RUN: llvm-strip --strip-unneeded %t.exe +# RUN: llvm-bolt %t.exe -o %t.out --lite=0 -v=1 -print-cfg 2>&1 | FileCheck %s -check-prefix=CHECK-NOSTRIP + +# CHECK-NOSTRIP-NOT: unclaimed PC-relative relocations left in data +# CHECK-NOSTRIP: BOLT-INFO: marking main.cold.1 as a fragment of main +# CHECK-NOSTRIP: Ignoring main +# CHECK-NOSTRIP: Ignoring main.cold.1 +# CHECK-NOSTRIP: Binary Function "main" after building cfg +# CHECK-NOSTRIP: PIC Jump table JUMP_TABLE for function main at {{.*}} with a total count of 0 +# CHECK-NOSTRIP-NEXT: 0x0000 : {{.+}} +# CHECK-NOSTRIP-NEXT: 0x0004 : [[LBB3:.+]] +# CHECK-NOSTRIP-NEXT: 0x0008 : {{.*}}main.cold.1{{.*}} +# CHECK-NOSTRIP-NEXT: 0x000c : [[LBB3]] +# CHECK-NOSTRIP: Binary Function "main.cold.1" after building cfg + + +# RUN: cp %t.exe %t.tmp.exe +# RUN: llvm-strip -s %t.tmp.exe +# RUN: llvm-bolt %t.tmp.exe -o %t.out --lite=0 -v=1 -print-cfg 2>&1 | FileCheck %s -check-prefix=CHECK-STRIP + +# CHECK-STRIP-NOT: unclaimed PC-relative relocations left in data +# CHECK-STRIP: BOLT-INFO: marking [[MAIN_COLD:.+]] as a fragment of [[MAIN:.+]] +# CHECK-STRIP: Ignoring [[MAIN]] +# CHECK-STRIP: Ignoring [[MAIN_COLD]] +# CHECK-STRIP: Binary Function "[[MAIN]]" after building cfg +# CHECK-STRIP: PIC Jump table JUMP_TABLE/[[MAIN]]{{.*}} for function [[MAIN]] at {{.*}} with a total count of 0 +# CHECK-STRIP-NEXT: 0x0000 : {{.+}} +# CHECK-STRIP-NEXT: 0x0004 : [[LBB3:.+]] +# CHECK-STRIP-NEXT: 0x0008 : {{.*}}[[MAIN_COLD]]{{.*}} +# CHECK-STRIP-NEXT: 0x000c : [[LBB3]] +# CHECK-STRIP: Binary Function "[[MAIN_COLD]]" after building cfg -# CHECK-NOT: unclaimed PC-relative relocations left in data -# CHECK: BOLT-INFO: marking main.cold.1 as a fragment of main .text .globl main .type main, %function .p2align 2 main: LBB0: + .cfi_startproc andl $0xf, %ecx cmpb $0x4, %cl # exit through ret @@ -35,6 +65,7 @@ LBB3: addq $0x8, %rsp ret + .cfi_endproc .size main, .-main # cold fragment is only reachable through jump table @@ -42,11 +73,13 @@ .type main.cold.1, %function .p2align 2 main.cold.1: + .cfi_startproc # load bearing nop: pad LBB4 so that it can't be treated # as __builtin_unreachable by analyzeJumpTable nop LBB4: callq abort +.cfi_endproc .size main.cold.1, .-main.cold.1 .rodata diff --git a/bolt/test/X86/split-func-jump-table-fragment.s b/bolt/test/X86/split-func-jump-table-fragment.s --- a/bolt/test/X86/split-func-jump-table-fragment.s +++ b/bolt/test/X86/split-func-jump-table-fragment.s @@ -1,21 +1,53 @@ # This reproduces a bug with jump table identification where jump table has # entries pointing to code in function and its cold fragment. +# This test verifies support for both stripped and non-stripped binaries. # REQUIRES: system-linux + + # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o -# RUN: llvm-strip --strip-unneeded %t.o # RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -# RUN: llvm-bolt %t.exe -o %t.out --lite=0 -v=1 2>&1 | FileCheck %s +# RUN: llvm-strip --strip-unneeded %t.exe +# RUN: llvm-bolt %t.exe -o %t.out --lite=0 -v=1 -print-cfg 2>&1 | FileCheck %s -check-prefix=CHECK-NOSTRIP + +# CHECK-NOSTRIP-NOT: unclaimed PC-relative relocations left in data +# CHECK-NOSTRIP: BOLT-INFO: marking main.cold.1 as a fragment of main +# CHECK-NOSTRIP: Ignoring main +# CHECK-NOSTRIP: Ignoring main.cold.1 +# CHECK-NOSTRIP: Binary Function "main" after building cfg +# CHECK-NOSTRIP: PIC Jump table JUMP_TABLE for function main at {{.*}} with a total count of 0 +# CHECK-NOSTRIP-NEXT: 0x0000 : {{.+}} +# CHECK-NOSTRIP-NEXT: 0x0004 : [[LBB3:.+]] +# CHECK-NOSTRIP-NEXT: 0x0008 : {{.*}}main.cold.1{{.*}} +# CHECK-NOSTRIP-NEXT: 0x000c : [[LBB3]] +# CHECK-NOSTRIP: Binary Function "main.cold.1" after building cfg + + + +# RUN: cp %t.exe %t.tmp.exe +# RUN: llvm-strip -s %t.tmp.exe +# RUN: llvm-bolt %t.tmp.exe -o %t.out --lite=0 -v=1 -print-cfg 2>&1 | FileCheck %s -check-prefix=CHECK-STRIP + +# CHECK-STRIP-NOT: unclaimed PC-relative relocations left in data +# CHECK-STRIP: BOLT-INFO: marking [[MAIN_COLD:.+]] as a fragment of [[MAIN:.+]] +# CHECK-STRIP: Ignoring [[MAIN]] +# CHECK-STRIP: Ignoring [[MAIN_COLD]] +# CHECK-STRIP: Binary Function "[[MAIN]]" after building cfg +# CHECK-STRIP: PIC Jump table JUMP_TABLE/[[MAIN]]{{.*}} for function [[MAIN]] at {{.*}} with a total count of 0 +# CHECK-STRIP-NEXT: 0x0000 : {{.+}} +# CHECK-STRIP-NEXT: 0x0004 : [[LBB3:.+]] +# CHECK-STRIP-NEXT: 0x0008 : {{.*}}[[MAIN_COLD]]{{.*}} +# CHECK-STRIP-NEXT: 0x000c : [[LBB3]] +# CHECK-STRIP: Binary Function "[[MAIN_COLD]]" after building cfg -# CHECK-NOT: unclaimed PC-relative relocations left in data -# CHECK: BOLT-INFO: marking main.cold.1 as a fragment of main .text .globl main .type main, %function .p2align 2 main: LBB0: + .cfi_startproc andl $0xf, %ecx cmpb $0x4, %cl # exit through abort in main.cold.1, registers cold fragment the regular way @@ -34,6 +66,7 @@ LBB3: addq $0x8, %rsp ret + .cfi_endproc .size main, .-main .globl main.cold.1 @@ -42,9 +75,11 @@ main.cold.1: # load bearing nop: pad LBB4 so that it can't be treated # as __builtin_unreachable by analyzeJumpTable + .cfi_startproc nop LBB4: callq abort + .cfi_endproc .size main.cold.1, .-main.cold.1 .rodata diff --git a/bolt/test/X86/split-landing-pad.s b/bolt/test/X86/split-landing-pad.s --- a/bolt/test/X86/split-landing-pad.s +++ b/bolt/test/X86/split-landing-pad.s @@ -8,6 +8,8 @@ # This test is written to ensure BOLT safely handle these targets, e.g., by # marking them as non-simple. # +# This test is for both stripped and non-stripped binaries +# # Steps to write this test: # - Create a copy of Inputs/src/unreachable.cpp # - Simplify bar(), focus on throw an exception @@ -25,10 +27,17 @@ # REQUIRES: system-linux # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o # RUN: %clang++ %cxxflags %t.o -o %t.exe -Wl,-q -# RUN: llvm-bolt -v=3 %t.exe -o %t.out 2>&1 | FileCheck %s +# RUN: llvm-bolt -v=3 %t.exe -o %t.out -print-exceptions -print-cfg 2>&1 | FileCheck %s + +# RUN: cp %t.exe %t.tmp.exe +# RUN: llvm-strip -s %t.tmp.exe +# RUN: llvm-bolt -v=3 %t.tmp.exe -o %t.out -print-exceptions -print-cfg 2>&1 | FileCheck %s -# CHECK: BOLT-WARNING: Ignoring foo -# CHECK: BOLT-WARNING: Ignoring foo.cold.1 +# CHECK: BOLT-INFO: marking [[FOO_COLD:.+]] as a fragment of [[FOO:.+]] +# CHECK: landing pad label: [[TMP2:.+]] +# CHECK: landing pad label: [[TMP5:.+]] +# CHECK: BOLT-WARNING: Ignoring [[FOO]] +# CHECK: BOLT-WARNING: Ignoring [[FOO_COLD]] # CHECK: BOLT-WARNING: skipped 2 functions due to cold fragments .text