diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -1157,8 +1157,8 @@ /// Return the number of emitted instructions for this function. uint32_t getNumNonPseudos() const { uint32_t N = 0; - for (BinaryBasicBlock *const &BB : layout()) - N += BB->getNumNonPseudos(); + for (const BinaryBasicBlock &BB : blocks()) + N += BB.getNumNonPseudos(); return N; } @@ -2335,13 +2335,13 @@ size_t estimateHotSize(const bool UseSplitSize = true) const { size_t Estimate = 0; if (UseSplitSize && isSplit()) { - for (const BinaryBasicBlock *BB : BasicBlocksLayout) - if (!BB->isCold()) - Estimate += BC.computeCodeSize(BB->begin(), BB->end()); + for (const BinaryBasicBlock &BB : blocks()) + if (!BB.isCold()) + Estimate += BC.computeCodeSize(BB.begin(), BB.end()); } else { - for (const BinaryBasicBlock *BB : BasicBlocksLayout) - if (BB->getKnownExecutionCount() != 0) - Estimate += BC.computeCodeSize(BB->begin(), BB->end()); + for (const BinaryBasicBlock &BB : blocks()) + if (BB.getKnownExecutionCount() != 0) + Estimate += BC.computeCodeSize(BB.begin(), BB.end()); } return Estimate; } @@ -2350,16 +2350,16 @@ if (!isSplit()) return estimateSize(); size_t Estimate = 0; - for (const BinaryBasicBlock *BB : BasicBlocksLayout) - if (BB->isCold()) - Estimate += BC.computeCodeSize(BB->begin(), BB->end()); + for (const BinaryBasicBlock &BB : blocks()) + if (BB.isCold()) + Estimate += BC.computeCodeSize(BB.begin(), BB.end()); return Estimate; } size_t estimateSize() const { size_t Estimate = 0; - for (const BinaryBasicBlock *BB : BasicBlocksLayout) - Estimate += BC.computeCodeSize(BB->begin(), BB->end()); + for (const BinaryBasicBlock &BB : blocks()) + Estimate += BC.computeCodeSize(BB.begin(), BB.end()); return Estimate; } diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -293,8 +293,8 @@ void BinaryFunction::markUnreachableBlocks() { std::stack Stack; - for (BinaryBasicBlock *BB : layout()) - BB->markValid(false); + for (BinaryBasicBlock &BB : blocks()) + BB.markValid(false); // Add all entries and landing pads as roots. for (BinaryBasicBlock *BB : BasicBlocks) { @@ -1750,14 +1750,14 @@ BinaryBasicBlock *LastIndirectJumpBB = nullptr; uint64_t LastJT = 0; uint16_t LastJTIndexReg = BC.MIB->getNoRegister(); - for (BinaryBasicBlock *BB : layout()) { - for (MCInst &Instr : *BB) { + for (BinaryBasicBlock &BB : blocks()) { + for (MCInst &Instr : BB) { if (!BC.MIB->isIndirectBranch(Instr)) continue; // If there's an indirect branch in a single-block function - // it must be a tail call. - if (layout_size() == 1) { + if (BasicBlocks.size() == 1) { BC.MIB->convertJmpToTailCall(Instr); return true; } @@ -1779,7 +1779,7 @@ const MCExpr *DispExpr; MCInst *PCRelBaseInstr; IndirectBranchType Type = BC.MIB->analyzeIndirectBranch( - Instr, BB->begin(), BB->end(), PtrSize, MemLocInstr, BaseRegNum, + Instr, BB.begin(), BB.end(), PtrSize, MemLocInstr, BaseRegNum, IndexRegNum, DispValue, DispExpr, PCRelBaseInstr); if (Type != IndirectBranchType::UNKNOWN || MemLocInstr != nullptr) continue; @@ -1791,7 +1791,7 @@ BC.MIB->convertTailCallToJmp(Instr); } else { LastIndirectJump = &Instr; - LastIndirectJumpBB = BB; + LastIndirectJumpBB = &BB; LastJT = BC.MIB->getJumpTable(Instr); LastJTIndexReg = BC.MIB->getJumpTableIndexReg(Instr); BC.MIB->unsetJumpTable(Instr); @@ -1807,7 +1807,7 @@ } } - addUnknownControlFlow(*BB); + addUnknownControlFlow(BB); continue; } @@ -1815,7 +1815,7 @@ // then most likely it's a tail call. Otherwise, we cannot tell for sure // what it is and conservatively reject the function's CFG. bool IsEpilogue = false; - for (const MCInst &Instr : *BB) { + for (const MCInst &Instr : BB) { if (BC.MIB->isLeave(Instr) || BC.MIB->isPop(Instr)) { IsEpilogue = true; break; @@ -1823,22 +1823,22 @@ } if (IsEpilogue) { BC.MIB->convertJmpToTailCall(Instr); - BB->removeAllSuccessors(); + BB.removeAllSuccessors(); continue; } if (opts::Verbosity >= 2) { outs() << "BOLT-INFO: rejected potential indirect tail call in " - << "function " << *this << " in basic block " << BB->getName() + << "function " << *this << " in basic block " << BB.getName() << ".\n"; - LLVM_DEBUG(BC.printInstructions(dbgs(), BB->begin(), BB->end(), - BB->getOffset(), this, true)); + LLVM_DEBUG(BC.printInstructions(dbgs(), BB.begin(), BB.end(), + BB.getOffset(), this, true)); } if (!opts::StrictMode) return false; - addUnknownControlFlow(*BB); + addUnknownControlFlow(BB); } } @@ -2180,8 +2180,8 @@ // later. This has no cost, since annotations are allocated by a bumpptr // allocator and won't be released anyway until late in the pipeline. if (!requiresAddressTranslation() && !opts::Instrument) { - for (BinaryBasicBlock *BB : layout()) - for (MCInst &Inst : *BB) + for (BinaryBasicBlock &BB : blocks()) + for (MCInst &Inst : BB) BC.MIB->clearOffset(Inst); } @@ -2192,9 +2192,9 @@ void BinaryFunction::calculateMacroOpFusionStats() { if (!getBinaryContext().isX86()) return; - for (BinaryBasicBlock *BB : layout()) { - auto II = BB->getMacroOpFusionPair(); - if (II == BB->end()) + for (const BinaryBasicBlock &BB : blocks()) { + auto II = BB.getMacroOpFusionPair(); + if (II == BB.end()) continue; // Check offset of the second instruction. @@ -2206,9 +2206,9 @@ LLVM_DEBUG(dbgs() << "\nmissed macro-op fusion at address 0x" << Twine::utohexstr(getAddress() + Offset) << " in function " << *this << "; executed " - << BB->getKnownExecutionCount() << " times.\n"); + << BB.getKnownExecutionCount() << " times.\n"); ++BC.MissedMacroFusionPairs; - BC.MissedMacroFusionExecCount += BB->getKnownExecutionCount(); + BC.MissedMacroFusionExecCount += BB.getKnownExecutionCount(); } } @@ -2308,11 +2308,11 @@ } uint64_t TotalScore = 0ULL; - for (BinaryBasicBlock *BB : layout()) { - uint64_t BBExecCount = BB->getExecutionCount(); + for (const BinaryBasicBlock &BB : blocks()) { + uint64_t BBExecCount = BB.getExecutionCount(); if (BBExecCount == BinaryBasicBlock::COUNT_NO_PROFILE) continue; - TotalScore += BBExecCount * BB->getNumNonPseudos(); + TotalScore += BBExecCount * BB.getNumNonPseudos(); } FunctionScore = TotalScore; return FunctionScore; @@ -2818,12 +2818,12 @@ } LLVM_DEBUG(dbgs() << "\n"); - for (BinaryBasicBlock *BB : BasicBlocksLayout) { - for (auto II = BB->begin(); II != BB->end();) { + for (BinaryBasicBlock &BB : blocks()) { + for (auto II = BB.begin(); II != BB.end();) { const MCCFIInstruction *CFI = getCFIFor(*II); if (CFI && (CFI->getOperation() == MCCFIInstruction::OpRememberState || CFI->getOperation() == MCCFIInstruction::OpRestoreState)) { - II = BB->eraseInstruction(II); + II = BB.eraseInstruction(II); } else { ++II; } @@ -2839,8 +2839,8 @@ uint64_t BinaryFunction::getInstructionCount() const { uint64_t Count = 0; - for (BinaryBasicBlock *const &Block : BasicBlocksLayout) - Count += Block->getNumNonPseudos(); + for (const BinaryBasicBlock &BB : blocks()) + Count += BB.getNumNonPseudos(); return Count; } @@ -3337,39 +3337,39 @@ void BinaryFunction::postProcessBranches() { if (!isSimple()) return; - for (BinaryBasicBlock *BB : BasicBlocksLayout) { - auto LastInstrRI = BB->getLastNonPseudo(); - if (BB->succ_size() == 1) { - if (LastInstrRI != BB->rend() && + for (BinaryBasicBlock &BB : blocks()) { + auto LastInstrRI = BB.getLastNonPseudo(); + if (BB.succ_size() == 1) { + if (LastInstrRI != BB.rend() && BC.MIB->isConditionalBranch(*LastInstrRI)) { // __builtin_unreachable() could create a conditional branch that // falls-through into the next function - hence the block will have only // one valid successor. Such behaviour is undefined and thus we remove // the conditional branch while leaving a valid successor. - BB->eraseInstruction(std::prev(LastInstrRI.base())); + BB.eraseInstruction(std::prev(LastInstrRI.base())); LLVM_DEBUG(dbgs() << "BOLT-DEBUG: erasing conditional branch in " - << BB->getName() << " in function " << *this << '\n'); + << BB.getName() << " in function " << *this << '\n'); } - } else if (BB->succ_size() == 0) { + } else if (BB.succ_size() == 0) { // Ignore unreachable basic blocks. - if (BB->pred_size() == 0 || BB->isLandingPad()) + if (BB.pred_size() == 0 || BB.isLandingPad()) continue; // If it's the basic block that does not end up with a terminator - we // insert a return instruction unless it's a call instruction. - if (LastInstrRI == BB->rend()) { + if (LastInstrRI == BB.rend()) { LLVM_DEBUG( dbgs() << "BOLT-DEBUG: at least one instruction expected in BB " - << BB->getName() << " in function " << *this << '\n'); + << BB.getName() << " in function " << *this << '\n'); continue; } if (!BC.MIB->isTerminator(*LastInstrRI) && !BC.MIB->isCall(*LastInstrRI)) { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding return to basic block " - << BB->getName() << " in function " << *this << '\n'); + << BB.getName() << " in function " << *this << '\n'); MCInst ReturnInstr; BC.MIB->createReturn(ReturnInstr); - BB->addInstruction(ReturnInstr); + BB.addInstruction(ReturnInstr); } } } @@ -3502,20 +3502,28 @@ return Status; } -BinaryFunction::BasicBlockOrderType BinaryFunction::dfs() const { - BasicBlockOrderType DFS; +BinaryFunction::BasicBlockListType BinaryFunction::dfs() const { + BasicBlockListType DFS; unsigned Index = 0; std::stack Stack; // Push entry points to the stack in reverse order. // // NB: we rely on the original order of entries to match. - for (auto BBI = layout_rbegin(); BBI != layout_rend(); ++BBI) { - BinaryBasicBlock *BB = *BBI; - if (isEntryPoint(*BB)) - Stack.push(BB); - BB->setLayoutIndex(BinaryBasicBlock::InvalidIndex); - } + SmallVector EntryPoints; + llvm::copy_if(BasicBlocks, std::back_inserter(EntryPoints), + [&](const BinaryBasicBlock *const BB) { return isEntryPoint(*BB); }); + // Sort entry points by their offset to make sure we got them in the right + // order. + llvm::stable_sort(EntryPoints, [](const BinaryBasicBlock *const A, + const BinaryBasicBlock *const B) { + return A->getOffset() < B->getOffset(); + }); + for (BinaryBasicBlock *const BB : reverse(EntryPoints)) + Stack.push(BB); + + for (BinaryBasicBlock &BB : blocks()) + BB.setLayoutIndex(BinaryBasicBlock::InvalidIndex); while (!Stack.empty()) { BinaryBasicBlock *BB = Stack.top(); @@ -3951,8 +3959,8 @@ if (AdjustmentRatio < 0.0) AdjustmentRatio = 0.0; - for (BinaryBasicBlock *&BB : layout()) - BB->adjustExecutionCount(AdjustmentRatio); + for (BinaryBasicBlock &BB : blocks()) + BB.adjustExecutionCount(AdjustmentRatio); ExecutionCount -= Count; } diff --git a/bolt/lib/Passes/ADRRelaxationPass.cpp b/bolt/lib/Passes/ADRRelaxationPass.cpp --- a/bolt/lib/Passes/ADRRelaxationPass.cpp +++ b/bolt/lib/Passes/ADRRelaxationPass.cpp @@ -29,8 +29,8 @@ void ADRRelaxationPass::runOnFunction(BinaryFunction &BF) { BinaryContext &BC = BF.getBinaryContext(); - for (BinaryBasicBlock *BB : BF.layout()) { - for (auto It = BB->begin(); It != BB->end(); ++It) { + for (BinaryBasicBlock &BB : BF) { + for (auto It = BB.begin(); It != BB.end(); ++It) { MCInst &Inst = *It; if (!BC.MIB->isADR(Inst)) continue; @@ -54,7 +54,7 @@ int64_t Addend = BC.MIB->getTargetAddend(Inst); InstructionListType Addr = BC.MIB->materializeAddress(Symbol, BC.Ctx.get(), Reg, Addend); - It = BB->replaceInstruction(It, Addr); + It = BB.replaceInstruction(It, Addr); } } } diff --git a/bolt/lib/Passes/Aligner.cpp b/bolt/lib/Passes/Aligner.cpp --- a/bolt/lib/Passes/Aligner.cpp +++ b/bolt/lib/Passes/Aligner.cpp @@ -83,11 +83,11 @@ size_t HotSize = 0; size_t ColdSize = 0; - for (const BinaryBasicBlock *BB : Function.layout()) - if (BB->isCold()) - ColdSize += BC.computeCodeSize(BB->begin(), BB->end(), Emitter); + for (const BinaryBasicBlock &BB : Function) + if (BB.isCold()) + ColdSize += BC.computeCodeSize(BB.begin(), BB.end(), Emitter); else - HotSize += BC.computeCodeSize(BB->begin(), BB->end(), Emitter); + HotSize += BC.computeCodeSize(BB.begin(), BB.end(), Emitter); Function.setAlignment(opts::AlignFunctions); if (HotSize > 0) diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp --- a/bolt/lib/Passes/BinaryPasses.cpp +++ b/bolt/lib/Passes/BinaryPasses.cpp @@ -339,9 +339,9 @@ uint64_t Bytes; Function.markUnreachableBlocks(); LLVM_DEBUG({ - for (BinaryBasicBlock *BB : Function.layout()) { - if (!BB->isValid()) { - dbgs() << "BOLT-INFO: UCE found unreachable block " << BB->getName() + for (BinaryBasicBlock &BB : Function) { + if (!BB.isValid()) { + dbgs() << "BOLT-INFO: UCE found unreachable block " << BB.getName() << " in function " << Function << "\n"; Function.dump(); } diff --git a/bolt/lib/Passes/CacheMetrics.cpp b/bolt/lib/Passes/CacheMetrics.cpp --- a/bolt/lib/Passes/CacheMetrics.cpp +++ b/bolt/lib/Passes/CacheMetrics.cpp @@ -43,15 +43,15 @@ for (BinaryFunction *BF : BinaryFunctions) { const BinaryContext &BC = BF->getBinaryContext(); - for (BinaryBasicBlock *BB : BF->layout()) { + for (BinaryBasicBlock &BB : *BF) { if (BF->isSimple() || BC.HasRelocations) { // Use addresses/sizes as in the output binary - BBAddr[BB] = BB->getOutputAddressRange().first; - BBSize[BB] = BB->getOutputSize(); + BBAddr[&BB] = BB.getOutputAddressRange().first; + BBSize[&BB] = BB.getOutputSize(); } else { // Output ranges should match the input if the body hasn't changed - BBAddr[BB] = BB->getInputAddressRange().first + BF->getAddress(); - BBSize[BB] = BB->getOriginalSize(); + BBAddr[&BB] = BB.getInputAddressRange().first + BF->getAddress(); + BBSize[&BB] = BB.getOriginalSize(); } } } @@ -68,11 +68,12 @@ for (BinaryFunction *BF : BinaryFunctions) { if (!BF->hasProfile()) continue; - for (BinaryBasicBlock *SrcBB : BF->layout()) { - auto BI = SrcBB->branch_info_begin(); - for (BinaryBasicBlock *DstBB : SrcBB->successors()) { - if (SrcBB != DstBB && BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && - BBAddr.at(SrcBB) + BBSize.at(SrcBB) == BBAddr.at(DstBB)) + for (BinaryBasicBlock &SrcBB : *BF) { + auto BI = SrcBB.branch_info_begin(); + for (BinaryBasicBlock *DstBB : SrcBB.successors()) { + if (&SrcBB != DstBB && + BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && + BBAddr.at(&SrcBB) + BBSize.at(&SrcBB) == BBAddr.at(DstBB)) Score += BI->Count; ++BI; } @@ -92,12 +93,13 @@ for (BinaryFunction *BF : BinaryFunctions) { if (!BF->hasProfile()) continue; - for (BinaryBasicBlock *SrcBB : BF->layout()) { - auto BI = SrcBB->branch_info_begin(); - for (BinaryBasicBlock *DstBB : SrcBB->successors()) { - if (DstBB != SrcBB) - Score += CacheMetrics::extTSPScore(BBAddr.at(SrcBB), BBSize.at(SrcBB), - BBAddr.at(DstBB), BI->Count); + for (BinaryBasicBlock &SrcBB : *BF) { + auto BI = SrcBB.branch_info_begin(); + for (BinaryBasicBlock *DstBB : SrcBB.successors()) { + if (DstBB != &SrcBB) + Score += + CacheMetrics::extTSPScore(BBAddr.at(&SrcBB), BBSize.at(&SrcBB), + BBAddr.at(DstBB), BI->Count); ++BI; } } @@ -262,13 +264,13 @@ NumProfiledFunctions++; if (BF->hasValidIndex()) NumHotFunctions++; - for (BinaryBasicBlock *BB : BF->layout()) { + for (const BinaryBasicBlock &BB : *BF) { NumBlocks++; - size_t BBAddrMin = BB->getOutputAddressRange().first; - size_t BBAddrMax = BB->getOutputAddressRange().second; + size_t BBAddrMin = BB.getOutputAddressRange().first; + size_t BBAddrMax = BB.getOutputAddressRange().second; TotalCodeMinAddr = std::min(TotalCodeMinAddr, BBAddrMin); TotalCodeMaxAddr = std::max(TotalCodeMaxAddr, BBAddrMax); - if (BF->hasValidIndex() && !BB->isCold()) { + if (BF->hasValidIndex() && !BB.isCold()) { NumHotBlocks++; HotCodeMinAddr = std::min(HotCodeMinAddr, BBAddrMin); HotCodeMaxAddr = std::max(HotCodeMaxAddr, BBAddrMax); diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp --- a/bolt/lib/Passes/IndirectCallPromotion.cpp +++ b/bolt/lib/Passes/IndirectCallPromotion.cpp @@ -162,11 +162,11 @@ BinaryFunction &BF = BFI.second; if (!BF.isSimple()) continue; - for (BinaryBasicBlock *BB : BF.layout()) { - auto BI = BB->branch_info_begin(); - for (BinaryBasicBlock *SuccBB : BB->successors()) { + for (const BinaryBasicBlock &BB : BF) { + auto BI = BB.branch_info_begin(); + for (BinaryBasicBlock *SuccBB : BB.successors()) { if (BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && BI->Count > 0) { - if (BB->getKnownExecutionCount() == 0 || + if (BB.getKnownExecutionCount() == 0 || SuccBB->getKnownExecutionCount() == 0) { errs() << "BOLT-WARNING: profile verification failed after ICP for " "function " diff --git a/bolt/lib/Passes/Inliner.cpp b/bolt/lib/Passes/Inliner.cpp --- a/bolt/lib/Passes/Inliner.cpp +++ b/bolt/lib/Passes/Inliner.cpp @@ -165,8 +165,8 @@ return INL_NONE; const MCPhysReg SPReg = BC.MIB->getStackPointer(); - for (const BinaryBasicBlock *BB : BF.layout()) { - for (const MCInst &Inst : *BB) { + for (const BinaryBasicBlock &BB : BF) { + for (const MCInst &Inst : BB) { // Tail calls are marked as implicitly using the stack pointer and they // could be inlined. if (BC.MIB->isTailCall(Inst)) diff --git a/bolt/lib/Passes/PLTCall.cpp b/bolt/lib/Passes/PLTCall.cpp --- a/bolt/lib/Passes/PLTCall.cpp +++ b/bolt/lib/Passes/PLTCall.cpp @@ -57,11 +57,11 @@ Function.getExecutionCount() == BinaryFunction::COUNT_NO_PROFILE) continue; - for (BinaryBasicBlock *BB : Function.layout()) { - if (opts::PLT == OT_HOT && !BB->getKnownExecutionCount()) + for (BinaryBasicBlock &BB : Function) { + if (opts::PLT == OT_HOT && !BB.getKnownExecutionCount()) continue; - for (MCInst &Instr : *BB) { + for (MCInst &Instr : BB) { if (!BC.MIB->isCall(Instr)) continue; const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr); diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp --- a/bolt/lib/Passes/SplitFunctions.cpp +++ b/bolt/lib/Passes/SplitFunctions.cpp @@ -90,8 +90,8 @@ return false; bool AllCold = true; - for (BinaryBasicBlock *BB : BF.layout()) { - const uint64_t ExecCount = BB->getExecutionCount(); + for (const BinaryBasicBlock &BB : BF) { + const uint64_t ExecCount = BB.getExecutionCount(); if (ExecCount == BinaryBasicBlock::COUNT_NO_PROFILE) return false; if (ExecCount != 0) diff --git a/bolt/lib/Passes/ThreeWayBranch.cpp b/bolt/lib/Passes/ThreeWayBranch.cpp --- a/bolt/lib/Passes/ThreeWayBranch.cpp +++ b/bolt/lib/Passes/ThreeWayBranch.cpp @@ -19,9 +19,8 @@ bool ThreeWayBranch::shouldRunOnFunction(BinaryFunction &Function) { BinaryContext &BC = Function.getBinaryContext(); - BinaryFunction::BasicBlockOrderType BlockLayout = Function.getLayout(); - for (BinaryBasicBlock *BB : BlockLayout) - for (MCInst &Inst : *BB) + for (const BinaryBasicBlock &BB : Function) + for (const MCInst &Inst : BB) if (BC.MIB->isPacked(Inst)) return false; return true;