diff --git a/llvm/lib/CodeGen/BranchRelaxation.cpp b/llvm/lib/CodeGen/BranchRelaxation.cpp --- a/llvm/lib/CodeGen/BranchRelaxation.cpp +++ b/llvm/lib/CodeGen/BranchRelaxation.cpp @@ -79,6 +79,7 @@ }; SmallVector BlockInfo; + MachineBasicBlock *TrampolineInsertionPoint; std::unique_ptr RS; LivePhysRegs LiveRegs; @@ -166,6 +167,7 @@ void BranchRelaxation::scanFunction() { BlockInfo.clear(); BlockInfo.resize(MF->getNumBlockIDs()); + TrampolineInsertionPoint = nullptr; // First thing, compute the size of all basic blocks, and see if the function // has any inline assembly in it. If so, we have to be conservative about @@ -176,6 +178,19 @@ // Compute block offsets and known bits. adjustBlockOffsets(*MF->begin()); + + // Place the trampoline insertion point at the end of the hot portion of the + // function. + for (MachineBasicBlock &MBB : reverse(*MF)) { + if (MBB.getSectionID() != MBBSectionID::ColdSectionID) { + TrampolineInsertionPoint = &MBB; + break; + } + } + if (TrampolineInsertionPoint == nullptr) { + LLVM_DEBUG(dbgs() << " No suitable trampoline insertion point found in " + << MF->getName()); + } } /// computeBlockSize - Compute the size for MBB. @@ -376,6 +391,49 @@ assert(!Fail && "branches to be relaxed must be analyzable"); (void)Fail; + // For cross-section branches whose destination is in the cold section, add a + // "trampoline branch", which unconditionally branches to the branch + // destination. Place the trampoline branch at the end of the function and + // retarget the conditional branch to the trampoline. + // tbz L1 + // => + // tbz L1Trampoline + // ... + // L1Trampoline: b L1 + if (MBB->getSectionID() != TBB->getSectionID() && + TBB->getSectionID() == MBBSectionID::ColdSectionID && + TrampolineInsertionPoint != nullptr) { + // If the insertion point is out of range, we can't put a trampoline there. + NewBB = + createNewBlockAfter(*TrampolineInsertionPoint, MBB->getBasicBlock()); + + if (isBlockInRange(MI, *NewBB)) { + LLVM_DEBUG(dbgs() << " Retarget destination to trampoline at " + << NewBB->back()); + + insertUncondBranch(NewBB, TBB); + + // Update the successor lists to include the trampoline. + MBB->replaceSuccessor(TBB, NewBB); + NewBB->addSuccessor(TBB); + + // Replace branch in the current (MBB) block. + removeBranch(MBB); + insertBranch(MBB, NewBB, FBB, Cond); + + TrampolineInsertionPoint = NewBB; + finalizeBlockChanges(MBB, NewBB); + return true; + } + + LLVM_DEBUG( + dbgs() << " Trampoline insertion point out of range for Bcc from " + << printMBBReference(*MBB) << " to " << printMBBReference(*TBB) + << ".\n"); + TrampolineInsertionPoint->setIsEndSection(NewBB->isEndSection()); + MF->erase(NewBB); + } + // Add an unconditional branch to the destination and invert the branch // condition to jump over it: // tbz L1 @@ -665,6 +723,7 @@ LLVM_DEBUG(dbgs() << " Basic blocks after relaxation\n\n"; dumpBBs()); BlockInfo.clear(); + TrampolineInsertionPoint = nullptr; return MadeChange; }