Index: lib/CodeGen/BranchFolding.h =================================================================== --- lib/CodeGen/BranchFolding.h +++ lib/CodeGen/BranchFolding.h @@ -29,9 +29,13 @@ public: class MBFIWrapper; - explicit BranchFolder(bool defaultEnableTailMerge, bool CommonHoist, + explicit BranchFolder(bool defaultEnableTailMerge, + bool CommonHoist, MBFIWrapper &MBFI, - const MachineBranchProbabilityInfo &MBPI); + const MachineBranchProbabilityInfo &MBPI, + // Min tail length to merge. Defaults to commandline + // flag. Ignored for optsize. + unsigned MinCommonTailLength = 0); bool OptimizeFunction(MachineFunction &MF, const TargetInstrInfo *tii, const TargetRegisterInfo *tri, MachineModuleInfo *mmi, @@ -99,6 +103,7 @@ bool EnableTailMerge; bool EnableHoistCommonCode; bool UpdateLiveIns; + unsigned MinCommonTailLength; const TargetInstrInfo *TII; const TargetRegisterInfo *TRI; MachineModuleInfo *MMI; @@ -129,7 +134,8 @@ bool TailMergeBlocks(MachineFunction &MF); bool TryTailMergeBlocks(MachineBasicBlock* SuccBB, - MachineBasicBlock* PredBB); + MachineBasicBlock* PredBB, + unsigned MinCommonTailLength); void setCommonTailEdgeWeights(MachineBasicBlock &TailMBB); void computeLiveIns(MachineBasicBlock &MBB); void ReplaceTailWithBranchTo(MachineBasicBlock::iterator OldInst, Index: lib/CodeGen/BranchFolding.cpp =================================================================== --- lib/CodeGen/BranchFolding.cpp +++ lib/CodeGen/BranchFolding.cpp @@ -110,9 +110,12 @@ BranchFolder::BranchFolder(bool defaultEnableTailMerge, bool CommonHoist, MBFIWrapper &FreqInfo, - const MachineBranchProbabilityInfo &ProbInfo) - : EnableHoistCommonCode(CommonHoist), MBBFreqInfo(FreqInfo), - MBPI(ProbInfo) { + const MachineBranchProbabilityInfo &ProbInfo, + unsigned MinTailLength) + : EnableHoistCommonCode(CommonHoist), MinCommonTailLength(MinTailLength), + MBBFreqInfo(FreqInfo), MBPI(ProbInfo) { + if (MinCommonTailLength == 0) + MinCommonTailLength = TailMergeSize; switch (FlagEnableTailMerge) { case cl::BOU_UNSET: EnableTailMerge = defaultEnableTailMerge; break; case cl::BOU_TRUE: EnableTailMerge = true; break; @@ -591,9 +594,21 @@ /// and decide if it would be profitable to merge those tails. Return the /// length of the common tail and iterators to the first common instruction /// in each block. +/// MBB1, MBB2 The blocks to check +/// MinCommonTailLength Minimum size of tail block to be merged. +/// CommonTailLen Out parameter to record the size of the shared tail between +/// MBB1 and MBB2 +/// I1, I2 Iterator references that will be changed to point to the first +/// instruction in the common tail shared by MBB1,MBB2 +/// SuccBB A common successor of MBB1, MBB2 which are in a canonical form +/// relative to SuccBB +/// PredBB The layout predecessor of SuccBB, if any. +/// FuncletMembership map from block to funclet #. +/// AfterPlacement True if we are merging blocks after layout. Stricter +/// thresholds apply to prevent undoing tail-duplication. static bool ProfitableToMerge(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2, - unsigned minCommonTailLength, unsigned &CommonTailLen, + unsigned MinCommonTailLength, unsigned &CommonTailLen, MachineBasicBlock::iterator &I1, MachineBasicBlock::iterator &I2, MachineBasicBlock *SuccBB, MachineBasicBlock *PredBB, @@ -651,7 +666,7 @@ ++EffectiveTailLen; // Check if the common tail is long enough to be worthwhile. - if (EffectiveTailLen >= minCommonTailLength) + if (EffectiveTailLen >= MinCommonTailLength) return true; // If we are optimizing for code size, 2 instructions in common is enough if @@ -674,7 +689,7 @@ /// those blocks appear in MergePotentials (where they are not necessarily /// consecutive). unsigned BranchFolder::ComputeSameTails(unsigned CurHash, - unsigned minCommonTailLength, + unsigned MinCommonTailLength, MachineBasicBlock *SuccBB, MachineBasicBlock *PredBB) { unsigned maxCommonTailLength = 0U; @@ -687,7 +702,7 @@ for (MPIterator I = std::prev(CurMPIter); I->getHash() == CurHash; --I) { unsigned CommonTailLen; if (ProfitableToMerge(CurMPIter->getBlock(), I->getBlock(), - minCommonTailLength, + MinCommonTailLength, CommonTailLen, TrialBBI1, TrialBBI2, SuccBB, PredBB, FuncletMembership, @@ -832,14 +847,13 @@ // branch to Succ added (but the predecessor/successor lists need no // adjustment). The lone predecessor of Succ that falls through into Succ, // if any, is given in PredBB. +// MinCommonTailLength - Except for the special cases below, tail-merge if +// there are at least this many instructions in common. bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, - MachineBasicBlock *PredBB) { + MachineBasicBlock *PredBB, + unsigned MinCommonTailLength) { bool MadeChange = false; - // Except for the special cases below, tail-merge if there are at least - // this many instructions in common. - unsigned minCommonTailLength = TailMergeSize; - DEBUG(dbgs() << "\nTryTailMergeBlocks: "; for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i) dbgs() << "BB#" << MergePotentials[i].getBlock()->getNumber() @@ -852,8 +866,8 @@ << PredBB->getNumber() << "\n"; } dbgs() << "Looking for common tails of at least " - << minCommonTailLength << " instruction" - << (minCommonTailLength == 1 ? "" : "s") << '\n'; + << MinCommonTailLength << " instruction" + << (MinCommonTailLength == 1 ? "" : "s") << '\n'; ); // Sort by hash value so that blocks with identical end sequences sort @@ -867,10 +881,10 @@ // Build SameTails, identifying the set of blocks with this hash code // and with the maximum number of instructions in common. unsigned maxCommonTailLength = ComputeSameTails(CurHash, - minCommonTailLength, + MinCommonTailLength, SuccBB, PredBB); - // If we didn't find any pair that has at least minCommonTailLength + // If we didn't find any pair that has at least MinCommonTailLength // instructions in common, remove all blocks with this hash code and retry. if (SameTails.empty()) { RemoveBlocksWithHash(CurHash, SuccBB, PredBB); @@ -976,7 +990,7 @@ // See if we can do any tail merging on those. if (MergePotentials.size() >= 2) - MadeChange |= TryTailMergeBlocks(nullptr, nullptr); + MadeChange |= TryTailMergeBlocks(nullptr, nullptr, MinCommonTailLength); } // Look at blocks (IBB) with multiple predecessors (PBB). @@ -1110,7 +1124,7 @@ TriedMerging.insert(MergePotentials[i].getBlock()); if (MergePotentials.size() >= 2) - MadeChange |= TryTailMergeBlocks(IBB, PredBB); + MadeChange |= TryTailMergeBlocks(IBB, PredBB, MinCommonTailLength); // Reinsert an unconditional branch if needed. The 1 below can occur as a // result of removing blocks in TryTailMergeBlocks. Index: lib/CodeGen/MachineBlockPlacement.cpp =================================================================== --- lib/CodeGen/MachineBlockPlacement.cpp +++ lib/CodeGen/MachineBlockPlacement.cpp @@ -1747,8 +1747,10 @@ BranchFoldPlacement; // No tail merging opportunities if the block number is less than four. if (MF.size() > 3 && EnableTailMerge) { + // Default to the standard tail-merge-size option. + unsigned TailMergeSize = 0; BranchFolder BF(/*EnableTailMerge=*/true, /*CommonHoist=*/false, *MBFI, - *MBPI); + *MBPI, TailMergeSize); if (BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(), getAnalysisIfAvailable(), MLI,