[BOLT] Make exception handling fragment aware

[FPar]

Summary

This adds basic fragment awareness in the exception handling passes and
generates the necessary symbols for fragments.

[FPar]

Improve error messages

Removed check in BinaryEmitter, because it is too noisy and probably not even necessary.

[FPar]

Rebase

[FPar]

Cleanup

[FPar]

Rebase

[rafauler]

LGTM

bolt/lib/Core/BinaryFunction.cpp

1576–1579

Why are we not putting all end labels here?

2848–2851

Why are we not putting all end labels here?

[FPar]

Mark all end labels undefined

bolt/include/bolt/Core/BinaryBasicBlock.h

private:
   mutable unsigned LayoutIndex{InvalidIndex};
 
   /// Number of pseudo instructions in this block.
   uint32_t NumPseudos{0};
 
   /// CFI state at the entry to this basic block.
   int32_t CFIState{-1};
 
-  /// In cases where the parent function has been split, IsCold == true means
-  /// this BB will be allocated outside its parent function.
-  bool IsCold{false};
+  /// In cases where the parent function has been split, FragmentNum > 0 means
+  /// this BB will be allocated in a fragment outside its parent function.
+  FragmentNum Fragment;
 
   /// Indicates if the block could be outlined.
   bool CanOutline{true};
 
   /// Flag to indicate whether this block is valid or not.  Invalid
   /// blocks may contain out of date or incorrect information.
   bool IsValid{true};
 
public:
   /// Return true if the basic block is an entry point into the function
   /// (either primary or secondary).
   bool isEntryPoint() const;
 
   bool isValid() const { return IsValid; }
 
   void markValid(const bool Valid) { IsValid = Valid; }
 
-  FragmentNum getFragmentNum() const {
-    return IsCold ? FragmentNum::cold() : FragmentNum::hot();
-  }
+  FragmentNum getFragmentNum() const { return Fragment; }
+
+  void setFragmentNum(const FragmentNum Value) { Fragment = Value; }
 
-  bool isCold() const { return IsCold; }
+  bool isSplit() const { return Fragment != FragmentNum::main(); }
 
-  void setIsCold(const bool Flag) { IsCold = Flag; }
+  bool isCold() const {
+    assert(Fragment.get() < 2 &&
+           "Function is split into more than two (hot/cold)-fragments");
+    return Fragment == FragmentNum::cold();
+  }
+
+  void setIsCold(const bool Flag) {
+    Fragment = Flag ? FragmentNum::cold() : FragmentNum::hot();
+  }
 
   /// Return true if the block can be outlined. At the moment we disallow
   /// outlining of blocks that can potentially throw exceptions or are
   /// the beginning of a landing pad. The entry basic block also can
   /// never be outlined.
   bool canOutline() const { return CanOutline; }
 
   void setCanOutline(const bool Flag) { CanOutline = Flag; }

bolt/include/bolt/Core/BinaryFunction.h

bool operator()(const BasicBlockOffset &A,
                     const BasicBlockOffset &B) const {
       return A.first < B.first;
     }
   };
   SmallVector<BasicBlockOffset, 0> BasicBlockOffsets;
 
   SmallVector<MCSymbol *, 0> ColdSymbols;
 
-  /// Symbol at the end of the function.
-  mutable MCSymbol *FunctionEndLabel{nullptr};
-
-  /// Symbol at the end of the cold part of split function.
-  mutable MCSymbol *FunctionColdEndLabel{nullptr};
+  /// Symbol at the end of each fragment of a split function.
+  mutable SmallVector<MCSymbol *, 0> FunctionEndLabels;
 
   /// Unique number associated with the function.
   uint64_t FunctionNumber;
 
   /// Count the number of functions created.
   static uint64_t Count;
 
   /// Map offsets of special instructions to addresses in the output.
public:
   ///
   /// Pass an offset of the entry point in the input binary and a corresponding
   /// global symbol to the callback function.
   ///
   /// Return true of all callbacks returned true, false otherwise.
   bool forEachEntryPoint(EntryPointCallbackTy Callback) const;
 
   /// Return MC symbol associated with the end of the function.
-  MCSymbol *getFunctionEndLabel() const {
+  MCSymbol *
+  getFunctionEndLabel(const FragmentNum Fragment = FragmentNum::main()) const {
     assert(BC.Ctx && "cannot be called with empty context");
+
+    size_t LabelIndex = Fragment.get();
+    if (LabelIndex >= FunctionEndLabels.size()) {
+      FunctionEndLabels.resize(LabelIndex + 1);
+    }
+
+    MCSymbol *&FunctionEndLabel = FunctionEndLabels[LabelIndex];
     if (!FunctionEndLabel) {
       std::unique_lock<std::shared_timed_mutex> Lock(BC.CtxMutex);
+      if (Fragment == FragmentNum::main())
         FunctionEndLabel = BC.Ctx->createNamedTempSymbol("func_end");
+      else
+        FunctionEndLabel = BC.Ctx->createNamedTempSymbol(
+            formatv("func_cold_end.{0}", Fragment.get() - 1));
     }
     return FunctionEndLabel;
   }
 
-  /// Return MC symbol associated with the end of the cold part of the function.
-  MCSymbol *getFunctionColdEndLabel() const {
-    if (!FunctionColdEndLabel) {
-      std::unique_lock<std::shared_timed_mutex> Lock(BC.CtxMutex);
-      FunctionColdEndLabel = BC.Ctx->createNamedTempSymbol("func_cold_end");
-    }
-    return FunctionColdEndLabel;
-  }
-
   /// Return a label used to identify where the constant island was emitted
   /// (AArch only). This is used to update the symbol table accordingly,
   /// emitting data marker symbols as required by the ABI.
   MCSymbol *getFunctionConstantIslandLabel() const {
     assert(Islands && "function expected to have constant islands");
 
     if (!Islands->FunctionConstantIslandLabel) {
       Islands->FunctionConstantIslandLabel =
MCSymbol *getLSDASymbol() {
 
     LSDASymbol = BC.Ctx->getOrCreateSymbol(
         Twine("GCC_except_table") + Twine::utohexstr(getFunctionNumber()));
 
     return LSDASymbol;
   }
 
   /// Return symbol pointing to function's LSDA for the cold part.
-  MCSymbol *getColdLSDASymbol() {
+  MCSymbol *getColdLSDASymbol(const FragmentNum Fragment) {
     if (ColdLSDASymbol)
       return ColdLSDASymbol;
     if (ColdCallSites.empty())
       return nullptr;
 
-    ColdLSDASymbol = BC.Ctx->getOrCreateSymbol(
-        Twine("GCC_cold_except_table") + Twine::utohexstr(getFunctionNumber()));
+    SmallString<8> SymbolSuffix;
+    if (Fragment != FragmentNum::cold())
+      SymbolSuffix = formatv(".{0}", Fragment.get());
+    ColdLSDASymbol = BC.Ctx->getOrCreateSymbol(formatv(
+        "GCC_cold_except_table{0:x-}{1}", getFunctionNumber(), SymbolSuffix));
 
     return ColdLSDASymbol;
   }
 
   void setOutputDataAddress(uint64_t Address) { OutputDataOffset = Address; }
 
   uint64_t getOutputDataAddress() const { return OutputDataOffset; }
 

bolt/include/bolt/Core/FunctionLayout.h

public:
   constexpr unsigned get() const { return Value; }
   constexpr bool operator==(const FragmentNum Other) const {
     return Value == Other.Value;
   }
   constexpr bool operator!=(const FragmentNum Other) const {
     return !(*this == Other);
   }
 
+  static constexpr FragmentNum main() { return FragmentNum(0); }
   static constexpr FragmentNum hot() { return FragmentNum(0); }
   static constexpr FragmentNum cold() { return FragmentNum(1); }
 };
 
 /// A freestanding subset of contiguous blocks of a function.
 class FunctionFragment {
   using BasicBlockListType = SmallVector<BinaryBasicBlock *, 0>;
   using FragmentListType = SmallVector<unsigned, 0>;

bolt/lib/Core/BinaryBasicBlock.cpp

for (BinaryBasicBlock *Succ : Successors) {
                << " not contained in the jump table.\n";
       }
     }
     // If there are any leftover entries in the jump table, they
     // must be one of the function end labels.
     if (Valid) {
       for (const MCSymbol *Sym : UniqueSyms) {
         Valid &= (Sym == Function->getFunctionEndLabel() ||
-                  Sym == Function->getFunctionColdEndLabel());
+                  Sym == Function->getFunctionEndLabel(getFragmentNum()));
         if (!Valid) {
           errs() << "BOLT-WARNING: Jump table contains illegal entry: "
                  << Sym->getName() << "\n";
         }
       }
     }
   } else {
     // Unknown control flow.

bolt/lib/Core/BinaryEmitter.cpp

if (FF.isMainFragment()) {
     }
   } else {
     Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
     Streamer.emitLabel(StartSymbol);
   }
 
   // Emit CFI start
   if (Function.hasCFI()) {
-    assert(Function.getLayout().isHotColdSplit() &&
-           "Exceptions supported only with hot/cold splitting.");
     Streamer.emitCFIStartProc(/*IsSimple=*/false);
     if (Function.getPersonalityFunction() != nullptr)
       Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
                                   Function.getPersonalityEncoding());
-    MCSymbol *LSDASymbol = FF.isSplitFragment() ? Function.getColdLSDASymbol()
-                                                : Function.getLSDASymbol();
+    MCSymbol *LSDASymbol = FF.isSplitFragment()
+                               ? Function.getColdLSDASymbol(FF.getFragmentNum())
     if (LSDASymbol)
       Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
     else
       Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);
     // Emit CFI instructions relative to the CIE
     for (const MCCFIInstruction &CFIInstr : Function.cie()) {
       // Only write CIE CFI insns that LLVM will not already emit
       const std::vector<MCCFIInstruction> &FrameInstrs =
bool BinaryEmitter::emitFunction(BinaryFunction &Function,
 
   if (opts::MarkFuncs)
     Streamer.emitIntValue(BC.MIB->getTrapFillValue(), 1);
 
   // Emit CFI end
   if (Function.hasCFI())
     Streamer.emitCFIEndProc();
 
-  MCSymbol *EndSymbol = FF.isSplitFragment()
-                            ? Function.getFunctionColdEndLabel()
-                            : Function.getFunctionEndLabel();
+  MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
   Streamer.emitLabel(EndSymbol);
 
   if (MAI->hasDotTypeDotSizeDirective()) {
     const MCExpr *SizeExpr = MCBinaryExpr::createSub(
         MCSymbolRefExpr::create(EndSymbol, Context),
         MCSymbolRefExpr::create(StartSymbol, Context), Context);
     Streamer.emitELFSize(StartSymbol, SizeExpr);
   }
void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
   const unsigned TTypeEncoding = BC.TTypeEncoding;
   const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
   const uint16_t TTypeAlignment = 4;
 
   // Type tables have to be aligned at 4 bytes.
   Streamer.emitValueToAlignment(TTypeAlignment);
 
   // Emit the LSDA label.
-  MCSymbol *LSDASymbol =
-      EmitColdPart ? BF.getColdLSDASymbol() : BF.getLSDASymbol();
+  MCSymbol *LSDASymbol = EmitColdPart
+                             ? BF.getColdLSDASymbol(FragmentNum::cold())
+                             : BF.getLSDASymbol();
   assert(LSDASymbol && "no LSDA symbol set");
   Streamer.emitLabel(LSDASymbol);
 
   // Corresponding FDE start.
   const MCSymbol *StartSymbol =
       EmitColdPart ? BF.getSymbol(FragmentNum::cold()) : BF.getSymbol();
 
   // Emit the LSDA header.

bolt/lib/Core/BinaryFunction.cpp

 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSymbol.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
if (!DisassemblyFailed)
     for (Relocation &Rel : FunctionRelocations)
       getOriginSection()->addPendingRelocation(Rel);
 
   // Inform BinaryContext that this function symbols will not be defined and
   // relocations should not be created against them.
   if (BC.HasRelocations) {
     for (std::pair<const uint32_t, MCSymbol *> &LI : Labels)
       BC.UndefinedSymbols.insert(LI.second);
-    if (FunctionEndLabel)
-      BC.UndefinedSymbols.insert(FunctionEndLabel);
+    for (MCSymbol *const EndLabel : FunctionEndLabels)
+      if (EndLabel)
+        BC.UndefinedSymbols.insert(EndLabel);
   }
 
   clearList(Relocations);
   clearList(ExternallyReferencedOffsets);
 
   if (Success && BC.HasRelocations)
     HasExternalRefRelocations = true;
 
   if (opts::Verbosity >= 1 && !Success)

 void BinaryFunction::clearDisasmState() {
   clearList(Instructions);
   clearList(IgnoredBranches);
   clearList(TakenBranches);
 
   if (BC.HasRelocations) {
     for (std::pair<const uint32_t, MCSymbol *> &LI : Labels)
       BC.UndefinedSymbols.insert(LI.second);
-    if (FunctionEndLabel)
-      BC.UndefinedSymbols.insert(FunctionEndLabel);
+    for (MCSymbol *const EndLabel : FunctionEndLabels)
+      if (EndLabel)
+        BC.UndefinedSymbols.insert(EndLabel);
   }
 }
 
 void BinaryFunction::setTrapOnEntry() {
   clearDisasmState();
 
   auto addTrapAtOffset = [&](uint64_t Offset) {
     MCInst TrapInstr;
     BC.MIB->createTrap(TrapInstr);
if (hasConstantIsland()) {
       const uint64_t DataOffset =
           Layout.getSymbolOffset(*getFunctionConstantIslandLabel());
       setOutputDataAddress(BaseAddress + DataOffset);
     }
     if (isSplit()) {
       const MCSymbol *ColdStartSymbol = getSymbol(FragmentNum::cold());
       assert(ColdStartSymbol && ColdStartSymbol->isDefined() &&
              "split function should have defined cold symbol");
-      const MCSymbol *ColdEndSymbol = getFunctionColdEndLabel();
+      const MCSymbol *ColdEndSymbol = getFunctionEndLabel(FragmentNum::cold());
       assert(ColdEndSymbol && ColdEndSymbol->isDefined() &&
              "split function should have defined cold end symbol");
       const uint64_t ColdStartOffset = Layout.getSymbolOffset(*ColdStartSymbol);
       const uint64_t ColdEndOffset = Layout.getSymbolOffset(*ColdEndSymbol);
       cold().setAddress(ColdBaseAddress + ColdStartOffset);
       cold().setImageSize(ColdEndOffset - ColdStartOffset);
       if (hasConstantIsland()) {
         const uint64_t DataOffset =

bolt/lib/Core/Exceptions.cpp

void BinaryFunction::updateEHRanges() {
   assert(CurrentState == State::CFG_Finalized && "unexpected state");
 
   // Build call sites table.
   struct EHInfo {
     const MCSymbol *LP; // landing pad
     uint64_t Action;
   };
 
+  for (const FunctionFragment FF : getLayout().fragments()) {
+    // Sites to update - either regular or cold.
+    CallSitesType &Sites = FF.isMainFragment() ? CallSites : ColdCallSites;
+
     // If previous call can throw, this is its exception handler.
     EHInfo PreviousEH = {nullptr, 0};
 
     // Marker for the beginning of exceptions range.
     const MCSymbol *StartRange = nullptr;
 
-  // Indicates whether the start range is located in a cold part.
-  bool IsStartInCold = false;
-
-  // Have we crossed hot/cold border for split functions?
-  bool SeenCold = false;
-
-  // Sites to update - either regular or cold.
-  CallSitesType *Sites = &CallSites;
-
-  for (BinaryBasicBlock *BB : getLayout().blocks()) {
-
-    if (BB->isCold() && !SeenCold) {
-      SeenCold = true;
-
-      // Close the range (if any) and change the target call sites.
-      if (StartRange) {
-        Sites->emplace_back(CallSite{StartRange, getFunctionEndLabel(),
-                                     PreviousEH.LP, PreviousEH.Action});
-      }
-      Sites = &ColdCallSites;
-
-      // Reset the range.
-      StartRange = nullptr;
-      PreviousEH = {nullptr, 0};
-    }
-
+    for (BinaryBasicBlock *const BB : FF) {
       for (auto II = BB->begin(); II != BB->end(); ++II) {
         if (!BC.MIB->isCall(*II))
           continue;
 
         // Instruction can throw an exception that should be handled.
         const bool Throws = BC.MIB->isInvoke(*II);
 
         // Ignore the call if it's a continuation of a no-throw gap.
         if (!Throws && !StartRange)
           continue;
 
+        assert(getLayout().isHotColdSplit() &&
+               "Exceptions only supported for hot/cold splitting");
+
         // Extract exception handling information from the instruction.
         const MCSymbol *LP = nullptr;
         uint64_t Action = 0;
-      if (const Optional<MCPlus::MCLandingPad> EHInfo = BC.MIB->getEHInfo(*II))
+        if (const Optional<MCPlus::MCLandingPad> EHInfo =
+                BC.MIB->getEHInfo(*II))
           std::tie(LP, Action) = *EHInfo;
 
         // No action if the exception handler has not changed.
         if (Throws && StartRange && PreviousEH.LP == LP &&
             PreviousEH.Action == Action)
           continue;
 
         // Same symbol is used for the beginning and the end of the range.
         const MCSymbol *EHSymbol;
         MCInst EHLabel;
         {
           std::unique_lock<std::shared_timed_mutex> Lock(BC.CtxMutex);
           EHSymbol = BC.Ctx->createNamedTempSymbol("EH");
           BC.MIB->createEHLabel(EHLabel, EHSymbol, BC.Ctx.get());
         }
 
         II = std::next(BB->insertPseudoInstr(II, EHLabel));
 
         // At this point we could be in one of the following states:
         //
         // I. Exception handler has changed and we need to close previous range
         //    and start a new one.
         //
         // II. Start a new exception range after the gap.
         //
         // III. Close current exception range and start a new gap.
         const MCSymbol *EndRange;
         if (StartRange) {
           // I, III:
           EndRange = EHSymbol;
         } else {
           // II:
           StartRange = EHSymbol;
-        IsStartInCold = SeenCold;
           EndRange = nullptr;
         }
 
         // Close the previous range.
         if (EndRange) {
-        Sites->emplace_back(
+          Sites.emplace_back(
               CallSite{StartRange, EndRange, PreviousEH.LP, PreviousEH.Action});
         }
 
         if (Throws) {
           // I, II:
           StartRange = EHSymbol;
-        IsStartInCold = SeenCold;
           PreviousEH = EHInfo{LP, Action};
         } else {
           StartRange = nullptr;
         }
       }
     }
 
     // Check if we need to close the range.
     if (StartRange) {
-    assert((!isSplit() || Sites == &ColdCallSites) && "sites mismatch");
-    const MCSymbol *EndRange =
-        IsStartInCold ? getFunctionColdEndLabel() : getFunctionEndLabel();
-    Sites->emplace_back(
+      assert((FF.isMainFragment() || &Sites == &ColdCallSites) &&
+             "sites mismatch");
+      const MCSymbol *EndRange = getFunctionEndLabel(FF.getFragmentNum());
+      Sites.emplace_back(
           CallSite{StartRange, EndRange, PreviousEH.LP, PreviousEH.Action});
     }
   }
+}
 
 const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;
 
 CFIReaderWriter::CFIReaderWriter(const DWARFDebugFrame &EHFrame) {
   // Prepare FDEs for fast lookup
   for (const dwarf::FrameEntry &Entry : EHFrame.entries()) {
     const auto *CurFDE = dyn_cast<dwarf::FDE>(&Entry);
     // Skip CIEs.

bolt/lib/Passes/Aligner.cpp

 // -- the size of the function
 // -- the specified number of bytes
 void alignCompact(BinaryFunction &Function, const MCCodeEmitter *Emitter) {
   const BinaryContext &BC = Function.getBinaryContext();
   size_t HotSize = 0;
   size_t ColdSize = 0;
 
   for (const BinaryBasicBlock &BB : Function)
-    if (BB.isCold())
+    if (BB.isSplit())
       ColdSize += BC.computeCodeSize(BB.begin(), BB.end(), Emitter);
     else
       HotSize += BC.computeCodeSize(BB.begin(), BB.end(), Emitter);
 
   Function.setAlignment(opts::AlignFunctions);
   if (HotSize > 0)
     Function.setMaxAlignmentBytes(
       std::min(size_t(opts::AlignFunctionsMaxBytes), HotSize));

bolt/lib/Passes/BinaryPasses.cpp

bool CheckLargeFunctions::shouldOptimize(const BinaryFunction &BF) const {
   return BF.isSimple() && !BF.isIgnored();
 }
 
 void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
   std::vector<std::pair<MCInst *, uint32_t>> PreservedOffsetAnnotations;
 
   for (auto &It : BC.getBinaryFunctions()) {
     BinaryFunction &BF = It.second;
-    int64_t CurrentGnuArgsSize = 0;
-
-    // Have we crossed hot/cold border for split functions?
-    bool SeenCold = false;
 
-    for (BinaryBasicBlock *BB : BF.getLayout().blocks()) {
-      if (BB->isCold() && !SeenCold) {
-        SeenCold = true;
-        CurrentGnuArgsSize = 0;
-      }
+    for (const FunctionFragment FF : BF.getLayout().fragments()) {
+      int64_t CurrentGnuArgsSize = 0;
 
-      // First convert GnuArgsSize annotations into CFIs. This may change instr
-      // pointers, so do it before recording ptrs for preserved annotations
+      for (BinaryBasicBlock *const BB : FF) {
+        // First convert GnuArgsSize annotations into CFIs. This may change
+        // instr pointers, so do it before recording ptrs for preserved
+        // annotations
         if (BF.usesGnuArgsSize()) {
           for (auto II = BB->begin(); II != BB->end(); ++II) {
             if (!BC.MIB->isInvoke(*II))
               continue;
             const int64_t NewGnuArgsSize = BC.MIB->getGnuArgsSize(*II);
-          assert(NewGnuArgsSize >= 0 && "expected non-negative GNU_args_size");
+            assert(NewGnuArgsSize >= 0 &&
+                   "expected non-negative GNU_args_size");
             if (NewGnuArgsSize != CurrentGnuArgsSize) {
               auto InsertII = BF.addCFIInstruction(
                   BB, II,
                   MCCFIInstruction::createGnuArgsSize(nullptr, NewGnuArgsSize));
               CurrentGnuArgsSize = NewGnuArgsSize;
               II = std::next(InsertII);
             }
           }
         }
 
-      // Now record preserved annotations separately and then strip annotations.
+        // Now record preserved annotations separately and then strip
+        // annotations.
         for (auto II = BB->begin(); II != BB->end(); ++II) {
           if (BF.requiresAddressTranslation() && BC.MIB->getOffset(*II))
             PreservedOffsetAnnotations.emplace_back(&(*II),
                                                     *BC.MIB->getOffset(*II));
           BC.MIB->stripAnnotations(*II);
         }
       }
     }
+  }
   for (BinaryFunction *BF : BC.getInjectedBinaryFunctions())
     for (BinaryBasicBlock &BB : *BF)
       for (MCInst &Instruction : BB)
         BC.MIB->stripAnnotations(Instruction);
 
   // Release all memory taken by annotations
   BC.MIB->freeAnnotations();
 

bolt/lib/Passes/IndirectCallPromotion.cpp

const JumpTable::JumpInfo *JI =
         JT->Counts.empty() ? &DefaultJI : &JT->Counts[Range.first];
     const size_t JIAdj = JT->Counts.empty() ? 0 : 1;
     assert(JT->Type == JumpTable::JTT_PIC ||
            JT->EntrySize == BC.AsmInfo->getCodePointerSize());
     for (size_t I = Range.first; I < Range.second; ++I, JI += JIAdj) {
       MCSymbol *Entry = JT->Entries[I];
       assert(BF.getBasicBlockForLabel(Entry) ||
              Entry == BF.getFunctionEndLabel() ||
-             Entry == BF.getFunctionColdEndLabel());
+             Entry == BF.getFunctionEndLabel(FragmentNum::cold()));
       if (Entry == BF.getFunctionEndLabel() ||
-          Entry == BF.getFunctionColdEndLabel())
+          Entry == BF.getFunctionEndLabel(FragmentNum::cold()))
         continue;
       const Location To(Entry);
       const BinaryBasicBlock::BinaryBranchInfo &BI = BB.getBranchInfo(Entry);
       Targets.emplace_back(From, To, BI.MispredictedCount, BI.Count,
                            I - Range.first);
     }
 
     // Sort by symbol then addr.