[IR] add new callbrpad instruction

[nickdesaulniers]

Summary

In order to properly support outputs along indirect edges of callbr, it
would be helpful to have distinct SSA values to refer to so that during
instruction selection, we can mark outputs that might come from specific
physregs as live-in and place copies to virtregs.

Please see the corresponding RFC accompanying this patch.

Link: https://github.com/llvm/llvm-project/issues/53562
Link: https://discourse.llvm.org/t/rfc-syncing-asm-goto-with-outputs-with-gcc/65453
Link: https://discourse.llvm.org/t/rfc-asm-goto-with-output-constraints/52354/4

[nickdesaulniers]

arcanist never does _what I want_

llvm/docs/LangRef.rst

 
 Example:
 """"""""
 
 .. code-block:: text
 
       %tok = cleanuppad within %cs []
 
+.. _i_callbrpad:
+
+'``callbrpad``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+     <resultval> = callbrpad <type> <value>
+
+Overview:
+"""""""""
+
+The '``callbrpad``' instruction corresponds 1:1 with a :ref:`callbr <i_callbr>`
+instruction, which should be the lone value to the '``callbrpad``'. It is
+required in order to consume outputs (if any) from the :ref:`callbr <i_callbr>`
+in the indirect branches of the :ref:`callbr <i_callbr>`.
+
+Arguments:
+""""""""""
+
+The '``callbrpad``' instruction takes one argument, which must be a
+`callbr <i_callbr>`. The type of the '``callbrpad``' corresponds to the type of
+the `callbr <i_callbr>` argument.
+
+Semantics:
+""""""""""
+
+The '``callbrpad``' instruction is consumed during codegen as a place for
+physical registers specified in output constraints of a
+:ref:`callbr <i_callbr`> to be copied back to virtual registers.
+
+Example:
+""""""""
+
+
+.. code-block:: llvm
+
+      %1 = callbrpad i32 %0
+
+
 .. _intrinsics:
 
 Intrinsic Functions
 ===================
 
 LLVM supports the notion of an "intrinsic function". These functions
 have well known names and semantics and are required to follow certain
 restrictions. Overall, these intrinsics represent an extension mechanism

llvm/docs/ReleaseNotes.rst

* The ``readnone``, ``readonly``, ``writeonly``, ``argmemonly``,
   * ``inaccessiblemem_or_argmemonly writeonly`` ->
     ``memory(argmem: write, inaccessiblemem: write)``
 
 * The constant expression variants of the following instructions has been
   removed:
 
   * ``fneg``
 
+* The ``callbrpad`` instruction was introduced for supporting outputs along
+  indirect edges of callbr.
+
 Changes to building LLVM
 ------------------------
 
 Changes to TableGen
 -------------------
 
 Changes to the AArch64 Backend
 ------------------------------

llvm/include/llvm-c/Core.h

typedef enum {
 
   /* Exception Handling Operators */
   LLVMResume         = 58,
   LLVMLandingPad     = 59,
   LLVMCleanupRet     = 61,
   LLVMCatchRet       = 62,
   LLVMCatchPad       = 63,
   LLVMCleanupPad     = 64,
-  LLVMCatchSwitch    = 65
+  LLVMCatchSwitch    = 65,
+  LLVMCallBrPad      = 69,
 } LLVMOpcode;
 
 typedef enum {
   LLVMVoidTypeKind,      /**< type with no size */
   LLVMHalfTypeKind,      /**< 16 bit floating point type */
   LLVMFloatTypeKind,     /**< 32 bit floating point type */
   LLVMDoubleTypeKind,    /**< 64 bit floating point type */
   LLVMX86_FP80TypeKind,  /**< 80 bit floating point type (X87) */

llvm/include/llvm/AsmParser/LLParser.h

#include "llvm/IR/Metadata.def"
     bool parseCast(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc);
     bool parseSelect(Instruction *&Inst, PerFunctionState &PFS);
     bool parseVAArg(Instruction *&Inst, PerFunctionState &PFS);
     bool parseExtractElement(Instruction *&Inst, PerFunctionState &PFS);
     bool parseInsertElement(Instruction *&Inst, PerFunctionState &PFS);
     bool parseShuffleVector(Instruction *&Inst, PerFunctionState &PFS);
     int parsePHI(Instruction *&Inst, PerFunctionState &PFS);
     bool parseLandingPad(Instruction *&Inst, PerFunctionState &PFS);
+    bool parseCallBrPad(Instruction *&Inst, PerFunctionState &PFS);
     bool parseCall(Instruction *&Inst, PerFunctionState &PFS,
                    CallInst::TailCallKind TCK);
     int parseAlloc(Instruction *&Inst, PerFunctionState &PFS);
     int parseLoad(Instruction *&Inst, PerFunctionState &PFS);
     int parseStore(Instruction *&Inst, PerFunctionState &PFS);
     int parseCmpXchg(Instruction *&Inst, PerFunctionState &PFS);
     int parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS);
     int parseFence(Instruction *&Inst, PerFunctionState &PFS);

llvm/include/llvm/AsmParser/LLToken.h

#include "llvm/IR/Attributes.inc"
   kw_resume,
   kw_unreachable,
   kw_cleanupret,
   kw_catchswitch,
   kw_catchret,
   kw_catchpad,
   kw_cleanuppad,
   kw_callbr,
+  kw_callbrpad,
 
   kw_alloca,
   kw_load,
   kw_store,
   kw_fence,
   kw_cmpxchg,
   kw_atomicrmw,
   kw_getelementptr,

llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h

bool findUnwindDestinations(
       SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
           &UnwindDests);
 
   bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
 
   bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
 
   bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
+  bool translateCallBrPad(const User &U, MachineIRBuilder &MIRBuilder);
 
   /// Translate one of LLVM's cast instructions into MachineInstrs, with the
   /// given generic Opcode.
   bool translateCast(unsigned Opcode, const User &U,
                      MachineIRBuilder &MIRBuilder);
 
   /// Translate a phi instruction.
   bool translatePHI(const User &U, MachineIRBuilder &MIRBuilder);

llvm/include/llvm/IR/IRBuilder.h

Value *CreateInsertValue(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
     return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
   }
 
   LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses,
                                    const Twine &Name = "") {
     return Insert(LandingPadInst::Create(Ty, NumClauses), Name);
   }
 
+  CallBrPadInst *CreateCallBrPad(CallBrInst *CBR) {
+    return Insert(CallBrPadInst::Create(CBR));
+  }
+
   Value *CreateFreeze(Value *V, const Twine &Name = "") {
     return Insert(new FreezeInst(V), Name);
   }
 
   //===--------------------------------------------------------------------===//
   // Utility creation methods
   //===--------------------------------------------------------------------===//
 

llvm/include/llvm/IR/InstVisitor.h

#include "llvm/IR/Instruction.def"
   RetTy visitSelectInst(SelectInst &I)            { DELEGATE(Instruction);}
   RetTy visitVAArgInst(VAArgInst   &I)            { DELEGATE(UnaryInstruction);}
   RetTy visitExtractElementInst(ExtractElementInst &I) { DELEGATE(Instruction);}
   RetTy visitInsertElementInst(InsertElementInst &I) { DELEGATE(Instruction);}
   RetTy visitShuffleVectorInst(ShuffleVectorInst &I) { DELEGATE(Instruction);}
   RetTy visitExtractValueInst(ExtractValueInst &I){ DELEGATE(UnaryInstruction);}
   RetTy visitInsertValueInst(InsertValueInst &I)  { DELEGATE(Instruction); }
   RetTy visitLandingPadInst(LandingPadInst &I)    { DELEGATE(Instruction); }
+  RetTy visitCallBrPadInst(CallBrPadInst &I)      { DELEGATE(Instruction); }
   RetTy visitFuncletPadInst(FuncletPadInst &I) { DELEGATE(Instruction); }
   RetTy visitCleanupPadInst(CleanupPadInst &I) { DELEGATE(FuncletPadInst); }
   RetTy visitCatchPadInst(CatchPadInst &I)     { DELEGATE(FuncletPadInst); }
   RetTy visitFreezeInst(FreezeInst &I)         { DELEGATE(Instruction); }
 
   // Handle the special intrinsic instruction classes.
   RetTy visitDbgDeclareInst(DbgDeclareInst &I)    { DELEGATE(DbgVariableIntrinsic);}
   RetTy visitDbgValueInst(DbgValueInst &I)        { DELEGATE(DbgVariableIntrinsic);}

llvm/include/llvm/IR/Instruction.h

public:
 
   /// Return true if the instruction will return (unwinding is considered as
   /// a form of returning control flow here).
   bool willReturn() const LLVM_READONLY;
 
   /// Return true if the instruction is a variety of EH-block.
   bool isEHPad() const {
     switch (getOpcode()) {
-    case Instruction::CatchSwitch:
+    case Instruction::CallBrPad:
     case Instruction::CatchPad:
+    case Instruction::CatchSwitch:
     case Instruction::CleanupPad:
     case Instruction::LandingPad:
       return true;
     default:
       return false;
     }
   }
 

llvm/include/llvm/IR/Instruction.def

 HANDLE_OTHER_INST(60, VAArg  , VAArgInst  )  // vaarg instruction
 HANDLE_OTHER_INST(61, ExtractElement, ExtractElementInst)// extract from vector
 HANDLE_OTHER_INST(62, InsertElement, InsertElementInst)  // insert into vector
 HANDLE_OTHER_INST(63, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
 HANDLE_OTHER_INST(64, ExtractValue, ExtractValueInst)// extract from aggregate
 HANDLE_OTHER_INST(65, InsertValue, InsertValueInst)  // insert into aggregate
 HANDLE_OTHER_INST(66, LandingPad, LandingPadInst)  // Landing pad instruction.
 HANDLE_OTHER_INST(67, Freeze, FreezeInst) // Freeze instruction.
-  LAST_OTHER_INST(67)
+HANDLE_OTHER_INST(68, CallBrPad, CallBrPadInst) // Landing pad for callbr instructions.
+  LAST_OTHER_INST(68)
 
 #undef  FIRST_TERM_INST
 #undef HANDLE_TERM_INST
 #undef   LAST_TERM_INST
 
 #undef  FIRST_UNARY_INST
 #undef HANDLE_UNARY_INST
 #undef   LAST_UNARY_INST

llvm/include/llvm/IR/Instructions.h

 
 template <>
 struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<1> {
 };
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value)
 
 //===----------------------------------------------------------------------===//
+//                           CallBrPadInst Class
+//===----------------------------------------------------------------------===//
+
+//===---------------------------------------------------------------------------
+/// The callbrpad instruction holds all of the information necessary to
+/// generate physreg to virtreg copies for callbr with inline asm. The
+/// callbrpad instruction cannot be moved from the top of a callbrpad block,
+/// which itself is accessible only from the indirect edge of a callbr.
+class CallBrPadInst : public UnaryInstruction {
+  CallBrPadInst() = delete;
+  explicit CallBrPadInst(Value *V)
+      : UnaryInstruction(V->getType(), Instruction::CallBrPad, V) {}
+
+public:
+  static CallBrPadInst *Create(Value *V) { return new CallBrPadInst(V); }
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static bool classof(const Instruction *I) {
+    return I->getOpcode() == Instruction::CallBrPad;
+  }
+  static bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+};
+
+//===----------------------------------------------------------------------===//
 //                               ReturnInst Class
 //===----------------------------------------------------------------------===//
 
 //===---------------------------------------------------------------------------
 /// Return a value (possibly void), from a function.  Execution
 /// does not continue in this function any longer.
 ///
 class ReturnInst : public Instruction {

llvm/lib/AsmParser/LLLexer.cpp

#define INSTKEYWORD(STR, Enum)                                                 \
   INSTKEYWORD(getelementptr, GetElementPtr);
 
   INSTKEYWORD(extractelement, ExtractElement);
   INSTKEYWORD(insertelement,  InsertElement);
   INSTKEYWORD(shufflevector,  ShuffleVector);
   INSTKEYWORD(extractvalue,   ExtractValue);
   INSTKEYWORD(insertvalue,    InsertValue);
   INSTKEYWORD(landingpad,     LandingPad);
+  INSTKEYWORD(callbrpad,      CallBrPad);
   INSTKEYWORD(cleanupret,     CleanupRet);
   INSTKEYWORD(catchret,       CatchRet);
   INSTKEYWORD(catchswitch,  CatchSwitch);
   INSTKEYWORD(catchpad,     CatchPad);
   INSTKEYWORD(cleanuppad,   CleanupPad);
 
   INSTKEYWORD(freeze,       Freeze);
 

llvm/lib/AsmParser/LLParser.cpp

if (FMF.any()) {
         return error(Loc, "fast-math-flags specified for phi without "
                           "floating-point scalar or vector return type");
       Inst->setFastMathFlags(FMF);
     }
     return 0;
   }
   case lltok::kw_landingpad:
     return parseLandingPad(Inst, PFS);
+  case lltok::kw_callbrpad:
+    return parseCallBrPad(Inst, PFS);
   case lltok::kw_freeze:
     return parseFreeze(Inst, PFS);
   // Call.
   case lltok::kw_call:
     return parseCall(Inst, PFS, CallInst::TCK_None);
   case lltok::kw_tail:
     return parseCall(Inst, PFS, CallInst::TCK_Tail);
   case lltok::kw_musttail:
for (auto I : IdToIndexMap) {
     }
   }
 
   if (parseToken(lltok::rparen, "expected ')' in callsites"))
     return true;
 
   return false;
 }
+
+/// CallBrPad
+///   ::= 'callbrpad' TypeAndValue
+bool LLParser::parseCallBrPad(Instruction *&Inst, PerFunctionState &PFS) {
+  Value *V;
+  if (parseTypeAndValue(V, PFS))
+    return true;
+  Inst = CallBrPadInst::Create(V);
+  return false;
+}

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

assert(&MF->front() == &NewEntryBB &&
          "New entry wasn't next in the list of basic block!");
 
   // Initialize stack protector information.
   StackProtector &SP = getAnalysis<StackProtector>();
   SP.copyToMachineFrameInfo(MF->getFrameInfo());
 
   return false;
 }
+
+bool IRTranslator::translateCallBrPad(const User &U,
+                                      MachineIRBuilder &MIRBuilder) {
+  return false;
+}

llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp

for (const BasicBlock &BB : *Fn) {
     // Don't create MachineBasicBlocks for imaginary EH pad blocks. These blocks
     // are really data, and no instructions can live here.
     if (BB.isEHPad()) {
       const Instruction *PadInst = BB.getFirstNonPHI();
       // If this is a non-landingpad EH pad, mark this function as using
       // funclets.
       // FIXME: SEH catchpads do not create EH scope/funclets, so we could avoid
       // setting this in such cases in order to improve frame layout.
-      if (!isa<LandingPadInst>(PadInst)) {
+      if (!isa<LandingPadInst>(PadInst) && !isa<CallBrPadInst>(PadInst)) {
         MF->setHasEHScopes(true);
         MF->setHasEHFunclets(true);
         MF->getFrameInfo().setHasOpaqueSPAdjustment(true);
       }
       if (isa<CatchSwitchInst>(PadInst)) {
         assert(&*BB.begin() == PadInst &&
                "WinEHPrepare failed to remove PHIs from imaginary BBs");
         continue;

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

private:
 
   void visitExtractElement(const User &I);
   void visitInsertElement(const User &I);
   void visitShuffleVector(const User &I);
 
   void visitExtractValue(const ExtractValueInst &I);
   void visitInsertValue(const InsertValueInst &I);
   void visitLandingPad(const LandingPadInst &LP);
+  void visitCallBrPad(const CallBrPadInst &LP);
 
   void visitGetElementPtr(const User &I);
   void visitSelect(const User &I);
 
   void visitAlloca(const AllocaInst &I);
   void visitLoad(const LoadInst &I);
   void visitStore(const StoreInst &I);
   void visitMaskedLoad(const CallInst &I, bool IsExpanding = false);

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
   uint64_t Idx = (NumElts + Imm) % NumElts;
 
   // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
   SmallVector<int, 8> Mask;
   for (unsigned i = 0; i < NumElts; ++i)
     Mask.push_back(Idx + i);
   setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
 }
+
+// Consider the following MIR after SelectionDAG, which produces output in
+// phyregs in the first case or virtregs in the second case.
+//
+// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
+// %5:gr32 = COPY $ebx
+// %6:gr32 = COPY $edx
+// %1:gr32 = COPY %6:gr32
+// %0:gr32 = COPY %5:gr32
+//
+// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
+// %1:gr32 = COPY %6:gr32
+// %0:gr32 = COPY %5:gr32
+//
+// Given %0, we'd like to return $ebx in the first case and %5 in the second.
+// Given %1, we'd like to return $edx in the first case and %6 in the second.
+//
+// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
+// to a single virtreg (such as %0). The remaining ouputs monotonically
+// increase in virtreg number from there. If a callbr has no outputs, then it
+// should not have a corresponding callbrpad; in fact, the callbrpad would not
+// even be able to refer to such a callbr.
+static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
+  MachineOperand *MO = &*MRI.def_begin(Reg);
+  MachineInstr *MI = MO->getParent();
+  if (MI->getOpcode() == TargetOpcode::COPY)
+    return FollowCopyChain(MRI, MI->getOperand(1).getReg());
+  if (MI->getOpcode() == TargetOpcode::INLINEASM_BR)
+    return Reg;
+  return {};
+}
+
+static size_t PopulateResultValues(const CallBrInst &CBR, SelectionDAG &DAG,
+                                   SmallVectorImpl<EVT> &ResultVTs,
+                                   SmallVectorImpl<SDValue> &ResultValues) {
+  Type *CBRType = CBR.getType();
+  assert(!CBRType->isVoidTy() &&
+         "callbr landing pad for callbr without returns");
+  ArrayRef<Type *> ResultTypes = CBRType->isStructTy()
+                                     ? cast<StructType>(CBRType)->elements()
+                                     : makeArrayRef(CBRType);
+  size_t NumReturnValues = ResultTypes.size();
+  ResultVTs.reserve(NumReturnValues);
+  ResultValues.reserve(NumReturnValues);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  for (Type *T : ResultTypes)
+    ResultVTs.emplace_back(TLI.getValueType(DAG.getDataLayout(), T));
+  return NumReturnValues;
+}
+
+void SelectionDAGBuilder::visitCallBrPad(const CallBrPadInst &LP) {
+  SmallVector<EVT, 8> ResultVTs;
+  SmallVector<SDValue, 8> ResultValues;
+  CallBrInst *CBR = cast<CallBrInst>(LP.getOperand(0));
+  const size_t NumReturnValues =
+      PopulateResultValues(*CBR, DAG, ResultVTs, ResultValues);
+  MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+  unsigned Reg = FuncInfo.ValueMap[CBR];
+  SDValue Chain = DAG.getRoot();
+
+  for (size_t i = 0; i != NumReturnValues; ++i, ++Reg) {
+    Register Output = FollowCopyChain(MRI, Reg);
+
+    if (Register::isPhysicalRegister(Output))
+      FuncInfo.MBB->addLiveIn(Output);
+
+    // TODO: should we mark the copied-from register as killed by this COPY (if
+    // physreg)?
+    SDValue V = DAG.getCopyFromReg(Chain, getCurSDLoc(), Output, ResultVTs[i]);
+    ResultValues.push_back(V);
+  }
+  SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
+                          DAG.getVTList(ResultVTs), ResultValues);
+  setValue(&LP, V);
+}

llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
     (void)IntrFound;
   }
 }
 
 /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and
 /// do other setup for EH landing-pad blocks.
 bool SelectionDAGISel::PrepareEHLandingPad() {
   MachineBasicBlock *MBB = FuncInfo->MBB;
-  const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
   const BasicBlock *LLVMBB = MBB->getBasicBlock();
-  const TargetRegisterClass *PtrRC =
-      TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout()));
+  const Instruction *FirstNonPhi = LLVMBB->getFirstNonPHI();
+
+  if (isa<CallBrPadInst>(FirstNonPhi))
+    return true;
 
+  const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
   auto Pers = classifyEHPersonality(PersonalityFn);
+  const TargetRegisterClass *PtrRC =
+      TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout()));
 
   // Catchpads have one live-in register, which typically holds the exception
   // pointer or code.
   if (isFuncletEHPersonality(Pers)) {
-    if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {
+    if (const auto *CPI = dyn_cast<CatchPadInst>(FirstNonPhi)) {
       if (hasExceptionPointerOrCodeUser(CPI)) {
         // Get or create the virtual register to hold the pointer or code.  Mark
         // the live in physreg and copy into the vreg.
         MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
         assert(EHPhysReg && "target lacks exception pointer register");
         MBB->addLiveIn(EHPhysReg);
         unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
         BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),
bool SelectionDAGISel::PrepareEHLandingPad() {
 
   // If the unwinder does not preserve all registers, ensure that the
   // function marks the clobbered registers as used.
   const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
   if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
     MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
 
   if (Pers == EHPersonality::Wasm_CXX) {
-    if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI()))
+    if (const auto *CPI = dyn_cast<CatchPadInst>(FirstNonPhi))
       mapWasmLandingPadIndex(MBB, CPI);
   } else {
     // Assign the call site to the landing pad's begin label.
     MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
     // Mark exception register as live in.
     if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))
       FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);
     // Mark exception selector register as live in.

llvm/lib/CodeGen/TargetLoweringBase.cpp

#include "llvm/IR/Instruction.def"
   case UserOp2:        return 0;
   case VAArg:          return 0;
   case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
   case InsertElement:  return ISD::INSERT_VECTOR_ELT;
   case ShuffleVector:  return ISD::VECTOR_SHUFFLE;
   case ExtractValue:   return ISD::MERGE_VALUES;
   case InsertValue:    return ISD::MERGE_VALUES;
   case LandingPad:     return 0;
+  case CallBrPad:      return 0;
   case Freeze:         return ISD::FREEZE;
   }
 
   llvm_unreachable("Unknown instruction type encountered!");
 }
 
 Value *
 TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilderBase &IRB,

llvm/lib/IR/AsmWriter.cpp

for (unsigned i = 0, e = LPI->getNumClauses(); i != e; ++i) {
       if (i != 0 || LPI->isCleanup()) Out << "\n";
       if (LPI->isCatch(i))
         Out << "          catch ";
       else
         Out << "          filter ";
 
       writeOperand(LPI->getClause(i), true);
     }
+  } else if (const auto *Pad = dyn_cast<CallBrPadInst>(&I)) {
+    Out << ' ';
+    writeOperand(Pad->getOperand(0), true);
   } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(&I)) {
     Out << " within ";
     writeOperand(CatchSwitch->getParentPad(), /*PrintType=*/false);
     Out << " [";
     unsigned Op = 0;
     for (const BasicBlock *PadBB : CatchSwitch->handlers()) {
       if (Op > 0)
         Out << ", ";

llvm/lib/IR/Dominators.cpp

bool DominatorTree::dominates(const Instruction *Def,
   // Invoke results are only usable in the normal destination, not in the
   // exceptional destination.
   if (const auto *II = dyn_cast<InvokeInst>(Def)) {
     BasicBlock *NormalDest = II->getNormalDest();
     BasicBlockEdge E(DefBB, NormalDest);
     return dominates(E, UseBB);
   }
 
-  // Callbr results are similarly only usable in the default destination.
-  if (const auto *CBI = dyn_cast<CallBrInst>(Def)) {
-    BasicBlock *NormalDest = CBI->getDefaultDest();
-    BasicBlockEdge E(DefBB, NormalDest);
-    return dominates(E, UseBB);
-  }
-
   return dominates(DefBB, UseBB);
 }
 
 bool DominatorTree::dominates(const BasicBlockEdge &BBE,
                               const BasicBlock *UseBB) const {
   // If the BB the edge ends in doesn't dominate the use BB, then the
   // edge also doesn't.
   const BasicBlock *Start = BBE.getStart();
bool DominatorTree::dominates(const Value *DefV, const Use &U) const {
   // their own block, except possibly a phi, so we don't need to
   // walk the block in any case.
   if (const InvokeInst *II = dyn_cast<InvokeInst>(Def)) {
     BasicBlock *NormalDest = II->getNormalDest();
     BasicBlockEdge E(DefBB, NormalDest);
     return dominates(E, U);
   }
 
-  // Callbr results are similarly only usable in the default destination.
+  // Callbr results are similarly only usable in the default destination,
+  // unless fed into a callbrpad on the indirect edge.
   if (const auto *CBI = dyn_cast<CallBrInst>(Def)) {
     BasicBlock *NormalDest = CBI->getDefaultDest();
+    if (UseBB != NormalDest && isa<CallBrPadInst>(UserInst))
+      return dominates(DefBB, UseBB);
     BasicBlockEdge E(DefBB, NormalDest);
     return dominates(E, U);
   }
 
   // If the def and use are in different blocks, do a simple CFG dominator
   // tree query.
   if (DefBB != UseBB)
     return dominates(DefBB, UseBB);

llvm/lib/IR/Instruction.cpp

const char *Instruction::getOpcodeName(unsigned OpCode) {
   case AShr:           return "ashr";
   case VAArg:          return "va_arg";
   case ExtractElement: return "extractelement";
   case InsertElement:  return "insertelement";
   case ShuffleVector:  return "shufflevector";
   case ExtractValue:   return "extractvalue";
   case InsertValue:    return "insertvalue";
   case LandingPad:     return "landingpad";
+  case CallBrPad:      return "callbrpad";
   case CleanupPad:     return "cleanuppad";
   case Freeze:         return "freeze";
 
   default: return "<Invalid operator> ";
   }
 }
 
 /// Return true if both instructions have the same special state. This must be

llvm/lib/IR/Verifier.cpp

#include "llvm/IR/Metadata.def"
   void visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI);
   void visitAtomicRMWInst(AtomicRMWInst &RMWI);
   void visitFenceInst(FenceInst &FI);
   void visitAllocaInst(AllocaInst &AI);
   void visitExtractValueInst(ExtractValueInst &EVI);
   void visitInsertValueInst(InsertValueInst &IVI);
   void visitEHPadPredecessors(Instruction &I);
   void visitLandingPadInst(LandingPadInst &LPI);
+  void visitCallBrPadInst(CallBrPadInst &CBP);
   void visitResumeInst(ResumeInst &RI);
   void visitCatchPadInst(CatchPadInst &CPI);
   void visitCatchReturnInst(CatchReturnInst &CatchReturn);
   void visitCleanupPadInst(CleanupPadInst &CPI);
   void visitFuncletPadInst(FuncletPadInst &FPI);
   void visitCatchSwitchInst(CatchSwitchInst &CatchSwitch);
   void visitCleanupReturnInst(CleanupReturnInst &CRI);
 
if (LPI.isCatch(i)) {
       Check(LPI.isFilter(i), "Clause is neither catch nor filter!", &LPI);
       Check(isa<ConstantArray>(Clause) || isa<ConstantAggregateZero>(Clause),
             "Filter operand is not an array of constants!", &LPI);
     }
   }
 
   visitInstruction(LPI);
 }
+void Verifier::visitCallBrPadInst(CallBrPadInst &CBP) {
+  const auto *CBR = dyn_cast<CallBrInst>(CBP.getOperand(0));
+  Check(CBR, "instruction requires callbr operand", &CBP);
+  if (!CBR)
+    return;
+  const BasicBlock *LandingPadBB = CBP.getParent();
+  const BasicBlock *CallBrBB = CBR->getParent();
+
+  // Is the callbr in a predecessor basic block?
+  Check(llvm::any_of(
+            predecessors(LandingPadBB),
+            [CallBrBB](const BasicBlock *Pred) { return Pred == CallBrBB; }),
+        "callbr operand must be immediate predecessor", &CBP);
+
+  Check(isa<CallBrPadInst>(LandingPadBB->getFirstNonPHI()),
+      "callbrpad must be first non phi in basic block", &CBP);
+}
 
 void Verifier::visitResumeInst(ResumeInst &RI) {
   Check(RI.getFunction()->hasPersonalityFn(),
         "ResumeInst needs to be in a function with a personality.", &RI);
 
   if (!LandingPadResultTy)
     LandingPadResultTy = RI.getValue()->getType();
   else

llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll

+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu %s -o - -stop-after=finalize-isel | \
+; RUN:  FileCheck %s
+
+; One virtual register, w/o phi
+define i32 @test0() nounwind {
+  ; CHECK-LABEL: name: test0
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000), %bb.2(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %1, 13 /* imm */, %bb.2
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY %1
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.cleanup:
+  ; CHECK-NEXT:   [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
+  ; CHECK-NEXT:   $eax = COPY [[MOV32ri]]
+  ; CHECK-NEXT:   RET 0, $eax
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   $eax = COPY %1
+  ; CHECK-NEXT:   RET 0, $eax
+  %direct = callbr i32 asm "", "=r,!i"()
+          to label %cleanup [label %z.split]
+
+cleanup:
+  ret i32 42
+z.split:
+  %indirect = callbrpad i32 %direct
+  ret i32 %indirect
+}
+
+; One virtual register, w/ phi
+define i32 @test1() nounwind {
+  ; CHECK-LABEL: name: test1
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000), %bb.1(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %4, 13 /* imm */, %bb.1
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY %4
+  ; CHECK-NEXT:   JMP_1 %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY %4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.cleanup:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY1]], %bb.1
+  ; CHECK-NEXT:   $eax = COPY [[PHI]]
+  ; CHECK-NEXT:   RET 0, $eax
+entry:
+  %direct = callbr i32 asm "", "=r,!i"()
+          to label %cleanup [label %z.split]
+
+z.split:
+  %indirect = callbrpad i32 %direct
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i32 [ %indirect, %z.split ], [ 42, %entry ]
+  ret i32 %retval.0
+}
+
+; Two virtual registers
+define i32 @test2() nounwind {
+  ; CHECK-LABEL: name: test2
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000), %bb.1(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %5, 2359306 /* regdef:GR32 */, def %6, 13 /* imm */, %bb.1
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY %6
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY %5
+  ; CHECK-NEXT:   JMP_1 %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY %5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.cleanup:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY2]], %bb.1
+  ; CHECK-NEXT:   $eax = COPY [[PHI]]
+  ; CHECK-NEXT:   RET 0, $eax
+entry:
+  %direct = callbr { i32, i32 } asm "", "=r,=r,!i"()
+          to label %cleanup [label %z.split]
+
+z.split:
+  %indirect = callbrpad { i32, i32 } %direct
+  %asmresult2 = extractvalue { i32, i32 } %indirect, 0
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i32 [ %asmresult2, %z.split ], [ 42, %entry ]
+  ret i32 %retval.0
+}
+
+; One physical register
+define i32 @test3() nounwind {
+  ; CHECK-LABEL: name: test3
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000), %bb.1(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 13 /* imm */, %bb.1
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]]
+  ; CHECK-NEXT:   JMP_1 %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $ebx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY $ebx
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY2]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.cleanup:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY3]], %bb.1
+  ; CHECK-NEXT:   $eax = COPY [[PHI]]
+  ; CHECK-NEXT:   RET 0, $eax
+entry:
+  %direct = callbr i32 asm "", "={bx},!i"()
+          to label %cleanup [label %z.split]
+
+z.split:
+  %indirect = callbrpad i32 %direct
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i32 [ %indirect, %z.split ], [ 42, %entry ]
+  ret i32 %retval.0
+}
+
+; Two physical registers
+define i32 @test4() nounwind {
+  ; CHECK-LABEL: name: test4
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000), %bb.1(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 10 /* regdef */, implicit-def $edx, 13 /* imm */, %bb.1
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY $edx
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY]]
+  ; CHECK-NEXT:   JMP_1 %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $ebx, $edx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gr32 = COPY $ebx
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:gr32 = COPY [[COPY4]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.cleanup:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY5]], %bb.1
+  ; CHECK-NEXT:   $eax = COPY [[PHI]]
+  ; CHECK-NEXT:   RET 0, $eax
+entry:
+  %direct = callbr { i32, i32 } asm "", "={bx},={dx},!i"()
+          to label %cleanup [label %z.split]
+
+z.split:
+  %indirect = callbrpad { i32, i32 } %direct
+  %asmresult2 = extractvalue { i32, i32 } %indirect, 0
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i32 [ %asmresult2, %z.split ], [ 42, %entry ]
+  ret i32 %retval.0
+}
+
+; Test the same destination appearing in the direct/fallthrough branch as the
+; indirect branch.
+define i32 @test5() nounwind {
+  ; CHECK-LABEL: name: test5
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   INLINEASM_BR &"# $0", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 13 /* imm */, %bb.1
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.cleanup (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   liveins: $ebx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY $ebx
+  ; CHECK-NEXT:   $eax = COPY [[COPY2]]
+  ; CHECK-NEXT:   RET 0, $eax
+entry:
+  %direct = callbr i32 asm "# $0", "={bx},!i"()
+          to label %cleanup [label %cleanup]
+
+cleanup:
+  %indirect = callbrpad i32 %direct
+  ret i32 %indirect
+}
+
+; "The Devil's cross" (i.e. two asm goto with conflicting physreg constraints
+; going to the same destination) as expressed by clang.
+define i64 @test6() nounwind {
+  ; CHECK-LABEL: name: test6
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000), %bb.3(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   INLINEASM_BR &"# $0 $1", 0 /* attdialect */, 10 /* regdef */, implicit-def $rdx, 13 /* imm */, %bb.3
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY $rdx
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.asm.fallthrough:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000), %bb.4(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   INLINEASM_BR &"# $0 $1", 0 /* attdialect */, 10 /* regdef */, implicit-def $rbx, 13 /* imm */, %bb.4
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY $rbx
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY2]]
+  ; CHECK-NEXT:   JMP_1 %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.foo:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gr64 = PHI %3, %bb.3, [[COPY3]], %bb.1, %4, %bb.4
+  ; CHECK-NEXT:   $rax = COPY [[PHI]]
+  ; CHECK-NEXT:   RET 0, $rax
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3.foo.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $rdx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gr64 = COPY $rdx
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:gr64 = COPY [[COPY4]]
+  ; CHECK-NEXT:   JMP_1 %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4.foo.split2 (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $rbx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:gr64 = COPY $rbx
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:gr64 = COPY [[COPY6]]
+  ; CHECK-NEXT:   JMP_1 %bb.2
+entry:
+  %0 = callbr i64 asm "# $0 $1", "={dx},!i"()
+          to label %asm.fallthrough [label %foo.split]
+
+asm.fallthrough:
+  %1 = callbr i64 asm "# $0 $1", "={bx},!i"()
+          to label %foo [label %foo.split2]
+
+foo:
+  %x.0 = phi i64 [ %3, %foo.split2 ], [ %2, %foo.split ], [ %1, %asm.fallthrough ]
+  ret i64 %x.0
+
+foo.split:
+  %2 = callbrpad i64 %0
+  br label %foo
+
+foo.split2:
+  %3 = callbrpad i64 %1
+  br label %foo
+}
+
+; Test a callbr looping back on itself.
+define i32 @test7() nounwind {
+  ; CHECK-LABEL: name: test7
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gr32 = IMPLICIT_DEF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.retry:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000), %bb.3(0x00000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gr32 = PHI [[DEF]], %bb.0, %2, %bb.3
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY [[PHI]]
+  ; CHECK-NEXT:   INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $edx, 2147483657 /* reguse tiedto:$0 */, [[COPY]](tied-def 3), 13 /* imm */, %bb.3
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY $edx
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   JMP_1 %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.asm.fallthrough:
+  ; CHECK-NEXT:   $eax = COPY [[COPY2]]
+  ; CHECK-NEXT:   RET 0, $eax
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3.retry.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $edx
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr32 = COPY $edx
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gr32 = COPY [[COPY3]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+entry:
+  br label %retry
+
+retry:
+  %x.0 = phi i32 [ undef, %entry ], [ %1, %retry.split ]
+  %0 = callbr i32 asm "", "={dx},0,!i"(i32 %x.0)
+          to label %asm.fallthrough [label %retry.split]
+
+asm.fallthrough:
+  ret i32 %0
+
+retry.split:
+  %1 = callbrpad i32 %0
+  br label %retry
+}

llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s
+; RUN: opt < %s -S -passes="simplifycfg<sink-common-insts>" -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s
 
 define void @callbr_duplicate_dest() {
 ; CHECK-LABEL: @callbr_duplicate_dest(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    callbr void asm sideeffect "", "!i"()
 ; CHECK-NEXT:    to label [[BB:%.*]] [label %bb]
 ; CHECK:       bb:
 ; CHECK-NEXT:    ret void
bb1:
   ret void
 
 bb2:
   ret void
 
 bb3:
   ret void
 }
+
+; Validate that callbrpad instructions do not get merged.
+
+define i32 @callbr_landingpad_nomerge() {
+; CHECK-LABEL: @callbr_landingpad_nomerge(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = callbr { i32, i32 } asm "", "=r,=r,!i"()
+; CHECK-NEXT:    to label [[ASM_FALLTHROUGH:%.*]] [label %z.split]
+; CHECK:       asm.fallthrough:
+; CHECK-NEXT:    [[TMP1:%.*]] = callbr { i32, i32 } asm "", "=r,=r,!i"()
+; CHECK-NEXT:    to label [[Z:%.*]] [label %z.split9]
+; CHECK:       z:
+; CHECK-NEXT:    [[DOTPN:%.*]] = phi { i32, i32 } [ [[TMP3:%.*]], [[Z_SPLIT9:%.*]] ], [ [[TMP2:%.*]], [[Z_SPLIT:%.*]] ], [ [[TMP1]], [[ASM_FALLTHROUGH]] ]
+; CHECK-NEXT:    [[RET_0:%.*]] = extractvalue { i32, i32 } [[DOTPN]], 0
+; CHECK-NEXT:    ret i32 [[RET_0]]
+; CHECK:       z.split:
+; CHECK-NEXT:    [[TMP2]] = callbrpad { i32, i32 } [[TMP0]]
+; CHECK-NEXT:    br label [[Z]]
+; CHECK:       z.split9:
+; CHECK-NEXT:    [[TMP3]] = callbrpad { i32, i32 } [[TMP1]]
+; CHECK-NEXT:    br label [[Z]]
+;
+entry:
+  %0 = callbr { i32, i32 } asm "", "=r,=r,!i"()
+  to label %asm.fallthrough [label %z.split]
+
+asm.fallthrough:
+  %1 = callbr { i32, i32 } asm "", "=r,=r,!i"()
+  to label %z [label %z.split9]
+
+z:
+  %.pn = phi { i32, i32 } [ %3, %z.split9 ], [ %2, %z.split ], [ %1, %asm.fallthrough ]
+  %ret.0 = extractvalue { i32, i32 } %.pn, 0
+  ret i32 %ret.0
+
+z.split:
+  %2 = callbrpad { i32, i32 } %0
+  br label %z
+
+z.split9:
+  %3 = callbrpad { i32, i32 } %1
+  br label %z
+}

llvm/test/Verifier/callbr.ll

define void @callbr_without_label_constraint() {
   callbr void asm sideeffect "#callbr_without_label_constraint", ""()
   to label %1 [label %2]
 1:
   ret void
 2:
   ret void
 }
 
-;; Ensure you cannot use the return value of a callbr in indirect targets.
+;; Ensure you cannot use the return value of a callbr in indirect targets,
+;; unless using a callbrpad.
 ; CHECK: Instruction does not dominate all uses!
 ; CHECK-NEXT: #test4
 define i32 @test4(i1 %var) {
 entry:
   %ret = callbr i32 asm sideeffect "#test4", "=r,!i"() to label %normal [label %abnormal]
 
 normal:
   ret i32 0
 
 abnormal:
   ret i32 %ret
 }
+
+; CHECK-NOT: Instruction does not dominate all uses!
+define i32 @test5(i1 %var) {
+entry:
+  %ret = callbr i32 asm "", "=r,!i"() to label %normal [label %abnormal]
+
+normal:
+  ret i32 0
+
+abnormal:
+  %indirect = callbrpad i32 %ret
+  ret i32 %indirect
+}
+
+define void @test_callbrpad0(i32 %foo) {
+;; Only accept callbr values as inputs. No Phis allowed!
+; CHECK: instruction requires callbr operand
+; CHECK-NEXT: %res2 = callbrpad i32 %foo
+  %res2 = callbrpad i32 %foo
+  ret void
+}
+
+define i64 @test_callbrpad_non_immediate_pred() {
+  %out = callbr i64 asm "", "=r,!i"() to label %ft [label %indirect]
+ft:
+  ret i64 42
+indirect:
+  br label %foo
+foo:
+;; Check that the landingpad is an immediate successor of the basic block that
+;; defined the value from the callbr.
+; CHECK: callbr operand must be immediate predecessor
+; CHECK-NEXT: %res = callbrpad i64 %out
+  %res = callbrpad i64 %out
+  ret i64 %res
+}
+
+declare i32 @x()
+define void @test_callbrpad_first_non_phi() {
+entry:
+  %x = callbr i32 asm "", "=r,!i"() to label %out [label %foo]
+foo:
+; CHECK: callbrpad must be first non phi in basic block
+; CHECK-NEXT: %0 = callbrpad i32 %x
+  %bar = call i32 @x()
+  %0 = callbrpad i32 %x
+  br label %out
+out:
+  ret void
+}

llvm/utils/vim/syntax/llvm.vim

 syn keyword llvmType label metadata x86_mmx x86_amx
 syn keyword llvmType type label opaque token ptr
 syn match   llvmType /\<i\d\+\>/
 
 " Instructions.
 " The true and false tokens can be used for comparison opcodes, but it's
 " much more common for these tokens to be used for boolean constants.
 syn keyword llvmStatement add addrspacecast alloca and arcp ashr atomicrmw
-syn keyword llvmStatement bitcast br catchpad catchswitch catchret call callbr
+syn keyword llvmStatement bitcast br catchpad catchswitch catchret call callbr callbrpad
 syn keyword llvmStatement cleanuppad cleanupret cmpxchg eq exact extractelement
 syn keyword llvmStatement extractvalue fadd fast fcmp fdiv fence fmul fneg fpext
 syn keyword llvmStatement fptosi fptoui fptrunc free freeze frem fsub
 syn keyword llvmStatement getelementptr icmp inbounds indirectbr insertelement
 syn keyword llvmStatement insertvalue inttoptr invoke landingpad load lshr
 syn keyword llvmStatement malloc max min mul nand ne ninf nnan nsw nsz nuw oeq
 syn keyword llvmStatement oge ogt ole olt one or ord phi ptrtoint resume ret
 syn keyword llvmStatement sdiv select sext sge sgt shl shufflevector sitofp