diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -12134,6 +12134,48 @@ %tok = cleanuppad within %cs [] +.. _i_callbrpad: + +'``callbrpad``' Instruction +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + = callbrpad + +Overview: +""""""""" + +The '``callbrpad``' instruction corresponds 1:1 with a :ref:`callbr ` +instruction, which should be the lone value to the '``callbrpad``'. It is +required in order to consume outputs (if any) from the :ref:`callbr ` +in the indirect branches of the :ref:`callbr `. + +Arguments: +"""""""""" + +The '``callbrpad``' instruction takes one argument, which must be a +`callbr `. The type of the '``callbrpad``' corresponds to the type of +the `callbr ` argument. + +Semantics: +"""""""""" + +The '``callbrpad``' instruction is consumed during codegen as a place for +physical registers specified in output constraints of a +:ref:`callbr to be copied back to virtual registers. + +Example: +"""""""" + + +.. code-block:: llvm + + %1 = callbrpad i32 %0 + + .. _intrinsics: Intrinsic Functions diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -93,6 +93,9 @@ * ``fneg`` +* The ``callbrpad`` instruction was introduced for supporting outputs along + indirect edges of callbr. + Changes to building LLVM ------------------------ diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -142,7 +142,8 @@ LLVMCatchRet = 62, LLVMCatchPad = 63, LLVMCleanupPad = 64, - LLVMCatchSwitch = 65 + LLVMCatchSwitch = 65, + LLVMCallBrPad = 69, } LLVMOpcode; typedef enum { diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h --- a/llvm/include/llvm/AsmParser/LLParser.h +++ b/llvm/include/llvm/AsmParser/LLParser.h @@ -619,6 +619,7 @@ bool parseShuffleVector(Instruction *&Inst, PerFunctionState &PFS); int parsePHI(Instruction *&Inst, PerFunctionState &PFS); bool parseLandingPad(Instruction *&Inst, PerFunctionState &PFS); + bool parseCallBrPad(Instruction *&Inst, PerFunctionState &PFS); bool parseCall(Instruction *&Inst, PerFunctionState &PFS, CallInst::TailCallKind TCK); int parseAlloc(Instruction *&Inst, PerFunctionState &PFS); diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -298,6 +298,7 @@ kw_catchpad, kw_cleanuppad, kw_callbr, + kw_callbrpad, kw_alloca, kw_load, diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -274,6 +274,7 @@ bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder); + bool translateCallBrPad(const User &U, MachineIRBuilder &MIRBuilder); /// Translate one of LLVM's cast instructions into MachineInstrs, with the /// given generic Opcode. diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -2396,6 +2396,10 @@ return Insert(LandingPadInst::Create(Ty, NumClauses), Name); } + CallBrPadInst *CreateCallBrPad(CallBrInst *CBR) { + return Insert(CallBrPadInst::Create(CBR)); + } + Value *CreateFreeze(Value *V, const Twine &Name = "") { return Insert(new FreezeInst(V), Name); } diff --git a/llvm/include/llvm/IR/InstVisitor.h b/llvm/include/llvm/IR/InstVisitor.h --- a/llvm/include/llvm/IR/InstVisitor.h +++ b/llvm/include/llvm/IR/InstVisitor.h @@ -194,6 +194,7 @@ RetTy visitExtractValueInst(ExtractValueInst &I){ DELEGATE(UnaryInstruction);} RetTy visitInsertValueInst(InsertValueInst &I) { DELEGATE(Instruction); } RetTy visitLandingPadInst(LandingPadInst &I) { DELEGATE(Instruction); } + RetTy visitCallBrPadInst(CallBrPadInst &I) { DELEGATE(Instruction); } RetTy visitFuncletPadInst(FuncletPadInst &I) { DELEGATE(Instruction); } RetTy visitCleanupPadInst(CleanupPadInst &I) { DELEGATE(FuncletPadInst); } RetTy visitCatchPadInst(CatchPadInst &I) { DELEGATE(FuncletPadInst); } diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h --- a/llvm/include/llvm/IR/Instruction.h +++ b/llvm/include/llvm/IR/Instruction.h @@ -668,8 +668,9 @@ /// Return true if the instruction is a variety of EH-block. bool isEHPad() const { switch (getOpcode()) { - case Instruction::CatchSwitch: + case Instruction::CallBrPad: case Instruction::CatchPad: + case Instruction::CatchSwitch: case Instruction::CleanupPad: case Instruction::LandingPad: return true; diff --git a/llvm/include/llvm/IR/Instruction.def b/llvm/include/llvm/IR/Instruction.def --- a/llvm/include/llvm/IR/Instruction.def +++ b/llvm/include/llvm/IR/Instruction.def @@ -218,7 +218,8 @@ HANDLE_OTHER_INST(65, InsertValue, InsertValueInst) // insert into aggregate HANDLE_OTHER_INST(66, LandingPad, LandingPadInst) // Landing pad instruction. HANDLE_OTHER_INST(67, Freeze, FreezeInst) // Freeze instruction. - LAST_OTHER_INST(67) +HANDLE_OTHER_INST(68, CallBrPad, CallBrPadInst) // Landing pad for callbr instructions. + LAST_OTHER_INST(68) #undef FIRST_TERM_INST #undef HANDLE_TERM_INST diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -3039,6 +3039,31 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value) +//===----------------------------------------------------------------------===// +// CallBrPadInst Class +//===----------------------------------------------------------------------===// + +//===--------------------------------------------------------------------------- +/// The callbrpad instruction holds all of the information necessary to +/// generate physreg to virtreg copies for callbr with inline asm. The +/// callbrpad instruction cannot be moved from the top of a callbrpad block, +/// which itself is accessible only from the indirect edge of a callbr. +class CallBrPadInst : public UnaryInstruction { + CallBrPadInst() = delete; + explicit CallBrPadInst(Value *V) + : UnaryInstruction(V->getType(), Instruction::CallBrPad, V) {} + +public: + static CallBrPadInst *Create(Value *V) { return new CallBrPadInst(V); } + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Instruction *I) { + return I->getOpcode() == Instruction::CallBrPad; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + //===----------------------------------------------------------------------===// // ReturnInst Class //===----------------------------------------------------------------------===// diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -880,6 +880,7 @@ INSTKEYWORD(extractvalue, ExtractValue); INSTKEYWORD(insertvalue, InsertValue); INSTKEYWORD(landingpad, LandingPad); + INSTKEYWORD(callbrpad, CallBrPad); INSTKEYWORD(cleanupret, CleanupRet); INSTKEYWORD(catchret, CatchRet); INSTKEYWORD(catchswitch, CatchSwitch); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -6335,6 +6335,8 @@ } case lltok::kw_landingpad: return parseLandingPad(Inst, PFS); + case lltok::kw_callbrpad: + return parseCallBrPad(Inst, PFS); case lltok::kw_freeze: return parseFreeze(Inst, PFS); // Call. @@ -9911,3 +9913,13 @@ return false; } + +/// CallBrPad +/// ::= 'callbrpad' TypeAndValue +bool LLParser::parseCallBrPad(Instruction *&Inst, PerFunctionState &PFS) { + Value *V; + if (parseTypeAndValue(V, PFS)) + return true; + Inst = CallBrPadInst::Create(V); + return false; +} diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3606,3 +3606,8 @@ return false; } + +bool IRTranslator::translateCallBrPad(const User &U, + MachineIRBuilder &MIRBuilder) { + return false; +} diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -244,7 +244,7 @@ // funclets. // FIXME: SEH catchpads do not create EH scope/funclets, so we could avoid // setting this in such cases in order to improve frame layout. - if (!isa(PadInst)) { + if (!isa(PadInst) && !isa(PadInst)) { MF->setHasEHScopes(true); MF->setHasEHFunclets(true); MF->getFrameInfo().setHasOpaqueSPAdjustment(true); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -550,6 +550,7 @@ void visitExtractValue(const ExtractValueInst &I); void visitInsertValue(const InsertValueInst &I); void visitLandingPad(const LandingPadInst &LP); + void visitCallBrPad(const CallBrPadInst &LP); void visitGetElementPtr(const User &I); void visitSelect(const User &I); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -11536,3 +11536,78 @@ Mask.push_back(Idx + i); setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask)); } + +// Consider the following MIR after SelectionDAG, which produces output in +// phyregs in the first case or virtregs in the second case. +// +// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx +// %5:gr32 = COPY $ebx +// %6:gr32 = COPY $edx +// %1:gr32 = COPY %6:gr32 +// %0:gr32 = COPY %5:gr32 +// +// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32 +// %1:gr32 = COPY %6:gr32 +// %0:gr32 = COPY %5:gr32 +// +// Given %0, we'd like to return $ebx in the first case and %5 in the second. +// Given %1, we'd like to return $edx in the first case and %6 in the second. +// +// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap +// to a single virtreg (such as %0). The remaining ouputs monotonically +// increase in virtreg number from there. If a callbr has no outputs, then it +// should not have a corresponding callbrpad; in fact, the callbrpad would not +// even be able to refer to such a callbr. +static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) { + MachineOperand *MO = &*MRI.def_begin(Reg); + MachineInstr *MI = MO->getParent(); + if (MI->getOpcode() == TargetOpcode::COPY) + return FollowCopyChain(MRI, MI->getOperand(1).getReg()); + if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) + return Reg; + return {}; +} + +static size_t PopulateResultValues(const CallBrInst &CBR, SelectionDAG &DAG, + SmallVectorImpl &ResultVTs, + SmallVectorImpl &ResultValues) { + Type *CBRType = CBR.getType(); + assert(!CBRType->isVoidTy() && + "callbr landing pad for callbr without returns"); + ArrayRef ResultTypes = CBRType->isStructTy() + ? cast(CBRType)->elements() + : makeArrayRef(CBRType); + size_t NumReturnValues = ResultTypes.size(); + ResultVTs.reserve(NumReturnValues); + ResultValues.reserve(NumReturnValues); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + for (Type *T : ResultTypes) + ResultVTs.emplace_back(TLI.getValueType(DAG.getDataLayout(), T)); + return NumReturnValues; +} + +void SelectionDAGBuilder::visitCallBrPad(const CallBrPadInst &LP) { + SmallVector ResultVTs; + SmallVector ResultValues; + CallBrInst *CBR = cast(LP.getOperand(0)); + const size_t NumReturnValues = + PopulateResultValues(*CBR, DAG, ResultVTs, ResultValues); + MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); + unsigned Reg = FuncInfo.ValueMap[CBR]; + SDValue Chain = DAG.getRoot(); + + for (size_t i = 0; i != NumReturnValues; ++i, ++Reg) { + Register Output = FollowCopyChain(MRI, Reg); + + if (Register::isPhysicalRegister(Output)) + FuncInfo.MBB->addLiveIn(Output); + + // TODO: should we mark the copied-from register as killed by this COPY (if + // physreg)? + SDValue V = DAG.getCopyFromReg(Chain, getCurSDLoc(), Output, ResultVTs[i]); + ResultValues.push_back(V); + } + SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), + DAG.getVTList(ResultVTs), ResultValues); + setValue(&LP, V); +} diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -1223,17 +1223,21 @@ /// do other setup for EH landing-pad blocks. bool SelectionDAGISel::PrepareEHLandingPad() { MachineBasicBlock *MBB = FuncInfo->MBB; - const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn(); const BasicBlock *LLVMBB = MBB->getBasicBlock(); - const TargetRegisterClass *PtrRC = - TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout())); + const Instruction *FirstNonPhi = LLVMBB->getFirstNonPHI(); + + if (isa(FirstNonPhi)) + return true; + const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn(); auto Pers = classifyEHPersonality(PersonalityFn); + const TargetRegisterClass *PtrRC = + TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout())); // Catchpads have one live-in register, which typically holds the exception // pointer or code. if (isFuncletEHPersonality(Pers)) { - if (const auto *CPI = dyn_cast(LLVMBB->getFirstNonPHI())) { + if (const auto *CPI = dyn_cast(FirstNonPhi)) { if (hasExceptionPointerOrCodeUser(CPI)) { // Get or create the virtual register to hold the pointer or code. Mark // the live in physreg and copy into the vreg. @@ -1264,7 +1268,7 @@ MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask); if (Pers == EHPersonality::Wasm_CXX) { - if (const auto *CPI = dyn_cast(LLVMBB->getFirstNonPHI())) + if (const auto *CPI = dyn_cast(FirstNonPhi)) mapWasmLandingPadIndex(MBB, CPI); } else { // Assign the call site to the landing pad's begin label. diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -1855,6 +1855,7 @@ case ExtractValue: return ISD::MERGE_VALUES; case InsertValue: return ISD::MERGE_VALUES; case LandingPad: return 0; + case CallBrPad: return 0; case Freeze: return ISD::FREEZE; } diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -4150,6 +4150,9 @@ writeOperand(LPI->getClause(i), true); } + } else if (const auto *Pad = dyn_cast(&I)) { + Out << ' '; + writeOperand(Pad->getOperand(0), true); } else if (const auto *CatchSwitch = dyn_cast(&I)) { Out << " within "; writeOperand(CatchSwitch->getParentPad(), /*PrintType=*/false); diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp --- a/llvm/lib/IR/Dominators.cpp +++ b/llvm/lib/IR/Dominators.cpp @@ -194,13 +194,6 @@ return dominates(E, UseBB); } - // Callbr results are similarly only usable in the default destination. - if (const auto *CBI = dyn_cast(Def)) { - BasicBlock *NormalDest = CBI->getDefaultDest(); - BasicBlockEdge E(DefBB, NormalDest); - return dominates(E, UseBB); - } - return dominates(DefBB, UseBB); } @@ -311,9 +304,12 @@ return dominates(E, U); } - // Callbr results are similarly only usable in the default destination. + // Callbr results are similarly only usable in the default destination, + // unless fed into a callbrpad on the indirect edge. if (const auto *CBI = dyn_cast(Def)) { BasicBlock *NormalDest = CBI->getDefaultDest(); + if (UseBB != NormalDest && isa(UserInst)) + return dominates(DefBB, UseBB); BasicBlockEdge E(DefBB, NormalDest); return dominates(E, U); } diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -459,6 +459,7 @@ case ExtractValue: return "extractvalue"; case InsertValue: return "insertvalue"; case LandingPad: return "landingpad"; + case CallBrPad: return "callbrpad"; case CleanupPad: return "cleanuppad"; case Freeze: return "freeze"; diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -544,6 +544,7 @@ void visitInsertValueInst(InsertValueInst &IVI); void visitEHPadPredecessors(Instruction &I); void visitLandingPadInst(LandingPadInst &LPI); + void visitCallBrPadInst(CallBrPadInst &CBP); void visitResumeInst(ResumeInst &RI); void visitCatchPadInst(CatchPadInst &CPI); void visitCatchReturnInst(CatchReturnInst &CatchReturn); @@ -4177,6 +4178,23 @@ visitInstruction(LPI); } +void Verifier::visitCallBrPadInst(CallBrPadInst &CBP) { + const auto *CBR = dyn_cast(CBP.getOperand(0)); + Check(CBR, "instruction requires callbr operand", &CBP); + if (!CBR) + return; + const BasicBlock *LandingPadBB = CBP.getParent(); + const BasicBlock *CallBrBB = CBR->getParent(); + + // Is the callbr in a predecessor basic block? + Check(llvm::any_of( + predecessors(LandingPadBB), + [CallBrBB](const BasicBlock *Pred) { return Pred == CallBrBB; }), + "callbr operand must be immediate predecessor", &CBP); + + Check(isa(LandingPadBB->getFirstNonPHI()), + "callbrpad must be first non phi in basic block", &CBP); +} void Verifier::visitResumeInst(ResumeInst &RI) { Check(RI.getFunction()->hasPersonalityFn(), diff --git a/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll b/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll @@ -0,0 +1,307 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +; RUN: llc -mtriple=x86_64-linux-gnu %s -o - -stop-after=finalize-isel | \ +; RUN: FileCheck %s + +; One virtual register, w/o phi +define i32 @test0() nounwind { + ; CHECK-LABEL: name: test0 + ; CHECK: bb.0 (%ir-block.0): + ; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %1, 13 /* imm */, %bb.2 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY %1 + ; CHECK-NEXT: JMP_1 %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.cleanup: + ; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42 + ; CHECK-NEXT: $eax = COPY [[MOV32ri]] + ; CHECK-NEXT: RET 0, $eax + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: $eax = COPY %1 + ; CHECK-NEXT: RET 0, $eax + %direct = callbr i32 asm "", "=r,!i"() + to label %cleanup [label %z.split] + +cleanup: + ret i32 42 +z.split: + %indirect = callbrpad i32 %direct + ret i32 %indirect +} + +; One virtual register, w/ phi +define i32 @test1() nounwind { + ; CHECK-LABEL: name: test1 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %4, 13 /* imm */, %bb.1 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY %4 + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY %4 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.cleanup: + ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY1]], %bb.1 + ; CHECK-NEXT: $eax = COPY [[PHI]] + ; CHECK-NEXT: RET 0, $eax +entry: + %direct = callbr i32 asm "", "=r,!i"() + to label %cleanup [label %z.split] + +z.split: + %indirect = callbrpad i32 %direct + br label %cleanup + +cleanup: + %retval.0 = phi i32 [ %indirect, %z.split ], [ 42, %entry ] + ret i32 %retval.0 +} + +; Two virtual registers +define i32 @test2() nounwind { + ; CHECK-LABEL: name: test2 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %5, 2359306 /* regdef:GR32 */, def %6, 13 /* imm */, %bb.1 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY %6 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY %5 + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY %5 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.cleanup: + ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY2]], %bb.1 + ; CHECK-NEXT: $eax = COPY [[PHI]] + ; CHECK-NEXT: RET 0, $eax +entry: + %direct = callbr { i32, i32 } asm "", "=r,=r,!i"() + to label %cleanup [label %z.split] + +z.split: + %indirect = callbrpad { i32, i32 } %direct + %asmresult2 = extractvalue { i32, i32 } %indirect, 0 + br label %cleanup + +cleanup: + %retval.0 = phi i32 [ %asmresult2, %z.split ], [ 42, %entry ] + ret i32 %retval.0 +} + +; One physical register +define i32 @test3() nounwind { + ; CHECK-LABEL: name: test3 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 13 /* imm */, %bb.1 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]] + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $ebx + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $ebx + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY2]] + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.cleanup: + ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY3]], %bb.1 + ; CHECK-NEXT: $eax = COPY [[PHI]] + ; CHECK-NEXT: RET 0, $eax +entry: + %direct = callbr i32 asm "", "={bx},!i"() + to label %cleanup [label %z.split] + +z.split: + %indirect = callbrpad i32 %direct + br label %cleanup + +cleanup: + %retval.0 = phi i32 [ %indirect, %z.split ], [ 42, %entry ] + ret i32 %retval.0 +} + +; Two physical registers +define i32 @test4() nounwind { + ; CHECK-LABEL: name: test4 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42 + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 10 /* regdef */, implicit-def $edx, 13 /* imm */, %bb.1 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $edx + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]] + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY]] + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $ebx, $edx + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr32 = COPY $ebx + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr32 = COPY [[COPY4]] + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.cleanup: + ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY5]], %bb.1 + ; CHECK-NEXT: $eax = COPY [[PHI]] + ; CHECK-NEXT: RET 0, $eax +entry: + %direct = callbr { i32, i32 } asm "", "={bx},={dx},!i"() + to label %cleanup [label %z.split] + +z.split: + %indirect = callbrpad { i32, i32 } %direct + %asmresult2 = extractvalue { i32, i32 } %indirect, 0 + br label %cleanup + +cleanup: + %retval.0 = phi i32 [ %asmresult2, %z.split ], [ 42, %entry ] + ret i32 %retval.0 +} + +; Test the same destination appearing in the direct/fallthrough branch as the +; indirect branch. +define i32 @test5() nounwind { + ; CHECK-LABEL: name: test5 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM_BR &"# $0", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 13 /* imm */, %bb.1 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]] + ; CHECK-NEXT: JMP_1 %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.cleanup (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: liveins: $ebx + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $ebx + ; CHECK-NEXT: $eax = COPY [[COPY2]] + ; CHECK-NEXT: RET 0, $eax +entry: + %direct = callbr i32 asm "# $0", "={bx},!i"() + to label %cleanup [label %cleanup] + +cleanup: + %indirect = callbrpad i32 %direct + ret i32 %indirect +} + +; "The Devil's cross" (i.e. two asm goto with conflicting physreg constraints +; going to the same destination) as expressed by clang. +define i64 @test6() nounwind { + ; CHECK-LABEL: name: test6 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.3(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM_BR &"# $0 $1", 0 /* attdialect */, 10 /* regdef */, implicit-def $rdx, 13 /* imm */, %bb.3 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdx + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]] + ; CHECK-NEXT: JMP_1 %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.asm.fallthrough: + ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.4(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM_BR &"# $0 $1", 0 /* attdialect */, 10 /* regdef */, implicit-def $rbx, 13 /* imm */, %bb.4 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rbx + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY2]] + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.foo: + ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr64 = PHI %3, %bb.3, [[COPY3]], %bb.1, %4, %bb.4 + ; CHECK-NEXT: $rax = COPY [[PHI]] + ; CHECK-NEXT: RET 0, $rax + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3.foo.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $rdx + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $rdx + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY [[COPY4]] + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.4.foo.split2 (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $rbx + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr64 = COPY $rbx + ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gr64 = COPY [[COPY6]] + ; CHECK-NEXT: JMP_1 %bb.2 +entry: + %0 = callbr i64 asm "# $0 $1", "={dx},!i"() + to label %asm.fallthrough [label %foo.split] + +asm.fallthrough: + %1 = callbr i64 asm "# $0 $1", "={bx},!i"() + to label %foo [label %foo.split2] + +foo: + %x.0 = phi i64 [ %3, %foo.split2 ], [ %2, %foo.split ], [ %1, %asm.fallthrough ] + ret i64 %x.0 + +foo.split: + %2 = callbrpad i64 %0 + br label %foo + +foo.split2: + %3 = callbrpad i64 %1 + br label %foo +} + +; Test a callbr looping back on itself. +define i32 @test7() nounwind { + ; CHECK-LABEL: name: test7 + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[DEF:%[0-9]+]]:gr32 = IMPLICIT_DEF + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.retry: + ; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.3(0x00000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[DEF]], %bb.0, %2, %bb.3 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY [[PHI]] + ; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $edx, 2147483657 /* reguse tiedto:$0 */, [[COPY]](tied-def 3), 13 /* imm */, %bb.3 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $edx + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]] + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.asm.fallthrough: + ; CHECK-NEXT: $eax = COPY [[COPY2]] + ; CHECK-NEXT: RET 0, $eax + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3.retry.split (machine-block-address-taken, landing-pad, inlineasm-br-indirect-target): + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: liveins: $edx + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr32 = COPY $edx + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr32 = COPY [[COPY3]] + ; CHECK-NEXT: JMP_1 %bb.1 +entry: + br label %retry + +retry: + %x.0 = phi i32 [ undef, %entry ], [ %1, %retry.split ] + %0 = callbr i32 asm "", "={dx},0,!i"(i32 %x.0) + to label %asm.fallthrough [label %retry.split] + +asm.fallthrough: + ret i32 %0 + +retry.split: + %1 = callbrpad i32 %0 + br label %retry +} diff --git a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll --- a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll +++ b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s +; RUN: opt < %s -S -passes="simplifycfg" -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s define void @callbr_duplicate_dest() { ; CHECK-LABEL: @callbr_duplicate_dest( @@ -57,3 +57,46 @@ bb3: ret void } + +; Validate that callbrpad instructions do not get merged. + +define i32 @callbr_landingpad_nomerge() { +; CHECK-LABEL: @callbr_landingpad_nomerge( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = callbr { i32, i32 } asm "", "=r,=r,!i"() +; CHECK-NEXT: to label [[ASM_FALLTHROUGH:%.*]] [label %z.split] +; CHECK: asm.fallthrough: +; CHECK-NEXT: [[TMP1:%.*]] = callbr { i32, i32 } asm "", "=r,=r,!i"() +; CHECK-NEXT: to label [[Z:%.*]] [label %z.split9] +; CHECK: z: +; CHECK-NEXT: [[DOTPN:%.*]] = phi { i32, i32 } [ [[TMP3:%.*]], [[Z_SPLIT9:%.*]] ], [ [[TMP2:%.*]], [[Z_SPLIT:%.*]] ], [ [[TMP1]], [[ASM_FALLTHROUGH]] ] +; CHECK-NEXT: [[RET_0:%.*]] = extractvalue { i32, i32 } [[DOTPN]], 0 +; CHECK-NEXT: ret i32 [[RET_0]] +; CHECK: z.split: +; CHECK-NEXT: [[TMP2]] = callbrpad { i32, i32 } [[TMP0]] +; CHECK-NEXT: br label [[Z]] +; CHECK: z.split9: +; CHECK-NEXT: [[TMP3]] = callbrpad { i32, i32 } [[TMP1]] +; CHECK-NEXT: br label [[Z]] +; +entry: + %0 = callbr { i32, i32 } asm "", "=r,=r,!i"() + to label %asm.fallthrough [label %z.split] + +asm.fallthrough: + %1 = callbr { i32, i32 } asm "", "=r,=r,!i"() + to label %z [label %z.split9] + +z: + %.pn = phi { i32, i32 } [ %3, %z.split9 ], [ %2, %z.split ], [ %1, %asm.fallthrough ] + %ret.0 = extractvalue { i32, i32 } %.pn, 0 + ret i32 %ret.0 + +z.split: + %2 = callbrpad { i32, i32 } %0 + br label %z + +z.split9: + %3 = callbrpad { i32, i32 } %1 + br label %z +} diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll --- a/llvm/test/Verifier/callbr.ll +++ b/llvm/test/Verifier/callbr.ll @@ -56,7 +56,8 @@ ret void } -;; Ensure you cannot use the return value of a callbr in indirect targets. +;; Ensure you cannot use the return value of a callbr in indirect targets, +;; unless using a callbrpad. ; CHECK: Instruction does not dominate all uses! ; CHECK-NEXT: #test4 define i32 @test4(i1 %var) { @@ -69,3 +70,53 @@ abnormal: ret i32 %ret } + +; CHECK-NOT: Instruction does not dominate all uses! +define i32 @test5(i1 %var) { +entry: + %ret = callbr i32 asm "", "=r,!i"() to label %normal [label %abnormal] + +normal: + ret i32 0 + +abnormal: + %indirect = callbrpad i32 %ret + ret i32 %indirect +} + +define void @test_callbrpad0(i32 %foo) { +;; Only accept callbr values as inputs. No Phis allowed! +; CHECK: instruction requires callbr operand +; CHECK-NEXT: %res2 = callbrpad i32 %foo + %res2 = callbrpad i32 %foo + ret void +} + +define i64 @test_callbrpad_non_immediate_pred() { + %out = callbr i64 asm "", "=r,!i"() to label %ft [label %indirect] +ft: + ret i64 42 +indirect: + br label %foo +foo: +;; Check that the landingpad is an immediate successor of the basic block that +;; defined the value from the callbr. +; CHECK: callbr operand must be immediate predecessor +; CHECK-NEXT: %res = callbrpad i64 %out + %res = callbrpad i64 %out + ret i64 %res +} + +declare i32 @x() +define void @test_callbrpad_first_non_phi() { +entry: + %x = callbr i32 asm "", "=r,!i"() to label %out [label %foo] +foo: +; CHECK: callbrpad must be first non phi in basic block +; CHECK-NEXT: %0 = callbrpad i32 %x + %bar = call i32 @x() + %0 = callbrpad i32 %x + br label %out +out: + ret void +} diff --git a/llvm/utils/vim/syntax/llvm.vim b/llvm/utils/vim/syntax/llvm.vim --- a/llvm/utils/vim/syntax/llvm.vim +++ b/llvm/utils/vim/syntax/llvm.vim @@ -23,7 +23,7 @@ " The true and false tokens can be used for comparison opcodes, but it's " much more common for these tokens to be used for boolean constants. syn keyword llvmStatement add addrspacecast alloca and arcp ashr atomicrmw -syn keyword llvmStatement bitcast br catchpad catchswitch catchret call callbr +syn keyword llvmStatement bitcast br catchpad catchswitch catchret call callbr callbrpad syn keyword llvmStatement cleanuppad cleanupret cmpxchg eq exact extractelement syn keyword llvmStatement extractvalue fadd fast fcmp fdiv fence fmul fneg fpext syn keyword llvmStatement fptosi fptoui fptrunc free freeze frem fsub