Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -6513,6 +6513,7 @@ The terminator instructions are: ':ref:`ret `', ':ref:`br `', ':ref:`switch `', ':ref:`indirectbr `', ':ref:`invoke `', +':ref:`callbr `' ':ref:`resume `', ':ref:`catchswitch `', ':ref:`catchret `', ':ref:`cleanupret `', @@ -6837,6 +6838,85 @@ %retval = invoke coldcc i32 %Testfnptr(i32 15) to label %Continue unwind label %TestCleanup ; i32:retval set +.. _i_callbr: + +'``callbr``' Instruction +^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + = callbr [cconv] [ret attrs] [addrspace()] [| () [fn attrs] + [operand bundles] to label or jump [other labels] + +Overview: +""""""""" + +The '``callbr``' instruction causes control to transfer to a specified +function, with the possibility of control flow transfer to either the +'``normal``' label or one of the '``other``' labels. + +This instruction should only be used to implement the "goto" feature of gcc +style inline assembly. Any other usage is an error in the IR verifier. + +Arguments: +"""""""""" + +This instruction requires several arguments: + +#. The optional "cconv" marker indicates which :ref:`calling + convention ` the call should use. If none is + specified, the call defaults to using C calling conventions. +#. The optional :ref:`Parameter Attributes ` list for return + values. Only '``zeroext``', '``signext``', and '``inreg``' attributes + are valid here. +#. The optional addrspace attribute can be used to indicate the address space + of the called function. If it is not specified, the program address space + from the :ref:`datalayout string` will be used. +#. '``ty``': the type of the call instruction itself which is also the + type of the return value. Functions that return no value are marked + ``void``. +#. '``fnty``': shall be the signature of the function being called. The + argument types must match the types implied by this signature. This + type can be omitted if the function is not varargs. +#. '``fnptrval``': An LLVM value containing a pointer to a function to + be called. In most cases, this is a direct function call, but + indirect ``callbr``'s are just as possible, calling an arbitrary pointer + to function value. +#. '``function args``': argument list whose types match the function + signature argument types and parameter attributes. All arguments must + be of :ref:`first class ` type. If the function signature + indicates the function accepts a variable number of arguments, the + extra arguments can be specified. +#. '``normal label``': the label reached when the called function + executes a '``ret``' instruction. +#. '``other labels``': the labels reached when a callee transfers control + to a location other than the normal '``normal label``' +#. The optional :ref:`function attributes ` list. +#. The optional :ref:`operand bundles ` list. + +Semantics: +"""""""""" + +This instruction is designed to operate as a standard '``call``' +instruction in most regards. The primary difference is that it +establishes an association with additional labels to define where control +flow goes after the call. + +The only use of this today is to implement the "goto" feature of gcc inline +assembly where additional labels can be provided as locations for the inline +assembly to return to. + +Example: +"""""""" + +.. code-block:: llvm + + callbr void asm "", "r,x"(i32 %x, i8 *blockaddress(@foo, %fail)) + to label %normal or jump [label %fail] + .. _i_resume: '``resume``' Instruction Index: include/llvm-c/Core.h =================================================================== --- include/llvm-c/Core.h +++ include/llvm-c/Core.h @@ -65,6 +65,7 @@ LLVMInvoke = 5, /* removed 6 due to API changes */ LLVMUnreachable = 7, + LLVMCallBr = 67, /* Standard Unary Operators */ LLVMFNeg = 66, Index: include/llvm/Analysis/SparsePropagation.h =================================================================== --- include/llvm/Analysis/SparsePropagation.h +++ include/llvm/Analysis/SparsePropagation.h @@ -329,12 +329,9 @@ return; } - if (TI.isExceptionalTerminator()) { - Succs.assign(Succs.size(), true); - return; - } - - if (isa(TI)) { + if (TI.isExceptionalTerminator() || + isa(TI) || + isa(TI)) { Succs.assign(Succs.size(), true); return; } Index: include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- include/llvm/Bitcode/LLVMBitCodes.h +++ include/llvm/Bitcode/LLVMBitCodes.h @@ -475,7 +475,8 @@ FUNC_CODE_INST_BR = 11, // BR: [bb#, bb#, cond] or [bb#] FUNC_CODE_INST_SWITCH = 12, // SWITCH: [opty, op0, op1, ...] FUNC_CODE_INST_INVOKE = 13, // INVOKE: [attr, fnty, op0,op1, ...] - // 14 is unused. + FUNC_CODE_INST_CALLBR = 14, // CALLBR: [attr, cc, norm, transfs, + // fnty, fnid, args...] FUNC_CODE_INST_UNREACHABLE = 15, // UNREACHABLE FUNC_CODE_INST_PHI = 16, // PHI: [ty, val0,bb0, ...] Index: include/llvm/CodeGen/GlobalISel/IRTranslator.h =================================================================== --- include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -241,6 +241,8 @@ bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder); + bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); + bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder); /// Translate one of LLVM's cast instructions into MachineInstrs, with the Index: include/llvm/CodeGen/MachineBasicBlock.h =================================================================== --- include/llvm/CodeGen/MachineBasicBlock.h +++ include/llvm/CodeGen/MachineBasicBlock.h @@ -125,6 +125,9 @@ /// Indicate that this basic block is the entry block of a cleanup funclet. bool IsCleanupFuncletEntry = false; + /// Indicate that this basic block is the target of an asm-goto. + bool IsAsmGotoTarget = false; + /// since getSymbol is a relatively heavy-weight operation, the symbol /// is only computed once and is cached. mutable MCSymbol *CachedMCSymbol = nullptr; @@ -399,6 +402,16 @@ /// Indicates if this is the entry block of a cleanup funclet. void setIsCleanupFuncletEntry(bool V = true) { IsCleanupFuncletEntry = V; } + /// Returns true if the block is an asm-goto target. That is the block might + /// be jumped to by inline assembly. + bool isAsmGotoTarget() const { return IsAsmGotoTarget; } + + /// Indicates this block an asm-goto target. That is the block might be + /// jumped to by inline assembly. + void setIsAsmGotoTarget(bool V = true) { IsAsmGotoTarget = V; } + + bool hasAsmGotoTargetSuccessor() const; + /// Returns true if it is legal to hoist instructions into this block. bool isLegalToHoistInto() const; Index: include/llvm/IR/CallSite.h =================================================================== --- include/llvm/IR/CallSite.h +++ include/llvm/IR/CallSite.h @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// // // This file defines the CallSite class, which is a handy wrapper for code that -// wants to treat Call and Invoke instructions in a generic way. When in non- -// mutation context (e.g. an analysis) ImmutableCallSite should be used. +// wants to treat Call, Invoke and CallBr instructions in a generic way. When +// in non-mutation context (e.g. an analysis) ImmutableCallSite should be used. // Finally, when some degree of customization is necessary between these two // extremes, CallSiteBase<> can be supplied with fine-tuned parameters. // @@ -17,7 +17,7 @@ // They are efficiently copyable, assignable and constructable, with cost // equivalent to copying a pointer (notice that they have only a single data // member). The internal representation carries a flag which indicates which of -// the two variants is enclosed. This allows for cheaper checks when various +// the three variants is enclosed. This allows for cheaper checks when various // accessors of CallSite are employed. // //===----------------------------------------------------------------------===// @@ -48,45 +48,50 @@ enum ID : unsigned; } -template class CallSiteBase { protected: - PointerIntPair I; + PointerIntPair I; CallSiteBase() = default; - CallSiteBase(CallTy *CI) : I(CI, true) { assert(CI); } - CallSiteBase(InvokeTy *II) : I(II, false) { assert(II); } + CallSiteBase(CallTy *CI) : I(CI, 1) { assert(CI); } + CallSiteBase(InvokeTy *II) : I(II, 0) { assert(II); } + CallSiteBase(CallBrTy *CBI) : I(CBI, 2) { assert(CBI); } explicit CallSiteBase(ValTy *II) { *this = get(II); } private: /// This static method is like a constructor. It will create an appropriate - /// call site for a Call or Invoke instruction, but it can also create a null - /// initialized CallSiteBase object for something which is NOT a call site. + /// call site for a Call, Invoke or CallBr instruction, but it can also create + /// a null initialized CallSiteBase object for something which is NOT a call + /// site. static CallSiteBase get(ValTy *V) { if (InstrTy *II = dyn_cast(V)) { if (II->getOpcode() == Instruction::Call) return CallSiteBase(static_cast(II)); - else if (II->getOpcode() == Instruction::Invoke) + if (II->getOpcode() == Instruction::Invoke) return CallSiteBase(static_cast(II)); + if (II->getOpcode() == Instruction::CallBr) + return CallSiteBase(static_cast(II)); } return CallSiteBase(); } public: - /// Return true if a CallInst is enclosed. Note that !isCall() does not mean - /// an InvokeInst is enclosed. It may also signify a NULL instruction pointer. - bool isCall() const { return I.getInt(); } + /// Return true if a CallInst is enclosed. + bool isCall() const { return I.getInt() == 1; } - /// Return true if a InvokeInst is enclosed. - bool isInvoke() const { return getInstruction() && !I.getInt(); } + /// Return true if a InvokeInst is enclosed. !I.getInt() may also signify a + /// NULL instruction pointer, so check that. + bool isInvoke() const { return getInstruction() && I.getInt() == 0; } + + /// Return true if a CallBrInst is enclosed. + bool isCallBr() const { return I.getInt() == 2; } InstrTy *getInstruction() const { return I.getPointer(); } InstrTy *operator->() const { return I.getPointer(); } @@ -97,7 +102,7 @@ /// Return the pointer to function that is being called. ValTy *getCalledValue() const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); return *getCallee(); } @@ -114,17 +119,16 @@ return false; if (isa(V) || isa(V)) return false; - if (const CallInst *CI = dyn_cast(getInstruction())) { - if (CI->isInlineAsm()) + if (const CallBase *CB = dyn_cast(getInstruction())) + if (CB->isInlineAsm()) return false; - } return true; } /// Set the callee to the specified value. Unlike the function of the same /// name on CallBase, does not modify the type! void setCalledFunction(Value *V) { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, callbr, or invoke instruction!"); assert(cast(V->getType())->getElementType() == cast(getInstruction())->getFunctionType() && "New callee type does not match FunctionType on call"); @@ -192,7 +196,7 @@ } void setArgument(unsigned ArgNo, Value* newVal) { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(arg_begin() + ArgNo < arg_end() && "Argument # out of range!"); getInstruction()->setOperand(ArgNo, newVal); } @@ -206,7 +210,7 @@ /// Given a use for an argument, get the argument number that corresponds to /// it. unsigned getArgumentNo(const Use *U) const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(isArgOperand(U) && "Argument # out of range!"); return U - arg_begin(); } @@ -230,7 +234,7 @@ /// Given a use for a data operand, get the data operand number that /// corresponds to it. unsigned getDataOperandNo(const Use *U) const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(isDataOperand(U) && "Data operand # out of range!"); return U - data_operands_begin(); } @@ -240,10 +244,11 @@ using data_operand_iterator = IterTy; /// data_operands_begin/data_operands_end - Return iterators iterating over - /// the call / invoke argument list and bundle operands. For invokes, this is - /// the set of instruction operands except the invoke target and the two - /// successor blocks; and for calls this is the set of instruction operands - /// except the call target. + /// the call / invoke / callbr argument list and bundle operands. For invokes, + /// this is the set of instruction operands except the invoke target and the + /// two successor blocks; for calls this is the set of instruction operands + /// except the call target; for callbrs the number of labels to skip must be + /// determined first. IterTy data_operands_begin() const { assert(getInstruction() && "Not a call or invoke instruction!"); @@ -280,17 +285,19 @@ return isCall() && cast(getInstruction())->isTailCall(); } -#define CALLSITE_DELEGATE_GETTER(METHOD) \ - InstrTy *II = getInstruction(); \ - return isCall() \ - ? cast(II)->METHOD \ - : cast(II)->METHOD +#define CALLSITE_DELEGATE_GETTER(METHOD) \ + InstrTy *II = getInstruction(); \ + return isCall() ? cast(II)->METHOD \ + : isCallBr() ? cast(II)->METHOD \ + : cast(II)->METHOD -#define CALLSITE_DELEGATE_SETTER(METHOD) \ - InstrTy *II = getInstruction(); \ - if (isCall()) \ - cast(II)->METHOD; \ - else \ +#define CALLSITE_DELEGATE_SETTER(METHOD) \ + InstrTy *II = getInstruction(); \ + if (isCall()) \ + cast(II)->METHOD; \ + else if (isCallBr()) \ + cast(II)->METHOD; \ + else \ cast(II)->METHOD unsigned getNumArgOperands() const { @@ -306,9 +313,7 @@ } bool isInlineAsm() const { - if (isCall()) - return cast(getInstruction())->isInlineAsm(); - return false; + return cast(getInstruction())->isInlineAsm(); } /// Get the calling convention of the call. @@ -392,10 +397,10 @@ /// Return true if the data operand at index \p i directly or indirectly has /// the attribute \p A. /// - /// Normal call or invoke arguments have per operand attributes, as specified - /// in the attribute set attached to this instruction, while operand bundle - /// operands may have some attributes implied by the type of its containing - /// operand bundle. + /// Normal call, invoke or callbr arguments have per operand attributes, as + /// specified in the attribute set attached to this instruction, while operand + /// bundle operands may have some attributes implied by the type of its + /// containing operand bundle. bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const { CALLSITE_DELEGATE_GETTER(dataOperandHasImpliedAttr(i, Kind)); } @@ -661,12 +666,13 @@ class CallSite : public CallSiteBase { + CallBrInst, User::op_iterator> { public: CallSite() = default; CallSite(CallSiteBase B) : CallSiteBase(B) {} CallSite(CallInst *CI) : CallSiteBase(CI) {} CallSite(InvokeInst *II) : CallSiteBase(II) {} + CallSite(CallBrInst *CBI) : CallSiteBase(CBI) {} explicit CallSite(Instruction *II) : CallSiteBase(II) {} explicit CallSite(Value *V) : CallSiteBase(V) {} @@ -888,6 +894,7 @@ ImmutableCallSite() = default; ImmutableCallSite(const CallInst *CI) : CallSiteBase(CI) {} ImmutableCallSite(const InvokeInst *II) : CallSiteBase(II) {} + ImmutableCallSite(const CallBrInst *CBI) : CallSiteBase(CBI) {} explicit ImmutableCallSite(const Instruction *II) : CallSiteBase(II) {} explicit ImmutableCallSite(const Value *V) : CallSiteBase(V) {} ImmutableCallSite(CallSite CS) : CallSiteBase(CS.getInstruction()) {} Index: include/llvm/IR/IRBuilder.h =================================================================== --- include/llvm/IR/IRBuilder.h +++ include/llvm/IR/IRBuilder.h @@ -943,6 +943,42 @@ Callee, NormalDest, UnwindDest, Args, Name); } + /// \brief Create a callbr instruction. + CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, + BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args = None, + const Twine &Name = "") { + return Insert(CallBrInst::Create(Ty, Callee, Fallthrough, IndirectDests, + Args), Name); + } + CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, + BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef OpBundles, + const Twine &Name = "") { + return Insert( + CallBrInst::Create(Ty, Callee, Fallthrough, IndirectDests, Args, + OpBundles), Name); + } + + CallBrInst *CreateCallBr(Function *Callee, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args = None, + const Twine &Name = "") { + return CreateCallBr(Callee->getFunctionType(), Callee, Fallthrough, + IndirectDests, Args, Name); + } + CallBrInst *CreateCallBr(Function *Callee, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef OpBundles, + const Twine &Name = "") { + return CreateCallBr(Callee->getFunctionType(), Callee, Fallthrough, + IndirectDests, Args, Name); + } + ResumeInst *CreateResume(Value *Exn) { return Insert(ResumeInst::Create(Exn)); } Index: include/llvm/IR/InstVisitor.h =================================================================== --- include/llvm/IR/InstVisitor.h +++ include/llvm/IR/InstVisitor.h @@ -217,14 +217,17 @@ RetTy visitVACopyInst(VACopyInst &I) { DELEGATE(IntrinsicInst); } RetTy visitIntrinsicInst(IntrinsicInst &I) { DELEGATE(CallInst); } - // Call and Invoke are slightly different as they delegate first through - // a generic CallSite visitor. + // Call, Invoke and CallBr are slightly different as they delegate first + // through a generic CallSite visitor. RetTy visitCallInst(CallInst &I) { return static_cast(this)->visitCallSite(&I); } RetTy visitInvokeInst(InvokeInst &I) { return static_cast(this)->visitCallSite(&I); } + RetTy visitCallBrInst(CallBrInst &I) { + return static_cast(this)->visitCallSite(&I); + } // While terminators don't have a distinct type modeling them, we support // intercepting them with dedicated a visitor callback. @@ -270,14 +273,14 @@ // The next level delegation for `CallBase` is slightly more complex in order // to support visiting cases where the call is also a terminator. RetTy visitCallBase(CallBase &I) { - if (isa(I)) + if (isa(I) || isa(I)) return static_cast(this)->visitTerminator(I); DELEGATE(Instruction); } - // Provide a legacy visitor for a 'callsite' that visits both calls and - // invokes. + // Provide a legacy visitor for a 'callsite' that visits calls, invokes, + // and calbrs. // // Prefer overriding the type system based `CallBase` instead. RetTy visitCallSite(CallSite CS) { Index: include/llvm/IR/InstrTypes.h =================================================================== --- include/llvm/IR/InstrTypes.h +++ include/llvm/IR/InstrTypes.h @@ -1033,16 +1033,23 @@ return 0; case Instruction::Invoke: return 2; + case Instruction::CallBr: + return getNumSubclassExtraOperandsDynamic(); } llvm_unreachable("Invalid opcode!"); } + /// Get the number of extra operands for instructions that don't have a fixed + /// number of extra operands. + unsigned getNumSubclassExtraOperandsDynamic() const; + public: using Instruction::getContext; static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Call || - I->getOpcode() == Instruction::Invoke; + I->getOpcode() == Instruction::Invoke || + I->getOpcode() == Instruction::CallBr; } static bool classof(const Value *V) { return isa(V) && classof(cast(V)); Index: include/llvm/IR/Instruction.h =================================================================== --- include/llvm/IR/Instruction.h +++ include/llvm/IR/Instruction.h @@ -135,6 +135,9 @@ bool isExceptionalTerminator() const { return isExceptionalTerminator(getOpcode()); } + bool isIndirectTerminator() const { + return isIndirectTerminator(getOpcode()); + } static const char* getOpcodeName(unsigned OpCode); @@ -202,6 +205,17 @@ } } + /// Returns true if the OpCode is a terminator with indirect targets. + static inline bool isIndirectTerminator(unsigned OpCode) { + switch (OpCode) { + case Instruction::IndirectBr: + case Instruction::CallBr: + return true; + default: + return false; + } + } + //===--------------------------------------------------------------------===// // Metadata manipulation. //===--------------------------------------------------------------------===// Index: include/llvm/IR/Instruction.def =================================================================== --- include/llvm/IR/Instruction.def +++ include/llvm/IR/Instruction.def @@ -134,89 +134,90 @@ HANDLE_TERM_INST ( 8, CleanupRet , CleanupReturnInst) HANDLE_TERM_INST ( 9, CatchRet , CatchReturnInst) HANDLE_TERM_INST (10, CatchSwitch , CatchSwitchInst) - LAST_TERM_INST (10) +HANDLE_TERM_INST (11, CallBr , CallBrInst) // A call-site terminator + LAST_TERM_INST (11) // Standard unary operators... - FIRST_UNARY_INST(11) -HANDLE_UNARY_INST(11, FNeg , UnaryOperator) - LAST_UNARY_INST(11) + FIRST_UNARY_INST(12) +HANDLE_UNARY_INST(12, FNeg , UnaryOperator) + LAST_UNARY_INST(12) // Standard binary operators... - FIRST_BINARY_INST(12) -HANDLE_BINARY_INST(12, Add , BinaryOperator) -HANDLE_BINARY_INST(13, FAdd , BinaryOperator) -HANDLE_BINARY_INST(14, Sub , BinaryOperator) -HANDLE_BINARY_INST(15, FSub , BinaryOperator) -HANDLE_BINARY_INST(16, Mul , BinaryOperator) -HANDLE_BINARY_INST(17, FMul , BinaryOperator) -HANDLE_BINARY_INST(18, UDiv , BinaryOperator) -HANDLE_BINARY_INST(19, SDiv , BinaryOperator) -HANDLE_BINARY_INST(20, FDiv , BinaryOperator) -HANDLE_BINARY_INST(21, URem , BinaryOperator) -HANDLE_BINARY_INST(22, SRem , BinaryOperator) -HANDLE_BINARY_INST(23, FRem , BinaryOperator) + FIRST_BINARY_INST(13) +HANDLE_BINARY_INST(13, Add , BinaryOperator) +HANDLE_BINARY_INST(14, FAdd , BinaryOperator) +HANDLE_BINARY_INST(15, Sub , BinaryOperator) +HANDLE_BINARY_INST(16, FSub , BinaryOperator) +HANDLE_BINARY_INST(17, Mul , BinaryOperator) +HANDLE_BINARY_INST(18, FMul , BinaryOperator) +HANDLE_BINARY_INST(19, UDiv , BinaryOperator) +HANDLE_BINARY_INST(20, SDiv , BinaryOperator) +HANDLE_BINARY_INST(21, FDiv , BinaryOperator) +HANDLE_BINARY_INST(22, URem , BinaryOperator) +HANDLE_BINARY_INST(23, SRem , BinaryOperator) +HANDLE_BINARY_INST(24, FRem , BinaryOperator) // Logical operators (integer operands) -HANDLE_BINARY_INST(24, Shl , BinaryOperator) // Shift left (logical) -HANDLE_BINARY_INST(25, LShr , BinaryOperator) // Shift right (logical) -HANDLE_BINARY_INST(26, AShr , BinaryOperator) // Shift right (arithmetic) -HANDLE_BINARY_INST(27, And , BinaryOperator) -HANDLE_BINARY_INST(28, Or , BinaryOperator) -HANDLE_BINARY_INST(29, Xor , BinaryOperator) - LAST_BINARY_INST(29) +HANDLE_BINARY_INST(25, Shl , BinaryOperator) // Shift left (logical) +HANDLE_BINARY_INST(26, LShr , BinaryOperator) // Shift right (logical) +HANDLE_BINARY_INST(27, AShr , BinaryOperator) // Shift right (arithmetic) +HANDLE_BINARY_INST(28, And , BinaryOperator) +HANDLE_BINARY_INST(29, Or , BinaryOperator) +HANDLE_BINARY_INST(30, Xor , BinaryOperator) + LAST_BINARY_INST(30) // Memory operators... - FIRST_MEMORY_INST(30) -HANDLE_MEMORY_INST(30, Alloca, AllocaInst) // Stack management -HANDLE_MEMORY_INST(31, Load , LoadInst ) // Memory manipulation instrs -HANDLE_MEMORY_INST(32, Store , StoreInst ) -HANDLE_MEMORY_INST(33, GetElementPtr, GetElementPtrInst) -HANDLE_MEMORY_INST(34, Fence , FenceInst ) -HANDLE_MEMORY_INST(35, AtomicCmpXchg , AtomicCmpXchgInst ) -HANDLE_MEMORY_INST(36, AtomicRMW , AtomicRMWInst ) - LAST_MEMORY_INST(36) + FIRST_MEMORY_INST(31) +HANDLE_MEMORY_INST(31, Alloca, AllocaInst) // Stack management +HANDLE_MEMORY_INST(32, Load , LoadInst ) // Memory manipulation instrs +HANDLE_MEMORY_INST(33, Store , StoreInst ) +HANDLE_MEMORY_INST(34, GetElementPtr, GetElementPtrInst) +HANDLE_MEMORY_INST(35, Fence , FenceInst ) +HANDLE_MEMORY_INST(36, AtomicCmpXchg , AtomicCmpXchgInst ) +HANDLE_MEMORY_INST(37, AtomicRMW , AtomicRMWInst ) + LAST_MEMORY_INST(37) // Cast operators ... // NOTE: The order matters here because CastInst::isEliminableCastPair // NOTE: (see Instructions.cpp) encodes a table based on this ordering. - FIRST_CAST_INST(37) -HANDLE_CAST_INST(37, Trunc , TruncInst ) // Truncate integers -HANDLE_CAST_INST(38, ZExt , ZExtInst ) // Zero extend integers -HANDLE_CAST_INST(39, SExt , SExtInst ) // Sign extend integers -HANDLE_CAST_INST(40, FPToUI , FPToUIInst ) // floating point -> UInt -HANDLE_CAST_INST(41, FPToSI , FPToSIInst ) // floating point -> SInt -HANDLE_CAST_INST(42, UIToFP , UIToFPInst ) // UInt -> floating point -HANDLE_CAST_INST(43, SIToFP , SIToFPInst ) // SInt -> floating point -HANDLE_CAST_INST(44, FPTrunc , FPTruncInst ) // Truncate floating point -HANDLE_CAST_INST(45, FPExt , FPExtInst ) // Extend floating point -HANDLE_CAST_INST(46, PtrToInt, PtrToIntInst) // Pointer -> Integer -HANDLE_CAST_INST(47, IntToPtr, IntToPtrInst) // Integer -> Pointer -HANDLE_CAST_INST(48, BitCast , BitCastInst ) // Type cast -HANDLE_CAST_INST(49, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast - LAST_CAST_INST(49) - - FIRST_FUNCLETPAD_INST(50) -HANDLE_FUNCLETPAD_INST(50, CleanupPad, CleanupPadInst) -HANDLE_FUNCLETPAD_INST(51, CatchPad , CatchPadInst) - LAST_FUNCLETPAD_INST(51) + FIRST_CAST_INST(38) +HANDLE_CAST_INST(38, Trunc , TruncInst ) // Truncate integers +HANDLE_CAST_INST(39, ZExt , ZExtInst ) // Zero extend integers +HANDLE_CAST_INST(40, SExt , SExtInst ) // Sign extend integers +HANDLE_CAST_INST(41, FPToUI , FPToUIInst ) // floating point -> UInt +HANDLE_CAST_INST(42, FPToSI , FPToSIInst ) // floating point -> SInt +HANDLE_CAST_INST(43, UIToFP , UIToFPInst ) // UInt -> floating point +HANDLE_CAST_INST(44, SIToFP , SIToFPInst ) // SInt -> floating point +HANDLE_CAST_INST(45, FPTrunc , FPTruncInst ) // Truncate floating point +HANDLE_CAST_INST(46, FPExt , FPExtInst ) // Extend floating point +HANDLE_CAST_INST(47, PtrToInt, PtrToIntInst) // Pointer -> Integer +HANDLE_CAST_INST(48, IntToPtr, IntToPtrInst) // Integer -> Pointer +HANDLE_CAST_INST(49, BitCast , BitCastInst ) // Type cast +HANDLE_CAST_INST(50, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast + LAST_CAST_INST(50) + + FIRST_FUNCLETPAD_INST(51) +HANDLE_FUNCLETPAD_INST(51, CleanupPad, CleanupPadInst) +HANDLE_FUNCLETPAD_INST(52, CatchPad , CatchPadInst) + LAST_FUNCLETPAD_INST(52) // Other operators... - FIRST_OTHER_INST(52) -HANDLE_OTHER_INST(52, ICmp , ICmpInst ) // Integer comparison instruction -HANDLE_OTHER_INST(53, FCmp , FCmpInst ) // Floating point comparison instr. -HANDLE_OTHER_INST(54, PHI , PHINode ) // PHI node instruction -HANDLE_OTHER_INST(55, Call , CallInst ) // Call a function -HANDLE_OTHER_INST(56, Select , SelectInst ) // select instruction -HANDLE_USER_INST (57, UserOp1, Instruction) // May be used internally in a pass -HANDLE_USER_INST (58, UserOp2, Instruction) // Internal to passes only -HANDLE_OTHER_INST(59, VAArg , VAArgInst ) // vaarg instruction -HANDLE_OTHER_INST(60, ExtractElement, ExtractElementInst)// extract from vector -HANDLE_OTHER_INST(61, InsertElement, InsertElementInst) // insert into vector -HANDLE_OTHER_INST(62, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. -HANDLE_OTHER_INST(63, ExtractValue, ExtractValueInst)// extract from aggregate -HANDLE_OTHER_INST(64, InsertValue, InsertValueInst) // insert into aggregate -HANDLE_OTHER_INST(65, LandingPad, LandingPadInst) // Landing pad instruction. - LAST_OTHER_INST(65) + FIRST_OTHER_INST(53) +HANDLE_OTHER_INST(53, ICmp , ICmpInst ) // Integer comparison instruction +HANDLE_OTHER_INST(54, FCmp , FCmpInst ) // Floating point comparison instr. +HANDLE_OTHER_INST(55, PHI , PHINode ) // PHI node instruction +HANDLE_OTHER_INST(56, Call , CallInst ) // Call a function +HANDLE_OTHER_INST(57, Select , SelectInst ) // select instruction +HANDLE_USER_INST (58, UserOp1, Instruction) // May be used internally in a pass +HANDLE_USER_INST (59, UserOp2, Instruction) // Internal to passes only +HANDLE_OTHER_INST(60, VAArg , VAArgInst ) // vaarg instruction +HANDLE_OTHER_INST(61, ExtractElement, ExtractElementInst)// extract from vector +HANDLE_OTHER_INST(62, InsertElement, InsertElementInst) // insert into vector +HANDLE_OTHER_INST(63, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. +HANDLE_OTHER_INST(64, ExtractValue, ExtractValueInst)// extract from aggregate +HANDLE_OTHER_INST(65, InsertValue, InsertValueInst) // insert into aggregate +HANDLE_OTHER_INST(66, LandingPad, LandingPadInst) // Landing pad instruction. + LAST_OTHER_INST(66) #undef FIRST_TERM_INST #undef HANDLE_TERM_INST Index: include/llvm/IR/Instructions.h =================================================================== --- include/llvm/IR/Instructions.h +++ include/llvm/IR/Instructions.h @@ -3887,6 +3887,249 @@ } //===----------------------------------------------------------------------===// +// CallBrInst Class +//===----------------------------------------------------------------------===// + +/// CallBr instruction, tracking function calls that may not return control but +/// instead transfer it to a third location. The SubclassData field is used to +/// hold the calling convention of the call. +/// +class CallBrInst : public CallBase { + + unsigned NumIndirectDests; + + CallBrInst(const CallBrInst &BI); + + /// Construct a CallBrInst given a range of arguments. + /// + /// Construct a CallBrInst from a range of arguments + inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, Instruction *InsertBefore); + + inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, BasicBlock *InsertAtEnd); + + void init(FunctionType *FTy, Value *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr); + + /// Compute the number of operands to allocate. + static int ComputeNumOperands(int NumArgs, int NumIndirectDests, + int NumBundleInputs = 0) { + // We need one operand for the called function, plus our extra operands and + // the input operand counts provided. + return 2 + NumIndirectDests + NumArgs + NumBundleInputs; + } + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + + CallBrInst *cloneImpl() const; + +public: + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size()); + return new (NumOperands) + CallBrInst(Ty, Func, Fallthrough, IndirectDests, Args, None, + NumOperands, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)); + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (NumOperands, DescriptorBytes) + CallBrInst(Ty, Func, Fallthrough, IndirectDests, Args, Bundles, + NumOperands, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + BasicBlock *InsertAtEnd) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size()); + return new (NumOperands) + CallBrInst(Ty, Func, Fallthrough, IndirectDests, Args, None, + NumOperands, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)); + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (NumOperands, DescriptorBytes) + CallBrInst(Ty, Func, Fallthrough, IndirectDests, Args, Bundles, + NumOperands, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(Function *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + return Create(Func->getFunctionType(), Func, Fallthrough, IndirectDests, + Args, NameStr, InsertBefore); + } + + static CallBrInst *Create(Function *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + return Create(Func->getFunctionType(), Func, Fallthrough, IndirectDests, + Args, Bundles, NameStr, InsertBefore); + } + + static CallBrInst *Create(Function *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + BasicBlock *InsertAtEnd) { + return Create(Func->getFunctionType(), Func, Fallthrough, IndirectDests, + Args, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(Function *Func, + BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + return Create(Func->getFunctionType(), Func, Fallthrough, IndirectDests, + Args, Bundles, NameStr, InsertAtEnd); + } + + /// Create a clone of \p CBI with a different set of operand bundles and + /// insert it before \p InsertPt. + /// + /// The returned callbr instruction is identical to \p CBI in every way + /// except that the operand bundles for the new instruction are set to the + /// operand bundles in \p Bundles. + static CallBrInst *Create(CallBrInst *CBI, + ArrayRef Bundles, + Instruction *InsertPt = nullptr); + + /// Return the number of callbr indirect dest labels. + /// + unsigned getNumIndirectDests() const { return NumIndirectDests; } + + /// getIndirectDestLabel - Return the i-th indirect dest label. + /// + Value *getIndirectDestLabel(unsigned i) const { + assert(i < getNumIndirectDests() && "Out of bounds!"); + return getOperand(i + getNumArgOperands() + getNumTotalBundleOperands() + + 1); + } + + Value *getIndirectDestLabelUse(unsigned i) const { + assert(i < getNumIndirectDests() && "Out of bounds!"); + return getOperandUse(i + getNumArgOperands() + getNumTotalBundleOperands() + + 1); + } + + // Return the destination basic blocks... + BasicBlock *getDefaultDest() const { + return cast(*(&Op<-1>() - getNumIndirectDests() - 1)); + } + BasicBlock *getIndirectDest(unsigned i) const { + return cast(*(&Op<-1>() - getNumIndirectDests() + i)); + } + SmallVector getIndirectDests() const { + SmallVector IndirectDests; + for (unsigned i = 0, e = getNumIndirectDests(); i < e; ++i) + IndirectDests.push_back(getIndirectDest(i)); + return IndirectDests; + } + void setDefaultDest(BasicBlock *B) { + *(&Op<-1>() - getNumIndirectDests() - 1) = reinterpret_cast(B); + } + void setIndirectDest(unsigned i, BasicBlock *B) { + *(&Op<-1>() - getNumIndirectDests() + i) = reinterpret_cast(B); + } + + BasicBlock *getSuccessor(unsigned i) const { + assert(i < getNumSuccessors() + 1 && + "Successor # out of range for callbr!"); + return i == 0 ? getDefaultDest() : getIndirectDest(i - 1); + } + + void setSuccessor(unsigned idx, BasicBlock *NewSucc) { + assert(idx < getNumIndirectDests() + 1 && + "Successor # out of range for callbr!"); + *(&Op<-1>() - getNumIndirectDests() -1 + idx) = + reinterpret_cast(NewSucc); + } + + unsigned getNumSuccessors() const { return getNumIndirectDests() + 1; } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Instruction *I) { + return (I->getOpcode() == Instruction::CallBr); + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + +private: + + // Shadow Instruction::setInstructionSubclassData with a private forwarding + // method so that subclasses cannot accidentally use it. + void setInstructionSubclassData(unsigned short D) { + Instruction::setInstructionSubclassData(D); + } +}; + +CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, Instruction *InsertBefore) + : CallBase(Ty->getReturnType(), Instruction::CallBr, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertBefore) { + init(Ty, Func, Fallthrough, IndirectDests, Args, Bundles, NameStr); +} + +CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, BasicBlock *InsertAtEnd) + : CallBase( + cast( + cast(Func->getType())->getElementType()) + ->getReturnType(), + Instruction::CallBr, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertAtEnd) { + init(Ty, Func, Fallthrough, IndirectDests, Args, Bundles, NameStr); +} + +//===----------------------------------------------------------------------===// // ResumeInst Class //===----------------------------------------------------------------------===// Index: lib/Analysis/ValueTracking.cpp =================================================================== --- lib/Analysis/ValueTracking.cpp +++ lib/Analysis/ValueTracking.cpp @@ -3922,6 +3922,7 @@ case Instruction::VAArg: case Instruction::Alloca: case Instruction::Invoke: + case Instruction::CallBr: case Instruction::PHI: case Instruction::Store: case Instruction::Ret: Index: lib/AsmParser/LLLexer.cpp =================================================================== --- lib/AsmParser/LLLexer.cpp +++ lib/AsmParser/LLLexer.cpp @@ -858,6 +858,7 @@ INSTKEYWORD(invoke, Invoke); INSTKEYWORD(resume, Resume); INSTKEYWORD(unreachable, Unreachable); + INSTKEYWORD(callbr, CallBr); INSTKEYWORD(alloca, Alloca); INSTKEYWORD(load, Load); Index: lib/AsmParser/LLParser.h =================================================================== --- lib/AsmParser/LLParser.h +++ lib/AsmParser/LLParser.h @@ -570,6 +570,7 @@ bool ParseCatchSwitch(Instruction *&Inst, PerFunctionState &PFS); bool ParseCatchPad(Instruction *&Inst, PerFunctionState &PFS); bool ParseCleanupPad(Instruction *&Inst, PerFunctionState &PFS); + bool ParseCallBr(Instruction *&Inst, PerFunctionState &PFS); bool ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc, unsigned OperandType); Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -163,6 +163,14 @@ AS = AS.addAttributes(Context, AttributeList::FunctionIndex, AttributeSet::get(Context, FnAttrs)); II->setAttributes(AS); + } else if (CallBrInst *CBI = dyn_cast(V)) { + AttributeList AS = CBI->getAttributes(); + AttrBuilder FnAttrs(AS.getFnAttributes()); + AS = AS.removeAttributes(Context, AttributeList::FunctionIndex); + FnAttrs.merge(B); + AS = AS.addAttributes(Context, AttributeList::FunctionIndex, + AttributeSet::get(Context, FnAttrs)); + CBI->setAttributes(AS); } else if (auto *GV = dyn_cast(V)) { AttrBuilder Attrs(GV->getAttributes()); Attrs.merge(B); @@ -5566,6 +5574,7 @@ case lltok::kw_catchswitch: return ParseCatchSwitch(Inst, PFS); case lltok::kw_catchpad: return ParseCatchPad(Inst, PFS); case lltok::kw_cleanuppad: return ParseCleanupPad(Inst, PFS); + case lltok::kw_callbr: return ParseCallBr(Inst, PFS); // Unary Operators. case lltok::kw_fneg: { FastMathFlags FMF = EatFastMathFlagsIfPresent(); @@ -6184,6 +6193,124 @@ return false; } +/// ParseCallBr +/// ::= 'callbr' OptionalCallingConv OptionalAttrs Type Value ParamList +/// OptionalAttrs OptionalOperandBundles 'to' TypeAndValue +/// '[' LabelList ']' +bool LLParser::ParseCallBr(Instruction *&Inst, PerFunctionState &PFS) { + LocTy CallLoc = Lex.getLoc(); + AttrBuilder RetAttrs, FnAttrs; + std::vector FwdRefAttrGrps; + LocTy NoBuiltinLoc; + unsigned CC; + Type *RetType = nullptr; + LocTy RetTypeLoc; + ValID CalleeID; + SmallVector ArgList; + SmallVector BundleList; + + BasicBlock *DefaultDest; + if (ParseOptionalCallingConv(CC) || ParseOptionalReturnAttrs(RetAttrs) || + ParseType(RetType, RetTypeLoc, true /*void allowed*/) || + ParseValID(CalleeID) || ParseParameterList(ArgList, PFS) || + ParseFnAttributeValuePairs(FnAttrs, FwdRefAttrGrps, false, + NoBuiltinLoc) || + ParseOptionalOperandBundles(BundleList, PFS) || + ParseToken(lltok::kw_to, "expected 'to' in callbr") || + ParseTypeAndBasicBlock(DefaultDest, PFS) || + ParseToken(lltok::lsquare, "expected '[' in callbr")) + return true; + + // Parse the destination list. + SmallVector IndirectDests; + + if (Lex.getKind() != lltok::rsquare) { + BasicBlock *DestBB; + if (ParseTypeAndBasicBlock(DestBB, PFS)) + return true; + IndirectDests.push_back(DestBB); + + while (EatIfPresent(lltok::comma)) { + if (ParseTypeAndBasicBlock(DestBB, PFS)) + return true; + IndirectDests.push_back(DestBB); + } + } + + if (ParseToken(lltok::rsquare, "expected ']' at end of block list")) + return true; + + // If RetType is a non-function pointer type, then this is the short syntax + // for the call, which means that RetType is just the return type. Infer the + // rest of the function argument types from the arguments that are present. + FunctionType *Ty = dyn_cast(RetType); + if (!Ty) { + // Pull out the types of all of the arguments... + std::vector ParamTypes; + for (unsigned i = 0, e = ArgList.size(); i != e; ++i) + ParamTypes.push_back(ArgList[i].V->getType()); + + if (!FunctionType::isValidReturnType(RetType)) + return Error(RetTypeLoc, "Invalid result type for LLVM function"); + + Ty = FunctionType::get(RetType, ParamTypes, false); + } + + CalleeID.FTy = Ty; + + // Look up the callee. + Value *Callee; + if (ConvertValIDToValue(PointerType::getUnqual(Ty), CalleeID, Callee, &PFS, + /*IsCall=*/true)) + return true; + + if (isa(Callee) && !Ty->getReturnType()->isVoidTy()) + return Error(RetTypeLoc, "asm-goto outputs not supported"); + + // Set up the Attribute for the function. + SmallVector Args; + SmallVector ArgAttrs; + + // Loop through FunctionType's arguments and ensure they are specified + // correctly. Also, gather any parameter attributes. + FunctionType::param_iterator I = Ty->param_begin(); + FunctionType::param_iterator E = Ty->param_end(); + for (unsigned i = 0, e = ArgList.size(); i != e; ++i) { + Type *ExpectedTy = nullptr; + if (I != E) { + ExpectedTy = *I++; + } else if (!Ty->isVarArg()) { + return Error(ArgList[i].Loc, "too many arguments specified"); + } + + if (ExpectedTy && ExpectedTy != ArgList[i].V->getType()) + return Error(ArgList[i].Loc, "argument is not of expected type '" + + getTypeString(ExpectedTy) + "'"); + Args.push_back(ArgList[i].V); + ArgAttrs.push_back(ArgList[i].Attrs); + } + + if (I != E) + return Error(CallLoc, "not enough parameters specified for call"); + + if (FnAttrs.hasAlignmentAttr()) + return Error(CallLoc, "callbr instructions may not have an alignment"); + + // Finish off the Attribute and check them + AttributeList PAL = + AttributeList::get(Context, AttributeSet::get(Context, FnAttrs), + AttributeSet::get(Context, RetAttrs), ArgAttrs); + + CallBrInst *CBI = + CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args, + BundleList); + CBI->setCallingConv(CC); + CBI->setAttributes(PAL); + ForwardRefAttrGroups[CBI] = FwdRefAttrGrps; + Inst = CBI; + return false; +} + //===----------------------------------------------------------------------===// // Binary Operators. //===----------------------------------------------------------------------===// Index: lib/AsmParser/LLToken.h =================================================================== --- lib/AsmParser/LLToken.h +++ lib/AsmParser/LLToken.h @@ -327,6 +327,7 @@ kw_catchret, kw_catchpad, kw_cleanuppad, + kw_callbr, kw_alloca, kw_load, Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -4231,6 +4231,74 @@ InstructionList.push_back(I); break; } + case bitc::FUNC_CODE_INST_CALLBR: { + // CALLBR: [attr, cc, norm, transfs, fty, fnid, args] + unsigned OpNum = 0; + AttributeList PAL = getAttributes(Record[OpNum++]); + unsigned CCInfo = Record[OpNum++]; + + BasicBlock *DefaultDest = getBasicBlock(Record[OpNum++]); + unsigned NumIndirectDests = Record[OpNum++]; + SmallVector IndirectDests; + for (unsigned i = 0, e = NumIndirectDests; i != e; ++i) + IndirectDests.push_back(getBasicBlock(Record[OpNum++])); + + FunctionType *FTy = nullptr; + if (CCInfo >> bitc::CALL_EXPLICIT_TYPE & 1 && + !(FTy = dyn_cast(getTypeByID(Record[OpNum++])))) + return error("Explicit call type is not a function type"); + + Value *Callee; + if (getValueTypePair(Record, OpNum, NextValueNo, Callee)) + return error("Invalid record"); + + PointerType *OpTy = dyn_cast(Callee->getType()); + if (!OpTy) + return error("Callee is not a pointer type"); + if (!FTy) { + FTy = dyn_cast(OpTy->getElementType()); + if (!FTy) + return error("Callee is not of pointer to function type"); + } else if (OpTy->getElementType() != FTy) + return error("Explicit call type does not match pointee type of " + "callee operand"); + if (Record.size() < FTy->getNumParams() + OpNum) + return error("Insufficient operands to call"); + + SmallVector Args; + // Read the fixed params. + for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) { + if (FTy->getParamType(i)->isLabelTy()) + Args.push_back(getBasicBlock(Record[OpNum])); + else + Args.push_back(getValue(Record, OpNum, NextValueNo, + FTy->getParamType(i))); + if (!Args.back()) + return error("Invalid record"); + } + + // Read type/value pairs for varargs params. + if (!FTy->isVarArg()) { + if (OpNum != Record.size()) + return error("Invalid record"); + } else { + while (OpNum != Record.size()) { + Value *Op; + if (getValueTypePair(Record, OpNum, NextValueNo, Op)) + return error("Invalid record"); + Args.push_back(Op); + } + } + + I = CallBrInst::Create(FTy, Callee, DefaultDest, IndirectDests, Args, + OperandBundles); + OperandBundles.clear(); + InstructionList.push_back(I); + cast(I)->setCallingConv( + static_cast((0x7ff & CCInfo) >> bitc::CALL_CCONV)); + cast(I)->setAttributes(PAL); + break; + } case bitc::FUNC_CODE_INST_UNREACHABLE: // UNREACHABLE I = new UnreachableInst(Context); InstructionList.push_back(I); Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -2777,6 +2777,41 @@ Vals.push_back(VE.getValueID(CatchSwitch.getUnwindDest())); break; } + case Instruction::CallBr: { + const CallBrInst *CBI = cast(&I); + const Value *Callee = CBI->getCalledValue(); + FunctionType *FTy = CBI->getFunctionType(); + + if (CBI->hasOperandBundles()) + writeOperandBundles(CBI, InstID); + + Code = bitc::FUNC_CODE_INST_CALLBR; + + Vals.push_back(VE.getAttributeListID(CBI->getAttributes())); + + Vals.push_back(CBI->getCallingConv() << bitc::CALL_CCONV | + 1 << bitc::CALL_EXPLICIT_TYPE); + + Vals.push_back(VE.getValueID(CBI->getDefaultDest())); + Vals.push_back(CBI->getNumIndirectDests()); + for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) + Vals.push_back(VE.getValueID(CBI->getIndirectDest(i))); + + Vals.push_back(VE.getTypeID(FTy)); + pushValueAndType(Callee, InstID, Vals); + + // Emit value #'s for the fixed parameters. + for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) + pushValue(I.getOperand(i), InstID, Vals); // fixed param. + + // Emit type/value pairs for varargs params. + if (FTy->isVarArg()) { + for (unsigned i = FTy->getNumParams(), e = CBI->getNumArgOperands(); + i != e; ++i) + pushValueAndType(I.getOperand(i), InstID, Vals); // vararg + } + break; + } case Instruction::Unreachable: Code = bitc::FUNC_CODE_INST_UNREACHABLE; AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV; Index: lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2935,8 +2935,9 @@ bool AsmPrinter:: isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const { // If this is a landing pad, it isn't a fall through. If it has no preds, - // then nothing falls through to it. - if (MBB->isEHPad() || MBB->pred_empty()) + // then nothing falls through to it. If it is an asm-goto target it isn't + // a fallthrough. + if (MBB->isEHPad() || MBB->pred_empty() || MBB->isAsmGotoTarget()) return false; // If there isn't exactly one predecessor, it can't be a fall through. Index: lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp =================================================================== --- lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -434,8 +434,14 @@ if (Modifier[0] == 'l') { // Labels are target independent. // FIXME: What if the operand isn't an MBB, report error? - const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol(); - Sym->print(OS, AP->MAI); + if (MI->getOperand(OpNo).isBlockAddress()) { + const BlockAddress *BA = MI->getOperand(OpNo).getBlockAddress(); + MCSymbol *Sym =AP->GetBlockAddressSymbol(BA); + Sym->print(OS, AP->MAI); + } else { + const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol(); + Sym->print(OS, AP->MAI); + } } else { if (InlineAsm::isMemKind(OpFlags)) { Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant, Index: lib/CodeGen/BranchFolding.cpp =================================================================== --- lib/CodeGen/BranchFolding.cpp +++ lib/CodeGen/BranchFolding.cpp @@ -1155,7 +1155,9 @@ continue; // Skip blocks which may jump to a landing pad. Can't tail merge these. - if (PBB->hasEHPadSuccessor()) + // Same for asm goto. + // FIXME: Can we make asm-goto work? + if (PBB->hasEHPadSuccessor() || PBB->hasAsmGotoTargetSuccessor()) continue; // After block placement, only consider predecessors that belong to the @@ -1759,7 +1761,7 @@ // see if it has a fall-through into its successor. bool CurFallsThru = MBB->canFallThrough(); - if (!MBB->isEHPad()) { + if (!MBB->isEHPad() && !MBB->isAsmGotoTarget()) { // Check all the predecessors of this block. If one of them has no fall // throughs, move this block right after it. for (MachineBasicBlock *PredBB : MBB->predecessors()) { @@ -1800,11 +1802,11 @@ // If this block doesn't already fall-through to that successor, and if // the succ doesn't already have a block that can fall through into it, - // and if the successor isn't an EH destination, we can arrange for the - // fallthrough to happen. + // and if the successor isn't an EH destination or asm-goto target, we + // can arrange for the fallthrough to happen. if (SuccBB != MBB && &*SuccPrev != MBB && !SuccPrev->canFallThrough() && !CurUnAnalyzable && - !SuccBB->isEHPad()) { + !SuccBB->isEHPad() && !SuccBB->isAsmGotoTarget()) { MBB->moveBefore(SuccBB); MadeChange = true; goto ReoptimizeBlock; Index: lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- lib/CodeGen/CodeGenPrepare.cpp +++ lib/CodeGen/CodeGenPrepare.cpp @@ -653,6 +653,15 @@ BB->getSinglePredecessor()->getSingleSuccessor())) return false; + // Skip merging if the block's successor is also a successor to any callbr + // that leads to this block. + for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) { + if (auto *CBI = dyn_cast((*PI)->getTerminator())) + for (unsigned i = 0, e = CBI->getNumSuccessors(); i != e; ++i) + if (DestBB == CBI->getSuccessor(i)) + return false; + } + // Try to skip merging if the unique predecessor of BB is terminated by a // switch or indirect branch instruction, and BB is used as an incoming block // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to Index: lib/CodeGen/GlobalISel/IRTranslator.cpp =================================================================== --- lib/CodeGen/GlobalISel/IRTranslator.cpp +++ lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1281,6 +1281,39 @@ return true; } +bool IRTranslator::translateCallBr(const User &U, + MachineIRBuilder &MIRBuilder) { + const CallBrInst &I = cast(U); + + const Value *Callee = I.getCalledValue(); + const Function *Fn = dyn_cast(Callee); + + if (Fn && Fn->isIntrinsic()) + return false; + + // FIXME: support whatever these are. + if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) + return false; + + unsigned Res = I.getType()->isVoidTy() ? 0 : getOrCreateVReg(I); + SmallVector Args; + for (auto &Arg : I.arg_operands()) + Args.push_back(getOrCreateVReg(*Arg)); + + if (!CLI->lowerCall(MIRBuilder, &I, Res, Args, + [&]() { return getOrCreateVReg(*I.getCalledValue()); })) + return false; + + // FIXME: track probabilities. + MachineBasicBlock &ReturnMBB = getMBB(*I.getDefaultDest()); + MIRBuilder.getMBB().addSuccessor(&ReturnMBB); + for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) + MIRBuilder.getMBB().addSuccessor(&getMBB(*I.getIndirectDest(i))); + MIRBuilder.buildBr(ReturnMBB); + + return true; +} + bool IRTranslator::translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder) { const LandingPadInst &LP = cast(U); Index: lib/CodeGen/IndirectBrExpandPass.cpp =================================================================== --- lib/CodeGen/IndirectBrExpandPass.cpp +++ lib/CodeGen/IndirectBrExpandPass.cpp @@ -148,11 +148,9 @@ ConstantInt *BBIndexC = ConstantInt::get(ITy, BBIndex); // Now rewrite the blockaddress to an integer constant based on the index. - // FIXME: We could potentially preserve the uses as arguments to inline asm. - // This would allow some uses such as diagnostic information in crashes to - // have higher quality even when this transform is enabled, but would break - // users that round-trip blockaddresses through inline assembly and then - // back into an indirectbr. + // FIXME: this part doesn't properly recognize other uses of blockaddress + // expressions, for instance, where they are used to pass labels to + // asm-goto. This part of the pass needs a rework. BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(BBIndexC, BA->getType())); } Index: lib/CodeGen/MachineBasicBlock.cpp =================================================================== --- lib/CodeGen/MachineBasicBlock.cpp +++ lib/CodeGen/MachineBasicBlock.cpp @@ -235,6 +235,13 @@ return false; } +bool MachineBasicBlock::hasAsmGotoTargetSuccessor() const { + for (const_succ_iterator I = succ_begin(), E = succ_end(); I != E; ++I) + if ((*I)->isAsmGotoTarget()) + return true; + return false; +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void MachineBasicBlock::dump() const { print(dbgs()); @@ -320,6 +327,11 @@ OS << "landing-pad"; HasAttributes = true; } + if (isAsmGotoTarget()) { + OS << (HasAttributes ? ", " : " ("); + OS << "asm-goto-target"; + HasAttributes = true; + } if (getAlignment()) { OS << (HasAttributes ? ", " : " ("); OS << "align " << getAlignment(); @@ -546,7 +558,7 @@ // layout successor, insert a branch. First we have to locate the only // non-landing-pad successor, as that is the fallthrough block. for (succ_iterator SI = succ_begin(), SE = succ_end(); SI != SE; ++SI) { - if ((*SI)->isEHPad()) + if ((*SI)->isEHPad() || (*SI)->isAsmGotoTarget()) continue; assert(!TBB && "Found more than one non-landing-pad successor!"); TBB = *SI; @@ -1201,9 +1213,13 @@ assert(Old != New && "Cannot replace self with self!"); MachineBasicBlock::instr_iterator I = instr_end(); + int Stop = 0; while (I != instr_begin()) { --I; - if (!I->isTerminator()) break; + if (!I->isTerminator() && !I->isInlineAsm()) ++Stop; + // FIXME: this is crude, but the asm-goto sometimes seems to be the second- + // to-last instruction. Find a prettier way to check this. + if (Stop > 1 || I->isPHI()) break; // Scan the operands of this machine instruction, replacing any uses of Old // with New. @@ -1259,13 +1275,14 @@ } // Remove superfluous edges. I.e., those which aren't destinations of this - // basic block, duplicate edges, or landing pads. + // basic block, duplicate edges, landing pads, or asm-goto targets. SmallPtrSet SeenMBBs; MachineBasicBlock::succ_iterator SI = succ_begin(); while (SI != succ_end()) { const MachineBasicBlock *MBB = *SI; if (!SeenMBBs.insert(MBB).second || - (MBB != DestA && MBB != DestB && !MBB->isEHPad())) { + (MBB != DestA && MBB != DestB && !MBB->isEHPad() && + !MBB->isAsmGotoTarget())) { // This is a superfluous edge, remove it. SI = removeSuccessor(SI); Changed = true; Index: lib/CodeGen/MachineBlockPlacement.cpp =================================================================== --- lib/CodeGen/MachineBlockPlacement.cpp +++ lib/CodeGen/MachineBlockPlacement.cpp @@ -637,7 +637,8 @@ auto AdjustedSumProb = BranchProbability::getOne(); for (MachineBasicBlock *Succ : BB->successors()) { bool SkipSucc = false; - if (Succ->isEHPad() || (BlockFilter && !BlockFilter->count(Succ))) { + if (Succ->isEHPad() || Succ->isAsmGotoTarget() || + (BlockFilter && !BlockFilter->count(Succ))) { SkipSucc = true; } else { BlockChain *SuccChain = BlockToChain[Succ]; Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -46,6 +46,7 @@ class BasicBlock; class BranchInst; class CallInst; +class CallBrInst; class CatchPadInst; class CatchReturnInst; class CatchSwitchInst; @@ -851,6 +852,7 @@ private: // These all get lowered before this pass. void visitInvoke(const InvokeInst &I); + void visitCallBr(const CallBrInst &I); void visitResume(const ResumeInst &I); void visitUnary(const User &I, unsigned Opcode); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2548,6 +2548,37 @@ InvokeMBB->normalizeSuccProbs(); // Drop into normal successor. + DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), + DAG.getBasicBlock(Return))); +} + +void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { + MachineBasicBlock *CallBrMBB = FuncInfo.MBB; + + // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't + // have to do anything here to lower funclet bundles. + assert(!I.hasOperandBundlesOtherThan( + {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && + "Cannot lower callbrs with arbitrary operand bundles yet!"); + + assert(isa(I.getCalledValue()) && + "Only know how to handle inlineasm callbr"); + visitInlineAsm(&I); + + // Retrieve successors. Look through artificial IR level blocks like + // catchswitch for successors. + MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()]; + + // Update successor info. + addSuccessorWithProb(CallBrMBB, Return); + for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { + MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)]; + Target->setIsAsmGotoTarget(); + addSuccessorWithProb(CallBrMBB, Target); + } + CallBrMBB->normalizeSuccProbs(); + + // Drop into normal successor. DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), DAG.getBasicBlock(Return))); @@ -7578,7 +7609,14 @@ // Process the call argument. BasicBlocks are labels, currently appearing // only in asm's. - if (const BasicBlock *BB = dyn_cast(OpInfo.CallOperandVal)) { + const Instruction *I = CS.getInstruction(); + if (isa(I) && + (ArgNo - 1) >= (cast(I)->getNumArgOperands() - + cast(I)->getNumIndirectDests())) { + const auto *BA = cast(OpInfo.CallOperandVal); + EVT VT = TLI.getValueType(DAG.getDataLayout(), BA->getType(), true); + OpInfo.CallOperand = DAG.getTargetBlockAddress(BA, VT); + } else if (const auto *BB = dyn_cast(OpInfo.CallOperandVal)) { OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); } else { OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); Index: lib/CodeGen/SelectionDAG/TargetLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -3244,7 +3244,8 @@ switch (ConstraintLetter) { default: break; case 'X': // Allows any operand; labels (basic block) use this. - if (Op.getOpcode() == ISD::BasicBlock) { + if (Op.getOpcode() == ISD::BasicBlock || + Op.getOpcode() == ISD::TargetBlockAddress) { Ops.push_back(Op); return; } @@ -3731,6 +3732,9 @@ return; } + if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress) + return; + // Otherwise, try to resolve it to something we know about by looking at // the actual operand type. if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) { Index: lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- lib/CodeGen/TargetLoweringBase.cpp +++ lib/CodeGen/TargetLoweringBase.cpp @@ -1455,6 +1455,7 @@ case Switch: return 0; case IndirectBr: return 0; case Invoke: return 0; + case CallBr: return 0; case Resume: return 0; case Unreachable: return 0; case CleanupRet: return 0; Index: lib/IR/AsmWriter.cpp =================================================================== --- lib/IR/AsmWriter.cpp +++ lib/IR/AsmWriter.cpp @@ -3836,6 +3836,51 @@ writeOperand(II->getNormalDest(), true); Out << " unwind "; writeOperand(II->getUnwindDest(), true); + } else if (const CallBrInst *CBI = dyn_cast(&I)) { + Operand = CBI->getCalledValue(); + FunctionType *FTy = CBI->getFunctionType(); + Type *RetTy = FTy->getReturnType(); + const AttributeList &PAL = CBI->getAttributes(); + + // Print the calling convention being used. + if (CBI->getCallingConv() != CallingConv::C) { + Out << " "; + PrintCallingConv(CBI->getCallingConv(), Out); + } + + if (PAL.hasAttributes(AttributeList::ReturnIndex)) + Out << ' ' << PAL.getAsString(AttributeList::ReturnIndex); + + // If possible, print out the short form of the callbr instruction. We can + // only do this if the first argument is a pointer to a nonvararg function, + // and if the return type is not a pointer to a function. + // + Out << ' '; + TypePrinter.print(FTy->isVarArg() ? FTy : RetTy, Out); + Out << ' '; + writeOperand(Operand, false); + Out << '('; + for (unsigned op = 0, Eop = CBI->getNumArgOperands(); op < Eop; ++op) { + if (op) + Out << ", "; + writeParamOperand(CBI->getArgOperand(op), PAL.getParamAttributes(op)); + } + + Out << ')'; + if (PAL.hasAttributes(AttributeList::FunctionIndex)) + Out << " #" << Machine.getAttributeGroupSlot(PAL.getFnAttributes()); + + writeOperandBundles(CBI); + + Out << "\n to "; + writeOperand(CBI->getDefaultDest(), true); + Out << " ["; + for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) { + if (i != 0) + Out << ", "; + writeOperand(CBI->getIndirectDest(i), true); + } + Out << ']'; } else if (const AllocaInst *AI = dyn_cast(&I)) { Out << ' '; if (AI->isUsedWithInAlloca()) Index: lib/IR/Instruction.cpp =================================================================== --- lib/IR/Instruction.cpp +++ lib/IR/Instruction.cpp @@ -301,6 +301,7 @@ case CatchRet: return "catchret"; case CatchPad: return "catchpad"; case CatchSwitch: return "catchswitch"; + case CallBr: return "callbr"; // Standard unary operators... case FNeg: return "fneg"; @@ -405,6 +406,10 @@ return CI->getCallingConv() == cast(I2)->getCallingConv() && CI->getAttributes() == cast(I2)->getAttributes() && CI->hasIdenticalOperandBundleSchema(*cast(I2)); + if (const CallBrInst *CI = dyn_cast(I1)) + return CI->getCallingConv() == cast(I2)->getCallingConv() && + CI->getAttributes() == cast(I2)->getAttributes() && + CI->hasIdenticalOperandBundleSchema(*cast(I2)); if (const InsertValueInst *IVI = dyn_cast(I1)) return IVI->getIndices() == cast(I2)->getIndices(); if (const ExtractValueInst *EVI = dyn_cast(I1)) @@ -516,6 +521,7 @@ return true; case Instruction::Call: case Instruction::Invoke: + case Instruction::CallBr: return !cast(this)->doesNotAccessMemory(); case Instruction::Store: return !cast(this)->isUnordered(); @@ -535,6 +541,7 @@ return true; case Instruction::Call: case Instruction::Invoke: + case Instruction::CallBr: return !cast(this)->onlyReadsMemory(); case Instruction::Load: return !cast(this)->isUnordered(); @@ -772,8 +779,9 @@ } void Instruction::setProfWeight(uint64_t W) { - assert((isa(this) || isa(this)) && - "Can only set weights for call and invoke instrucitons"); + assert((isa(this) || isa(this) || + isa(this)) && + "Can only set weights for call, invoke and callbr instructions"); SmallVector Weights; Weights.push_back(W); MDBuilder MDB(getContext()); Index: lib/IR/Instructions.cpp =================================================================== --- lib/IR/Instructions.cpp +++ lib/IR/Instructions.cpp @@ -256,6 +256,11 @@ Function *CallBase::getCaller() { return getParent()->getParent(); } +unsigned CallBase::getNumSubclassExtraOperandsDynamic() const { + assert(getOpcode() == Instruction::CallBr && "Unexpected opcode!"); + return cast(this)->getNumIndirectDests() + 1; +} + bool CallBase::isIndirectCall() const { const Value *V = getCalledValue(); if (isa(V) || isa(V)) @@ -727,6 +732,76 @@ } //===----------------------------------------------------------------------===// +// CallBrInst Implementation +//===----------------------------------------------------------------------===// + +void CallBrInst::init(FunctionType *FTy, Value *Fn, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr) { + this->FTy = FTy; + + assert((int)getNumOperands() == + ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)) && + "NumOperands not set up?"); + NumIndirectDests = IndirectDests.size(); + setDefaultDest(Fallthrough); + for (unsigned i = 0; i != NumIndirectDests; ++i) + setIndirectDest(i, IndirectDests[i]); + setCalledOperand(Fn); + +#ifndef NDEBUG + assert(((Args.size() == FTy->getNumParams()) || + (FTy->isVarArg() && Args.size() > FTy->getNumParams())) && + "Calling a function with bad signature"); + + for (unsigned i = 0, e = Args.size(); i != e; i++) + assert((i >= FTy->getNumParams() || + FTy->getParamType(i) == Args[i]->getType()) && + "Calling a function with a bad signature!"); +#endif + + std::copy(Args.begin(), Args.end(), op_begin()); + + auto It = populateBundleOperandInfos(Bundles, Args.size()); + (void)It; + assert(It + 2 + IndirectDests.size() == op_end() && "Should add up!"); + + setName(NameStr); +} + +CallBrInst::CallBrInst(const CallBrInst &CBI) + : CallBase(CBI.Attrs, CBI.FTy, CBI.getType(), Instruction::CallBr, + OperandTraits::op_end(this) - CBI.getNumOperands(), + CBI.getNumOperands()) { + setCallingConv(CBI.getCallingConv()); + std::copy(CBI.op_begin(), CBI.op_end(), op_begin()); + std::copy(CBI.bundle_op_info_begin(), CBI.bundle_op_info_end(), + bundle_op_info_begin()); + SubclassOptionalData = CBI.SubclassOptionalData; + NumIndirectDests = CBI.NumIndirectDests; +} + +CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef OpB, + Instruction *InsertPt) { + std::vector Args(CBI->arg_begin(), CBI->arg_end()); + + auto *NewCBI = CallBrInst::Create(CBI->getFunctionType(), + CBI->getCalledValue(), + CBI->getDefaultDest(), + CBI->getIndirectDests(), + Args, OpB, CBI->getName(), InsertPt); + NewCBI->setCallingConv(CBI->getCallingConv()); + NewCBI->SubclassOptionalData = CBI->SubclassOptionalData; + NewCBI->setAttributes(CBI->getAttributes()); + NewCBI->setDebugLoc(CBI->getDebugLoc()); + NewCBI->NumIndirectDests = CBI->NumIndirectDests; + return NewCBI; +} + +//===----------------------------------------------------------------------===// // ReturnInst Implementation //===----------------------------------------------------------------------===// @@ -3996,6 +4071,14 @@ return new(getNumOperands()) InvokeInst(*this); } +CallBrInst *CallBrInst::cloneImpl() const { + if (hasOperandBundles()) { + unsigned DescriptorBytes = getNumOperandBundles() * sizeof(BundleOpInfo); + return new (getNumOperands(), DescriptorBytes) CallBrInst(*this); + } + return new (getNumOperands()) CallBrInst(*this); +} + ResumeInst *ResumeInst::cloneImpl() const { return new (1) ResumeInst(*this); } CleanupReturnInst *CleanupReturnInst::cloneImpl() const { Index: lib/IR/Value.cpp =================================================================== --- lib/IR/Value.cpp +++ lib/IR/Value.cpp @@ -57,7 +57,8 @@ // FIXME: Why isn't this in the subclass gunk?? // Note, we cannot call isa before the CallInst has been // constructed. - if (SubclassID == Instruction::Call || SubclassID == Instruction::Invoke) + if (SubclassID == Instruction::Call || SubclassID == Instruction::Invoke || + SubclassID == Instruction::CallBr) assert((VTy->isFirstClassType() || VTy->isVoidTy() || VTy->isStructTy()) && "invalid CallInst type!"); else if (SubclassID != BasicBlockVal && Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -466,6 +466,7 @@ void visitReturnInst(ReturnInst &RI); void visitSwitchInst(SwitchInst &SI); void visitIndirectBrInst(IndirectBrInst &BI); + void visitCallBrInst(CallBrInst &CBI); void visitSelectInst(SelectInst &SI); void visitUserOp1(Instruction &I); void visitUserOp2(Instruction &I) { visitUserOp1(I); } @@ -2450,6 +2451,26 @@ visitTerminator(BI); } +void Verifier::visitCallBrInst(CallBrInst &CBI) { + Assert(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", + &CBI); + Assert(CBI.getType()->isVoidTy(), "Callbr return value is not supported!", + &CBI); + for (unsigned i = 0, e = CBI.getNumSuccessors(); i != e; ++i) + Assert(CBI.getSuccessor(i)->getType()->isLabelTy(), + "Callbr successors must all have pointer type!", &CBI); + for (unsigned i = 0, e = CBI.getNumOperands(); i != e; ++i) { + Assert(i >= CBI.getNumArgOperands() || !isa(CBI.getOperand(i)), + "Using an unescaped label as a callbr argument!", &CBI); + if (isa(CBI.getOperand(i))) + for (unsigned j = i + 1; j != e; ++j) + Assert(CBI.getOperand(i) != CBI.getOperand(j), + "Duplicate callbr destination!", &CBI); + } + + visitTerminator(CBI); +} + void Verifier::visitSelectInst(SelectInst &SI) { Assert(!SelectInst::areInvalidOperands(SI.getOperand(0), SI.getOperand(1), SI.getOperand(2)), Index: lib/Target/X86/X86AsmPrinter.cpp =================================================================== --- lib/Target/X86/X86AsmPrinter.cpp +++ lib/Target/X86/X86AsmPrinter.cpp @@ -253,6 +253,11 @@ printSymbolOperand(P, MO, O); break; } + case MachineOperand::MO_BlockAddress: { + MCSymbol *Sym = P.GetBlockAddressSymbol(MO.getBlockAddress()); + Sym->print(O, P.MAI); + break; + } } } Index: lib/Target/X86/X86InstrInfo.cpp =================================================================== --- lib/Target/X86/X86InstrInfo.cpp +++ lib/Target/X86/X86InstrInfo.cpp @@ -2579,8 +2579,20 @@ // Working from the bottom, when we see a non-terminator instruction, we're // done. - if (!isUnpredicatedTerminator(*I)) - break; + if (!isUnpredicatedTerminator(*I)) { + // If the non-terminator is an inline assembly block, we pessimistically + // assume it's an asm-goto that can't be handled by this analysis. + // In cases of non-void returns, the asm may be the second-to-last + // non-terminator instead. + // TODO: make a more precise check for asm-goto with outputs to reduce + // false positives + if (I->getOpcode() == X86::INLINEASM || + (I != MBB.begin() && I->getOpcode() == TargetOpcode::COPY && + (--I)->getOpcode() == X86::INLINEASM)) + return true; + else + break; + } // A terminator that isn't a branch can't easily be handled by this // analysis. Index: lib/Transforms/InstCombine/InstCombineCalls.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineCalls.cpp +++ lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// This file implements the visitCall and visitInvoke functions. +// This file implements the visitCall, visitInvoke and visitCallBr functions. // //===----------------------------------------------------------------------===// @@ -1834,8 +1834,8 @@ IntrinsicInst *II = dyn_cast(&CI); if (!II) return visitCallBase(CI); - // Intrinsics cannot occur in an invoke, so handle them here instead of in - // visitCallBase. + // Intrinsics cannot occur in an invoke or a callbr, so handle them here + // instead of in visitCallBase. if (auto *MI = dyn_cast(II)) { bool Changed = false; @@ -4017,6 +4017,11 @@ return visitCallBase(II); } +// CallBrInst simplification +Instruction *InstCombiner::visitCallBrInst(CallBrInst &CBI) { + return visitCallBase(CBI); +} + /// If this cast does not affect the value passed through the varargs area, we /// can eliminate the use of the cast. static bool isSafeToEliminateVarargsCast(const CallBase &Call, @@ -4145,7 +4150,7 @@ return nullptr; } -/// Improvements for call and invoke instructions. +/// Improvements for call, callbr and invoke instructions. Instruction *InstCombiner::visitCallBase(CallBase &Call) { if (isAllocLikeFn(&Call, &TLI)) return visitAllocSite(Call); @@ -4178,7 +4183,7 @@ } // If the callee is a pointer to a function, attempt to move any casts to the - // arguments of the call/invoke. + // arguments of the call/callbr/invoke. Value *Callee = Call.getCalledValue(); if (!isa(Callee) && transformConstExprCastCall(Call)) return nullptr; @@ -4211,9 +4216,9 @@ if (isa(OldCall)) return eraseInstFromFunction(*OldCall); - // We cannot remove an invoke, because it would change the CFG, just - // change the callee to a null pointer. - cast(OldCall)->setCalledFunction( + // We cannot remove an invoke or a callbr, because it would change thexi + // CFG, just change the callee to a null pointer. + cast(OldCall)->setCalledFunction( CalleeF->getFunctionType(), Constant::getNullValue(CalleeF->getType())); return nullptr; @@ -4228,8 +4233,8 @@ if (!Call.getType()->isVoidTy()) replaceInstUsesWith(Call, UndefValue::get(Call.getType())); - if (isa(Call)) { - // Can't remove an invoke because we cannot change the CFG. + if (Call.isTerminator()) { + // Can't remove an invoke or callbr because we cannot change the CFG. return nullptr; } @@ -4282,7 +4287,7 @@ } /// If the callee is a constexpr cast of a function, attempt to move the cast to -/// the arguments of the call/invoke. +/// the arguments of the call/callbr/invoke. bool InstCombiner::transformConstExprCastCall(CallBase &Call) { auto *Callee = dyn_cast(Call.getCalledValue()->stripPointerCasts()); if (!Callee) @@ -4333,17 +4338,17 @@ return false; // Attribute not compatible with transformed value. } - // If the callbase is an invoke instruction, and the return value is used by - // a PHI node in a successor, we cannot change the return type of the call - // because there is no place to put the cast instruction (without breaking - // the critical edge). Bail out in this case. + // If the callbase is an invoke/callbr instruction, and the return value is + // used by a PHI node in a successor, we cannot change the return type of + // the call because there is no place to put the cast instruction (without + // breaking the critical edge). Bail out in this case. if (!Caller->use_empty()) - if (InvokeInst *II = dyn_cast(Caller)) - for (User *U : II->users()) + if (Caller->isTerminator()) + for (User *U : Caller->users()) if (PHINode *PN = dyn_cast(U)) - if (PN->getParent() == II->getNormalDest() || - PN->getParent() == II->getUnwindDest()) - return false; + for (int i = 0, e = Caller->getNumSuccessors(); i != e; ++i) + if (PN->getParent() == Caller->getSuccessor(i)) + return false; } unsigned NumActualArgs = Call.arg_size(); @@ -4497,6 +4502,9 @@ if (InvokeInst *II = dyn_cast(Caller)) { NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(), II->getUnwindDest(), Args, OpBundles); + } else if (CallBrInst *CBI = dyn_cast(Caller)) { + NewCall = Builder.CreateCallBr(Callee, CBI->getDefaultDest(), + CBI->getIndirectDests(), Args, OpBundles); } else { NewCall = Builder.CreateCall(Callee, Args, OpBundles); cast(NewCall)->setTailCallKind( @@ -4520,11 +4528,14 @@ NV = NC = CastInst::CreateBitOrPointerCast(NC, OldRetTy); NC->setDebugLoc(Caller->getDebugLoc()); - // If this is an invoke instruction, we should insert it after the first - // non-phi, instruction in the normal successor block. + // If this is an invoke/callbr instruction, we should insert it after the + // first non-phi instruction in the normal successor block. if (InvokeInst *II = dyn_cast(Caller)) { BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt(); InsertNewInstBefore(NC, *I); + } else if (CallBrInst *CBI = dyn_cast(Caller)) { + BasicBlock::iterator I = CBI->getDefaultDest()->getFirstInsertionPt(); + InsertNewInstBefore(NC, *I); } else { // Otherwise, it's a call, just insert cast right after the call. InsertNewInstBefore(NC, *Caller); @@ -4673,6 +4684,12 @@ NewArgs, OpBundles); cast(NewCaller)->setCallingConv(II->getCallingConv()); cast(NewCaller)->setAttributes(NewPAL); + } else if (CallBrInst *CBI = dyn_cast(&Call)) { + NewCaller = + CallBrInst::Create(NewFTy, NewCallee, CBI->getDefaultDest(), + CBI->getIndirectDests(), NewArgs, OpBundles); + cast(NewCaller)->setCallingConv(CBI->getCallingConv()); + cast(NewCaller)->setAttributes(NewPAL); } else { NewCaller = CallInst::Create(NewFTy, NewCallee, NewArgs, OpBundles); cast(NewCaller)->setTailCallKind( Index: lib/Transforms/InstCombine/InstCombineInternal.h =================================================================== --- lib/Transforms/InstCombine/InstCombineInternal.h +++ lib/Transforms/InstCombine/InstCombineInternal.h @@ -392,6 +392,7 @@ Instruction *visitSelectInst(SelectInst &SI); Instruction *visitCallInst(CallInst &CI); Instruction *visitInvokeInst(InvokeInst &II); + Instruction *visitCallBrInst(CallBrInst &CBI); Instruction *SliceUpIllegalIntegerPHI(PHINode &PN); Instruction *visitPHINode(PHINode &PN); Index: lib/Transforms/InstCombine/InstructionCombining.cpp =================================================================== --- lib/Transforms/InstCombine/InstructionCombining.cpp +++ lib/Transforms/InstCombine/InstructionCombining.cpp @@ -921,8 +921,8 @@ // If the InVal is an invoke at the end of the pred block, then we can't // insert a computation after it without breaking the edge. - if (InvokeInst *II = dyn_cast(InVal)) - if (II->getParent() == NonConstBB) + if (isa(InVal)) + if (cast(InVal)->getParent() == NonConstBB) return nullptr; // If the incoming non-constant value is in I's block, we will remove one Index: lib/Transforms/Scalar/GVN.cpp =================================================================== --- lib/Transforms/Scalar/GVN.cpp +++ lib/Transforms/Scalar/GVN.cpp @@ -1131,6 +1131,14 @@ return false; } + // FIXME: Can we support the fallthrough edge? + if (isa(Pred->getTerminator())) { + LLVM_DEBUG( + dbgs() << "COULD NOT PRE LOAD BECAUSE OF CALLBR CRITICAL EDGE '" + << Pred->getName() << "': " << *LI << '\n'); + return false; + } + if (LoadBB->isEHPad()) { LLVM_DEBUG( dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '" @@ -2167,8 +2175,8 @@ return false; // We don't currently value number ANY inline asm calls. - if (CallInst *CallI = dyn_cast(CurInst)) - if (CallI->isInlineAsm()) + if (auto *CallB = dyn_cast(CurInst)) + if (CallB->isInlineAsm()) return false; uint32_t ValNo = VN.lookup(CurInst); @@ -2251,6 +2259,11 @@ if (isa(PREPred->getTerminator())) return false; + // Don't do PRE across callbr. + // FIXME: Can we do this across the fallthrough edge? + if (isa(PREPred->getTerminator())) + return false; + // We can't do PRE safely on a critical edge, so instead we schedule // the edge to be split and perform the PRE the next time we iterate // on the function. Index: lib/Transforms/Scalar/JumpThreading.cpp =================================================================== --- lib/Transforms/Scalar/JumpThreading.cpp +++ lib/Transforms/Scalar/JumpThreading.cpp @@ -1055,7 +1055,7 @@ Condition = IB->getAddress()->stripPointerCasts(); Preference = WantBlockAddress; } else { - return false; // Must be an invoke. + return false; // Must be an invoke or callbr. } // Run constant folding to see if we can reduce the condition to a simple @@ -1428,7 +1428,9 @@ // Add all the unavailable predecessors to the PredsToSplit list. for (BasicBlock *P : predecessors(LoadBB)) { // If the predecessor is an indirect goto, we can't split the edge. - if (isa(P->getTerminator())) + // Same for CallBr. + if (isa(P->getTerminator()) || + isa(P->getTerminator())) return false; if (!AvailablePredSet.count(P)) @@ -1641,8 +1643,9 @@ ++PredWithKnownDest; // If the predecessor ends with an indirect goto, we can't change its - // destination. - if (isa(Pred->getTerminator())) + // destination. Same for CallBr. + if (isa(Pred->getTerminator()) || + isa(Pred->getTerminator())) continue; PredToDestList.push_back(std::make_pair(Pred, DestBB)); Index: lib/Transforms/Scalar/SCCP.cpp =================================================================== --- lib/Transforms/Scalar/SCCP.cpp +++ lib/Transforms/Scalar/SCCP.cpp @@ -638,6 +638,11 @@ visitTerminator(II); } + void visitCallBrInst (CallBrInst &CBI) { + visitCallSite(&CBI); + visitTerminator(CBI); + } + void visitCallSite (CallSite CS); void visitResumeInst (ResumeInst &I) { /*returns void*/ } void visitUnreachableInst(UnreachableInst &I) { /*returns void*/ } @@ -733,6 +738,13 @@ return; } + // In case of callbr, we pessimistically assume that all successors are + // feasible. + if (isa(&TI)) { + Succs.assign(TI.getNumSuccessors(), true); + return; + } + LLVM_DEBUG(dbgs() << "Unknown terminator instruction: " << TI << '\n'); llvm_unreachable("SCCP: Don't know how to handle this terminator!"); } @@ -1597,6 +1609,7 @@ return true; case Instruction::Call: case Instruction::Invoke: + case Instruction::CallBr: // There are two reasons a call can have an undef result // 1. It could be tracked. // 2. It could be constant-foldable. Index: lib/Transforms/Utils/BasicBlockUtils.cpp =================================================================== --- lib/Transforms/Utils/BasicBlockUtils.cpp +++ lib/Transforms/Utils/BasicBlockUtils.cpp @@ -542,6 +542,8 @@ // all BlockAddress uses would need to be updated. assert(!isa(Preds[i]->getTerminator()) && "Cannot split an edge from an IndirectBrInst"); + assert(!isa(Preds[i]->getTerminator()) && + "Cannot split an edge from a CallBrInst"); Preds[i]->getTerminator()->replaceUsesOfWith(BB, NewBB); } Index: lib/Transforms/Utils/BreakCriticalEdges.cpp =================================================================== --- lib/Transforms/Utils/BreakCriticalEdges.cpp +++ lib/Transforms/Utils/BreakCriticalEdges.cpp @@ -144,6 +144,10 @@ // it in this generic function. if (DestBB->isEHPad()) return nullptr; + // Don't split the non-fallthrough edge from a callbr. + if (isa(TI) && SuccNum > 0) + return nullptr; + // Create a new basic block, linking it into the CFG. BasicBlock *NewBB = BasicBlock::Create(TI->getContext(), TIBB->getName() + "." + DestBB->getName() + "_crit_edge"); Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -1504,6 +1504,10 @@ assert(TheCall->getParent() && TheCall->getFunction() && "Instruction not in function!"); + // FIXME: we don't inline callbr yet. + if (isa(TheCall)) + return false; + // If IFI has any state in it, zap it before we fill it in. IFI.reset(); @@ -1729,6 +1733,8 @@ Instruction *NewI = nullptr; if (isa(I)) NewI = CallInst::Create(cast(I), OpDefs, I); + else if (isa(I)) + NewI = CallBrInst::Create(cast(I), OpDefs, I); else NewI = InvokeInst::Create(cast(I), OpDefs, I); @@ -2031,6 +2037,8 @@ Instruction *NewInst; if (CS.isCall()) NewInst = CallInst::Create(cast(I), OpBundles, I); + else if (CS.isCallBr()) + NewInst = CallBrInst::Create(cast(I), OpBundles, I); else NewInst = InvokeInst::Create(cast(I), OpBundles, I); NewInst->takeName(I); Index: lib/Transforms/Utils/Local.cpp =================================================================== --- lib/Transforms/Utils/Local.cpp +++ lib/Transforms/Utils/Local.cpp @@ -996,6 +996,18 @@ } } + // We cannot fold the block if it's a branch to an already present callbr + // successor because that creates duplicate successors. + for (auto I = pred_begin(BB), E = pred_end(BB); I != E; ++I) { + if (auto *CBI = dyn_cast((*I)->getTerminator())) { + if (Succ == CBI->getDefaultDest()) + return false; + for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) + if (Succ == CBI->getIndirectDest(i)) + return false; + } + } + LLVM_DEBUG(dbgs() << "Killing Trivial BB: \n" << *BB); SmallVector Updates; Index: lib/Transforms/Utils/LoopSimplify.cpp =================================================================== --- lib/Transforms/Utils/LoopSimplify.cpp +++ lib/Transforms/Utils/LoopSimplify.cpp @@ -27,6 +27,9 @@ // to transform the loop and make these guarantees. Client code should check // that these conditions are true before relying on them. // +// Similar complications arise from callbr instructions, particularly in +// asm-goto where blockaddress expressions are used. +// // Note that the simplifycfg pass will clean up blocks which are split out but // end up being unnecessary, so usage of this pass should not pessimize // generated code. @@ -123,10 +126,11 @@ PI != PE; ++PI) { BasicBlock *P = *PI; if (!L->contains(P)) { // Coming in from outside the loop? - // If the loop is branched to from an indirect branch, we won't + // If the loop is branched to from an indirect terminator, we won't // be able to fully transform the loop, because it prohibits // edge splitting. - if (isa(P->getTerminator())) return nullptr; + if (P->getTerminator()->isIndirectTerminator()) + return nullptr; // Keep track of it. OutsideBlocks.push_back(P); @@ -235,8 +239,8 @@ for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { if (PN->getIncomingValue(i) != PN || !L->contains(PN->getIncomingBlock(i))) { - // We can't split indirectbr edges. - if (isa(PN->getIncomingBlock(i)->getTerminator())) + // We can't split indirect control flow edges. + if (PN->getIncomingBlock(i)->getTerminator()->isIndirectTerminator()) return nullptr; OuterLoopPreds.push_back(PN->getIncomingBlock(i)); } @@ -357,8 +361,8 @@ for (pred_iterator I = pred_begin(Header), E = pred_end(Header); I != E; ++I){ BasicBlock *P = *I; - // Indirectbr edges cannot be split, so we must fail if we find one. - if (isa(P->getTerminator())) + // Indirect edges cannot be split, so we must fail if we find one. + if (P->getTerminator()->isIndirectTerminator()) return nullptr; if (P != Preheader) BackedgeBlocks.push_back(P); Index: lib/Transforms/Utils/LoopUtils.cpp =================================================================== --- lib/Transforms/Utils/LoopUtils.cpp +++ lib/Transforms/Utils/LoopUtils.cpp @@ -65,6 +65,9 @@ if (isa(PredBB->getTerminator())) // We cannot rewrite exiting edges from an indirectbr. return false; + if (isa(PredBB->getTerminator())) + // We cannot rewrite exiting edges from a callbr. + return false; InLoopPredecessors.push_back(PredBB); } else { Index: lib/Transforms/Utils/SimplifyCFG.cpp =================================================================== --- lib/Transforms/Utils/SimplifyCFG.cpp +++ lib/Transforms/Utils/SimplifyCFG.cpp @@ -1265,8 +1265,10 @@ while (isa(I2)) I2 = &*BB2_Itr++; } + // FIXME: Can we define a safety predicate for CallBr? if (isa(I1) || !I1->isIdenticalToWhenDefined(I2) || - (isa(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))) + (isa(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) || + isa(I1)) return false; BasicBlock *BIParent = BI->getParent(); @@ -1349,9 +1351,14 @@ HoistTerminator: // It may not be possible to hoist an invoke. + // FIXME: Can we define a safety predicate for CallBr? if (isa(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) return Changed; + // TODO: callbr hoisting currently disabled pending further study. + if (isa(I1)) + return Changed; + for (BasicBlock *Succ : successors(BB1)) { for (PHINode &PN : Succ->phis()) { Value *BB1V = PN.getIncomingValueForBlock(BB1); @@ -1443,7 +1450,7 @@ // Conservatively return false if I is an inline-asm instruction. Sinking // and merging inline-asm instructions can potentially create arguments // that cannot satisfy the inline-asm constraints. - if (const auto *C = dyn_cast(I)) + if (const auto *C = dyn_cast(I)) if (C->isInlineAsm()) return false; @@ -1506,7 +1513,7 @@ // We can't create a PHI from this GEP. return false; // Don't create indirect calls! The called value is the final operand. - if ((isa(I0) || isa(I0)) && OI == OE - 1) { + if (isa(I0) && OI == OE - 1) { // FIXME: if the call was *already* indirect, we should do this. return false; } Index: test/Bitcode/callbr.ll =================================================================== --- /dev/null +++ test/Bitcode/callbr.ll @@ -0,0 +1,14 @@ +; RUN: llvm-dis < %s.bc | FileCheck %s + +; callbr.ll.bc was generated by passing this file to llvm-as. + +define i32 @test_asm_goto(i32 %x){ +entry: +; CHECK: callbr void asm "", "r,X"(i32 %x, i8* blockaddress(@test_asm_goto, %fail)) +; CHECK-NEXT: to label %normal [label %fail] + callbr void asm "", "r,X"(i32 %x, i8* blockaddress(@test_asm_goto, %fail)) to label %normal [label %fail] +normal: + ret i32 1 +fail: + ret i32 0 +} Index: test/CodeGen/X86/callbr-asm-destinations.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/callbr-asm-destinations.ll @@ -0,0 +1,15 @@ +; RUN: not llc -mtriple=i686-- < %s 2> %t +; RUN: FileCheck %s < %t + +; CHECK: Duplicate callbr destination + +; A test for asm-goto duplicate labels limitation + +define i32 @test(i32 %a) { +entry: + %0 = add i32 %a, 4 + callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail)) to label %fail [label %fail] + +fail: + ret i32 1 +} Index: test/CodeGen/X86/callbr-asm-errors.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/callbr-asm-errors.ll @@ -0,0 +1,18 @@ +; RUN: not llc -mtriple=i686-- < %s 2> %t +; RUN: FileCheck %s < %t + +; CHECK: Duplicate callbr destination + +; A test for asm-goto duplicate labels limitation + +define i32 @test(i32 %a) { +entry: + %0 = add i32 %a, 4 + callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail), i8* blockaddress(@test, %fail)) to label %normal [label %fail, label %fail] + +normal: + ret i32 %0 + +fail: + ret i32 1 +} Index: test/CodeGen/X86/callbr-asm-outputs.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/callbr-asm-outputs.ll @@ -0,0 +1,18 @@ +; RUN: not llc -mtriple=i686-- < %s 2> %t +; RUN: FileCheck %s < %t + +; CHECK: error: asm-goto outputs not supported + +; A test for asm-goto output prohibition + +define i32 @test(i32 %a) { +entry: + %0 = add i32 %a, 4 + %1 = callbr i32 asm "xorl $1, $1; jmp ${1:l}", "=&r,r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail)) to label %normal [label %fail] + +normal: + ret i32 %1 + +fail: + ret i32 1 +} Index: test/CodeGen/X86/callbr-asm.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/callbr-asm.ll @@ -0,0 +1,133 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=i686-- -O3 | FileCheck %s + +; Tests for using callbr as an asm-goto wrapper + +; Test 1 - fallthrough label gets removed, but the fallthrough code that is +; unreachable due to asm ending on a jmp is still left in. +define i32 @test1(i32 %a) { +; CHECK-LABEL: test1: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: addl $4, %eax +; CHECK-NEXT: #APP +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: jmp .Ltmp00 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: # %bb.1: # %normal +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl +; CHECK-NEXT: .Ltmp0: # Block address taken +; CHECK-NEXT: .LBB0_2: # %fail +; CHECK-NEXT: movl $1, %eax +; CHECK-NEXT: retl +entry: + %0 = add i32 %a, 4 + callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test1, %fail)) to label %normal [label %fail] + +normal: + ret i32 0 + +fail: + ret i32 1 +} + +; Test 2 - callbr terminates an unreachable block, function gets simplified +; to a trivial zero return. +define i32 @test2(i32 %a) { +; CHECK-LABEL: test2: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl +entry: + br label %normal + +unreachableasm: + %0 = add i32 %a, 4 + callbr void asm sideeffect "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test2, %fail)) to label %normal [label %fail] + +normal: + ret i32 0 + +fail: + ret i32 1 +} + + +; Test 3 - asm-goto implements a loop. The loop gets recognized, but many loop +; transforms fail due to canonicalization having callbr exceptions. Trivial +; blocks at labels 1 and 3 also don't get simplified due to callbr. +define dso_local i32 @test3(i32 %a) { +; CHECK-LABEL: test3: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: .Ltmp1: # Block address taken +; CHECK-NEXT: .LBB2_1: # %label01 +; CHECK-NEXT: # =>This Loop Header: Depth=1 +; CHECK-NEXT: # Child Loop BB2_2 Depth 2 +; CHECK-NEXT: # Child Loop BB2_3 Depth 3 +; CHECK-NEXT: # Child Loop BB2_4 Depth 4 +; CHECK-NEXT: .Ltmp2: # Block address taken +; CHECK-NEXT: .LBB2_2: # %label02 +; CHECK-NEXT: # Parent Loop BB2_1 Depth=1 +; CHECK-NEXT: # => This Loop Header: Depth=2 +; CHECK-NEXT: # Child Loop BB2_3 Depth 3 +; CHECK-NEXT: # Child Loop BB2_4 Depth 4 +; CHECK-NEXT: addl $4, {{[0-9]+}}(%esp) +; CHECK-NEXT: .Ltmp3: # Block address taken +; CHECK-NEXT: .LBB2_3: # %label03 +; CHECK-NEXT: # Parent Loop BB2_1 Depth=1 +; CHECK-NEXT: # Parent Loop BB2_2 Depth=2 +; CHECK-NEXT: # => This Loop Header: Depth=3 +; CHECK-NEXT: # Child Loop BB2_4 Depth 4 +; CHECK-NEXT: .p2align 4, 0x90 +; CHECK-NEXT: .Ltmp4: # Block address taken +; CHECK-NEXT: .LBB2_4: # %label04 +; CHECK-NEXT: # Parent Loop BB2_1 Depth=1 +; CHECK-NEXT: # Parent Loop BB2_2 Depth=2 +; CHECK-NEXT: # Parent Loop BB2_3 Depth=3 +; CHECK-NEXT: # => This Inner Loop Header: Depth=4 +; CHECK-NEXT: #APP +; CHECK-NEXT: jmp .Ltmp10 +; CHECK-NEXT: jmp .Ltmp20 +; CHECK-NEXT: jmp .Ltmp30 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: # %bb.5: # %normal0 +; CHECK-NEXT: # in Loop: Header=BB2_4 Depth=4 +; CHECK-NEXT: #APP +; CHECK-NEXT: jmp .Ltmp10 +; CHECK-NEXT: jmp .Ltmp20 +; CHECK-NEXT: jmp .Ltmp30 +; CHECK-NEXT: jmp .Ltmp40 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: # %bb.6: # %normal1 +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: retl +entry: + %a.addr = alloca i32, align 4 + store i32 %a, i32* %a.addr, align 4 + br label %label01 + +label01: ; preds = %normal0, %label04, %entry + br label %label02 + +label02: ; preds = %normal0, %label04, %label01 + %0 = load i32, i32* %a.addr, align 4 + %add = add nsw i32 %0, 4 + store i32 %add, i32* %a.addr, align 4 + br label %label03 + +label03: ; preds = %normal0, %label04, %label02 + br label %label04 + +label04: ; preds = %normal0, %label03 + callbr void asm sideeffect "jmp ${0:l}; jmp ${1:l}; jmp ${2:l}", "X,X,X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test3, %label01), i8* blockaddress(@test3, %label02), i8* blockaddress(@test3, %label03)) + to label %normal0 [label %label01, label %label02, label %label03] + +normal0: ; preds = %label04 + callbr void asm sideeffect "jmp ${0:l}; jmp ${1:l}; jmp ${2:l}; jmp ${3:l}", "X,X,X,X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test3, %label01), i8* blockaddress(@test3, %label02), i8* blockaddress(@test3, %label03), i8* blockaddress(@test3, %label04)) + to label %normal1 [label %label01, label %label02, label %label03, label %label04] + +normal1: ; preds = %normal0 + %1 = load i32, i32* %a.addr, align 4 + ret i32 %1 +} Index: test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll =================================================================== --- test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll +++ test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll @@ -63,14 +63,6 @@ resume { i8*, i32 } zeroinitializer } -define i8 @call_with_same_range() { -; CHECK-LABEL: @call_with_same_range -; CHECK: tail call i8 @call_with_range - bitcast i8 0 to i8 - %out = call i8 @dummy(), !range !0 - ret i8 %out -} - define i8 @invoke_with_same_range() personality i8* undef { ; CHECK-LABEL: @invoke_with_same_range() ; CHECK: tail call i8 @invoke_with_range() @@ -84,6 +76,13 @@ resume { i8*, i32 } zeroinitializer } +define i8 @call_with_same_range() { +; CHECK-LABEL: @call_with_same_range +; CHECK: tail call i8 @call_with_range + bitcast i8 0 to i8 + %out = call i8 @dummy(), !range !0 + ret i8 %out +} declare i8 @dummy(); Index: test/Transforms/MergeFunc/inline-asm.ll =================================================================== --- test/Transforms/MergeFunc/inline-asm.ll +++ test/Transforms/MergeFunc/inline-asm.ll @@ -3,13 +3,13 @@ ; CHECK-LABEL: @int_ptr_arg_different ; CHECK-NEXT: call void asm +; CHECK-LABEL: @int_ptr_null +; CHECK-NEXT: tail call void @float_ptr_null() + ; CHECK-LABEL: @int_ptr_arg_same ; CHECK-NEXT: %2 = bitcast i32* %0 to float* ; CHECK-NEXT: tail call void @float_ptr_arg_same(float* %2) -; CHECK-LABEL: @int_ptr_null -; CHECK-NEXT: tail call void @float_ptr_null() - ; Used to satisfy minimum size limit declare void @stuff()