Index: llvm/include/llvm/IR/CallSite.h =================================================================== --- llvm/include/llvm/IR/CallSite.h +++ llvm/include/llvm/IR/CallSite.h @@ -656,10 +656,7 @@ private: IterTy getCallee() const { - if (isCall()) // Skip Callee - return cast(getInstruction())->op_end() - 1; - else // Skip BB, BB, Callee - return cast(getInstruction())->op_end() - 3; + return cast(getInstruction())->op_end() - 1; } }; Index: llvm/include/llvm/IR/InstrTypes.h =================================================================== --- llvm/include/llvm/IR/InstrTypes.h +++ llvm/include/llvm/IR/InstrTypes.h @@ -25,6 +25,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Attributes.h" +#include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Instruction.h" @@ -904,76 +905,6 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CmpInst, Value) -//===----------------------------------------------------------------------===// -// FuncletPadInst Class -//===----------------------------------------------------------------------===// -class FuncletPadInst : public Instruction { -private: - FuncletPadInst(const FuncletPadInst &CPI); - - explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad, - ArrayRef Args, unsigned Values, - const Twine &NameStr, Instruction *InsertBefore); - explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad, - ArrayRef Args, unsigned Values, - const Twine &NameStr, BasicBlock *InsertAtEnd); - - void init(Value *ParentPad, ArrayRef Args, const Twine &NameStr); - -protected: - // Note: Instruction needs to be a friend here to call cloneImpl. - friend class Instruction; - friend class CatchPadInst; - friend class CleanupPadInst; - - FuncletPadInst *cloneImpl() const; - -public: - /// Provide fast operand accessors - DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - - /// getNumArgOperands - Return the number of funcletpad arguments. - /// - unsigned getNumArgOperands() const { return getNumOperands() - 1; } - - /// Convenience accessors - - /// Return the outer EH-pad this funclet is nested within. - /// - /// Note: This returns the associated CatchSwitchInst if this FuncletPadInst - /// is a CatchPadInst. - Value *getParentPad() const { return Op<-1>(); } - void setParentPad(Value *ParentPad) { - assert(ParentPad); - Op<-1>() = ParentPad; - } - - /// getArgOperand/setArgOperand - Return/set the i-th funcletpad argument. - /// - Value *getArgOperand(unsigned i) const { return getOperand(i); } - void setArgOperand(unsigned i, Value *v) { setOperand(i, v); } - - /// arg_operands - iteration adapter for range-for loops. - op_range arg_operands() { return op_range(op_begin(), op_end() - 1); } - - /// arg_operands - iteration adapter for range-for loops. - const_op_range arg_operands() const { - return const_op_range(op_begin(), op_end() - 1); - } - - // Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const Instruction *I) { return I->isFuncletPad(); } - static bool classof(const Value *V) { - return isa(V) && classof(cast(V)); - } -}; - -template <> -struct OperandTraits - : public VariadicOperandTraits {}; - -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(FuncletPadInst, Value) - /// A lightweight accessor for an operand bundle meant to be passed /// around by value. struct OperandBundleUse { @@ -1058,54 +989,468 @@ using OperandBundleDef = OperandBundleDefT; using ConstOperandBundleDef = OperandBundleDefT; -/// A mixin to add operand bundle functionality to llvm instruction -/// classes. -/// -/// OperandBundleUser uses the descriptor area co-allocated with the host User -/// to store some meta information about which operands are "normal" operands, -/// and which ones belong to some operand bundle. -/// -/// The layout of an operand bundle user is -/// -/// +-----------uint32_t End-------------------------------------+ -/// | | -/// | +--------uint32_t Begin--------------------+ | -/// | | | | -/// ^ ^ v v -/// |------|------|----|----|----|----|----|---------|----|---------|----|----- -/// | BOI0 | BOI1 | .. | DU | U0 | U1 | .. | BOI0_U0 | .. | BOI1_U0 | .. | Un -/// |------|------|----|----|----|----|----|---------|----|---------|----|----- -/// v v ^ ^ -/// | | | | -/// | +--------uint32_t Begin------------+ | -/// | | -/// +-----------uint32_t End-----------------------------+ -/// -/// -/// BOI0, BOI1 ... are descriptions of operand bundles in this User's use list. -/// These descriptions are installed and managed by this class, and they're all -/// instances of OperandBundleUser::BundleOpInfo. -/// -/// DU is an additional descriptor installed by User's 'operator new' to keep -/// track of the 'BOI0 ... BOIN' co-allocation. OperandBundleUser does not -/// access or modify DU in any way, it's an implementation detail private to -/// User. -/// -/// The regular Use& vector for the User starts at U0. The operand bundle uses -/// are part of the Use& vector, just like normal uses. In the diagram above, -/// the operand bundle uses start at BOI0_U0. Each instance of BundleOpInfo has -/// information about a contiguous set of uses constituting an operand bundle, -/// and the total set of operand bundle uses themselves form a contiguous set of -/// uses (i.e. there are no gaps between uses corresponding to individual -/// operand bundles). +//===----------------------------------------------------------------------===// +// CallBase Class +//===----------------------------------------------------------------------===// + +/// Base class for all callable instructions (InvokeInst and CallInst) +/// Holds everything related to calling a function. /// -/// This class does not know the location of the set of operand bundle uses -/// within the use list -- that is decided by the User using this class via the -/// BeginIdx argument in populateBundleOperandInfos. +/// All call-like instructions are required to use a common operand layout: +/// - Zero or more arguments to the call, +/// - Zero or more operand bundles with zero or more operand inputs each +/// bundle, +/// - Zero or more subclass controlled operands +/// - The called function. /// -/// Currently operand bundle users with hung-off operands are not supported. -template class OperandBundleUser { +/// This allows this base class to easily access the called function and the +/// start of the arguments without knowing how many other operands a particular +/// subclass requires. Note that accessing the end of the argument list isn't +/// as cheap as most other operations on the base class. +class CallBase : public Instruction { +protected: + /// The last operand is the called operand. + static constexpr int CalledOperandOpEndIdx = -1; + + AttributeList Attrs; ///< parameter attributes for callable + FunctionType *FTy; + + template + CallBase(AttributeList const &A, FunctionType *FT, ArgsTy &&... Args) + : Instruction(std::forward(Args)...), Attrs(A), FTy(FT) {} + + using Instruction::Instruction; + + bool hasDescriptor() const { return Value::HasDescriptor; } + + unsigned getNumSubclassExtraOperands() const { + switch (getOpcode()) { + case Instruction::Call: + return 0; + case Instruction::Invoke: + return 2; + } + llvm_unreachable("Invalid opcode!"); + } + public: + using Instruction::getContext; + + static bool classof(const Instruction *I) { + return I->getOpcode() == Instruction::Call || + I->getOpcode() == Instruction::Invoke; + } + + FunctionType *getFunctionType() const { return FTy; } + + void mutateFunctionType(FunctionType *FTy) { + Value::mutateType(FTy->getReturnType()); + this->FTy = FTy; + } + + /// Return the iterator pointing to the beginning of the argument list. + User::op_iterator arg_begin() { return op_begin(); } + User::const_op_iterator arg_begin() const { + return const_cast(this)->arg_begin(); + } + + /// Return the iterator pointing to the end of the argument list. + User::op_iterator arg_end() { + // Walk from the end of the operands over the called operand, the subclass + // operands, and any operands for bundles to find the end of the argument + // operands. + return op_end() - getNumTotalBundleOperands() - + getNumSubclassExtraOperands() - 1; + } + User::const_op_iterator arg_end() const { + return const_cast(this)->arg_end(); + } + + /// Iteration adapter for range-for loops. + iterator_range arg_operands() { + return make_range(arg_begin(), arg_end()); + } + iterator_range arg_operands() const { + return make_range(arg_begin(), arg_end()); + } + + unsigned getNumArgOperands() const { return arg_end() - arg_begin(); } + + Value *getArgOperand(unsigned i) const { + assert(i < getNumArgOperands() && "Out of bounds!"); + return getOperand(i); + } + + void setArgOperand(unsigned i, Value *v) { + assert(i < getNumArgOperands() && "Out of bounds!"); + setOperand(i, v); + } + + /// Wrappers for getting the \c Use of a call argument. + const Use &getArgOperandUse(unsigned i) const { + assert(i < getNumArgOperands() && "Out of bounds!"); + return User::getOperandUse(i); + } + Use &getArgOperandUse(unsigned i) { + assert(i < getNumArgOperands() && "Out of bounds!"); + return User::getOperandUse(i); + } + + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + Value *getCalledOperand() const { return Op(); } + + // DEPRECATED: This routine will be removed in favor of `getCalledOperand` in + // the near future. + Value *getCalledValue() const { return getCalledOperand(); } + + const Use &getCalledOperandUse() const { return Op(); } + Use &getCalledOperandUse() { return Op(); } + + /// Returns the function called, or null if this is an + /// indirect function invocation. + Function *getCalledFunction() const { + return dyn_cast_or_null(getCalledOperand()); + } + + void setCalledOperand(Value *V) { Op() = V; } + + /// Sets the function called, including updating the function type. + void setCalledFunction(Value *Fn) { + setCalledFunction( + cast(cast(Fn->getType())->getElementType()), + Fn); + } + + /// Sets the function called, including updating to the specified function + /// type. + void setCalledFunction(FunctionType *FTy, Value *Fn) { + this->FTy = FTy; + assert(FTy == cast( + cast(Fn->getType())->getElementType())); + setCalledOperand(Fn); + } + + CallingConv::ID getCallingConv() const { + return static_cast(getSubclassDataFromInstruction() >> 2); + } + + void setCallingConv(CallingConv::ID CC) { + auto ID = static_cast(CC); + assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention"); + setInstructionSubclassData((getSubclassDataFromInstruction() & 3) | + (ID << 2)); + } + + /// \name Attribute API + /// + /// These methods access and modify attributes on this call (including + /// looking through to the attributes on the called function when necessary). + ///@{ + + /// Return the parameter attributes for this call. + /// + AttributeList getAttributes() const { return Attrs; } + + /// Set the parameter attributes for this call. + /// + void setAttributes(AttributeList A) { Attrs = A; } + + /// Determine whether this call has the given attribute. + bool hasFnAttr(Attribute::AttrKind Kind) const { + assert(Kind != Attribute::NoBuiltin && + "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"); + return hasFnAttrImpl(Kind); + } + + /// Determine whether this call has the given attribute. + bool hasFnAttr(StringRef Kind) const { return hasFnAttrImpl(Kind); } + + /// adds the attribute to the list of attributes. + void addAttribute(unsigned i, Attribute::AttrKind Kind) { + AttributeList PAL = getAttributes(); + PAL = PAL.addAttribute(getContext(), i, Kind); + setAttributes(PAL); + } + + /// adds the attribute to the list of attributes. + void addAttribute(unsigned i, Attribute Attr) { + AttributeList PAL = getAttributes(); + PAL = PAL.addAttribute(getContext(), i, Attr); + setAttributes(PAL); + } + + /// Adds the attribute to the indicated argument + void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) { + assert(ArgNo < getNumArgOperands() && "Out of bounds"); + AttributeList PAL = getAttributes(); + PAL = PAL.addParamAttribute(getContext(), ArgNo, Kind); + setAttributes(PAL); + } + + /// Adds the attribute to the indicated argument + void addParamAttr(unsigned ArgNo, Attribute Attr) { + assert(ArgNo < getNumArgOperands() && "Out of bounds"); + AttributeList PAL = getAttributes(); + PAL = PAL.addParamAttribute(getContext(), ArgNo, Attr); + setAttributes(PAL); + } + + /// removes the attribute from the list of attributes. + void removeAttribute(unsigned i, Attribute::AttrKind Kind) { + AttributeList PAL = getAttributes(); + PAL = PAL.removeAttribute(getContext(), i, Kind); + setAttributes(PAL); + } + + /// removes the attribute from the list of attributes. + void removeAttribute(unsigned i, StringRef Kind) { + AttributeList PAL = getAttributes(); + PAL = PAL.removeAttribute(getContext(), i, Kind); + setAttributes(PAL); + } + + /// Removes the attribute from the given argument + void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) { + assert(ArgNo < getNumArgOperands() && "Out of bounds"); + AttributeList PAL = getAttributes(); + PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind); + setAttributes(PAL); + } + + /// Removes the attribute from the given argument + void removeParamAttr(unsigned ArgNo, StringRef Kind) { + assert(ArgNo < getNumArgOperands() && "Out of bounds"); + AttributeList PAL = getAttributes(); + PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind); + setAttributes(PAL); + } + + /// adds the dereferenceable attribute to the list of attributes. + void addDereferenceableAttr(unsigned i, uint64_t Bytes) { + AttributeList PAL = getAttributes(); + PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes); + setAttributes(PAL); + } + + /// adds the dereferenceable_or_null attribute to the list of + /// attributes. + void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) { + AttributeList PAL = getAttributes(); + PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes); + setAttributes(PAL); + } + + /// Determine whether the return value has the given attribute. + bool hasRetAttr(Attribute::AttrKind Kind) const; + + /// Determine whether the argument or parameter has the given attribute. + bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const; + + /// Get the attribute of a given kind at a position. + Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const { + return getAttributes().getAttribute(i, Kind); + } + + /// Get the attribute of a given kind at a position. + Attribute getAttribute(unsigned i, StringRef Kind) const { + return getAttributes().getAttribute(i, Kind); + } + + /// Get the attribute of a given kind from a given arg + Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const { + assert(ArgNo < getNumArgOperands() && "Out of bounds"); + return getAttributes().getParamAttr(ArgNo, Kind); + } + + /// Get the attribute of a given kind from a given arg + Attribute getParamAttr(unsigned ArgNo, StringRef Kind) const { + assert(ArgNo < getNumArgOperands() && "Out of bounds"); + return getAttributes().getParamAttr(ArgNo, Kind); + } + + /// Return true if the data operand at index \p i has the attribute \p + /// A. + /// + /// Data operands include call arguments and values used in operand bundles, + /// but does not include the callee operand. This routine dispatches to the + /// underlying AttributeList or the OperandBundleUser as appropriate. + /// + /// The index \p i is interpreted as + /// + /// \p i == Attribute::ReturnIndex -> the return value + /// \p i in [1, arg_size + 1) -> argument number (\p i - 1) + /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index + /// (\p i - 1) in the operand list. + bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const { + // Note that we have to add one because `i` isn't zero-indexed. + assert(i < (getNumArgOperands() + getNumTotalBundleOperands() + 1) && + "Data operand index out of bounds!"); + + // The attribute A can either be directly specified, if the operand in + // question is a call argument; or be indirectly implied by the kind of its + // containing operand bundle, if the operand is a bundle operand. + + if (i == AttributeList::ReturnIndex) + return hasRetAttr(Kind); + + // FIXME: Avoid these i - 1 calculations and update the API to use + // zero-based indices. + if (i < (getNumArgOperands() + 1)) + return paramHasAttr(i - 1, Kind); + + assert(hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && + "Must be either a call argument or an operand bundle!"); + return bundleOperandHasAttr(i - 1, Kind); + } + + /// Extract the alignment of the return value. + unsigned getRetAlignment() const { return Attrs.getRetAlignment(); } + + /// Extract the alignment for a call or parameter (0=unknown). + unsigned getParamAlignment(unsigned ArgNo) const { + return Attrs.getParamAlignment(ArgNo); + } + + /// Extract the number of dereferenceable bytes for a call or + /// parameter (0=unknown). + uint64_t getDereferenceableBytes(unsigned i) const { + return Attrs.getDereferenceableBytes(i); + } + + /// Extract the number of dereferenceable_or_null bytes for a call or + /// parameter (0=unknown). + uint64_t getDereferenceableOrNullBytes(unsigned i) const { + return Attrs.getDereferenceableOrNullBytes(i); + } + + /// Determine if the return value is marked with NoAlias attribute. + bool returnDoesNotAlias() const { + return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias); + } + + /// If one of the arguments has the 'returned' attribute, returns its + /// operand value. Otherwise, return nullptr. + Value *getReturnedArgOperand() const; + + /// Return true if the call should not be treated as a call to a + /// builtin. + bool isNoBuiltin() const { + return hasFnAttrImpl(Attribute::NoBuiltin) && + !hasFnAttrImpl(Attribute::Builtin); + } + + /// Determine if the call requires strict floating point semantics. + bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); } + + /// Return true if the call should not be inlined. + bool isNoInline() const { return hasFnAttr(Attribute::NoInline); } + void setIsNoInline() { + addAttribute(AttributeList::FunctionIndex, Attribute::NoInline); + } + /// Determine if the call does not access memory. + bool doesNotAccessMemory() const { return hasFnAttr(Attribute::ReadNone); } + void setDoesNotAccessMemory() { + addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone); + } + + /// Determine if the call does not access or only reads memory. + bool onlyReadsMemory() const { + return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly); + } + void setOnlyReadsMemory() { + addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly); + } + + /// Determine if the call does not access or only writes memory. + bool doesNotReadMemory() const { + return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly); + } + void setDoesNotReadMemory() { + addAttribute(AttributeList::FunctionIndex, Attribute::WriteOnly); + } + + /// Determine if the call can access memmory only using pointers based + /// on its arguments. + bool onlyAccessesArgMemory() const { + return hasFnAttr(Attribute::ArgMemOnly); + } + void setOnlyAccessesArgMemory() { + addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly); + } + + /// Determine if the function may only access memory that is + /// inaccessible from the IR. + bool onlyAccessesInaccessibleMemory() const { + return hasFnAttr(Attribute::InaccessibleMemOnly); + } + void setOnlyAccessesInaccessibleMemory() { + addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly); + } + + /// Determine if the function may only access memory that is + /// either inaccessible from the IR or pointed to by its arguments. + bool onlyAccessesInaccessibleMemOrArgMem() const { + return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly); + } + void setOnlyAccessesInaccessibleMemOrArgMem() { + addAttribute(AttributeList::FunctionIndex, + Attribute::InaccessibleMemOrArgMemOnly); + } + /// Determine if the call cannot return. + bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); } + void setDoesNotReturn() { + addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn); + } + + /// Determine if the call should not perform indirect branch tracking. + bool doesNoCfCheck() const { return hasFnAttr(Attribute::NoCfCheck); } + + /// Determine if the call cannot unwind. + bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); } + void setDoesNotThrow() { + addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind); + } + + /// Determine if the invoke cannot be duplicated. + bool cannotDuplicate() const { return hasFnAttr(Attribute::NoDuplicate); } + void setCannotDuplicate() { + addAttribute(AttributeList::FunctionIndex, Attribute::NoDuplicate); + } + + /// Determine if the invoke is convergent + bool isConvergent() const { return hasFnAttr(Attribute::Convergent); } + void setConvergent() { + addAttribute(AttributeList::FunctionIndex, Attribute::Convergent); + } + void setNotConvergent() { + removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent); + } + + /// Determine if the call returns a structure through first + /// pointer argument. + bool hasStructRetAttr() const { + if (getNumArgOperands() == 0) + return false; + + // Be friendly and also check the callee. + return paramHasAttr(0, Attribute::StructRet); + } + + /// Determine if any call argument is an aggregate passed by value. + bool hasByValArgument() const { + return Attrs.hasAttrSomewhere(Attribute::ByVal); + } + + ///@{ + // End of attribute API. + + /// \name Operand Bundle API + /// + /// This group of methods provides the API to access and manipulate operand + /// bundles on this call. + /// @{ + /// Return the number of operand bundles associated with this User. unsigned getNumOperandBundles() const { return std::distance(bundle_op_info_begin(), bundle_op_info_end()); @@ -1261,8 +1606,7 @@ /// Return true if \p Other has the same sequence of operand bundle /// tags with the same number of operands on each one of them as this /// OperandBundleUser. - bool hasIdenticalOperandBundleSchema( - const OperandBundleUser &Other) const { + bool hasIdenticalOperandBundleSchema(const CallBase &Other) const { if (getNumOperandBundles() != Other.getNumOperandBundles()) return false; @@ -1281,7 +1625,6 @@ return false; } -protected: /// Is the function attribute S disallowed by some operand bundle on /// this operand bundle user? bool isFnAttrDisallowedByOpBundle(StringRef S) const { @@ -1340,8 +1683,8 @@ /// OperandBundleUse. OperandBundleUse operandBundleFromBundleOpInfo(const BundleOpInfo &BOI) const { - auto op_begin = static_cast(this)->op_begin(); - ArrayRef Inputs(op_begin + BOI.Begin, op_begin + BOI.End); + auto begin = op_begin(); + ArrayRef Inputs(begin + BOI.Begin, begin + BOI.End); return OperandBundleUse(BOI.Tag, Inputs); } @@ -1350,37 +1693,79 @@ /// Return the start of the list of BundleOpInfo instances associated /// with this OperandBundleUser. + /// + /// OperandBundleUser uses the descriptor area co-allocated with the host User + /// to store some meta information about which operands are "normal" operands, + /// and which ones belong to some operand bundle. + /// + /// The layout of an operand bundle user is + /// + /// +-----------uint32_t End-------------------------------------+ + /// | | + /// | +--------uint32_t Begin--------------------+ | + /// | | | | + /// ^ ^ v v + /// |------|------|----|----|----|----|----|---------|----|---------|----|----- + /// | BOI0 | BOI1 | .. | DU | U0 | U1 | .. | BOI0_U0 | .. | BOI1_U0 | .. | Un + /// |------|------|----|----|----|----|----|---------|----|---------|----|----- + /// v v ^ ^ + /// | | | | + /// | +--------uint32_t Begin------------+ | + /// | | + /// +-----------uint32_t End-----------------------------+ + /// + /// + /// BOI0, BOI1 ... are descriptions of operand bundles in this User's use + /// list. These descriptions are installed and managed by this class, and + /// they're all instances of OperandBundleUser::BundleOpInfo. + /// + /// DU is an additional descriptor installed by User's 'operator new' to keep + /// track of the 'BOI0 ... BOIN' co-allocation. OperandBundleUser does not + /// access or modify DU in any way, it's an implementation detail private to + /// User. + /// + /// The regular Use& vector for the User starts at U0. The operand bundle + /// uses are part of the Use& vector, just like normal uses. In the diagram + /// above, the operand bundle uses start at BOI0_U0. Each instance of + /// BundleOpInfo has information about a contiguous set of uses constituting + /// an operand bundle, and the total set of operand bundle uses themselves + /// form a contiguous set of uses (i.e. there are no gaps between uses + /// corresponding to individual operand bundles). + /// + /// This class does not know the location of the set of operand bundle uses + /// within the use list -- that is decided by the User using this class via + /// the BeginIdx argument in populateBundleOperandInfos. + /// + /// Currently operand bundle users with hung-off operands are not supported. bundle_op_iterator bundle_op_info_begin() { - if (!static_cast(this)->hasDescriptor()) + if (!hasDescriptor()) return nullptr; - uint8_t *BytesBegin = static_cast(this)->getDescriptor().begin(); + uint8_t *BytesBegin = getDescriptor().begin(); return reinterpret_cast(BytesBegin); } /// Return the start of the list of BundleOpInfo instances associated /// with this OperandBundleUser. const_bundle_op_iterator bundle_op_info_begin() const { - auto *NonConstThis = - const_cast *>(this); + auto *NonConstThis = const_cast(this); return NonConstThis->bundle_op_info_begin(); } /// Return the end of the list of BundleOpInfo instances associated /// with this OperandBundleUser. bundle_op_iterator bundle_op_info_end() { - if (!static_cast(this)->hasDescriptor()) + if (!hasDescriptor()) return nullptr; - uint8_t *BytesEnd = static_cast(this)->getDescriptor().end(); + uint8_t *BytesEnd = getDescriptor().end(); return reinterpret_cast(BytesEnd); } /// Return the end of the list of BundleOpInfo instances associated /// with this OperandBundleUser. const_bundle_op_iterator bundle_op_info_end() const { - auto *NonConstThis = - const_cast *>(this); + auto *NonConstThis = const_cast(this); return NonConstThis->bundle_op_info_end(); } @@ -1400,30 +1785,8 @@ /// /// Each \p OperandBundleDef instance is tracked by a OperandBundleInfo /// instance allocated in this User's descriptor. - OpIteratorTy populateBundleOperandInfos(ArrayRef Bundles, - const unsigned BeginIndex) { - auto It = static_cast(this)->op_begin() + BeginIndex; - for (auto &B : Bundles) - It = std::copy(B.input_begin(), B.input_end(), It); - - auto *ContextImpl = static_cast(this)->getContext().pImpl; - auto BI = Bundles.begin(); - unsigned CurrentIndex = BeginIndex; - - for (auto &BOI : bundle_op_infos()) { - assert(BI != Bundles.end() && "Incorrect allocation?"); - - BOI.Tag = ContextImpl->getOrInsertBundleTag(BI->getTag()); - BOI.Begin = CurrentIndex; - BOI.End = CurrentIndex + BI->input_size(); - CurrentIndex = BOI.End; - BI++; - } - - assert(BI == Bundles.end() && "Incorrect allocation?"); - - return It; - } + op_iterator populateBundleOperandInfos(ArrayRef Bundles, + const unsigned BeginIndex); /// Return the BundleOpInfo for the operand at index OpIdx. /// @@ -1437,6 +1800,7 @@ llvm_unreachable("Did not find operand bundle for operand!"); } +protected: /// Return the total number of values used in \p Bundles. static unsigned CountBundleInputs(ArrayRef Bundles) { unsigned Total = 0; @@ -1444,8 +1808,102 @@ Total += B.input_size(); return Total; } + + /// @} + // End of operand bundle API. + +private: + bool hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const; + bool hasFnAttrOnCalledFunction(StringRef Kind) const; + + template bool hasFnAttrImpl(AttrKind Kind) const { + if (Attrs.hasAttribute(AttributeList::FunctionIndex, Kind)) + return true; + + // Operand bundles override attributes on the called function, but don't + // override attributes directly present on the call instruction. + if (isFnAttrDisallowedByOpBundle(Kind)) + return false; + + return hasFnAttrOnCalledFunction(Kind); + } +}; + +template <> +struct OperandTraits : public VariadicOperandTraits {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CallBase, Value) + +//===----------------------------------------------------------------------===// +// FuncletPadInst Class +//===----------------------------------------------------------------------===// +class FuncletPadInst : public Instruction { +private: + FuncletPadInst(const FuncletPadInst &CPI); + + explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad, + ArrayRef Args, unsigned Values, + const Twine &NameStr, Instruction *InsertBefore); + explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad, + ArrayRef Args, unsigned Values, + const Twine &NameStr, BasicBlock *InsertAtEnd); + + void init(Value *ParentPad, ArrayRef Args, const Twine &NameStr); + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + friend class CatchPadInst; + friend class CleanupPadInst; + + FuncletPadInst *cloneImpl() const; + +public: + /// Provide fast operand accessors + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + /// getNumArgOperands - Return the number of funcletpad arguments. + /// + unsigned getNumArgOperands() const { return getNumOperands() - 1; } + + /// Convenience accessors + + /// Return the outer EH-pad this funclet is nested within. + /// + /// Note: This returns the associated CatchSwitchInst if this FuncletPadInst + /// is a CatchPadInst. + Value *getParentPad() const { return Op<-1>(); } + void setParentPad(Value *ParentPad) { + assert(ParentPad); + Op<-1>() = ParentPad; + } + + /// getArgOperand/setArgOperand - Return/set the i-th funcletpad argument. + /// + Value *getArgOperand(unsigned i) const { return getOperand(i); } + void setArgOperand(unsigned i, Value *v) { setOperand(i, v); } + + /// arg_operands - iteration adapter for range-for loops. + op_range arg_operands() { return op_range(op_begin(), op_end() - 1); } + + /// arg_operands - iteration adapter for range-for loops. + const_op_range arg_operands() const { + return const_op_range(op_begin(), op_end() - 1); + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Instruction *I) { return I->isFuncletPad(); } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } }; +template <> +struct OperandTraits + : public VariadicOperandTraits {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(FuncletPadInst, Value) + } // end namespace llvm #endif // LLVM_IR_INSTRTYPES_H Index: llvm/include/llvm/IR/Instructions.h =================================================================== --- llvm/include/llvm/IR/Instructions.h +++ llvm/include/llvm/IR/Instructions.h @@ -1418,535 +1418,13 @@ } }; -class CallInst; -class InvokeInst; - -template struct CallBaseParent { using type = Instruction; }; - -//===----------------------------------------------------------------------===// -/// Base class for all callable instructions (InvokeInst and CallInst) -/// Holds everything related to calling a function, abstracting from the base -/// type @p BaseInstTy and the concrete instruction @p InstTy -/// -template -class CallBase : public CallBaseParent::type, - public OperandBundleUser { -protected: - AttributeList Attrs; ///< parameter attributes for callable - FunctionType *FTy; - using BaseInstTy = typename CallBaseParent::type; - - template - CallBase(AttributeList const &A, FunctionType *FT, ArgsTy &&... Args) - : BaseInstTy(std::forward(Args)...), Attrs(A), FTy(FT) {} - bool hasDescriptor() const { return Value::HasDescriptor; } - - using BaseInstTy::BaseInstTy; - - using OperandBundleUser::isFnAttrDisallowedByOpBundle; - using OperandBundleUser::getNumTotalBundleOperands; - using OperandBundleUser::bundleOperandHasAttr; - using Instruction::getSubclassDataFromInstruction; - using Instruction::setInstructionSubclassData; - -public: - using Instruction::getContext; - using OperandBundleUser::hasOperandBundles; - using OperandBundleUser::getBundleOperandsStartIndex; - - static bool classof(const Instruction *I) { - llvm_unreachable( - "CallBase is not meant to be used as part of the classof hierarchy"); - } - -public: - /// Return the parameter attributes for this call. - /// - AttributeList getAttributes() const { return Attrs; } - - /// Set the parameter attributes for this call. - /// - void setAttributes(AttributeList A) { Attrs = A; } - - FunctionType *getFunctionType() const { return FTy; } - - void mutateFunctionType(FunctionType *FTy) { - Value::mutateType(FTy->getReturnType()); - this->FTy = FTy; - } - - /// Return the number of call arguments. - /// - unsigned getNumArgOperands() const { - return getNumOperands() - getNumTotalBundleOperands() - InstTy::ArgOffset; - } - - /// getArgOperand/setArgOperand - Return/set the i-th call argument. - /// - Value *getArgOperand(unsigned i) const { - assert(i < getNumArgOperands() && "Out of bounds!"); - return getOperand(i); - } - void setArgOperand(unsigned i, Value *v) { - assert(i < getNumArgOperands() && "Out of bounds!"); - setOperand(i, v); - } - - /// Return the iterator pointing to the beginning of the argument list. - User::op_iterator arg_begin() { return op_begin(); } - - /// Return the iterator pointing to the end of the argument list. - User::op_iterator arg_end() { - // [ call args ], [ operand bundles ], callee - return op_end() - getNumTotalBundleOperands() - InstTy::ArgOffset; - } - - /// Iteration adapter for range-for loops. - iterator_range arg_operands() { - return make_range(arg_begin(), arg_end()); - } - - /// Return the iterator pointing to the beginning of the argument list. - User::const_op_iterator arg_begin() const { return op_begin(); } - - /// Return the iterator pointing to the end of the argument list. - User::const_op_iterator arg_end() const { - // [ call args ], [ operand bundles ], callee - return op_end() - getNumTotalBundleOperands() - InstTy::ArgOffset; - } - - /// Iteration adapter for range-for loops. - iterator_range arg_operands() const { - return make_range(arg_begin(), arg_end()); - } - - /// Wrappers for getting the \c Use of a call argument. - const Use &getArgOperandUse(unsigned i) const { - assert(i < getNumArgOperands() && "Out of bounds!"); - return User::getOperandUse(i); - } - Use &getArgOperandUse(unsigned i) { - assert(i < getNumArgOperands() && "Out of bounds!"); - return User::getOperandUse(i); - } - - /// If one of the arguments has the 'returned' attribute, return its - /// operand value. Otherwise, return nullptr. - Value *getReturnedArgOperand() const { - unsigned Index; - - if (Attrs.hasAttrSomewhere(Attribute::Returned, &Index) && Index) - return getArgOperand(Index - AttributeList::FirstArgIndex); - if (const Function *F = getCalledFunction()) - if (F->getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && - Index) - return getArgOperand(Index - AttributeList::FirstArgIndex); - - return nullptr; - } - - User::op_iterator op_begin() { - return OperandTraits::op_begin(this); - } - - User::const_op_iterator op_begin() const { - return OperandTraits::op_begin(const_cast(this)); - } - - User::op_iterator op_end() { return OperandTraits::op_end(this); } - - User::const_op_iterator op_end() const { - return OperandTraits::op_end(const_cast(this)); - } - - Value *getOperand(unsigned i_nocapture) const { - assert(i_nocapture < OperandTraits::operands(this) && - "getOperand() out of range!"); - return cast_or_null(OperandTraits::op_begin( - const_cast(this))[i_nocapture] - .get()); - } - - void setOperand(unsigned i_nocapture, Value *Val_nocapture) { - assert(i_nocapture < OperandTraits::operands(this) && - "setOperand() out of range!"); - OperandTraits::op_begin(this)[i_nocapture] = Val_nocapture; - } - - unsigned getNumOperands() const { - return OperandTraits::operands(this); - } - template Use &Op() { - return User::OpFrom(this); - } - template const Use &Op() const { - return User::OpFrom(this); - } - - /// Return the function called, or null if this is an - /// indirect function invocation. - /// - Function *getCalledFunction() const { - return dyn_cast_or_null(Op<-InstTy::ArgOffset>()); - } - - /// Determine whether this call has the given attribute. - bool hasFnAttr(Attribute::AttrKind Kind) const { - assert(Kind != Attribute::NoBuiltin && - "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin"); - return hasFnAttrImpl(Kind); - } - - /// Determine whether this call has the given attribute. - bool hasFnAttr(StringRef Kind) const { return hasFnAttrImpl(Kind); } - - /// getCallingConv/setCallingConv - Get or set the calling convention of this - /// function call. - CallingConv::ID getCallingConv() const { - return static_cast(getSubclassDataFromInstruction() >> 2); - } - void setCallingConv(CallingConv::ID CC) { - auto ID = static_cast(CC); - assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention"); - setInstructionSubclassData((getSubclassDataFromInstruction() & 3) | - (ID << 2)); - } - - - /// adds the attribute to the list of attributes. - void addAttribute(unsigned i, Attribute::AttrKind Kind) { - AttributeList PAL = getAttributes(); - PAL = PAL.addAttribute(getContext(), i, Kind); - setAttributes(PAL); - } - - /// adds the attribute to the list of attributes. - void addAttribute(unsigned i, Attribute Attr) { - AttributeList PAL = getAttributes(); - PAL = PAL.addAttribute(getContext(), i, Attr); - setAttributes(PAL); - } - - /// Adds the attribute to the indicated argument - void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) { - assert(ArgNo < getNumArgOperands() && "Out of bounds"); - AttributeList PAL = getAttributes(); - PAL = PAL.addParamAttribute(getContext(), ArgNo, Kind); - setAttributes(PAL); - } - - /// Adds the attribute to the indicated argument - void addParamAttr(unsigned ArgNo, Attribute Attr) { - assert(ArgNo < getNumArgOperands() && "Out of bounds"); - AttributeList PAL = getAttributes(); - PAL = PAL.addParamAttribute(getContext(), ArgNo, Attr); - setAttributes(PAL); - } - - /// removes the attribute from the list of attributes. - void removeAttribute(unsigned i, Attribute::AttrKind Kind) { - AttributeList PAL = getAttributes(); - PAL = PAL.removeAttribute(getContext(), i, Kind); - setAttributes(PAL); - } - - /// removes the attribute from the list of attributes. - void removeAttribute(unsigned i, StringRef Kind) { - AttributeList PAL = getAttributes(); - PAL = PAL.removeAttribute(getContext(), i, Kind); - setAttributes(PAL); - } - - /// Removes the attribute from the given argument - void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) { - assert(ArgNo < getNumArgOperands() && "Out of bounds"); - AttributeList PAL = getAttributes(); - PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind); - setAttributes(PAL); - } - - /// Removes the attribute from the given argument - void removeParamAttr(unsigned ArgNo, StringRef Kind) { - assert(ArgNo < getNumArgOperands() && "Out of bounds"); - AttributeList PAL = getAttributes(); - PAL = PAL.removeParamAttribute(getContext(), ArgNo, Kind); - setAttributes(PAL); - } - - /// adds the dereferenceable attribute to the list of attributes. - void addDereferenceableAttr(unsigned i, uint64_t Bytes) { - AttributeList PAL = getAttributes(); - PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes); - setAttributes(PAL); - } - - /// adds the dereferenceable_or_null attribute to the list of - /// attributes. - void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) { - AttributeList PAL = getAttributes(); - PAL = PAL.addDereferenceableOrNullAttr(getContext(), i, Bytes); - setAttributes(PAL); - } - - /// Determine whether the return value has the given attribute. - bool hasRetAttr(Attribute::AttrKind Kind) const { - if (Attrs.hasAttribute(AttributeList::ReturnIndex, Kind)) - return true; - - // Look at the callee, if available. - if (const Function *F = getCalledFunction()) - return F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Kind); - return false; - } - - /// Determine whether the argument or parameter has the given attribute. - bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const { - assert(ArgNo < getNumArgOperands() && "Param index out of bounds!"); - - if (Attrs.hasParamAttribute(ArgNo, Kind)) - return true; - if (const Function *F = getCalledFunction()) - return F->getAttributes().hasParamAttribute(ArgNo, Kind); - return false; - } - - /// Get the attribute of a given kind at a position. - Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const { - return getAttributes().getAttribute(i, Kind); - } - - /// Get the attribute of a given kind at a position. - Attribute getAttribute(unsigned i, StringRef Kind) const { - return getAttributes().getAttribute(i, Kind); - } - - /// Get the attribute of a given kind from a given arg - Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const { - assert(ArgNo < getNumArgOperands() && "Out of bounds"); - return getAttributes().getParamAttr(ArgNo, Kind); - } - - /// Get the attribute of a given kind from a given arg - Attribute getParamAttr(unsigned ArgNo, StringRef Kind) const { - assert(ArgNo < getNumArgOperands() && "Out of bounds"); - return getAttributes().getParamAttr(ArgNo, Kind); - } - /// Return true if the data operand at index \p i has the attribute \p - /// A. - /// - /// Data operands include call arguments and values used in operand bundles, - /// but does not include the callee operand. This routine dispatches to the - /// underlying AttributeList or the OperandBundleUser as appropriate. - /// - /// The index \p i is interpreted as - /// - /// \p i == Attribute::ReturnIndex -> the return value - /// \p i in [1, arg_size + 1) -> argument number (\p i - 1) - /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index - /// (\p i - 1) in the operand list. - bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const { - // There are getNumOperands() - (InstTy::ArgOffset - 1) data operands. - // The last operand is the callee. - assert(i < (getNumOperands() - InstTy::ArgOffset + 1) && - "Data operand index out of bounds!"); - - // The attribute A can either be directly specified, if the operand in - // question is a call argument; or be indirectly implied by the kind of its - // containing operand bundle, if the operand is a bundle operand. - - if (i == AttributeList::ReturnIndex) - return hasRetAttr(Kind); - - // FIXME: Avoid these i - 1 calculations and update the API to use - // zero-based indices. - if (i < (getNumArgOperands() + 1)) - return paramHasAttr(i - 1, Kind); - - assert(hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && - "Must be either a call argument or an operand bundle!"); - return bundleOperandHasAttr(i - 1, Kind); - } - - /// Extract the alignment of the return value. - unsigned getRetAlignment() const { return Attrs.getRetAlignment(); } - - /// Extract the alignment for a call or parameter (0=unknown). - unsigned getParamAlignment(unsigned ArgNo) const { - return Attrs.getParamAlignment(ArgNo); - } - - /// Extract the number of dereferenceable bytes for a call or - /// parameter (0=unknown). - uint64_t getDereferenceableBytes(unsigned i) const { - return Attrs.getDereferenceableBytes(i); - } - - /// Extract the number of dereferenceable_or_null bytes for a call or - /// parameter (0=unknown). - uint64_t getDereferenceableOrNullBytes(unsigned i) const { - return Attrs.getDereferenceableOrNullBytes(i); - } - - /// Determine if the return value is marked with NoAlias attribute. - bool returnDoesNotAlias() const { - return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias); - } - - /// Return true if the call should not be treated as a call to a - /// builtin. - bool isNoBuiltin() const { - return hasFnAttrImpl(Attribute::NoBuiltin) && - !hasFnAttrImpl(Attribute::Builtin); - } - - /// Determine if the call requires strict floating point semantics. - bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); } - - /// Return true if the call should not be inlined. - bool isNoInline() const { return hasFnAttr(Attribute::NoInline); } - void setIsNoInline() { - addAttribute(AttributeList::FunctionIndex, Attribute::NoInline); - } - /// Determine if the call does not access memory. - bool doesNotAccessMemory() const { - return hasFnAttr(Attribute::ReadNone); - } - void setDoesNotAccessMemory() { - addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone); - } - - /// Determine if the call does not access or only reads memory. - bool onlyReadsMemory() const { - return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly); - } - void setOnlyReadsMemory() { - addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly); - } - - /// Determine if the call does not access or only writes memory. - bool doesNotReadMemory() const { - return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly); - } - void setDoesNotReadMemory() { - addAttribute(AttributeList::FunctionIndex, Attribute::WriteOnly); - } - - /// Determine if the call can access memmory only using pointers based - /// on its arguments. - bool onlyAccessesArgMemory() const { - return hasFnAttr(Attribute::ArgMemOnly); - } - void setOnlyAccessesArgMemory() { - addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly); - } - - /// Determine if the function may only access memory that is - /// inaccessible from the IR. - bool onlyAccessesInaccessibleMemory() const { - return hasFnAttr(Attribute::InaccessibleMemOnly); - } - void setOnlyAccessesInaccessibleMemory() { - addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly); - } - - /// Determine if the function may only access memory that is - /// either inaccessible from the IR or pointed to by its arguments. - bool onlyAccessesInaccessibleMemOrArgMem() const { - return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly); - } - void setOnlyAccessesInaccessibleMemOrArgMem() { - addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOrArgMemOnly); - } - /// Determine if the call cannot return. - bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); } - void setDoesNotReturn() { - addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn); - } - - /// Determine if the call should not perform indirect branch tracking. - bool doesNoCfCheck() const { return hasFnAttr(Attribute::NoCfCheck); } - - /// Determine if the call cannot unwind. - bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); } - void setDoesNotThrow() { - addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind); - } - - /// Determine if the invoke cannot be duplicated. - bool cannotDuplicate() const {return hasFnAttr(Attribute::NoDuplicate); } - void setCannotDuplicate() { - addAttribute(AttributeList::FunctionIndex, Attribute::NoDuplicate); - } - - /// Determine if the invoke is convergent - bool isConvergent() const { return hasFnAttr(Attribute::Convergent); } - void setConvergent() { - addAttribute(AttributeList::FunctionIndex, Attribute::Convergent); - } - void setNotConvergent() { - removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent); - } - - /// Determine if the call returns a structure through first - /// pointer argument. - bool hasStructRetAttr() const { - if (getNumArgOperands() == 0) - return false; - - // Be friendly and also check the callee. - return paramHasAttr(0, Attribute::StructRet); - } - - /// Determine if any call argument is an aggregate passed by value. - bool hasByValArgument() const { - return Attrs.hasAttrSomewhere(Attribute::ByVal); - } - /// Get a pointer to the function that is invoked by this - /// instruction. - const Value *getCalledValue() const { return Op<-InstTy::ArgOffset>(); } - Value *getCalledValue() { return Op<-InstTy::ArgOffset>(); } - - /// Set the function called. - void setCalledFunction(Value* Fn) { - setCalledFunction( - cast(cast(Fn->getType())->getElementType()), - Fn); - } - void setCalledFunction(FunctionType *FTy, Value *Fn) { - this->FTy = FTy; - assert(FTy == cast( - cast(Fn->getType())->getElementType())); - Op<-InstTy::ArgOffset>() = Fn; - } - -protected: - template bool hasFnAttrImpl(AttrKind Kind) const { - if (Attrs.hasAttribute(AttributeList::FunctionIndex, Kind)) - return true; - - // Operand bundles override attributes on the called function, but don't - // override attributes directly present on the call instruction. - if (isFnAttrDisallowedByOpBundle(Kind)) - return false; - - if (const Function *F = getCalledFunction()) - return F->getAttributes().hasAttribute(AttributeList::FunctionIndex, - Kind); - return false; - } -}; - //===----------------------------------------------------------------------===// /// This class represents a function call, abstracting a target /// machine's calling convention. This class uses low bit of the SubClassData /// field to indicate whether or not this is a tail call. The rest of the bits /// hold the calling convention of the call. /// -class CallInst : public CallBase { - friend class OperandBundleUser; - +class CallInst : public CallBase { CallInst(const CallInst &CI); /// Construct a CallInst given a range of arguments. @@ -1986,6 +1464,13 @@ ArrayRef Bundles, const Twine &NameStr); void init(Value *Func, const Twine &NameStr); + /// Compute the number of operands to allocate. + static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) { + // We need one operand for the called function, plus the input operand + // counts provided. + return 1 + NumArgs + NumBundleInputs; + } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -1993,8 +1478,6 @@ CallInst *cloneImpl() const; public: - static constexpr int ArgOffset = 1; - static CallInst *Create(Value *Func, ArrayRef Args, ArrayRef Bundles = None, const Twine &NameStr = "", @@ -2015,7 +1498,7 @@ static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef Args, const Twine &NameStr, Instruction *InsertBefore = nullptr) { - return new (unsigned(Args.size() + 1)) + return new (ComputeNumOperands(Args.size())) CallInst(Ty, Func, Args, None, NameStr, InsertBefore); } @@ -2023,39 +1506,39 @@ ArrayRef Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { - const unsigned TotalOps = - unsigned(Args.size()) + CountBundleInputs(Bundles) + 1; + const int NumOperands = + ComputeNumOperands(Args.size(), CountBundleInputs(Bundles)); const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); - return new (TotalOps, DescriptorBytes) + return new (NumOperands, DescriptorBytes) CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore); } static CallInst *Create(Value *Func, ArrayRef Args, ArrayRef Bundles, const Twine &NameStr, BasicBlock *InsertAtEnd) { - const unsigned TotalOps = - unsigned(Args.size()) + CountBundleInputs(Bundles) + 1; + const int NumOperands = + ComputeNumOperands(Args.size(), CountBundleInputs(Bundles)); const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); - return new (TotalOps, DescriptorBytes) + return new (NumOperands, DescriptorBytes) CallInst(Func, Args, Bundles, NameStr, InsertAtEnd); } static CallInst *Create(Value *Func, ArrayRef Args, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return new (unsigned(Args.size() + 1)) + return new (ComputeNumOperands(Args.size())) CallInst(Func, Args, None, NameStr, InsertAtEnd); } static CallInst *Create(Value *F, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { - return new (1) CallInst(F, NameStr, InsertBefore); + return new (ComputeNumOperands(0)) CallInst(F, NameStr, InsertBefore); } static CallInst *Create(Value *F, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return new (1) CallInst(F, NameStr, InsertAtEnd); + return new (ComputeNumOperands(0)) CallInst(F, NameStr, InsertAtEnd); } /// Create a clone of \p CI with a different set of operand bundles and @@ -2146,7 +1629,7 @@ } /// Check if this call is an inline asm statement. - bool isInlineAsm() const { return isa(Op<-1>()); } + bool isInlineAsm() const { return isa(getCalledOperand()); } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { @@ -2164,32 +1647,28 @@ } }; -template <> -struct OperandTraits> - : public VariadicOperandTraits, 1> {}; - CallInst::CallInst(Value *Func, ArrayRef Args, ArrayRef Bundles, const Twine &NameStr, BasicBlock *InsertAtEnd) - : CallBase( - cast( - cast(Func->getType())->getElementType()) - ->getReturnType(), - Instruction::Call, - OperandTraits>::op_end(this) - - (Args.size() + CountBundleInputs(Bundles) + 1), - unsigned(Args.size() + CountBundleInputs(Bundles) + 1), InsertAtEnd) { + : CallBase(cast( + cast(Func->getType())->getElementType()) + ->getReturnType(), + Instruction::Call, + OperandTraits::op_end(this) - + (Args.size() + CountBundleInputs(Bundles) + 1), + unsigned(Args.size() + CountBundleInputs(Bundles) + 1), + InsertAtEnd) { init(Func, Args, Bundles, NameStr); } CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef Args, ArrayRef Bundles, const Twine &NameStr, Instruction *InsertBefore) - : CallBase(Ty->getReturnType(), Instruction::Call, - OperandTraits>::op_end(this) - - (Args.size() + CountBundleInputs(Bundles) + 1), - unsigned(Args.size() + CountBundleInputs(Bundles) + 1), - InsertBefore) { + : CallBase(Ty->getReturnType(), Instruction::Call, + OperandTraits::op_end(this) - + (Args.size() + CountBundleInputs(Bundles) + 1), + unsigned(Args.size() + CountBundleInputs(Bundles) + 1), + InsertBefore) { init(Ty, Func, Args, Bundles, NameStr); } @@ -4066,8 +3545,16 @@ /// Invoke instruction. The SubclassData field is used to hold the /// calling convention of the call. /// -class InvokeInst : public CallBase { - friend class OperandBundleUser; +class InvokeInst : public CallBase { + /// The number of operands for this call beyond the called function, + /// arguments, and operand bundles. + static constexpr int NumExtraOperands = 2; + + /// The index from the end of the operand array to the normal destination. + static constexpr int NormalDestOpEndIdx = -3; + + /// The index from the end of the operand array to the unwind destination. + static constexpr int UnwindDestOpEndIdx = -2; InvokeInst(const InvokeInst &BI); @@ -4076,26 +3563,25 @@ /// Construct an InvokeInst from a range of arguments inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, ArrayRef Bundles, - unsigned Values, const Twine &NameStr, + int NumOperands, const Twine &NameStr, Instruction *InsertBefore) : InvokeInst(cast( cast(Func->getType())->getElementType()), - Func, IfNormal, IfException, Args, Bundles, Values, NameStr, - InsertBefore) {} + Func, IfNormal, IfException, Args, Bundles, NumOperands, + NameStr, InsertBefore) {} inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, - ArrayRef Bundles, unsigned Values, + ArrayRef Bundles, int NumOperands, const Twine &NameStr, Instruction *InsertBefore); /// Construct an InvokeInst given a range of arguments. /// /// Construct an InvokeInst from a range of arguments inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, ArrayRef Bundles, - unsigned Values, const Twine &NameStr, + int NumOperands, const Twine &NameStr, BasicBlock *InsertAtEnd); - void init(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, ArrayRef Bundles, const Twine &NameStr) { @@ -4108,6 +3594,13 @@ BasicBlock *IfException, ArrayRef Args, ArrayRef Bundles, const Twine &NameStr); + /// Compute the number of operands to allocate. + static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) { + // We need one operand for the called function, plus our extra operands and + // the input operand counts provided. + return 1 + NumExtraOperands + NumArgs + NumBundleInputs; + } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -4115,7 +3608,6 @@ InvokeInst *cloneImpl() const; public: - static constexpr int ArgOffset = 3; static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, const Twine &NameStr, @@ -4141,9 +3633,10 @@ BasicBlock *IfException, ArrayRef Args, const Twine &NameStr, Instruction *InsertBefore = nullptr) { - unsigned Values = unsigned(Args.size()) + 3; - return new (Values) InvokeInst(Ty, Func, IfNormal, IfException, Args, None, - Values, NameStr, InsertBefore); + int NumOperands = ComputeNumOperands(Args.size()); + return new (NumOperands) + InvokeInst(Ty, Func, IfNormal, IfException, Args, None, NumOperands, + NameStr, InsertBefore); } static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, @@ -4151,11 +3644,12 @@ ArrayRef Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { - unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3; + int NumOperands = + ComputeNumOperands(Args.size(), CountBundleInputs(Bundles)); unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); - return new (Values, DescriptorBytes) - InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, Values, + return new (NumOperands, DescriptorBytes) + InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, NumOperands, NameStr, InsertBefore); } @@ -4163,21 +3657,22 @@ BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, const Twine &NameStr, BasicBlock *InsertAtEnd) { - unsigned Values = unsigned(Args.size()) + 3; - return new (Values) InvokeInst(Func, IfNormal, IfException, Args, None, - Values, NameStr, InsertAtEnd); + int NumOperands = ComputeNumOperands(Args.size()); + return new (NumOperands) InvokeInst(Func, IfNormal, IfException, Args, None, + NumOperands, NameStr, InsertAtEnd); } static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, ArrayRef Bundles, const Twine &NameStr, BasicBlock *InsertAtEnd) { - unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3; + int NumOperands = + ComputeNumOperands(Args.size(), CountBundleInputs(Bundles)); unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); - return new (Values, DescriptorBytes) - InvokeInst(Func, IfNormal, IfException, Args, Bundles, Values, NameStr, - InsertAtEnd); + return new (NumOperands, DescriptorBytes) + InvokeInst(Func, IfNormal, IfException, Args, Bundles, NumOperands, + NameStr, InsertAtEnd); } /// Create a clone of \p II with a different set of operand bundles and @@ -4198,43 +3693,18 @@ addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind); } - /// Return the function called, or null if this is an - /// indirect function invocation. - /// - Function *getCalledFunction() const { - return dyn_cast(Op<-3>()); - } - - /// Get a pointer to the function that is invoked by this - /// instruction - const Value *getCalledValue() const { return Op<-3>(); } - Value *getCalledValue() { return Op<-3>(); } - - /// Set the function called. - void setCalledFunction(Value* Fn) { - setCalledFunction( - cast(cast(Fn->getType())->getElementType()), - Fn); - } - void setCalledFunction(FunctionType *FTy, Value *Fn) { - this->FTy = FTy; - assert(FTy == cast( - cast(Fn->getType())->getElementType())); - Op<-3>() = Fn; - } - // get*Dest - Return the destination basic blocks... BasicBlock *getNormalDest() const { - return cast(Op<-2>()); + return cast(Op()); } BasicBlock *getUnwindDest() const { - return cast(Op<-1>()); + return cast(Op()); } void setNormalDest(BasicBlock *B) { - Op<-2>() = reinterpret_cast(B); + Op() = reinterpret_cast(B); } void setUnwindDest(BasicBlock *B) { - Op<-1>() = reinterpret_cast(B); + Op() = reinterpret_cast(B); } /// Get the landingpad instruction from the landing pad @@ -4246,9 +3716,12 @@ return i == 0 ? getNormalDest() : getUnwindDest(); } - void setSuccessor(unsigned idx, BasicBlock *NewSucc) { - assert(idx < 2 && "Successor # out of range for invoke!"); - *(&Op<-2>() + idx) = reinterpret_cast(NewSucc); + void setSuccessor(unsigned i, BasicBlock *NewSucc) { + assert(i < 2 && "Successor # out of range for invoke!"); + if (i == 0) + setNormalDest(NewSucc); + else + setUnwindDest(NewSucc); } unsigned getNumSuccessors() const { return 2; } @@ -4270,36 +3743,29 @@ } }; -template <> -struct OperandTraits> - : public VariadicOperandTraits, 3> {}; - InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, - ArrayRef Bundles, unsigned Values, + ArrayRef Bundles, int NumOperands, const Twine &NameStr, Instruction *InsertBefore) - : CallBase(Ty->getReturnType(), Instruction::Invoke, - OperandTraits>::op_end(this) - - Values, - Values, InsertBefore) { + : CallBase(Ty->getReturnType(), Instruction::Invoke, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertBefore) { init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr); } InvokeInst::InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, - ArrayRef Bundles, unsigned Values, + ArrayRef Bundles, int NumOperands, const Twine &NameStr, BasicBlock *InsertAtEnd) - : CallBase( - cast( - cast(Func->getType())->getElementType()) - ->getReturnType(), - Instruction::Invoke, - OperandTraits>::op_end(this) - Values, Values, - InsertAtEnd) { + : CallBase(cast( + cast(Func->getType())->getElementType()) + ->getReturnType(), + Instruction::Invoke, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertAtEnd) { init(Func, IfNormal, IfException, Args, Bundles, NameStr); } - //===----------------------------------------------------------------------===// // ResumeInst Class //===----------------------------------------------------------------------===// Index: llvm/lib/IR/Instructions.cpp =================================================================== --- llvm/lib/IR/Instructions.cpp +++ llvm/lib/IR/Instructions.cpp @@ -65,10 +65,7 @@ //===----------------------------------------------------------------------===// User::op_iterator CallSite::getCallee() const { - Instruction *II(getInstruction()); - return isCall() - ? cast(II)->op_end() - 1 // Skip Callee - : cast(II)->op_end() - 3; // Skip BB, BB, Callee + return cast(getInstruction())->op_end() - 1; } //===----------------------------------------------------------------------===// @@ -253,6 +250,82 @@ getOperandList()[OpNo] = Val; } +//===----------------------------------------------------------------------===// +// CallBase Implementation +//===----------------------------------------------------------------------===// + +Value *CallBase::getReturnedArgOperand() const { + unsigned Index; + + if (Attrs.hasAttrSomewhere(Attribute::Returned, &Index) && Index) + return getArgOperand(Index - AttributeList::FirstArgIndex); + if (const Function *F = getCalledFunction()) + if (F->getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && + Index) + return getArgOperand(Index - AttributeList::FirstArgIndex); + + return nullptr; +} + +bool CallBase::hasRetAttr(Attribute::AttrKind Kind) const { + if (Attrs.hasAttribute(AttributeList::ReturnIndex, Kind)) + return true; + + // Look at the callee, if available. + if (const Function *F = getCalledFunction()) + return F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Kind); + return false; +} + +/// Determine whether the argument or parameter has the given attribute. +bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const { + assert(ArgNo < getNumArgOperands() && "Param index out of bounds!"); + + if (Attrs.hasParamAttribute(ArgNo, Kind)) + return true; + if (const Function *F = getCalledFunction()) + return F->getAttributes().hasParamAttribute(ArgNo, Kind); + return false; +} + +bool CallBase::hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const { + if (const Function *F = getCalledFunction()) + return F->getAttributes().hasAttribute(AttributeList::FunctionIndex, Kind); + return false; +} + +bool CallBase::hasFnAttrOnCalledFunction(StringRef Kind) const { + if (const Function *F = getCalledFunction()) + return F->getAttributes().hasAttribute(AttributeList::FunctionIndex, Kind); + return false; +} + +CallBase::op_iterator +CallBase::populateBundleOperandInfos(ArrayRef Bundles, + const unsigned BeginIndex) { + auto It = op_begin() + BeginIndex; + for (auto &B : Bundles) + It = std::copy(B.input_begin(), B.input_end(), It); + + auto *ContextImpl = getContext().pImpl; + auto BI = Bundles.begin(); + unsigned CurrentIndex = BeginIndex; + + for (auto &BOI : bundle_op_infos()) { + assert(BI != Bundles.end() && "Incorrect allocation?"); + + BOI.Tag = ContextImpl->getOrInsertBundleTag(BI->getTag()); + BOI.Begin = CurrentIndex; + BOI.End = CurrentIndex + BI->input_size(); + CurrentIndex = BOI.End; + BI++; + } + + assert(BI == Bundles.end() && "Incorrect allocation?"); + + return It; +} + //===----------------------------------------------------------------------===// // CallInst Implementation //===----------------------------------------------------------------------===// @@ -262,7 +335,7 @@ this->FTy = FTy; assert(getNumOperands() == Args.size() + CountBundleInputs(Bundles) + 1 && "NumOperands not set up?"); - Op<-1>() = Func; + setCalledOperand(Func); #ifndef NDEBUG assert((Args.size() == FTy->getNumParams() || @@ -288,7 +361,7 @@ FTy = cast(cast(Func->getType())->getElementType()); assert(getNumOperands() == 1 && "NumOperands not set up?"); - Op<-1>() = Func; + setCalledOperand(Func); assert(FTy->getNumParams() == 0 && "Calling a function with bad signature"); @@ -296,31 +369,27 @@ } CallInst::CallInst(Value *Func, const Twine &Name, Instruction *InsertBefore) - : CallBase( - cast( - cast(Func->getType())->getElementType()) - ->getReturnType(), - Instruction::Call, - OperandTraits>::op_end(this) - 1, 1, - InsertBefore) { + : CallBase(cast( + cast(Func->getType())->getElementType()) + ->getReturnType(), + Instruction::Call, OperandTraits::op_end(this) - 1, 1, + InsertBefore) { init(Func, Name); } CallInst::CallInst(Value *Func, const Twine &Name, BasicBlock *InsertAtEnd) - : CallBase( - cast( - cast(Func->getType())->getElementType()) - ->getReturnType(), - Instruction::Call, - OperandTraits>::op_end(this) - 1, 1, InsertAtEnd) { + : CallBase(cast( + cast(Func->getType())->getElementType()) + ->getReturnType(), + Instruction::Call, OperandTraits::op_end(this) - 1, 1, + InsertAtEnd) { init(Func, Name); } CallInst::CallInst(const CallInst &CI) - : CallBase(CI.Attrs, CI.FTy, CI.getType(), Instruction::Call, - OperandTraits>::op_end(this) - - CI.getNumOperands(), - CI.getNumOperands()) { + : CallBase(CI.Attrs, CI.FTy, CI.getType(), Instruction::Call, + OperandTraits::op_end(this) - CI.getNumOperands(), + CI.getNumOperands()) { setTailCallKind(CI.getTailCallKind()); setCallingConv(CI.getCallingConv()); @@ -560,11 +629,12 @@ const Twine &NameStr) { this->FTy = FTy; - assert(getNumOperands() == 3 + Args.size() + CountBundleInputs(Bundles) && + assert((int)getNumOperands() == + ComputeNumOperands(Args.size(), CountBundleInputs(Bundles)) && "NumOperands not set up?"); - Op<-3>() = Fn; - Op<-2>() = IfNormal; - Op<-1>() = IfException; + setNormalDest(IfNormal); + setUnwindDest(IfException); + setCalledOperand(Fn); #ifndef NDEBUG assert(((Args.size() == FTy->getNumParams()) || @@ -587,10 +657,9 @@ } InvokeInst::InvokeInst(const InvokeInst &II) - : CallBase(II.Attrs, II.FTy, II.getType(), Instruction::Invoke, - OperandTraits>::op_end(this) - - II.getNumOperands(), - II.getNumOperands()) { + : CallBase(II.Attrs, II.FTy, II.getType(), Instruction::Invoke, + OperandTraits::op_end(this) - II.getNumOperands(), + II.getNumOperands()) { setCallingConv(II.getCallingConv()); std::copy(II.op_begin(), II.op_end(), op_begin()); std::copy(II.bundle_op_info_begin(), II.bundle_op_info_end(), Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -3922,6 +3922,10 @@ } } + // Get a pointer to the call base of the instruction if it is some form of + // call. + const CallBase *CBI = dyn_cast(&I); + for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { Assert(I.getOperand(i) != nullptr, "Instruction has null operand!", &I); @@ -3934,10 +3938,9 @@ if (Function *F = dyn_cast(I.getOperand(i))) { // Check to make sure that the "address of" an intrinsic function is never // taken. - Assert( - !F->isIntrinsic() || - i == (isa(I) ? e - 1 : isa(I) ? e - 3 : 0), - "Cannot take the address of an intrinsic!", &I); + Assert(!F->isIntrinsic() || + (CBI && &CBI->getCalledOperandUse() == &I.getOperandUse(i)), + "Cannot take the address of an intrinsic!", &I); Assert( !F->isIntrinsic() || isa(I) || F->getIntrinsicID() == Intrinsic::donothing || @@ -3963,8 +3966,7 @@ } else if (isa(I.getOperand(i))) { verifyDominatesUse(I, i); } else if (isa(I.getOperand(i))) { - Assert((i + 1 == e && isa(I)) || - (i + 3 == e && isa(I)), + Assert(CBI && &CBI->getCalledOperandUse() == &I.getOperandUse(i), "Cannot take the address of an inline asm!", &I); } else if (ConstantExpr *CE = dyn_cast(I.getOperand(i))) { if (CE->getType()->isPtrOrPtrVectorTy() ||