Index: include/llvm/IR/CallSite.h =================================================================== --- include/llvm/IR/CallSite.h +++ include/llvm/IR/CallSite.h @@ -41,6 +41,7 @@ typename BBTy = const BasicBlock, typename ValTy = const Value, typename UserTy = const User, + typename UseTy = const Use, typename InstrTy = const Instruction, typename CallTy = const CallInst, typename InvokeTy = const InvokeInst, @@ -315,6 +316,22 @@ CALLSITE_DELEGATE_SETTER(setDoesNotThrow()); } + int getNumOperandBundles() const { + CALLSITE_DELEGATE_GETTER(getNumOperandBundles()); + } + + bool hasOperandBundles() const { + CALLSITE_DELEGATE_GETTER(hasOperandBundles()); + } + + int getNumTotalBundleOperands() const { + CALLSITE_DELEGATE_GETTER(getNumTotalBundleOperands()); + } + + OperandBundleSetT getOperandBundle(unsigned Index) const { + CALLSITE_DELEGATE_GETTER(getOperandBundle(Index)); + } + #undef CALLSITE_DELEGATE_GETTER #undef CALLSITE_DELEGATE_SETTER @@ -379,10 +396,15 @@ private: unsigned getArgumentEndOffset() const { - if (isCall()) - return 1; // Skip Callee - else - return 3; // Skip BB, BB, Callee + if (isCall()) { + // Skip [ operand bundles ], Callee + auto *CI = cast(getInstruction()); + return 1 + CI->getNumTotalBundleOperands(); + } else { + // Skip [ operand bundles ], BB, BB, Callee + auto *II = cast(getInstruction()); + return 3 + II->getNumTotalBundleOperands(); + } } IterTy getCallee() const { @@ -393,7 +415,7 @@ } }; -class CallSite : public CallSiteBase { public: Index: include/llvm/IR/InstrTypes.h =================================================================== --- include/llvm/IR/InstrTypes.h +++ include/llvm/IR/InstrTypes.h @@ -1092,6 +1092,219 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CmpInst, Value) +/// \brief A lightweight accessor for a set of operand bundles meant to be +/// passed around by value. +/// +/// There are two distinct views of a set of operand bundles: a "use" view +/// (OperandBundleSetUse) and a "def" view (OperandBundleSetDef). The former is +/// more useful when viewing or editing the operand bundle set of an existing +/// instruction, while the latter is useful when viewing an operand bundle as a +/// distinct entity. +template struct OperandBundleSetT { + StringRef Tag; + ArrayRef Inputs; + + OperandBundleSetT() {} + explicit OperandBundleSetT(StringRef Tag, ArrayRef Inputs) + : Tag(Tag), Inputs(Inputs) {} +}; + +typedef OperandBundleSetT OperandBundleSetUse; +typedef OperandBundleSetT ConstOperandBundleSetUse; + +typedef OperandBundleSetT OperandBundleSetDef; +typedef OperandBundleSetT ConstOperandBundleSetDef; + +/// \brief A mixin to add operand bundle functionality to llvm instruction +/// classes. +/// +/// OperandBundleUser uses the descriptor area co-allocated with the host User +/// to store some meta information about which operands are "normal" operands, +/// and which ones belong to some operand bundle. +/// +/// The layout of an operand bundle user is +/// +/// +-------uint32_t End---------------------------------+ +/// / \ +/// / +------uint32_t Begin------------------+ \ +/// / / \ \ +/// ^ ^ v v +/// |------|------|----|----|----|----|----|---------|----|---------|----|----- +/// | BOI0 | BOI1 | .. | DU | U0 | U1 | .. | BOI0_U0 | .. | BOI1_U0 | .. | User +/// |------|------|----|----|----|----|----|---------|----|---------|----|----- +/// v v ^ ^ +/// \ \ / / +/// \ +------uint32_t Begin----------+ / +/// \ / +/// +-------uint32_t End-------------------------+ +/// +/// +/// BOI0, BOI1 ... are descriptions of operand bundles in this User's use list. +/// These descriptions are installed and managed by this class, and they're all +/// instances of OperandBundleUser::BundleOpInfo. +/// +/// DU is an additional descriptor installed by User's 'operator new' to keep +/// track of the additional 'BOI0 ... BOIN' co-allocation. OperandBundleUser +/// does not access or modify DU in any way, it's an implementation detail +/// private to User. +/// +/// The regular Use& vector for the User starts at U0. The operand bundle uses +/// are part of the Use& vector, just like normal uses. In the diagram above, +/// these start at OB1_U0. Each instance of BundleOpInfo has information about +/// a contiguous set of uses constituting an operand bundle. All of these uses +/// corresponding to bundle operands are appended *after* the normal uses. +/// +/// Currently operand bundle users with hung-off operands are not supported. +template class OperandBundleUser { +public: + /// \brief Return the number of operand bundles associated with this User. + unsigned getNumOperandBundles() const { + return std::distance(bundle_op_info_begin(), bundle_op_info_end()); + } + + /// \brief Return true if this User has any operand bundles. + bool hasOperandBundles() const { return getNumOperandBundles() != 0; } + + /// \brief Return the total number operands (not operand bundles) used by + /// every operand bundle in this OperandBundleUser. + unsigned getNumTotalBundleOperands() const { + if (!hasOperandBundles()) + return 0; + + auto *Begin = bundle_op_info_begin(); + auto *Back = bundle_op_info_end() - 1; + + assert(Begin <= Back && "hasOperandBundles() returned true!"); + + return Back->End - Begin->Begin; + } + + /// \brief Return the operand bundle at a specific index. + ConstOperandBundleSetUse getOperandBundle(unsigned Index) const { + assert(Index < getNumOperandBundles() && "Index out of bounds!"); + auto *BOI = bundle_op_info_begin() + Index; + auto op_begin = static_cast(this)->op_begin(); + ArrayRef Inputs(op_begin + BOI->Begin, op_begin + BOI->End); + return ConstOperandBundleSetUse(BOI->Tag->getKey(), Inputs); + } + + /// \brief Return the operand bundle at a specific index. + OperandBundleSetUse getOperandBundle(unsigned Index) { + assert(Index < getNumOperandBundles() && "Index out of bounds!"); + auto *BOI = bundle_op_info_begin() + Index; + auto op_begin = static_cast(this)->op_begin(); + ArrayRef Inputs(op_begin + BOI->Begin, op_begin + BOI->End); + return OperandBundleSetUse(BOI->Tag->getKey(), Inputs); + } + +protected: + /// \brief Used to keep track of an operand bundle. See the main comment on + /// OperandBundleUser above. + struct BundleOpInfo { + /// \brief The operand bundle tag, interned by + /// LLVMContextImpl::internBundleTag. + StringMapEntry *Tag; + + /// \brief The index in the Use& vector where operands for this operand + /// bundle starts. + uint32_t Begin; + + /// \brief The index in the Use& vector where operands for this operand + /// bundle ends. + uint32_t End; + }; + + typedef BundleOpInfo *bundle_op_iterator; + typedef const BundleOpInfo *const_bundle_op_iterator; + + /// \brief Return the start of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + bundle_op_iterator bundle_op_info_begin() { + if (!static_cast(this)->hasDescriptor()) + return nullptr; + + uint8_t *BytesBegin = static_cast(this)->getDescriptor().begin(); + return reinterpret_cast(BytesBegin); + } + + /// \brief Return the start of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + const_bundle_op_iterator bundle_op_info_begin() const { + auto *NonConstThis = + const_cast *>(this); + return NonConstThis->bundle_op_info_begin(); + } + + /// \brief Return the end of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + bundle_op_iterator bundle_op_info_end() { + if (!static_cast(this)->hasDescriptor()) + return nullptr; + + uint8_t *BytesEnd = static_cast(this)->getDescriptor().end(); + return reinterpret_cast(BytesEnd); + } + + /// \brief Return the end of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + const_bundle_op_iterator bundle_op_info_end() const { + auto *NonConstThis = + const_cast *>(this); + return NonConstThis->bundle_op_info_end(); + } + + /// \brief Return the range [\p bundle_op_info_begin, \p bundle_op_info_end). + iterator_range bundle_op_infos() { + return iterator_range(bundle_op_info_begin(), + bundle_op_info_end()); + } + + /// \brief Return the range [\p bundle_op_info_begin, \p bundle_op_info_end). + iterator_range bundle_op_infos() const { + return iterator_range(bundle_op_info_begin(), + bundle_op_info_end()); + } + + /// \brief Populate the BundleOpInfo instances and the Use& vector from \p + /// Bundles. Return the op_iterator pointing to the Use& one past the last + /// last bundle operand use. + /// + /// Each \p OperandBundleSetDef instance is tracked by a OperandBundleInfo + /// instance allocated in this User's descriptor. + OpIteratorTy populateBundleOperandInfos(ArrayRef Bundles, + const unsigned BeginIndex) { + auto It = static_cast(this)->op_begin() + BeginIndex; + for (auto &B : Bundles) + It = std::copy(B.Inputs.begin(), B.Inputs.end(), It); + + auto *ContextImpl = static_cast(this)->getContext().pImpl; + auto BI = Bundles.begin(); + unsigned CurrentIndex = BeginIndex; + + for (auto &BOI : bundle_op_infos()) { + assert(BI != Bundles.end() && "Incorrect allocation?"); + + BOI.Tag = ContextImpl->internBundleTag(BI->Tag); + BOI.Begin = CurrentIndex; + BOI.End = CurrentIndex + BI->Inputs.size(); + CurrentIndex = BOI.End; + BI++; + } + + assert(BI == Bundles.end() && "Incorrect allocation?"); + + return It; + } + + /// \brief Return the total number of values used in \p Bundles. + static unsigned CountBundleInputs(ArrayRef Bundles) { + unsigned Total = 0; + for (auto &B : Bundles) + Total += B.Inputs.size(); + return Total; + } +}; + } // End llvm namespace #endif Index: include/llvm/IR/Instructions.h =================================================================== --- include/llvm/IR/Instructions.h +++ include/llvm/IR/Instructions.h @@ -1350,62 +1350,102 @@ /// field to indicate whether or not this is a tail call. The rest of the bits /// hold the calling convention of the call. /// -class CallInst : public Instruction { +class CallInst : public Instruction, + public OperandBundleUser { AttributeSet AttributeList; ///< parameter attributes for call FunctionType *FTy; CallInst(const CallInst &CI); - void init(Value *Func, ArrayRef Args, const Twine &NameStr) { + void init(Value *Func, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr) { init(cast( cast(Func->getType())->getElementType()), - Func, Args, NameStr); + Func, Args, Bundles, NameStr); } void init(FunctionType *FTy, Value *Func, ArrayRef Args, - const Twine &NameStr); + ArrayRef Bundles, const Twine &NameStr); void init(Value *Func, const Twine &NameStr); /// Construct a CallInst given a range of arguments. /// \brief Construct a CallInst from a range of arguments inline CallInst(FunctionType *Ty, Value *Func, ArrayRef Args, - const Twine &NameStr, Instruction *InsertBefore); - inline CallInst(Value *Func, ArrayRef Args, const Twine &NameStr, + ArrayRef Bundles, const Twine &NameStr, + Instruction *InsertBefore); + inline CallInst(Value *Func, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr, Instruction *InsertBefore) : CallInst(cast( cast(Func->getType())->getElementType()), - Func, Args, NameStr, InsertBefore) {} + Func, Args, Bundles, NameStr, InsertBefore) {} + + inline CallInst(Value *Func, ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore) + : CallInst(Func, Args, None, NameStr, InsertBefore) {} /// Construct a CallInst given a range of arguments. /// \brief Construct a CallInst from a range of arguments inline CallInst(Value *Func, ArrayRef Args, - const Twine &NameStr, BasicBlock *InsertAtEnd); + ArrayRef Bundles, const Twine &NameStr, + BasicBlock *InsertAtEnd); explicit CallInst(Value *F, const Twine &NameStr, Instruction *InsertBefore); CallInst(Value *F, const Twine &NameStr, BasicBlock *InsertAtEnd); + + friend class OperandBundleUser; + bool hasDescriptor() const { return HasDescriptor; } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; CallInst *cloneImpl() const; public: - static CallInst *Create(Value *Func, - ArrayRef Args, + static CallInst *Create(Value *Func, ArrayRef Args, + ArrayRef Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { return Create(cast( cast(Func->getType())->getElementType()), - Func, Args, NameStr, InsertBefore); + Func, Args, Bundles, NameStr, InsertBefore); + } + static CallInst *Create(Value *Func, ArrayRef Args, + const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + return Create(cast( + cast(Func->getType())->getElementType()), + Func, Args, None, NameStr, InsertBefore); } static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef Args, - const Twine &NameStr = "", + const Twine &NameStr, Instruction *InsertBefore = nullptr) { return new (unsigned(Args.size() + 1)) - CallInst(Ty, Func, Args, NameStr, InsertBefore); + CallInst(Ty, Func, Args, None, NameStr, InsertBefore); + } + static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + const unsigned TotalOps = + unsigned(Args.size()) + CountBundleInputs(Bundles) + 1; + const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (TotalOps, DescriptorBytes) + CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore); } - static CallInst *Create(Value *Func, - ArrayRef Args, + static CallInst *Create(Value *Func, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return new(unsigned(Args.size() + 1)) - CallInst(Func, Args, NameStr, InsertAtEnd); + const unsigned TotalOps = + unsigned(Args.size()) + CountBundleInputs(Bundles) + 1; + const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (TotalOps, DescriptorBytes) + CallInst(Func, Args, Bundles, NameStr, InsertAtEnd); + } + static CallInst *Create(Value *Func, ArrayRef Args, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + return new (unsigned(Args.size() + 1)) + CallInst(Func, Args, None, NameStr, InsertAtEnd); } static CallInst *Create(Value *F, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { @@ -1469,7 +1509,9 @@ /// getNumArgOperands - Return the number of call arguments. /// - unsigned getNumArgOperands() const { return getNumOperands() - 1; } + unsigned getNumArgOperands() const { + return getNumOperands() - getNumTotalBundleOperands() - 1; + } /// getArgOperand/setArgOperand - Return/set the i-th call argument. /// @@ -1486,12 +1528,14 @@ iterator_range arg_operands() { // The last operand in the op list is the callee - it's not one of the args // so we don't want to iterate over it. - return iterator_range(op_begin(), op_end() - 1); + return iterator_range( + op_begin(), op_end() - getNumTotalBundleOperands() - 1); } /// arg_operands - iteration adapter for range-for loops. iterator_range arg_operands() const { - return iterator_range(op_begin(), op_end() - 1); + return iterator_range( + op_begin(), op_end() - getNumTotalBundleOperands() - 1); } /// \brief Wrappers for getting the \c Use of a call argument. @@ -1712,21 +1756,26 @@ }; CallInst::CallInst(Value *Func, ArrayRef Args, - const Twine &NameStr, BasicBlock *InsertAtEnd) - : Instruction(cast(cast(Func->getType()) - ->getElementType())->getReturnType(), - Instruction::Call, - OperandTraits::op_end(this) - (Args.size() + 1), - unsigned(Args.size() + 1), InsertAtEnd) { - init(Func, Args, NameStr); + ArrayRef Bundles, const Twine &NameStr, + BasicBlock *InsertAtEnd) + : Instruction( + cast(cast(Func->getType()) + ->getElementType())->getReturnType(), + Instruction::Call, OperandTraits::op_end(this) - + (Args.size() + CountBundleInputs(Bundles) + 1), + unsigned(Args.size() + CountBundleInputs(Bundles) + 1), InsertAtEnd) { + init(Func, Args, Bundles, NameStr); } CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef Args, - const Twine &NameStr, Instruction *InsertBefore) + ArrayRef Bundles, const Twine &NameStr, + Instruction *InsertBefore) : Instruction(Ty->getReturnType(), Instruction::Call, - OperandTraits::op_end(this) - (Args.size() + 1), - unsigned(Args.size() + 1), InsertBefore) { - init(Ty, Func, Args, NameStr); + OperandTraits::op_end(this) - + (Args.size() + CountBundleInputs(Bundles) + 1), + unsigned(Args.size() + CountBundleInputs(Bundles) + 1), + InsertBefore) { + init(Ty, Func, Args, Bundles, NameStr); } @@ -3204,70 +3253,112 @@ /// InvokeInst - Invoke instruction. The SubclassData field is used to hold the /// calling convention of the call. /// -class InvokeInst : public TerminatorInst { +class InvokeInst : public TerminatorInst, + public OperandBundleUser { AttributeSet AttributeList; FunctionType *FTy; InvokeInst(const InvokeInst &BI); void init(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef Args, const Twine &NameStr) { + ArrayRef Args, ArrayRef Bundles, + const Twine &NameStr) { init(cast( cast(Func->getType())->getElementType()), - Func, IfNormal, IfException, Args, NameStr); + Func, IfNormal, IfException, Args, Bundles, NameStr); } void init(FunctionType *FTy, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, - const Twine &NameStr); + ArrayRef Bundles, const Twine &NameStr); /// Construct an InvokeInst given a range of arguments. /// /// \brief Construct an InvokeInst from a range of arguments inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef Args, unsigned Values, + ArrayRef Args, + ArrayRef Bundles, unsigned Values, const Twine &NameStr, Instruction *InsertBefore) : InvokeInst(cast( cast(Func->getType())->getElementType()), - Func, IfNormal, IfException, Args, Values, NameStr, + Func, IfNormal, IfException, Args, Bundles, Values, NameStr, InsertBefore) {} inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, - unsigned Values, const Twine &NameStr, - Instruction *InsertBefore); + ArrayRef Bundles, unsigned Values, + const Twine &NameStr, Instruction *InsertBefore); /// Construct an InvokeInst given a range of arguments. /// /// \brief Construct an InvokeInst from a range of arguments inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef Args, unsigned Values, + ArrayRef Args, + ArrayRef Bundles, unsigned Values, const Twine &NameStr, BasicBlock *InsertAtEnd); + + friend class OperandBundleUser; + bool hasDescriptor() const { return HasDescriptor; } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; InvokeInst *cloneImpl() const; public: - static InvokeInst *Create(Value *Func, - BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef Args, const Twine &NameStr = "", + static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef Args, + const Twine &NameStr, Instruction *InsertBefore = nullptr) { return Create(cast( cast(Func->getType())->getElementType()), - Func, IfNormal, IfException, Args, NameStr, InsertBefore); + Func, IfNormal, IfException, Args, None, NameStr, + InsertBefore); } - static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, + static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, + ArrayRef Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { + return Create(cast( + cast(Func->getType())->getElementType()), + Func, IfNormal, IfException, Args, Bundles, NameStr, + InsertBefore); + } + static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef Args, + const Twine &NameStr, + Instruction *InsertBefore = nullptr) { unsigned Values = unsigned(Args.size()) + 3; - return new (Values) InvokeInst(Ty, Func, IfNormal, IfException, Args, + return new (Values) InvokeInst(Ty, Func, IfNormal, IfException, Args, None, Values, NameStr, InsertBefore); } + static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3; + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (Values, DescriptorBytes) + InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, Values, + NameStr, InsertBefore); + } static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, const Twine &NameStr, BasicBlock *InsertAtEnd) { unsigned Values = unsigned(Args.size()) + 3; - return new(Values) InvokeInst(Func, IfNormal, IfException, Args, - Values, NameStr, InsertAtEnd); + return new (Values) InvokeInst(Func, IfNormal, IfException, Args, None, + Values, NameStr, InsertAtEnd); + } + static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3; + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (Values, DescriptorBytes) + InvokeInst(Func, IfNormal, IfException, Args, Bundles, Values, NameStr, + InsertAtEnd); } /// Provide fast operand accessors @@ -3282,7 +3373,9 @@ /// getNumArgOperands - Return the number of invoke arguments. /// - unsigned getNumArgOperands() const { return getNumOperands() - 3; } + unsigned getNumArgOperands() const { + return getNumOperands() - getNumTotalBundleOperands() - 3; + } /// getArgOperand/setArgOperand - Return/set the i-th invoke argument. /// @@ -3297,12 +3390,14 @@ /// arg_operands - iteration adapter for range-for loops. iterator_range arg_operands() { - return iterator_range(op_begin(), op_end() - 3); + return iterator_range( + op_begin(), op_end() - getNumTotalBundleOperands() - 3); } /// arg_operands - iteration adapter for range-for loops. iterator_range arg_operands() const { - return iterator_range(op_begin(), op_end() - 3); + return iterator_range( + op_begin(), op_end() - getNumTotalBundleOperands() - 3); } /// \brief Wrappers for getting the \c Use of a invoke argument. @@ -3525,23 +3620,23 @@ InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, - unsigned Values, const Twine &NameStr, - Instruction *InsertBefore) + ArrayRef Bundles, unsigned Values, + const Twine &NameStr, Instruction *InsertBefore) : TerminatorInst(Ty->getReturnType(), Instruction::Invoke, OperandTraits::op_end(this) - Values, Values, InsertBefore) { - init(Ty, Func, IfNormal, IfException, Args, NameStr); + init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr); } -InvokeInst::InvokeInst(Value *Func, - BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef Args, unsigned Values, +InvokeInst::InvokeInst(Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef Args, + ArrayRef Bundles, unsigned Values, const Twine &NameStr, BasicBlock *InsertAtEnd) - : TerminatorInst(cast(cast(Func->getType()) - ->getElementType())->getReturnType(), - Instruction::Invoke, - OperandTraits::op_end(this) - Values, - Values, InsertAtEnd) { - init(Func, IfNormal, IfException, Args, NameStr); + : TerminatorInst( + cast(cast(Func->getType()) + ->getElementType())->getReturnType(), + Instruction::Invoke, OperandTraits::op_end(this) - Values, + Values, InsertAtEnd) { + init(Func, IfNormal, IfException, Args, Bundles, NameStr); } DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InvokeInst, Value) Index: lib/IR/Instructions.cpp =================================================================== --- lib/IR/Instructions.cpp +++ lib/IR/Instructions.cpp @@ -226,9 +226,11 @@ } void CallInst::init(FunctionType *FTy, Value *Func, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr) { this->FTy = FTy; - assert(getNumOperands() == Args.size() + 1 && "NumOperands not set up?"); + assert(getNumOperands() == Args.size() + CountBundleInputs(Bundles) + 1 && + "NumOperands not set up?"); Op<-1>() = Func; #ifndef NDEBUG @@ -243,6 +245,11 @@ #endif std::copy(Args.begin(), Args.end(), op_begin()); + + auto It = populateBundleOperandInfos(Bundles, Args.size()); + (void)It; + assert(It + 1 == op_end() && "Should add up!"); + setName(NameStr); } @@ -284,8 +291,10 @@ AttributeList(CI.AttributeList), FTy(CI.FTy) { setTailCallKind(CI.getTailCallKind()); setCallingConv(CI.getCallingConv()); - + std::copy(CI.op_begin(), CI.op_end(), op_begin()); + std::copy(CI.bundle_op_info_begin(), CI.bundle_op_info_end(), + bundle_op_info_begin()); SubclassOptionalData = CI.SubclassOptionalData; } @@ -499,10 +508,12 @@ void InvokeInst::init(FunctionType *FTy, Value *Fn, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr) { this->FTy = FTy; - assert(getNumOperands() == 3 + Args.size() && "NumOperands not set up?"); + assert(getNumOperands() == 3 + Args.size() + CountBundleInputs(Bundles) && + "NumOperands not set up?"); Op<-3>() = Fn; Op<-2>() = IfNormal; Op<-1>() = IfException; @@ -519,6 +530,11 @@ #endif std::copy(Args.begin(), Args.end(), op_begin()); + + auto It = populateBundleOperandInfos(Bundles, Args.size()); + (void)It; + assert(It + 3 == op_end() && "Should add up!"); + setName(NameStr); } @@ -530,6 +546,8 @@ AttributeList(II.AttributeList), FTy(II.FTy) { setCallingConv(II.getCallingConv()); std::copy(II.op_begin(), II.op_end(), op_begin()); + std::copy(II.bundle_op_info_begin(), II.bundle_op_info_end(), + bundle_op_info_begin()); SubclassOptionalData = II.SubclassOptionalData; } Index: lib/IR/LLVMContextImpl.h =================================================================== --- lib/IR/LLVMContextImpl.h +++ lib/IR/LLVMContextImpl.h @@ -986,6 +986,12 @@ int getOrAddScopeRecordIdxEntry(MDNode *N, int ExistingIdx); int getOrAddScopeInlinedAtIdxEntry(MDNode *Scope, MDNode *IA,int ExistingIdx); + /// \brief A set of interned tags for operand bundles. It currently maps all + /// tags to 0, but in the future we may consider using the uint32_t to + /// refcount the interned strings. + StringMap BundleTagCache; + StringMapEntry *internBundleTag(StringRef Tag); + LLVMContextImpl(LLVMContext &C); ~LLVMContextImpl(); Index: lib/IR/LLVMContextImpl.cpp =================================================================== --- lib/IR/LLVMContextImpl.cpp +++ lib/IR/LLVMContextImpl.cpp @@ -219,6 +219,19 @@ return hash_combine_range(Ops.begin(), Ops.end()); } +StringMapEntry *LLVMContextImpl::internBundleTag(StringRef Tag) { + auto I = BundleTagCache.find(Tag); + if (I != BundleTagCache.end()) + return &*I; + + auto *Entry = + StringMapEntry::Create(Tag, BundleTagCache.getAllocator(), 0); + bool WasInserted = BundleTagCache.insert(Entry); + (void)WasInserted; + assert(WasInserted && "Expected entry to be inserted"); + return Entry; +} + // ConstantsContext anchors void UnaryConstantExpr::anchor() { }