Index: include/llvm/IR/Function.h =================================================================== --- include/llvm/IR/Function.h +++ include/llvm/IR/Function.h @@ -61,10 +61,12 @@ /* * Value::SubclassData * - * bit 0 : HasLazyArguments - * bit 1 : HasPrefixData - * bit 2 : HasPrologueData - * bit 3-6: CallingConvention + * bit 0 : HasLazyArguments + * bit 1 : HasPrefixData + * bit 2 : HasPrologueData + * bit 3 : [reserved] + * bits 4-13 : CallingConvention + * bits 14-15 : [reserved] */ /// Bits from GlobalObject::GlobalObjectSubclassData. @@ -108,7 +110,7 @@ public: static Function *Create(FunctionType *Ty, LinkageTypes Linkage, const Twine &N = "", Module *M = nullptr) { - return new(1) Function(Ty, Linkage, N, M); + return new Function(Ty, Linkage, N, M); } ~Function() override; @@ -116,14 +118,6 @@ /// \brief Provide fast operand accessors DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - /// \brief Get the personality function associated with this function. - bool hasPersonalityFn() const { return getNumOperands() != 0; } - Constant *getPersonalityFn() const { - assert(hasPersonalityFn()); - return cast(Op<0>()); - } - void setPersonalityFn(Constant *C); - Type *getReturnType() const; // Return the type of the ret val FunctionType *getFunctionType() const; // Return the FunctionType for me @@ -158,11 +152,12 @@ /// calling convention of this function. The enum values for the known /// calling conventions are defined in CallingConv.h. CallingConv::ID getCallingConv() const { - return static_cast(getSubclassDataFromValue() >> 3); + return static_cast((getSubclassDataFromValue() >> 4) & + 0x3ff); } void setCallingConv(CallingConv::ID CC) { - setValueSubclassData((getSubclassDataFromValue() & 7) | - (static_cast(CC) << 3)); + setValueSubclassData((getSubclassDataFromValue() & 0xc00f) | + ((static_cast(CC) & 0x3ff) << 4)); } /// @brief Return the attribute list for this Function. @@ -491,17 +486,30 @@ size_t arg_size() const; bool arg_empty() const; + /// \brief Check whether this function has a personality function. + bool hasPersonalityFn() const { + return getSubclassDataFromValue() & (1<<15); + } + + /// \brief Get the personality function associated with this function. + Constant *getPersonalityFn() const; + void setPersonalityFn(Constant *Fn); + + /// \brief Check whether this function has prefix data. bool hasPrefixData() const { return getSubclassDataFromValue() & (1<<1); } + /// \brief Get the prefix data associated with this function. Constant *getPrefixData() const; void setPrefixData(Constant *PrefixData); + /// \brief Check whether this function has prologue data. bool hasPrologueData() const { return getSubclassDataFromValue() & (1<<2); } + /// \brief Get the prologue data associated with this function. Constant *getPrologueData() const; void setPrologueData(Constant *PrologueData); @@ -596,11 +604,15 @@ DISubprogram *getSubprogram() const; private: + void allocHungoffUselist(); + template void setHungoffOperand(Constant *C); + // Shadow Value::setValueSubclassData with a private forwarding method so that // subclasses cannot accidentally use it. void setValueSubclassData(unsigned short D) { Value::setValueSubclassData(D); } + void setValueSubclassDataBit(unsigned Bit, bool On); bool hasMetadataHashEntry() const { return getGlobalObjectSubClassData() & HasMetadataHashEntryBit; @@ -613,7 +625,7 @@ }; template <> -struct OperandTraits : public OptionalOperandTraits {}; +struct OperandTraits : public HungoffOperandTraits<3> {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(Function, Value) Index: include/llvm/IR/User.h =================================================================== --- include/llvm/IR/User.h +++ include/llvm/IR/User.h @@ -170,19 +170,6 @@ NumUserOperands = NumOps; } - /// Set the number of operands on a Function. - /// - /// Function always allocates space for a single operands, but - /// doesn't always use it. - /// - /// FIXME: As that the number of operands is used to find the start of - /// the allocated memory in operator delete, we need to always think we have - /// 1 operand before delete. - void setFunctionNumOperands(unsigned NumOps) { - assert(NumOps <= 1 && "Function can only have 0 or 1 operands"); - NumUserOperands = NumOps; - } - /// \brief Subclasses with hung off uses need to manage the operand count /// themselves. In these instances, the operand count isn't used to find the /// OperandList, so there's no issue in having the operand count change. Index: lib/Bitcode/Writer/ValueEnumerator.cpp =================================================================== --- lib/Bitcode/Writer/ValueEnumerator.cpp +++ lib/Bitcode/Writer/ValueEnumerator.cpp @@ -87,15 +87,9 @@ if (!isa(A.getAliasee())) orderValue(A.getAliasee(), OM); for (const Function &F : M) { - if (F.hasPrefixData()) - if (!isa(F.getPrefixData())) - orderValue(F.getPrefixData(), OM); - if (F.hasPrologueData()) - if (!isa(F.getPrologueData())) - orderValue(F.getPrologueData(), OM); - if (F.hasPersonalityFn()) - if (!isa(F.getPersonalityFn())) - orderValue(F.getPersonalityFn(), OM); + for (const Use &U : F.operands()) + if (!isa(U.get())) + orderValue(U.get(), OM); } OM.LastGlobalConstantID = OM.size(); @@ -273,12 +267,8 @@ for (const GlobalAlias &A : M.aliases()) predictValueUseListOrder(A.getAliasee(), nullptr, OM, Stack); for (const Function &F : M) { - if (F.hasPrefixData()) - predictValueUseListOrder(F.getPrefixData(), nullptr, OM, Stack); - if (F.hasPrologueData()) - predictValueUseListOrder(F.getPrologueData(), nullptr, OM, Stack); - if (F.hasPersonalityFn()) - predictValueUseListOrder(F.getPersonalityFn(), nullptr, OM, Stack); + for (const Use &U : F.operands()) + predictValueUseListOrder(U.get(), nullptr, OM, Stack); } return Stack; @@ -321,20 +311,10 @@ for (const GlobalAlias &GA : M.aliases()) EnumerateValue(GA.getAliasee()); - // Enumerate the prefix data constants. + // Enumerate any optional Function data. for (const Function &F : M) - if (F.hasPrefixData()) - EnumerateValue(F.getPrefixData()); - - // Enumerate the prologue data constants. - for (const Function &F : M) - if (F.hasPrologueData()) - EnumerateValue(F.getPrologueData()); - - // Enumerate the personality functions. - for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) - if (I->hasPersonalityFn()) - EnumerateValue(I->getPersonalityFn()); + for (const Use &U : F.operands()) + EnumerateValue(U.get()); // Enumerate the metadata type. // Index: lib/IR/AsmWriter.cpp =================================================================== --- lib/IR/AsmWriter.cpp +++ lib/IR/AsmWriter.cpp @@ -102,17 +102,9 @@ orderValue(&A, OM); } for (const Function &F : *M) { - if (F.hasPrefixData()) - if (!isa(F.getPrefixData())) - orderValue(F.getPrefixData(), OM); - - if (F.hasPrologueData()) - if (!isa(F.getPrologueData())) - orderValue(F.getPrologueData(), OM); - - if (F.hasPersonalityFn()) - if (!isa(F.getPersonalityFn())) - orderValue(F.getPersonalityFn(), OM); + for (const Use &U : F.operands()) + if (!isa(U.get())) + orderValue(U.get(), OM); orderValue(&F, OM); @@ -262,8 +254,8 @@ for (const GlobalAlias &A : M->aliases()) predictValueUseListOrder(A.getAliasee(), nullptr, OM, Stack); for (const Function &F : *M) - if (F.hasPrefixData()) - predictValueUseListOrder(F.getPrefixData(), nullptr, OM, Stack); + for (const Use &U : F.operands()) + predictValueUseListOrder(U.get(), nullptr, OM, Stack); return Stack; } Index: lib/IR/Function.cpp =================================================================== --- lib/IR/Function.cpp +++ lib/IR/Function.cpp @@ -279,9 +279,6 @@ // Remove the function from the on-the-side GC table. clearGC(); - - // FIXME: needed by operator delete - setFunctionNumOperands(1); } void Function::BuildLazyArguments() const { @@ -328,14 +325,16 @@ while (!BasicBlocks.empty()) BasicBlocks.begin()->eraseFromParent(); - // Prefix and prologue data are stored in a side table. - setPrefixData(nullptr); - setPrologueData(nullptr); + // Drop uses of any optional data (real or placeholder). + if (getNumOperands()) { + Op<0>().set(nullptr); + Op<1>().set(nullptr); + Op<2>().set(nullptr); + setNumHungOffUseOperands(0); + } // Metadata is stored in a side-table. clearMetadata(); - - setPersonalityFn(nullptr); } void Function::addAttribute(unsigned i, Attribute::AttrKind attr) { @@ -423,18 +422,12 @@ setGC(SrcF->getGC()); else clearGC(); + if (SrcF->hasPersonalityFn()) + setPersonalityFn(SrcF->getPersonalityFn()); if (SrcF->hasPrefixData()) setPrefixData(SrcF->getPrefixData()); - else - setPrefixData(nullptr); if (SrcF->hasPrologueData()) setPrologueData(SrcF->getPrologueData()); - else - setPrologueData(nullptr); - if (SrcF->hasPersonalityFn()) - setPersonalityFn(SrcF->getPersonalityFn()); - else - setPersonalityFn(nullptr); } /// \brief This does the actual lookup of an intrinsic ID which @@ -929,61 +922,64 @@ return false; } -static Constant * -getFunctionData(const Function *F, - const LLVMContextImpl::FunctionDataMapTy &Map) { - const auto &Entry = Map.find(F); - assert(Entry != Map.end()); - return cast(Entry->second->getReturnValue()); -} - -/// setFunctionData - Set "Map[F] = Data". Return an updated SubclassData value -/// in which Bit is low iff Data is null. -static unsigned setFunctionData(Function *F, - LLVMContextImpl::FunctionDataMapTy &Map, - Constant *Data, unsigned SCData, unsigned Bit) { - ReturnInst *&Holder = Map[F]; - if (Data) { - if (Holder) - Holder->setOperand(0, Data); - else - Holder = ReturnInst::Create(F->getContext(), Data); - return SCData | (1 << Bit); - } else { - delete Holder; - Map.erase(F); - return SCData & ~(1 << Bit); - } +Constant *Function::getPersonalityFn() const { + assert(hasPersonalityFn() && getNumOperands()); + return cast(Op<0>()); +} + +void Function::setPersonalityFn(Constant *Fn) { + setHungoffOperand<0>(Fn); + setValueSubclassDataBit(15, Fn != nullptr); } Constant *Function::getPrefixData() const { - assert(hasPrefixData()); - return getFunctionData(this, getContext().pImpl->PrefixDataMap); + assert(hasPrefixData() && getNumOperands()); + return cast(Op<1>()); } void Function::setPrefixData(Constant *PrefixData) { - if (!PrefixData && !hasPrefixData()) - return; - - unsigned SCData = getSubclassDataFromValue(); - SCData = setFunctionData(this, getContext().pImpl->PrefixDataMap, PrefixData, - SCData, /*Bit=*/1); - setValueSubclassData(SCData); + setHungoffOperand<1>(PrefixData); + setValueSubclassDataBit(1, PrefixData != nullptr); } Constant *Function::getPrologueData() const { - assert(hasPrologueData()); - return getFunctionData(this, getContext().pImpl->PrologueDataMap); + assert(hasPrologueData() && getNumOperands()); + return cast(Op<2>()); } void Function::setPrologueData(Constant *PrologueData) { - if (!PrologueData && !hasPrologueData()) + setHungoffOperand<2>(PrologueData); + setValueSubclassDataBit(2, PrologueData != nullptr); +} + +void Function::allocHungoffUselist() { + // If we've already allocated a uselist, stop here. + if (getNumOperands()) return; - unsigned SCData = getSubclassDataFromValue(); - SCData = setFunctionData(this, getContext().pImpl->PrologueDataMap, - PrologueData, SCData, /*Bit=*/2); - setValueSubclassData(SCData); + allocHungoffUses(3, /*IsPhi=*/ false); + setNumHungOffUseOperands(3); + + // Initialize the uselist with placeholder operands to allow traversal. + auto *CPN = ConstantPointerNull::get(Type::getInt1PtrTy(getContext(), 0)); + Op<0>().set(CPN); + Op<1>().set(CPN); + Op<2>().set(CPN); +} + +template +void Function::setHungoffOperand(Constant *C) { + allocHungoffUselist(); + Op().set( + C ? C : ConstantPointerNull::get(Type::getInt1PtrTy(getContext(), 0))); +} + +void Function::setValueSubclassDataBit(unsigned Bit, bool On) { + assert(Bit < 16 && "SubclassData contains only 16 bits"); + if (On) + setValueSubclassData(getSubclassDataFromValue() | (1 << Bit)); + else + setValueSubclassData(getSubclassDataFromValue() & ~(1 << Bit)); } void Function::setEntryCount(uint64_t Count) { @@ -1001,22 +997,3 @@ } return None; } - -void Function::setPersonalityFn(Constant *C) { - if (!C) { - if (hasPersonalityFn()) { - // Note, the num operands is used to compute the offset of the operand, so - // the order here matters. Clearing the operand then clearing the num - // operands ensures we have the correct offset to the operand. - Op<0>().set(nullptr); - setFunctionNumOperands(0); - } - } else { - // Note, the num operands is used to compute the offset of the operand, so - // the order here matters. We need to set num operands to 1 first so that - // we get the correct offset to the first operand when we set it. - if (!hasPersonalityFn()) - setFunctionNumOperands(1); - Op<0>().set(C); - } -} Index: lib/IR/LLVMContextImpl.h =================================================================== --- lib/IR/LLVMContextImpl.h +++ lib/IR/LLVMContextImpl.h @@ -972,17 +972,6 @@ /// instructions in different blocks at the same location. DenseMap, unsigned> DiscriminatorTable; - typedef DenseMap FunctionDataMapTy; - - /// \brief Mapping from a function to its prefix data, which is stored as the - /// operand of an unparented ReturnInst so that the prefix data has a Use. - FunctionDataMapTy PrefixDataMap; - - /// \brief Mapping from a function to its prologue data, which is stored as - /// the operand of an unparented ReturnInst so that the prologue data has a - /// Use. - FunctionDataMapTy PrologueDataMap; - int getOrAddScopeRecordIdxEntry(MDNode *N, int ExistingIdx); int getOrAddScopeInlinedAtIdxEntry(MDNode *Scope, MDNode *IA,int ExistingIdx); Index: lib/IR/TypeFinder.cpp =================================================================== --- lib/IR/TypeFinder.cpp +++ lib/IR/TypeFinder.cpp @@ -44,14 +44,8 @@ for (Module::const_iterator FI = M.begin(), E = M.end(); FI != E; ++FI) { incorporateType(FI->getType()); - if (FI->hasPrefixData()) - incorporateValue(FI->getPrefixData()); - - if (FI->hasPrologueData()) - incorporateValue(FI->getPrologueData()); - - if (FI->hasPersonalityFn()) - incorporateValue(FI->getPersonalityFn()); + for (const Use &U : FI->operands()) + incorporateValue(U.get()); // First incorporate the arguments. for (Function::const_arg_iterator AI = FI->arg_begin(), Index: lib/Transforms/IPO/GlobalDCE.cpp =================================================================== --- lib/Transforms/IPO/GlobalDCE.cpp +++ lib/Transforms/IPO/GlobalDCE.cpp @@ -215,14 +215,8 @@ // any globals used will be marked as needed. Function *F = cast(G); - if (F->hasPrefixData()) - MarkUsedGlobalsAsNeeded(F->getPrefixData()); - - if (F->hasPrologueData()) - MarkUsedGlobalsAsNeeded(F->getPrologueData()); - - if (F->hasPersonalityFn()) - MarkUsedGlobalsAsNeeded(F->getPersonalityFn()); + for (Use &U : F->operands()) + MarkUsedGlobalsAsNeeded(cast(U.get())); for (BasicBlock &BB : *F) for (Instruction &I : BB) Index: test/Bitcode/compatibility-3.6.ll =================================================================== --- test/Bitcode/compatibility-3.6.ll +++ test/Bitcode/compatibility-3.6.ll @@ -375,8 +375,8 @@ ; CHECK: declare x86_vectorcallcc void @f.cc80() declare x86_vectorcallcc void @f.x86_vectorcallcc() ; CHECK: declare x86_vectorcallcc void @f.x86_vectorcallcc() -declare cc8191 void @f.cc8191() -; CHECK: declare cc8191 void @f.cc8191() +declare cc1023 void @f.cc1023() +; CHECK: declare cc1023 void @f.cc1023() ; Functions -- ret attrs (Return attributes) declare zeroext i64 @f.zeroext() Index: test/Bitcode/compatibility-3.7.ll =================================================================== --- test/Bitcode/compatibility-3.7.ll +++ test/Bitcode/compatibility-3.7.ll @@ -375,8 +375,8 @@ ; CHECK: declare x86_vectorcallcc void @f.cc80() declare x86_vectorcallcc void @f.x86_vectorcallcc() ; CHECK: declare x86_vectorcallcc void @f.x86_vectorcallcc() -declare cc8191 void @f.cc8191() -; CHECK: declare cc8191 void @f.cc8191() +declare cc1023 void @f.cc1023() +; CHECK: declare cc1023 void @f.cc1023() ; Functions -- ret attrs (Return attributes) declare zeroext i64 @f.zeroext() Index: test/Bitcode/compatibility.ll =================================================================== --- test/Bitcode/compatibility.ll +++ test/Bitcode/compatibility.ll @@ -377,8 +377,8 @@ ; CHECK: declare x86_vectorcallcc void @f.cc80() declare x86_vectorcallcc void @f.x86_vectorcallcc() ; CHECK: declare x86_vectorcallcc void @f.x86_vectorcallcc() -declare cc8191 void @f.cc8191() -; CHECK: declare cc8191 void @f.cc8191() +declare cc1023 void @f.cc1023() +; CHECK: declare cc1023 void @f.cc1023() ; Functions -- ret attrs (Return attributes) declare zeroext i64 @f.zeroext() Index: test/Bitcode/tailcall.ll =================================================================== --- test/Bitcode/tailcall.ll +++ test/Bitcode/tailcall.ll @@ -3,16 +3,16 @@ ; Check that musttail and tail roundtrip. -declare cc8191 void @t1_callee() -define cc8191 void @t1() { -; CHECK: tail call cc8191 void @t1_callee() - tail call cc8191 void @t1_callee() +declare cc1023 void @t1_callee() +define cc1023 void @t1() { +; CHECK: tail call cc1023 void @t1_callee() + tail call cc1023 void @t1_callee() ret void } -declare cc8191 void @t2_callee() -define cc8191 void @t2() { -; CHECK: musttail call cc8191 void @t2_callee() - musttail call cc8191 void @t2_callee() +declare cc1023 void @t2_callee() +define cc1023 void @t2() { +; CHECK: musttail call cc1023 void @t2_callee() + musttail call cc1023 void @t2_callee() ret void }