diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -3799,6 +3799,41 @@ Finally, some targets may provide defined semantics when using the value as the operand to an inline assembly, but that is target specific. +.. _dso_local_equivalent: + +DSO Local Equivalent +-------------------- + +``dso_local_equivalent @func`` + +A '``dso_local_equivalent``' constant represents a function which is +functionally equivalent to a given function, but is always defined in the +current linkage unit. The resulting pointer has the same type as the underlying +function. The resulting pointer is permitted, but not required, to be different +from a pointer to the function, and it may have different values in different +translation units. + +The target function may not have ``extern_weak`` linkage. + +``dso_local_equivalent`` can be implemented as such: + +- If the function has local linkage, hidden visibility, or is + ``dso_local``, ``dso_local_equivalent`` can be implemented as simply a pointer + to the function. +- ``dso_local_equivalent`` can be implemented with a stub that tail-calls the + function. Many targets support relocations that resolve at link time to either + a function or a stub for it, depending on if the function is defined within the + linkage unit; LLVM will use this when available. (This is commonly called a + "PLT stub".) On other targets, the stub may need to be emitted explicitly. + +This can be used wherever a ``dso_local`` instance of a function is needed without +needing to explicitly make the original function ``dso_local``. An instance where +this can be used is for static offset calculations between a function and some other +``dso_local`` symbol. This is especially useful for the Relative VTables C++ ABI, +where dynamic relocations for function pointers in VTables can be replaced with +static relocations for offsets between the VTable and virtual functions which +may not be ``dso_local``. + .. _constantexprs: Constant Expressions diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -263,6 +263,7 @@ LLVMGlobalIFuncValueKind, LLVMGlobalVariableValueKind, LLVMBlockAddressValueKind, + LLVMDSOLocalEquivalentValueKind, LLVMConstantExprValueKind, LLVMConstantArrayValueKind, LLVMConstantStructValueKind, diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h --- a/llvm/include/llvm/Analysis/ConstantFolding.h +++ b/llvm/include/llvm/Analysis/ConstantFolding.h @@ -25,6 +25,7 @@ class CallBase; class Constant; class ConstantExpr; +class DSOLocalEquivalent; class DataLayout; class Function; class GlobalValue; @@ -34,8 +35,11 @@ /// If this constant is a constant offset from a global, return the global and /// the constant. Because of constantexprs, this function is recursive. +/// If the global is part of a dso_local_equivalent constant, return it through +/// `Equiv` if it is provided. bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, APInt &Offset, - const DataLayout &DL); + const DataLayout &DL, + DSOLocalEquivalent **DSOEquiv = nullptr); /// ConstantFoldInstruction - Try to constant fold the specified instruction. /// If successful, the constant result is returned, if not, null is returned. diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -416,6 +416,10 @@ /// evaluation. ModulePass *createPreISelIntrinsicLoweringPass(); + /// This pass lowers references to dso_local_equivalent to generic IR + /// implementations on targets that don't have a native lowering of them. + ModulePass *createDSOLocalEquivalentLoweringPass(const TargetMachine *TM); + /// GlobalMerge - This pass merges internal (by default) globals into structs /// to enable reuse of a base pointer by indexed addressing modes. /// It can also be configured to focus on size optimizations only. diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h --- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -38,7 +38,7 @@ const TargetMachine *TM = nullptr; public: - TargetLoweringObjectFileELF() = default; + TargetLoweringObjectFileELF(); ~TargetLoweringObjectFileELF() override = default; void Initialize(MCContext &Ctx, const TargetMachine &TM) override; @@ -97,6 +97,9 @@ const GlobalValue *RHS, const TargetMachine &TM) const override; + const MCExpr *lowerDSOLocalEquivalent(const DSOLocalEquivalent *Equiv, + const TargetMachine &TM) const override; + MCSection *getSectionForCommandLines() const override; }; diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -888,6 +888,42 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BlockAddress, Value) +/// Wrapper for a function that represents a value that +/// functionally represents the original function. This can be a function, +/// global alias to a function, or an ifunc. +class DSOLocalEquivalent final : public Constant { + friend class Constant; + + DSOLocalEquivalent(GlobalValue *GV); + + void *operator new(size_t s) { return User::operator new(s, 1); } + + void destroyConstantImpl(); + Value *handleOperandChangeImpl(Value *From, Value *To); + +public: + /// Return a DSOLocalEquivalent for the specified global value. + static DSOLocalEquivalent *get(GlobalValue *GV); + + /// Transparently provide more efficient getOperand methods. + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + GlobalValue *getGlobalValue() const { + return cast(Op<0>().get()); + } + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Value *V) { + return V->getValueID() == DSOLocalEquivalentVal; + } +}; + +template <> +struct OperandTraits + : public FixedNumOperandTraits {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(DSOLocalEquivalent, Value) + //===----------------------------------------------------------------------===// /// A constant value that is initialized with an expression using /// other constant values. diff --git a/llvm/include/llvm/IR/Value.def b/llvm/include/llvm/IR/Value.def --- a/llvm/include/llvm/IR/Value.def +++ b/llvm/include/llvm/IR/Value.def @@ -65,6 +65,7 @@ HANDLE_GLOBAL_VALUE(GlobalVariable) HANDLE_CONSTANT(BlockAddress) HANDLE_CONSTANT(ConstantExpr) +HANDLE_CONSTANT(DSOLocalEquivalent) // ConstantAggregate. HANDLE_CONSTANT(ConstantArray) diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h --- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h +++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h @@ -38,6 +38,7 @@ class SectionKind; class StringRef; class TargetMachine; +class DSOLocalEquivalent; class TargetLoweringObjectFile : public MCObjectFileInfo { /// Name-mangler for global names. @@ -47,6 +48,7 @@ bool SupportIndirectSymViaGOTPCRel = false; bool SupportGOTPCRelWithOffset = true; bool SupportDebugThreadLocalLocation = true; + bool SupportDSOLocalEquivalentLowering = false; /// PersonalityEncoding, LSDAEncoding, TTypeEncoding - Some encoding values /// for EH. @@ -180,6 +182,17 @@ return nullptr; } + /// Target supports a native lowering of a dso_local_equivalent constant + /// without needing to replace it with equivalent IR. + bool supportDSOLocalEquivalentLowering() const { + return SupportDSOLocalEquivalentLowering; + } + + virtual const MCExpr *lowerDSOLocalEquivalent(const DSOLocalEquivalent *Equiv, + const TargetMachine &TM) const { + return nullptr; + } + /// Target supports replacing a data "PC"-relative access to a symbol /// through another symbol, by accessing the later via a GOT entry instead? bool supportIndirectSymViaGOTPCRel() const { diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -295,7 +295,11 @@ /// If this constant is a constant offset from a global, return the global and /// the constant. Because of constantexprs, this function is recursive. bool llvm::IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, - APInt &Offset, const DataLayout &DL) { + APInt &Offset, const DataLayout &DL, + DSOLocalEquivalent **DSOEquiv) { + if (DSOEquiv) + *DSOEquiv = nullptr; + // Trivial case, constant is the global. if ((GV = dyn_cast(C))) { unsigned BitWidth = DL.getIndexTypeSizeInBits(GV->getType()); @@ -303,6 +307,15 @@ return true; } + if (auto *FoundDSOEquiv = dyn_cast(C)) { + if (DSOEquiv) + *DSOEquiv = FoundDSOEquiv; + GV = FoundDSOEquiv->getGlobalValue(); + unsigned BitWidth = DL.getIndexTypeSizeInBits(GV->getType()); + Offset = APInt(BitWidth, 0); + return true; + } + // Otherwise, if this isn't a constant expr, bail out. auto *CE = dyn_cast(C); if (!CE) return false; @@ -310,7 +323,8 @@ // Look through ptr->int and ptr->ptr casts. if (CE->getOpcode() == Instruction::PtrToInt || CE->getOpcode() == Instruction::BitCast) - return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, DL); + return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, DL, + DSOEquiv); // i32* getelementptr ([5 x i32]* @a, i32 0, i32 5) auto *GEP = dyn_cast(CE); @@ -321,7 +335,8 @@ APInt TmpOffset(BitWidth, 0); // If the base isn't a global+constant, we aren't either. - if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, TmpOffset, DL)) + if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, TmpOffset, DL, + DSOEquiv)) return false; // Otherwise, add any offset that our operands provide. diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -725,6 +725,7 @@ KEYWORD(vscale); KEYWORD(x); KEYWORD(blockaddress); + KEYWORD(dso_local_equivalent); // Metadata types. KEYWORD(distinct); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -3496,6 +3496,39 @@ return false; } + case lltok::kw_dso_local_equivalent: { + // ValID ::= 'dso_local_equivalent' @foo + Lex.Lex(); + + ValID Fn; + + if (parseValID(Fn)) + return true; + + if (Fn.Kind != ValID::t_GlobalID && Fn.Kind != ValID::t_GlobalName) + return error(Fn.Loc, + "expected global value name in dso_local_equivalent"); + + // Try to find the function (but skip it if it's forward-referenced). + GlobalValue *GV = nullptr; + if (Fn.Kind == ValID::t_GlobalID) { + if (Fn.UIntVal < NumberedVals.size()) + GV = NumberedVals[Fn.UIntVal]; + } else if (!ForwardRefVals.count(Fn.StrVal)) { + GV = M->getNamedValue(Fn.StrVal); + } + + assert(GV && "Could not find a corresponding global variable"); + + if (!GV->getValueType()->isFunctionTy()) + return error(Fn.Loc, "expected a function, alias to function, or ifunc " + "in dso_local_equivalent"); + + ID.ConstantVal = DSOLocalEquivalent::get(GV); + ID.Kind = ValID::t_Constant; + return false; + } + case lltok::kw_trunc: case lltok::kw_zext: case lltok::kw_sext: diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h --- a/llvm/lib/AsmParser/LLToken.h +++ b/llvm/lib/AsmParser/LLToken.h @@ -361,6 +361,7 @@ kw_extractvalue, kw_insertvalue, kw_blockaddress, + kw_dso_local_equivalent, kw_freeze, diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2326,6 +2326,9 @@ if (const BlockAddress *BA = dyn_cast(CV)) return MCSymbolRefExpr::create(GetBlockAddressSymbol(BA), Ctx); + if (const auto *Equiv = dyn_cast(CV)) + return getObjFileLowering().lowerDSOLocalEquivalent(Equiv, TM); + const ConstantExpr *CE = dyn_cast(CV); if (!CE) { llvm_unreachable("Unknown constant value to lower!"); @@ -2422,18 +2425,25 @@ case Instruction::Sub: { GlobalValue *LHSGV; APInt LHSOffset; + DSOLocalEquivalent *DSOEquiv; if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHSGV, LHSOffset, - getDataLayout())) { + getDataLayout(), &DSOEquiv)) { GlobalValue *RHSGV; APInt RHSOffset; if (IsConstantOffsetFromGlobal(CE->getOperand(1), RHSGV, RHSOffset, getDataLayout())) { const MCExpr *RelocExpr = getObjFileLowering().lowerRelativeReference(LHSGV, RHSGV, TM); - if (!RelocExpr) + if (!RelocExpr) { + const MCExpr *LHSExpr = + MCSymbolRefExpr::create(getSymbol(LHSGV), Ctx); + if (DSOEquiv && + getObjFileLowering().supportDSOLocalEquivalentLowering()) + LHSExpr = + getObjFileLowering().lowerDSOLocalEquivalent(DSOEquiv, TM); RelocExpr = MCBinaryExpr::createSub( - MCSymbolRefExpr::create(getSymbol(LHSGV), Ctx), - MCSymbolRefExpr::create(getSymbol(RHSGV), Ctx), Ctx); + LHSExpr, MCSymbolRefExpr::create(getSymbol(RHSGV), Ctx), Ctx); + } int64_t Addend = (LHSOffset - RHSOffset).getSExtValue(); if (Addend != 0) RelocExpr = MCBinaryExpr::createAdd( diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -20,6 +20,7 @@ DeadMachineInstructionElim.cpp DetectDeadLanes.cpp DFAPacketizer.cpp + DSOLocalEquivalentLowering.cpp DwarfEHPrepare.cpp EarlyIfConversion.cpp EdgeBundles.cpp diff --git a/llvm/lib/CodeGen/DSOLocalEquivalentLowering.cpp b/llvm/lib/CodeGen/DSOLocalEquivalentLowering.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/DSOLocalEquivalentLowering.cpp @@ -0,0 +1,143 @@ +//===- DSOLocalEquivalentLowering.cpp - dso_local_equivalent lowering pass ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// If this target cannot lower the dso_local_equivalent pass to an MCExpr, the +// expression is lowered with a generic IR implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/Triple.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/Target/TargetMachine.h" + +namespace llvm { + +static Function *getOrCreateRelativeStub(GlobalValue *GV) { + SmallString<16> StubName(GV->getName()); + StubName.append(".stub"); + + auto &M = *GV->getParent(); + Function *Stub = M.getFunction(StubName); + if (Stub) { + assert(Stub->isDSOLocal() && + "The previous definition of this stub should be dso_local."); + return Stub; + } + + if (const auto *F = dyn_cast(GV)) { + // Propagate function attributes. + Stub = Function::Create(F->getFunctionType(), F->getLinkage(), StubName, M); + Stub->setAttributes(F->getAttributes()); + } else if (isa(GV)) { + Stub = Function::Create(cast(GV->getValueType()), + GV->getLinkage(), StubName, M); + } else { + llvm_unreachable("Expected the global value to be a function, ifunc, or " + "alias to a function or ifunc."); + } + + Stub->setDSOLocal(true); + Stub->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + if (!Stub->hasLocalLinkage()) { + Stub->setVisibility(GlobalValue::HiddenVisibility); + if (Triple(M.getTargetTriple()).supportsCOMDAT()) + Stub->setComdat(M.getOrInsertComdat(StubName)); + } + + // Fill the Stub with a tail call that will be optimized. + BasicBlock *block = BasicBlock::Create(M.getContext(), "entry", Stub); + IRBuilder<> block_builder(block); + SmallVector args; + for (auto &arg : Stub->args()) + args.push_back(&arg); + + CallInst *call; + if (auto *F = dyn_cast(GV)) { + call = block_builder.CreateCall(F, args); + call->setAttributes(F->getAttributes()); + } else { + call = block_builder.CreateCall(cast(GV->getValueType()), GV, + args); + } + + call->setTailCall(); + if (call->getType()->isVoidTy()) + block_builder.CreateRetVoid(); + else + block_builder.CreateRet(call); + + return Stub; +} + +static Value *lowerDSOLocalEquivalent(DSOLocalEquivalent *Equiv) { + GlobalValue *GV = Equiv->getGlobalValue(); + if (GV->isDSOLocal() || GV->isImplicitDSOLocal()) + return GV; + return getOrCreateRelativeStub(GV); +} + +static bool checkValue(Value *Val) { + if (auto *Equiv = dyn_cast(Val)) { + Value *NewVal = lowerDSOLocalEquivalent(Equiv); + Equiv->replaceAllUsesWith(NewVal); + return true; + } + return false; +} + +static bool checkFunction(Function &F) { + bool Changed = false; + for (BasicBlock &BB : F) { + for (auto BBI = BB.begin(); BBI != BB.end(); ++BBI) { + for (Instruction &I : BB) { + for (auto ValIt = I.value_op_begin(); ValIt != I.value_op_end(); + ++ValIt) + Changed |= checkValue(*ValIt); + } + } + } + return Changed; +} + +namespace { + +class DSOLocalEquivalentLoweringLegacyPass : public ModulePass { +public: + static char ID; + const TargetMachine *TM; + + DSOLocalEquivalentLoweringLegacyPass(const TargetMachine *TM) + : ModulePass(ID), TM(TM) { + assert(!TM->getObjFileLowering()->supportDSOLocalEquivalentLowering() && + "This pass should not need to be run on targets that can natively " + "lower dso_local_equivalent"); + } + + bool runOnModule(Module &M) override { + bool Changed = false; + for (GlobalVariable &GV : M.globals()) { + if (GV.hasInitializer()) + Changed |= checkValue(GV.getInitializer()); + } + for (Function &F : M) + Changed |= checkFunction(F); + return Changed; + } +}; + +} // namespace + +char DSOLocalEquivalentLoweringLegacyPass::ID; + +ModulePass *createDSOLocalEquivalentLoweringPass(const TargetMachine *TM) { + return new DSOLocalEquivalentLoweringLegacyPass(TM); +} + +} // namespace llvm diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1514,6 +1514,9 @@ if (const BlockAddress *BA = dyn_cast(C)) return DAG.getBlockAddress(BA, VT); + if (const auto *Equiv = dyn_cast(C)) + return getValue(Equiv->getGlobalValue()); + VectorType *VecTy = cast(V->getType()); // Now that we know the number and type of the elements, get that number of diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -105,6 +105,11 @@ // ELF //===----------------------------------------------------------------------===// +TargetLoweringObjectFileELF::TargetLoweringObjectFileELF() + : TargetLoweringObjectFile() { + SupportDSOLocalEquivalentLowering = true; +} + void TargetLoweringObjectFileELF::Initialize(MCContext &Ctx, const TargetMachine &TgtM) { TargetLoweringObjectFile::Initialize(Ctx, TgtM); @@ -1007,6 +1012,20 @@ MCSymbolRefExpr::create(TM.getSymbol(RHS), getContext()), getContext()); } +const MCExpr *TargetLoweringObjectFileELF::lowerDSOLocalEquivalent( + const DSOLocalEquivalent *Equiv, const TargetMachine &TM) const { + assert(supportDSOLocalEquivalentLowering()); + + const auto *GV = Equiv->getGlobalValue(); + + // A PLT entry is not needed for dso_local globals. + if (GV->isDSOLocal() || GV->isImplicitDSOLocal()) + return MCSymbolRefExpr::create(TM.getSymbol(GV), getContext()); + + return MCSymbolRefExpr::create(TM.getSymbol(GV), PLTRelativeVariantKind, + getContext()); +} + MCSection *TargetLoweringObjectFileELF::getSectionForCommandLines() const { // Use ".GCC.command.line" since this feature is to support clang's // -frecord-gcc-switches which in turn attempts to mimic GCC's switch of the diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -41,6 +41,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/SaveAndRestore.h" #include "llvm/Support/Threading.h" +#include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils.h" @@ -882,6 +883,8 @@ bool TargetPassConfig::addISelPasses() { if (TM->useEmulatedTLS()) addPass(createLowerEmuTLSPass()); + if (!TM->getObjFileLowering()->supportDSOLocalEquivalentLowering()) + addPass(createDSOLocalEquivalentLoweringPass(TM)); addPass(createPreISelIntrinsicLoweringPass()); PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis())); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1455,6 +1455,13 @@ return; } + if (const auto *Equiv = dyn_cast(CV)) { + Out << "dso_local_equivalent "; + WriteAsOperandInternal(Out, Equiv->getGlobalValue(), &TypePrinter, Machine, + Context); + return; + } + if (const ConstantArray *CA = dyn_cast(CV)) { Type *ETy = CA->getType()->getElementType(); Out << '['; diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -509,6 +509,9 @@ case Constant::BlockAddressVal: delete static_cast(C); break; + case Constant::DSOLocalEquivalentVal: + delete static_cast(C); + break; case Constant::UndefValueVal: delete static_cast(C); break; @@ -654,10 +657,17 @@ return false; // Relative pointers do not need to be dynamically relocated. - if (auto *LHSGV = dyn_cast(LHSOp0->stripPointerCasts())) - if (auto *RHSGV = dyn_cast(RHSOp0->stripPointerCasts())) + if (auto *RHSGV = + dyn_cast(RHSOp0->stripInBoundsConstantOffsets())) { + auto *LHS = LHSOp0->stripInBoundsConstantOffsets(); + if (auto *LHSGV = dyn_cast(LHS)) { if (LHSGV->isDSOLocal() && RHSGV->isDSOLocal()) return false; + } else if (isa(LHS)) { + if (RHSGV->isDSOLocal()) + return false; + } + } } } } @@ -1763,6 +1773,54 @@ return nullptr; } +DSOLocalEquivalent *DSOLocalEquivalent::get(GlobalValue *GV) { + DSOLocalEquivalent *&Equiv = GV->getContext().pImpl->DSOLocalEquivalents[GV]; + if (!Equiv) + Equiv = new DSOLocalEquivalent(GV); + + assert(Equiv->getGlobalValue() == GV && + "DSOLocalFunction does not match the expected global value"); + return Equiv; +} + +DSOLocalEquivalent::DSOLocalEquivalent(GlobalValue *GV) + : Constant(GV->getType(), Value::DSOLocalEquivalentVal, &Op<0>(), 1) { + setOperand(0, GV); +} + +/// Remove the constant from the constant table. +void DSOLocalEquivalent::destroyConstantImpl() { + const GlobalValue *GV = getGlobalValue(); + GV->getContext().pImpl->DSOLocalEquivalents.erase(GV); +} + +Value *DSOLocalEquivalent::handleOperandChangeImpl(Value *From, Value *To) { + assert(From == getGlobalValue() && "Changing value does not match operand."); + assert(To->getType() == getType() && "Mismatched types"); + assert(isa(To) && "Can only replace the operands with a constant"); + + // The replacement is with another global value. + if (const auto *ToObj = dyn_cast(To)) { + DSOLocalEquivalent *&NewEquiv = + getContext().pImpl->DSOLocalEquivalents[ToObj]; + if (NewEquiv) + return NewEquiv; + } + + // The replacement could be a bitcast or an alias to another function. We can + // replace it with a bitcast to the dso_local_equivalent of that function. + auto *Func = cast(To->stripPointerCastsAndAliases()); + DSOLocalEquivalent *&NewEquiv = getContext().pImpl->DSOLocalEquivalents[Func]; + if (NewEquiv) + return llvm::ConstantExpr::getBitCast(NewEquiv, getType()); + + // Replace this with the new one. + getContext().pImpl->DSOLocalEquivalents.erase(getGlobalValue()); + NewEquiv = this; + setOperand(0, Func); + return nullptr; +} + //---- ConstantExpr::get() implementations. // diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -1380,6 +1380,9 @@ DenseMap, BlockAddress *> BlockAddresses; + + DenseMap DSOLocalEquivalents; + ConstantUniqueMap ExprConstants; ConstantUniqueMap InlineAsms; diff --git a/llvm/test/CodeGen/X86/dso_local_equivalent.ll b/llvm/test/CodeGen/X86/dso_local_equivalent.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/dso_local_equivalent.ll @@ -0,0 +1,156 @@ +; RUN: llc -mtriple=x86_64-linux-gnu -relocation-model=pic -data-sections -o - %s | FileCheck %s -check-prefixes=PLT,CHECK + +; Check that stubs are emitted for targets that do not have a native lowering for dso_local_equivalent. +; RUN: llc -mtriple=x86_64-apple-darwin -relocation-model=pic -data-sections -o - %s | FileCheck %s -check-prefixes=NO-PLT,CHECK + +; Check win32 (which uses COFF) to cover the case where stubs will need to be emitted, but comdats are supported and should be added to the stubs. +; RUN: llc -mtriple=i386-pc-win32 -relocation-model=pic -data-sections -o - %s | FileCheck %s -check-prefixes=NO-PLT-COMDAT + +; Just ensure that we can write to an object file without error. +; RUN: llc -filetype=obj -mtriple=x86_64-linux-gnu -relocation-model=pic -data-sections -o - %s +; RUN: llc -filetype=obj -mtriple=x86_64-apple-darwin -relocation-model=pic -data-sections -o - %s +; RUN: llc -filetype=obj -mtriple=i386-pc-win32 -relocation-model=pic -data-sections -o - %s + +declare void @extern_func() + +; CHECK: call_extern_func: +; PLT: callq extern_func@PLT +; NO-PLT: callq _extern_func.stub{{$}} +define void @call_extern_func() { + call void dso_local_equivalent @extern_func() + ret void +} + +declare hidden void @hidden_func() +declare protected void @protected_func() +declare dso_local void @dso_local_func() +define internal void @internal_func() { +entry: + ret void +} +define private void @private_func() { +entry: + ret void +} + +; CHECK: call_hidden_func: +; PLT: callq hidden_func{{$}} +; NO-PLT: callq _hidden_func{{$}} +define void @call_hidden_func() { + call void dso_local_equivalent @hidden_func() + ret void +} + +; CHECK: call_protected_func: +; PLT: callq protected_func{{$}} +; NO-PLT: callq _protected_func{{$}} +define void @call_protected_func() { + call void dso_local_equivalent @protected_func() + ret void +} + +; CHECK: call_dso_local_func: +; PLT: callq dso_local_func{{$}} +; NO-PLT: callq _dso_local_func{{$}} +define void @call_dso_local_func() { + call void dso_local_equivalent @dso_local_func() + ret void +} + +; CHECK: call_internal_func: +; PLT: callq internal_func{{$}} +; NO-PLT: callq _internal_func{{$}} +define void @call_internal_func() { + call void dso_local_equivalent @internal_func() + ret void +} + +; CHECK: call_private_func: +; PLT: callq .Lprivate_func{{$}} +; NO-PLT: callq l_private_func{{$}} +define void @call_private_func() { + call void dso_local_equivalent @private_func() + ret void +} + +define void @aliasee_func() { +entry: + ret void +} + +@alias_func = alias void (), void ()* @aliasee_func +@dso_local_alias_func = dso_local alias void (), void ()* @aliasee_func + +; CHECK: call_alias_func: +; PLT: callq alias_func@PLT +; NO-PLT: callq _alias_func.stub{{$}} +define void @call_alias_func() { + call void dso_local_equivalent @alias_func() + ret void +} + +; CHECK: call_dso_local_alias_func: +; PLT: callq .Ldso_local_alias_func$local{{$}} +; NO-PLT: callq _dso_local_alias_func{{$}} +define void @call_dso_local_alias_func() { + call void dso_local_equivalent @dso_local_alias_func() + ret void +} + +@ifunc_func = ifunc void (), i64 ()* @resolver +@dso_local_ifunc_func = dso_local ifunc void (), i64 ()* @resolver + +define internal i64 @resolver() { +entry: + ret i64 0 +} + +; If an ifunc is not dso_local already, then we should still emit a stub for it +; to ensure it will be dso_local. +; CHECK: call_ifunc_func: +; PLT: callq ifunc_func@PLT +; NO-PLT: callq _ifunc_func.stub{{$}} +define void @call_ifunc_func() { + call void dso_local_equivalent @ifunc_func() + ret void +} + +; CHECK: call_dso_local_ifunc_func: +; PLT: callq dso_local_ifunc_func{{$}} +; NO-PLT: callq _dso_local_ifunc_func{{$}} +define void @call_dso_local_ifunc_func() { + call void dso_local_equivalent @dso_local_ifunc_func() + ret void +} + +; NO-PLT: _extern_func.stub: +; NO-PLT: jmp _extern_func +; NO-PLT-NOT: _hidden_func.stub +; NO-PLT-NOT: _protected_func.stub +; NO-PLT-NOT: _dso_local_func.stub +; NO-PLT-NOT: _internal_func.stub +; NO-PLT-NOT: l_private_func.stub +; NO-PLT: _alias_func.stub: +; NO-PLT: jmp _alias_func +; NO-PLT-NOT: _dso_local_alias_func.stub: +; NO-PLT: _ifunc_func.stub: +; NO-PLT: jmp _ifunc_func +; NO-PLT-NOT: _dso_local_ifunc_func.stub: + +; Check comdat sections are emitted for stubs. +; NO-PLT-COMDAT: .section .text,"xr",discard,_extern_func.stub +; NO-PLT-COMDAT: .globl _extern_func.stub +; NO-PLT-COMDAT: _extern_func.stub: +; NO-PLT-COMDAT-NOT: _hidden_func.stub +; NO-PLT-COMDAT-NOT: _protected_func.stub +; NO-PLT-COMDAT-NOT: _dso_local_func.stub +; NO-PLT-COMDAT-NOT: _internal_func.stub +; NO-PLT-COMDAT-NOT: l_private_func.stub +; NO-PLT-COMDAT: .section .text,"xr",discard,_alias_func.stub +; NO-PLT-COMDAT: .globl _alias_func.stub +; NO-PLT-COMDAT: _alias_func.stub: +; NO-PLT-COMDAT-NOT: _dso_local_alias_func.stub: +; NO-PLT-COMDAT: .section .text,"xr",discard,_ifunc_func.stub +; NO-PLT-COMDAT: .globl _ifunc_func.stub +; NO-PLT-COMDAT:_ifunc_func.stub: +; NO-PLT-COMDAT-NOT: _dso_local_ifunc_func.stub: diff --git a/llvm/test/CodeGen/X86/relptr-rodata.ll b/llvm/test/CodeGen/X86/relptr-rodata.ll --- a/llvm/test/CodeGen/X86/relptr-rodata.ll +++ b/llvm/test/CodeGen/X86/relptr-rodata.ll @@ -19,3 +19,27 @@ ; CHECK: relro2: ; CHECK: .long hidden-relro2 @relro2 = constant i32 trunc (i64 sub (i64 ptrtoint (i8* @hidden to i64), i64 ptrtoint (i32* @relro2 to i64)) to i32) + +; CHECK: .section .rodata.cst8 +; CHECK-NEXT: .globl obj +; CHECK: obj: +; CHECK: .long 0 +; CHECK: .long (hidden_func-obj)-4 + +declare hidden void @hidden_func() + +; Ensure that inbound GEPs with constant offsets are also resolved. +@obj = dso_local unnamed_addr constant { { i32, i32 } } { + { i32, i32 } { + i32 0, + i32 trunc (i64 sub (i64 ptrtoint (void ()* dso_local_equivalent @hidden_func to i64), i64 ptrtoint (i32* getelementptr inbounds ({ { i32, i32 } }, { { i32, i32 } }* @obj, i32 0, i32 0, i32 1) to i64)) to i32) + } }, align 4 + +; CHECK: .section .rodata.rodata2 +; CHECK-NEXT: .globl rodata2 +; CHECK: rodata2: +; CHECK: .long extern_func@PLT-rodata2 + +declare void @extern_func() + +@rodata2 = dso_local constant i32 trunc (i64 sub (i64 ptrtoint (void ()* dso_local_equivalent @extern_func to i64), i64 ptrtoint (i32* @rodata2 to i64)) to i32)