diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -3799,6 +3799,41 @@
Finally, some targets may provide defined semantics when using the value
as the operand to an inline assembly, but that is target specific.
+.. _dso_local_equivalent:
+
+DSO Local Equivalent
+--------------------
+
+``dso_local_equivalent @func``
+
+A '``dso_local_equivalent``' constant represents a function which is
+functionally equivalent to a given function, but is always defined in the
+current linkage unit. The resulting pointer has the same type as the underlying
+function. The resulting pointer is permitted, but not required, to be different
+from a pointer to the function, and it may have different values in different
+translation units.
+
+The target function may not have ``extern_weak`` linkage.
+
+``dso_local_equivalent`` can be implemented as such:
+
+- If the function has local linkage, hidden visibility, or is
+ ``dso_local``, ``dso_local_equivalent`` can be implemented as simply a pointer
+ to the function.
+- ``dso_local_equivalent`` can be implemented with a stub that tail-calls the
+ function. Many targets support relocations that resolve at link time to either
+ a function or a stub for it, depending on if the function is defined within the
+ linkage unit; LLVM will use this when available. (This is commonly called a
+ "PLT stub".) On other targets, the stub may need to be emitted explicitly.
+
+This can be used wherever a ``dso_local`` instance of a function is needed without
+needing to explicitly make the original function ``dso_local``. An instance where
+this can be used is for static offset calculations between a function and some other
+``dso_local`` symbol. This is especially useful for the Relative VTables C++ ABI,
+where dynamic relocations for function pointers in VTables can be replaced with
+static relocations for offsets between the VTable and virtual functions which
+may not be ``dso_local``.
+
.. _constantexprs:
Constant Expressions
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -263,6 +263,7 @@
LLVMGlobalIFuncValueKind,
LLVMGlobalVariableValueKind,
LLVMBlockAddressValueKind,
+ LLVMDSOLocalEquivalentValueKind,
LLVMConstantExprValueKind,
LLVMConstantArrayValueKind,
LLVMConstantStructValueKind,
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -25,6 +25,7 @@
class CallBase;
class Constant;
class ConstantExpr;
+class DSOLocalEquivalent;
class DataLayout;
class Function;
class GlobalValue;
@@ -34,8 +35,11 @@
/// If this constant is a constant offset from a global, return the global and
/// the constant. Because of constantexprs, this function is recursive.
+/// If the global is part of a dso_local_equivalent constant, return it through
+/// `Equiv` if it is provided.
bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, APInt &Offset,
- const DataLayout &DL);
+ const DataLayout &DL,
+ DSOLocalEquivalent **DSOEquiv = nullptr);
/// ConstantFoldInstruction - Try to constant fold the specified instruction.
/// If successful, the constant result is returned, if not, null is returned.
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -416,6 +416,10 @@
/// evaluation.
ModulePass *createPreISelIntrinsicLoweringPass();
+ /// This pass lowers references to dso_local_equivalent to generic IR
+ /// implementations on targets that don't have a native lowering of them.
+ ModulePass *createDSOLocalEquivalentLoweringPass(const TargetMachine *TM);
+
/// GlobalMerge - This pass merges internal (by default) globals into structs
/// to enable reuse of a base pointer by indexed addressing modes.
/// It can also be configured to focus on size optimizations only.
diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
--- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -38,7 +38,7 @@
const TargetMachine *TM = nullptr;
public:
- TargetLoweringObjectFileELF() = default;
+ TargetLoweringObjectFileELF();
~TargetLoweringObjectFileELF() override = default;
void Initialize(MCContext &Ctx, const TargetMachine &TM) override;
@@ -97,6 +97,9 @@
const GlobalValue *RHS,
const TargetMachine &TM) const override;
+ const MCExpr *lowerDSOLocalEquivalent(const DSOLocalEquivalent *Equiv,
+ const TargetMachine &TM) const override;
+
MCSection *getSectionForCommandLines() const override;
};
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -888,6 +888,42 @@
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BlockAddress, Value)
+/// Wrapper for a function that represents a value that
+/// functionally represents the original function. This can be a function,
+/// global alias to a function, or an ifunc.
+class DSOLocalEquivalent final : public Constant {
+ friend class Constant;
+
+ DSOLocalEquivalent(GlobalValue *GV);
+
+ void *operator new(size_t s) { return User::operator new(s, 1); }
+
+ void destroyConstantImpl();
+ Value *handleOperandChangeImpl(Value *From, Value *To);
+
+public:
+ /// Return a DSOLocalEquivalent for the specified global value.
+ static DSOLocalEquivalent *get(GlobalValue *GV);
+
+ /// Transparently provide more efficient getOperand methods.
+ DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+ GlobalValue *getGlobalValue() const {
+ return cast<GlobalValue>(Op<0>().get());
+ }
+
+ /// Methods for support type inquiry through isa, cast, and dyn_cast:
+ static bool classof(const Value *V) {
+ return V->getValueID() == DSOLocalEquivalentVal;
+ }
+};
+
+template <>
+struct OperandTraits<DSOLocalEquivalent>
+ : public FixedNumOperandTraits<DSOLocalEquivalent, 1> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(DSOLocalEquivalent, Value)
+
//===----------------------------------------------------------------------===//
/// A constant value that is initialized with an expression using
/// other constant values.
diff --git a/llvm/include/llvm/IR/Value.def b/llvm/include/llvm/IR/Value.def
--- a/llvm/include/llvm/IR/Value.def
+++ b/llvm/include/llvm/IR/Value.def
@@ -65,6 +65,7 @@
HANDLE_GLOBAL_VALUE(GlobalVariable)
HANDLE_CONSTANT(BlockAddress)
HANDLE_CONSTANT(ConstantExpr)
+HANDLE_CONSTANT(DSOLocalEquivalent)
// ConstantAggregate.
HANDLE_CONSTANT(ConstantArray)
diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
--- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
@@ -38,6 +38,7 @@
class SectionKind;
class StringRef;
class TargetMachine;
+class DSOLocalEquivalent;
class TargetLoweringObjectFile : public MCObjectFileInfo {
/// Name-mangler for global names.
@@ -47,6 +48,7 @@
bool SupportIndirectSymViaGOTPCRel = false;
bool SupportGOTPCRelWithOffset = true;
bool SupportDebugThreadLocalLocation = true;
+ bool SupportDSOLocalEquivalentLowering = false;
/// PersonalityEncoding, LSDAEncoding, TTypeEncoding - Some encoding values
/// for EH.
@@ -180,6 +182,17 @@
return nullptr;
}
+ /// Target supports a native lowering of a dso_local_equivalent constant
+ /// without needing to replace it with equivalent IR.
+ bool supportDSOLocalEquivalentLowering() const {
+ return SupportDSOLocalEquivalentLowering;
+ }
+
+ virtual const MCExpr *lowerDSOLocalEquivalent(const DSOLocalEquivalent *Equiv,
+ const TargetMachine &TM) const {
+ return nullptr;
+ }
+
/// Target supports replacing a data "PC"-relative access to a symbol
/// through another symbol, by accessing the later via a GOT entry instead?
bool supportIndirectSymViaGOTPCRel() const {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -295,7 +295,11 @@
/// If this constant is a constant offset from a global, return the global and
/// the constant. Because of constantexprs, this function is recursive.
bool llvm::IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
- APInt &Offset, const DataLayout &DL) {
+ APInt &Offset, const DataLayout &DL,
+ DSOLocalEquivalent **DSOEquiv) {
+ if (DSOEquiv)
+ *DSOEquiv = nullptr;
+
// Trivial case, constant is the global.
if ((GV = dyn_cast<GlobalValue>(C))) {
unsigned BitWidth = DL.getIndexTypeSizeInBits(GV->getType());
@@ -303,6 +307,15 @@
return true;
}
+ if (auto *FoundDSOEquiv = dyn_cast<DSOLocalEquivalent>(C)) {
+ if (DSOEquiv)
+ *DSOEquiv = FoundDSOEquiv;
+ GV = FoundDSOEquiv->getGlobalValue();
+ unsigned BitWidth = DL.getIndexTypeSizeInBits(GV->getType());
+ Offset = APInt(BitWidth, 0);
+ return true;
+ }
+
// Otherwise, if this isn't a constant expr, bail out.
auto *CE = dyn_cast<ConstantExpr>(C);
if (!CE) return false;
@@ -310,7 +323,8 @@
// Look through ptr->int and ptr->ptr casts.
if (CE->getOpcode() == Instruction::PtrToInt ||
CE->getOpcode() == Instruction::BitCast)
- return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, DL);
+ return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, DL,
+ DSOEquiv);
// i32* getelementptr ([5 x i32]* @a, i32 0, i32 5)
auto *GEP = dyn_cast<GEPOperator>(CE);
@@ -321,7 +335,8 @@
APInt TmpOffset(BitWidth, 0);
// If the base isn't a global+constant, we aren't either.
- if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, TmpOffset, DL))
+ if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, TmpOffset, DL,
+ DSOEquiv))
return false;
// Otherwise, add any offset that our operands provide.
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -725,6 +725,7 @@
KEYWORD(vscale);
KEYWORD(x);
KEYWORD(blockaddress);
+ KEYWORD(dso_local_equivalent);
// Metadata types.
KEYWORD(distinct);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -3496,6 +3496,39 @@
return false;
}
+ case lltok::kw_dso_local_equivalent: {
+ // ValID ::= 'dso_local_equivalent' @foo
+ Lex.Lex();
+
+ ValID Fn;
+
+ if (parseValID(Fn))
+ return true;
+
+ if (Fn.Kind != ValID::t_GlobalID && Fn.Kind != ValID::t_GlobalName)
+ return error(Fn.Loc,
+ "expected global value name in dso_local_equivalent");
+
+ // Try to find the function (but skip it if it's forward-referenced).
+ GlobalValue *GV = nullptr;
+ if (Fn.Kind == ValID::t_GlobalID) {
+ if (Fn.UIntVal < NumberedVals.size())
+ GV = NumberedVals[Fn.UIntVal];
+ } else if (!ForwardRefVals.count(Fn.StrVal)) {
+ GV = M->getNamedValue(Fn.StrVal);
+ }
+
+ assert(GV && "Could not find a corresponding global variable");
+
+ if (!GV->getValueType()->isFunctionTy())
+ return error(Fn.Loc, "expected a function, alias to function, or ifunc "
+ "in dso_local_equivalent");
+
+ ID.ConstantVal = DSOLocalEquivalent::get(GV);
+ ID.Kind = ValID::t_Constant;
+ return false;
+ }
+
case lltok::kw_trunc:
case lltok::kw_zext:
case lltok::kw_sext:
diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h
--- a/llvm/lib/AsmParser/LLToken.h
+++ b/llvm/lib/AsmParser/LLToken.h
@@ -361,6 +361,7 @@
kw_extractvalue,
kw_insertvalue,
kw_blockaddress,
+ kw_dso_local_equivalent,
kw_freeze,
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2326,6 +2326,9 @@
if (const BlockAddress *BA = dyn_cast<BlockAddress>(CV))
return MCSymbolRefExpr::create(GetBlockAddressSymbol(BA), Ctx);
+ if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(CV))
+ return getObjFileLowering().lowerDSOLocalEquivalent(Equiv, TM);
+
const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV);
if (!CE) {
llvm_unreachable("Unknown constant value to lower!");
@@ -2422,18 +2425,25 @@
case Instruction::Sub: {
GlobalValue *LHSGV;
APInt LHSOffset;
+ DSOLocalEquivalent *DSOEquiv;
if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHSGV, LHSOffset,
- getDataLayout())) {
+ getDataLayout(), &DSOEquiv)) {
GlobalValue *RHSGV;
APInt RHSOffset;
if (IsConstantOffsetFromGlobal(CE->getOperand(1), RHSGV, RHSOffset,
getDataLayout())) {
const MCExpr *RelocExpr =
getObjFileLowering().lowerRelativeReference(LHSGV, RHSGV, TM);
- if (!RelocExpr)
+ if (!RelocExpr) {
+ const MCExpr *LHSExpr =
+ MCSymbolRefExpr::create(getSymbol(LHSGV), Ctx);
+ if (DSOEquiv &&
+ getObjFileLowering().supportDSOLocalEquivalentLowering())
+ LHSExpr =
+ getObjFileLowering().lowerDSOLocalEquivalent(DSOEquiv, TM);
RelocExpr = MCBinaryExpr::createSub(
- MCSymbolRefExpr::create(getSymbol(LHSGV), Ctx),
- MCSymbolRefExpr::create(getSymbol(RHSGV), Ctx), Ctx);
+ LHSExpr, MCSymbolRefExpr::create(getSymbol(RHSGV), Ctx), Ctx);
+ }
int64_t Addend = (LHSOffset - RHSOffset).getSExtValue();
if (Addend != 0)
RelocExpr = MCBinaryExpr::createAdd(
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -20,6 +20,7 @@
DeadMachineInstructionElim.cpp
DetectDeadLanes.cpp
DFAPacketizer.cpp
+ DSOLocalEquivalentLowering.cpp
DwarfEHPrepare.cpp
EarlyIfConversion.cpp
EdgeBundles.cpp
diff --git a/llvm/lib/CodeGen/DSOLocalEquivalentLowering.cpp b/llvm/lib/CodeGen/DSOLocalEquivalentLowering.cpp
new file mode 100644
--- /dev/null
+++ b/llvm/lib/CodeGen/DSOLocalEquivalentLowering.cpp
@@ -0,0 +1,143 @@
+//===- DSOLocalEquivalentLowering.cpp - dso_local_equivalent lowering pass ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// If this target cannot lower the dso_local_equivalent pass to an MCExpr, the
+// expression is lowered with a generic IR implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Triple.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/Target/TargetMachine.h"
+
+namespace llvm {
+
+static Function *getOrCreateRelativeStub(GlobalValue *GV) {
+ SmallString<16> StubName(GV->getName());
+ StubName.append(".stub");
+
+ auto &M = *GV->getParent();
+ Function *Stub = M.getFunction(StubName);
+ if (Stub) {
+ assert(Stub->isDSOLocal() &&
+ "The previous definition of this stub should be dso_local.");
+ return Stub;
+ }
+
+ if (const auto *F = dyn_cast<Function>(GV)) {
+ // Propagate function attributes.
+ Stub = Function::Create(F->getFunctionType(), F->getLinkage(), StubName, M);
+ Stub->setAttributes(F->getAttributes());
+ } else if (isa<GlobalIFunc, GlobalAlias>(GV)) {
+ Stub = Function::Create(cast<FunctionType>(GV->getValueType()),
+ GV->getLinkage(), StubName, M);
+ } else {
+ llvm_unreachable("Expected the global value to be a function, ifunc, or "
+ "alias to a function or ifunc.");
+ }
+
+ Stub->setDSOLocal(true);
+ Stub->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+ if (!Stub->hasLocalLinkage()) {
+ Stub->setVisibility(GlobalValue::HiddenVisibility);
+ if (Triple(M.getTargetTriple()).supportsCOMDAT())
+ Stub->setComdat(M.getOrInsertComdat(StubName));
+ }
+
+ // Fill the Stub with a tail call that will be optimized.
+ BasicBlock *block = BasicBlock::Create(M.getContext(), "entry", Stub);
+ IRBuilder<> block_builder(block);
+ SmallVector<Value *, 8> args;
+ for (auto &arg : Stub->args())
+ args.push_back(&arg);
+
+ CallInst *call;
+ if (auto *F = dyn_cast<Function>(GV)) {
+ call = block_builder.CreateCall(F, args);
+ call->setAttributes(F->getAttributes());
+ } else {
+ call = block_builder.CreateCall(cast<FunctionType>(GV->getValueType()), GV,
+ args);
+ }
+
+ call->setTailCall();
+ if (call->getType()->isVoidTy())
+ block_builder.CreateRetVoid();
+ else
+ block_builder.CreateRet(call);
+
+ return Stub;
+}
+
+static Value *lowerDSOLocalEquivalent(DSOLocalEquivalent *Equiv) {
+ GlobalValue *GV = Equiv->getGlobalValue();
+ if (GV->isDSOLocal() || GV->isImplicitDSOLocal())
+ return GV;
+ return getOrCreateRelativeStub(GV);
+}
+
+static bool checkValue(Value *Val) {
+ if (auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val)) {
+ Value *NewVal = lowerDSOLocalEquivalent(Equiv);
+ Equiv->replaceAllUsesWith(NewVal);
+ return true;
+ }
+ return false;
+}
+
+static bool checkFunction(Function &F) {
+ bool Changed = false;
+ for (BasicBlock &BB : F) {
+ for (auto BBI = BB.begin(); BBI != BB.end(); ++BBI) {
+ for (Instruction &I : BB) {
+ for (auto ValIt = I.value_op_begin(); ValIt != I.value_op_end();
+ ++ValIt)
+ Changed |= checkValue(*ValIt);
+ }
+ }
+ }
+ return Changed;
+}
+
+namespace {
+
+class DSOLocalEquivalentLoweringLegacyPass : public ModulePass {
+public:
+ static char ID;
+ const TargetMachine *TM;
+
+ DSOLocalEquivalentLoweringLegacyPass(const TargetMachine *TM)
+ : ModulePass(ID), TM(TM) {
+ assert(!TM->getObjFileLowering()->supportDSOLocalEquivalentLowering() &&
+ "This pass should not need to be run on targets that can natively "
+ "lower dso_local_equivalent");
+ }
+
+ bool runOnModule(Module &M) override {
+ bool Changed = false;
+ for (GlobalVariable &GV : M.globals()) {
+ if (GV.hasInitializer())
+ Changed |= checkValue(GV.getInitializer());
+ }
+ for (Function &F : M)
+ Changed |= checkFunction(F);
+ return Changed;
+ }
+};
+
+} // namespace
+
+char DSOLocalEquivalentLoweringLegacyPass::ID;
+
+ModulePass *createDSOLocalEquivalentLoweringPass(const TargetMachine *TM) {
+ return new DSOLocalEquivalentLoweringLegacyPass(TM);
+}
+
+} // namespace llvm
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1514,6 +1514,9 @@
if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
return DAG.getBlockAddress(BA, VT);
+ if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
+ return getValue(Equiv->getGlobalValue());
+
VectorType *VecTy = cast<VectorType>(V->getType());
// Now that we know the number and type of the elements, get that number of
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -105,6 +105,11 @@
// ELF
//===----------------------------------------------------------------------===//
+TargetLoweringObjectFileELF::TargetLoweringObjectFileELF()
+ : TargetLoweringObjectFile() {
+ SupportDSOLocalEquivalentLowering = true;
+}
+
void TargetLoweringObjectFileELF::Initialize(MCContext &Ctx,
const TargetMachine &TgtM) {
TargetLoweringObjectFile::Initialize(Ctx, TgtM);
@@ -1007,6 +1012,20 @@
MCSymbolRefExpr::create(TM.getSymbol(RHS), getContext()), getContext());
}
+const MCExpr *TargetLoweringObjectFileELF::lowerDSOLocalEquivalent(
+ const DSOLocalEquivalent *Equiv, const TargetMachine &TM) const {
+ assert(supportDSOLocalEquivalentLowering());
+
+ const auto *GV = Equiv->getGlobalValue();
+
+ // A PLT entry is not needed for dso_local globals.
+ if (GV->isDSOLocal() || GV->isImplicitDSOLocal())
+ return MCSymbolRefExpr::create(TM.getSymbol(GV), getContext());
+
+ return MCSymbolRefExpr::create(TM.getSymbol(GV), PLTRelativeVariantKind,
+ getContext());
+}
+
MCSection *TargetLoweringObjectFileELF::getSectionForCommandLines() const {
// Use ".GCC.command.line" since this feature is to support clang's
// -frecord-gcc-switches which in turn attempts to mimic GCC's switch of the
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -41,6 +41,7 @@
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Support/Threading.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils.h"
@@ -882,6 +883,8 @@
bool TargetPassConfig::addISelPasses() {
if (TM->useEmulatedTLS())
addPass(createLowerEmuTLSPass());
+ if (!TM->getObjFileLowering()->supportDSOLocalEquivalentLowering())
+ addPass(createDSOLocalEquivalentLoweringPass(TM));
addPass(createPreISelIntrinsicLoweringPass());
PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -1455,6 +1455,13 @@
return;
}
+ if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(CV)) {
+ Out << "dso_local_equivalent ";
+ WriteAsOperandInternal(Out, Equiv->getGlobalValue(), &TypePrinter, Machine,
+ Context);
+ return;
+ }
+
if (const ConstantArray *CA = dyn_cast<ConstantArray>(CV)) {
Type *ETy = CA->getType()->getElementType();
Out << '[';
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -509,6 +509,9 @@
case Constant::BlockAddressVal:
delete static_cast<BlockAddress *>(C);
break;
+ case Constant::DSOLocalEquivalentVal:
+ delete static_cast<DSOLocalEquivalent *>(C);
+ break;
case Constant::UndefValueVal:
delete static_cast<UndefValue *>(C);
break;
@@ -654,10 +657,17 @@
return false;
// Relative pointers do not need to be dynamically relocated.
- if (auto *LHSGV = dyn_cast<GlobalValue>(LHSOp0->stripPointerCasts()))
- if (auto *RHSGV = dyn_cast<GlobalValue>(RHSOp0->stripPointerCasts()))
+ if (auto *RHSGV =
+ dyn_cast<GlobalValue>(RHSOp0->stripInBoundsConstantOffsets())) {
+ auto *LHS = LHSOp0->stripInBoundsConstantOffsets();
+ if (auto *LHSGV = dyn_cast<GlobalValue>(LHS)) {
if (LHSGV->isDSOLocal() && RHSGV->isDSOLocal())
return false;
+ } else if (isa<DSOLocalEquivalent>(LHS)) {
+ if (RHSGV->isDSOLocal())
+ return false;
+ }
+ }
}
}
}
@@ -1763,6 +1773,54 @@
return nullptr;
}
+DSOLocalEquivalent *DSOLocalEquivalent::get(GlobalValue *GV) {
+ DSOLocalEquivalent *&Equiv = GV->getContext().pImpl->DSOLocalEquivalents[GV];
+ if (!Equiv)
+ Equiv = new DSOLocalEquivalent(GV);
+
+ assert(Equiv->getGlobalValue() == GV &&
+ "DSOLocalFunction does not match the expected global value");
+ return Equiv;
+}
+
+DSOLocalEquivalent::DSOLocalEquivalent(GlobalValue *GV)
+ : Constant(GV->getType(), Value::DSOLocalEquivalentVal, &Op<0>(), 1) {
+ setOperand(0, GV);
+}
+
+/// Remove the constant from the constant table.
+void DSOLocalEquivalent::destroyConstantImpl() {
+ const GlobalValue *GV = getGlobalValue();
+ GV->getContext().pImpl->DSOLocalEquivalents.erase(GV);
+}
+
+Value *DSOLocalEquivalent::handleOperandChangeImpl(Value *From, Value *To) {
+ assert(From == getGlobalValue() && "Changing value does not match operand.");
+ assert(To->getType() == getType() && "Mismatched types");
+ assert(isa<Constant>(To) && "Can only replace the operands with a constant");
+
+ // The replacement is with another global value.
+ if (const auto *ToObj = dyn_cast<GlobalValue>(To)) {
+ DSOLocalEquivalent *&NewEquiv =
+ getContext().pImpl->DSOLocalEquivalents[ToObj];
+ if (NewEquiv)
+ return NewEquiv;
+ }
+
+ // The replacement could be a bitcast or an alias to another function. We can
+ // replace it with a bitcast to the dso_local_equivalent of that function.
+ auto *Func = cast<Function>(To->stripPointerCastsAndAliases());
+ DSOLocalEquivalent *&NewEquiv = getContext().pImpl->DSOLocalEquivalents[Func];
+ if (NewEquiv)
+ return llvm::ConstantExpr::getBitCast(NewEquiv, getType());
+
+ // Replace this with the new one.
+ getContext().pImpl->DSOLocalEquivalents.erase(getGlobalValue());
+ NewEquiv = this;
+ setOperand(0, Func);
+ return nullptr;
+}
+
//---- ConstantExpr::get() implementations.
//
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -1380,6 +1380,9 @@
DenseMap<std::pair<const Function *, const BasicBlock *>, BlockAddress *>
BlockAddresses;
+
+ DenseMap<const GlobalValue *, DSOLocalEquivalent *> DSOLocalEquivalents;
+
ConstantUniqueMap<ConstantExpr> ExprConstants;
ConstantUniqueMap<InlineAsm> InlineAsms;
diff --git a/llvm/test/CodeGen/X86/dso_local_equivalent.ll b/llvm/test/CodeGen/X86/dso_local_equivalent.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/X86/dso_local_equivalent.ll
@@ -0,0 +1,156 @@
+; RUN: llc -mtriple=x86_64-linux-gnu -relocation-model=pic -data-sections -o - %s | FileCheck %s -check-prefixes=PLT,CHECK
+
+; Check that stubs are emitted for targets that do not have a native lowering for dso_local_equivalent.
+; RUN: llc -mtriple=x86_64-apple-darwin -relocation-model=pic -data-sections -o - %s | FileCheck %s -check-prefixes=NO-PLT,CHECK
+
+; Check win32 (which uses COFF) to cover the case where stubs will need to be emitted, but comdats are supported and should be added to the stubs.
+; RUN: llc -mtriple=i386-pc-win32 -relocation-model=pic -data-sections -o - %s | FileCheck %s -check-prefixes=NO-PLT-COMDAT
+
+; Just ensure that we can write to an object file without error.
+; RUN: llc -filetype=obj -mtriple=x86_64-linux-gnu -relocation-model=pic -data-sections -o - %s
+; RUN: llc -filetype=obj -mtriple=x86_64-apple-darwin -relocation-model=pic -data-sections -o - %s
+; RUN: llc -filetype=obj -mtriple=i386-pc-win32 -relocation-model=pic -data-sections -o - %s
+
+declare void @extern_func()
+
+; CHECK: call_extern_func:
+; PLT: callq extern_func@PLT
+; NO-PLT: callq _extern_func.stub{{$}}
+define void @call_extern_func() {
+ call void dso_local_equivalent @extern_func()
+ ret void
+}
+
+declare hidden void @hidden_func()
+declare protected void @protected_func()
+declare dso_local void @dso_local_func()
+define internal void @internal_func() {
+entry:
+ ret void
+}
+define private void @private_func() {
+entry:
+ ret void
+}
+
+; CHECK: call_hidden_func:
+; PLT: callq hidden_func{{$}}
+; NO-PLT: callq _hidden_func{{$}}
+define void @call_hidden_func() {
+ call void dso_local_equivalent @hidden_func()
+ ret void
+}
+
+; CHECK: call_protected_func:
+; PLT: callq protected_func{{$}}
+; NO-PLT: callq _protected_func{{$}}
+define void @call_protected_func() {
+ call void dso_local_equivalent @protected_func()
+ ret void
+}
+
+; CHECK: call_dso_local_func:
+; PLT: callq dso_local_func{{$}}
+; NO-PLT: callq _dso_local_func{{$}}
+define void @call_dso_local_func() {
+ call void dso_local_equivalent @dso_local_func()
+ ret void
+}
+
+; CHECK: call_internal_func:
+; PLT: callq internal_func{{$}}
+; NO-PLT: callq _internal_func{{$}}
+define void @call_internal_func() {
+ call void dso_local_equivalent @internal_func()
+ ret void
+}
+
+; CHECK: call_private_func:
+; PLT: callq .Lprivate_func{{$}}
+; NO-PLT: callq l_private_func{{$}}
+define void @call_private_func() {
+ call void dso_local_equivalent @private_func()
+ ret void
+}
+
+define void @aliasee_func() {
+entry:
+ ret void
+}
+
+@alias_func = alias void (), void ()* @aliasee_func
+@dso_local_alias_func = dso_local alias void (), void ()* @aliasee_func
+
+; CHECK: call_alias_func:
+; PLT: callq alias_func@PLT
+; NO-PLT: callq _alias_func.stub{{$}}
+define void @call_alias_func() {
+ call void dso_local_equivalent @alias_func()
+ ret void
+}
+
+; CHECK: call_dso_local_alias_func:
+; PLT: callq .Ldso_local_alias_func$local{{$}}
+; NO-PLT: callq _dso_local_alias_func{{$}}
+define void @call_dso_local_alias_func() {
+ call void dso_local_equivalent @dso_local_alias_func()
+ ret void
+}
+
+@ifunc_func = ifunc void (), i64 ()* @resolver
+@dso_local_ifunc_func = dso_local ifunc void (), i64 ()* @resolver
+
+define internal i64 @resolver() {
+entry:
+ ret i64 0
+}
+
+; If an ifunc is not dso_local already, then we should still emit a stub for it
+; to ensure it will be dso_local.
+; CHECK: call_ifunc_func:
+; PLT: callq ifunc_func@PLT
+; NO-PLT: callq _ifunc_func.stub{{$}}
+define void @call_ifunc_func() {
+ call void dso_local_equivalent @ifunc_func()
+ ret void
+}
+
+; CHECK: call_dso_local_ifunc_func:
+; PLT: callq dso_local_ifunc_func{{$}}
+; NO-PLT: callq _dso_local_ifunc_func{{$}}
+define void @call_dso_local_ifunc_func() {
+ call void dso_local_equivalent @dso_local_ifunc_func()
+ ret void
+}
+
+; NO-PLT: _extern_func.stub:
+; NO-PLT: jmp _extern_func
+; NO-PLT-NOT: _hidden_func.stub
+; NO-PLT-NOT: _protected_func.stub
+; NO-PLT-NOT: _dso_local_func.stub
+; NO-PLT-NOT: _internal_func.stub
+; NO-PLT-NOT: l_private_func.stub
+; NO-PLT: _alias_func.stub:
+; NO-PLT: jmp _alias_func
+; NO-PLT-NOT: _dso_local_alias_func.stub:
+; NO-PLT: _ifunc_func.stub:
+; NO-PLT: jmp _ifunc_func
+; NO-PLT-NOT: _dso_local_ifunc_func.stub:
+
+; Check comdat sections are emitted for stubs.
+; NO-PLT-COMDAT: .section .text,"xr",discard,_extern_func.stub
+; NO-PLT-COMDAT: .globl _extern_func.stub
+; NO-PLT-COMDAT: _extern_func.stub:
+; NO-PLT-COMDAT-NOT: _hidden_func.stub
+; NO-PLT-COMDAT-NOT: _protected_func.stub
+; NO-PLT-COMDAT-NOT: _dso_local_func.stub
+; NO-PLT-COMDAT-NOT: _internal_func.stub
+; NO-PLT-COMDAT-NOT: l_private_func.stub
+; NO-PLT-COMDAT: .section .text,"xr",discard,_alias_func.stub
+; NO-PLT-COMDAT: .globl _alias_func.stub
+; NO-PLT-COMDAT: _alias_func.stub:
+; NO-PLT-COMDAT-NOT: _dso_local_alias_func.stub:
+; NO-PLT-COMDAT: .section .text,"xr",discard,_ifunc_func.stub
+; NO-PLT-COMDAT: .globl _ifunc_func.stub
+; NO-PLT-COMDAT:_ifunc_func.stub:
+; NO-PLT-COMDAT-NOT: _dso_local_ifunc_func.stub:
diff --git a/llvm/test/CodeGen/X86/relptr-rodata.ll b/llvm/test/CodeGen/X86/relptr-rodata.ll
--- a/llvm/test/CodeGen/X86/relptr-rodata.ll
+++ b/llvm/test/CodeGen/X86/relptr-rodata.ll
@@ -19,3 +19,27 @@
; CHECK: relro2:
; CHECK: .long hidden-relro2
@relro2 = constant i32 trunc (i64 sub (i64 ptrtoint (i8* @hidden to i64), i64 ptrtoint (i32* @relro2 to i64)) to i32)
+
+; CHECK: .section .rodata.cst8
+; CHECK-NEXT: .globl obj
+; CHECK: obj:
+; CHECK: .long 0
+; CHECK: .long (hidden_func-obj)-4
+
+declare hidden void @hidden_func()
+
+; Ensure that inbound GEPs with constant offsets are also resolved.
+@obj = dso_local unnamed_addr constant { { i32, i32 } } {
+ { i32, i32 } {
+ i32 0,
+ i32 trunc (i64 sub (i64 ptrtoint (void ()* dso_local_equivalent @hidden_func to i64), i64 ptrtoint (i32* getelementptr inbounds ({ { i32, i32 } }, { { i32, i32 } }* @obj, i32 0, i32 0, i32 1) to i64)) to i32)
+ } }, align 4
+
+; CHECK: .section .rodata.rodata2
+; CHECK-NEXT: .globl rodata2
+; CHECK: rodata2:
+; CHECK: .long extern_func@PLT-rodata2
+
+declare void @extern_func()
+
+@rodata2 = dso_local constant i32 trunc (i64 sub (i64 ptrtoint (void ()* dso_local_equivalent @extern_func to i64), i64 ptrtoint (i32* @rodata2 to i64)) to i32)