Index: clang/lib/CodeGen/CGVTables.cpp =================================================================== --- clang/lib/CodeGen/CGVTables.cpp +++ clang/lib/CodeGen/CGVTables.cpp @@ -1313,7 +1313,10 @@ llvm::DenseSet Visited; llvm::GlobalObject::VCallVisibility TypeVis = GetVCallVisibilityLevel(RD, Visited); + + // Add a single range covering the whole global. if (TypeVis != llvm::GlobalObject::VCallVisibilityPublic) - VTable->setVCallVisibilityMetadata(TypeVis); + VTable->setVCallVisibility( + {{TypeVis, 0, std::numeric_limits::max()}}); } } Index: clang/lib/CodeGen/MicrosoftCXXABI.cpp =================================================================== --- clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -1662,8 +1662,11 @@ llvm::DenseSet Visited; llvm::GlobalObject::VCallVisibility TypeVis = CGM.GetVCallVisibilityLevel(RD, Visited); + + // Add a single range covering the whole global if (TypeVis != llvm::GlobalObject::VCallVisibilityPublic) - VTable->setVCallVisibilityMetadata(TypeVis); + VTable->setVCallVisibility( + {{TypeVis, 0, std::numeric_limits::max()}}); } // The location of the first virtual function pointer in the virtual table, Index: llvm/include/llvm/IR/GlobalObject.h =================================================================== --- llvm/include/llvm/IR/GlobalObject.h +++ llvm/include/llvm/IR/GlobalObject.h @@ -136,9 +136,16 @@ void copyMetadata(const GlobalObject *Src, unsigned Offset); void addTypeMetadata(unsigned Offset, Metadata *TypeID); - void setVCallVisibilityMetadata(VCallVisibility Visibility); - VCallVisibility getVCallVisibility() const; - std::pair getVTableOffsetRange() const; + + struct VCallVisibilityEntry { + GlobalObject::VCallVisibility Visibility; + uint64_t RangeStart; + uint64_t RangeEnd; + }; + using VCallVisibilityList = std::vector; + + void setVCallVisibility(VCallVisibilityList Visibility); + VCallVisibilityList getVCallVisibility() const; /// Returns true if the alignment of the value can be unilaterally /// increased. Index: llvm/lib/Analysis/ModuleSummaryAnalysis.cpp =================================================================== --- llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -622,9 +622,20 @@ !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() && !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass(); bool Constant = V.isConstant(); + + GlobalObject::VCallVisibilityList List = V.getVCallVisibility(); + // Find the maximum visibility from all !vcall_visibility attachments. + GlobalObject::VCallVisibility TypeVis = + GlobalObject::VCallVisibilityTranslationUnit; + for (auto Entry : List) { + TypeVis = std::min(TypeVis, Entry.Visibility); + } + if (List.empty()) + TypeVis = GlobalObject::VCallVisibilityPublic; + GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized, Constant ? false : CanBeInternalized, - Constant, V.getVCallVisibility()); + Constant, TypeVis); auto GVarSummary = std::make_unique(Flags, VarFlags, RefEdges.takeVector()); if (NonRenamableLocal) Index: llvm/lib/IR/Metadata.cpp =================================================================== --- llvm/lib/IR/Metadata.cpp +++ llvm/lib/IR/Metadata.cpp @@ -1513,42 +1513,56 @@ TypeID})); } -void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) { +void GlobalObject::setVCallVisibility(VCallVisibilityList Visibility) { // Remove any existing vcall visibility metadata first in case we are // updating. eraseMetadata(LLVMContext::MD_vcall_visibility); - addMetadata(LLVMContext::MD_vcall_visibility, - *MDNode::get(getContext(), - {ConstantAsMetadata::get(ConstantInt::get( - Type::getInt64Ty(getContext()), Visibility))})); + for (auto Entry : Visibility) { + addMetadata( + LLVMContext::MD_vcall_visibility, + *MDNode::get(getContext(), + { + ConstantAsMetadata::get(ConstantInt::get( + Type::getInt64Ty(getContext()), Entry.Visibility)), + ConstantAsMetadata::get(ConstantInt::get( + Type::getInt64Ty(getContext()), Entry.RangeStart)), + ConstantAsMetadata::get(ConstantInt::get( + Type::getInt64Ty(getContext()), Entry.RangeEnd)), + })); + } } -GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const { - if (MDNode *MD = getMetadata(LLVMContext::MD_vcall_visibility)) { +GlobalObject::VCallVisibilityList GlobalObject::getVCallVisibility() const { + VCallVisibilityList Result; + + SmallVector MDs; + getMetadata(LLVMContext::MD_vcall_visibility, MDs); + for (MDNode *MD : MDs) { uint64_t Val = cast( cast(MD->getOperand(0))->getValue()) ->getZExtValue(); assert(Val <= 2 && "unknown vcall visibility!"); - return (VCallVisibility)Val; - } - return VCallVisibility::VCallVisibilityPublic; -} -std::pair GlobalObject::getVTableOffsetRange() const { - if (MDNode *MD = getMetadata(LLVMContext::MD_vcall_visibility)) { + uint64_t RangeStart = 0; + uint64_t RangeEnd = std::numeric_limits::max(); if (MD->getNumOperands() >= 3) { - uint64_t RangeStart = - cast( - cast(MD->getOperand(1))->getValue()) - ->getZExtValue(); - uint64_t RangeEnd = - cast( - cast(MD->getOperand(2))->getValue()) - ->getZExtValue(); - return std::make_pair(RangeStart, RangeEnd); + RangeStart = cast( + cast(MD->getOperand(1))->getValue()) + ->getZExtValue(); + RangeEnd = cast( + cast(MD->getOperand(2))->getValue()) + ->getZExtValue(); } + + Result.push_back({VCallVisibility(Val), RangeStart, RangeEnd}); + } + + if (Result.empty()) { + Result.push_back({VCallVisibility::VCallVisibilityPublic, 0, + std::numeric_limits::max()}); } - return std::make_pair(0, std::numeric_limits::max()); + + return Result; } void Function::setSubprogram(DISubprogram *SP) { Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -776,7 +776,7 @@ auto *Op2Val = cast(MD->getOperand(2))->getValue(); Assert(isa(Op2Val), "bad !vcall_visibility attachment"); auto Op2Int = cast(Op2Val)->getValue(); - Assert(Op2Int.uge(0) && Op2Int.ult(std::numeric_limits::max()), + Assert(Op2Int.uge(0) && Op2Int.ule(std::numeric_limits::max()), "bad !vcall_visibility attachment"); Assert(Op1Int.ule(Op2Int), "bad !vcall_visibility attachment"); Index: llvm/lib/Transforms/IPO/GlobalDCE.cpp =================================================================== --- llvm/lib/Transforms/IPO/GlobalDCE.cpp +++ llvm/lib/Transforms/IPO/GlobalDCE.cpp @@ -159,15 +159,16 @@ } /// Recursively iterate over the (sub-)constants in the vtable and look for -/// vptrs, if their offset is within [RangeStart..RangeEnd), add them to VFuncs. +/// vptrs, if their offset is within ranges in List, add them to VFuncs. static void FindVirtualFunctionsInVTable(Module &M, Constant *C, - uint64_t RangeStart, uint64_t RangeEnd, + GlobalObject::VCallVisibilityList List, SmallPtrSet *VFuncs, uint64_t BaseOffset = 0) { if (auto *GV = dyn_cast(C)) { if (auto *F = dyn_cast(GV)) - if (RangeStart <= BaseOffset && BaseOffset < RangeEnd) - VFuncs->insert(F); + for (auto Entry : List) + if (Entry.RangeStart <= BaseOffset && BaseOffset < Entry.RangeEnd) + VFuncs->insert(F); // Do not recurse outside of the current global. return; @@ -179,22 +180,20 @@ for (auto EI : llvm::enumerate(STy->elements())) { auto Offset = SL->getElementOffset(EI.index()); unsigned Op = SL->getElementContainingOffset(Offset); - FindVirtualFunctionsInVTable(M, cast(S->getOperand(Op)), - RangeStart, RangeEnd, VFuncs, - BaseOffset + Offset); + FindVirtualFunctionsInVTable(M, cast(S->getOperand(Op)), List, + VFuncs, BaseOffset + Offset); } } else if (auto *A = dyn_cast(C)) { ArrayType *ATy = A->getType(); auto EltSize = M.getDataLayout().getTypeAllocSize(ATy->getElementType()); for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) { - FindVirtualFunctionsInVTable(M, cast(A->getOperand(i)), - RangeStart, RangeEnd, VFuncs, - BaseOffset + EltSize * i); + FindVirtualFunctionsInVTable(M, cast(A->getOperand(i)), List, + VFuncs, BaseOffset + EltSize * i); } } else { for (auto &Op : C->operands()) { - FindVirtualFunctionsInVTable(M, cast(Op), RangeStart, RangeEnd, - VFuncs, BaseOffset); + FindVirtualFunctionsInVTable(M, cast(Op), List, VFuncs, + BaseOffset); } } } @@ -233,18 +232,27 @@ // unit, we know that we can see all virtual functions which might use it, // so VFE is safe. if (auto GO = dyn_cast(&GV)) { - GlobalObject::VCallVisibility TypeVis = GO->getVCallVisibility(); + GlobalObject::VCallVisibilityList List = GO->getVCallVisibility(); + + // FIXME: For now, find the maximum visibility from all !vcall_visibility + // attachments, and conservatively treat all ranges with that visibility. + GlobalObject::VCallVisibility TypeVis = + GlobalObject::VCallVisibilityTranslationUnit; + for (auto Entry : List) { + TypeVis = std::min(TypeVis, Entry.Visibility); + } + if (List.empty()) + TypeVis = GlobalObject::VCallVisibilityPublic; + if (TypeVis == GlobalObject::VCallVisibilityTranslationUnit || (LTOPostLink && TypeVis == GlobalObject::VCallVisibilityLinkageUnit)) { LLVM_DEBUG(dbgs() << GV.getName() << " is safe for VFE\n"); - // Find and record all the vfunctions that are within the offset range - // specified in the !vcall_visibility attribute. - auto Range = GO->getVTableOffsetRange(); + // Find and record all the vfunctions that are within the offset ranges + // specified in the !vcall_visibility attributes. SmallPtrSet VFuncs; - FindVirtualFunctionsInVTable(M, GV.getInitializer(), std::get<0>(Range), - std::get<1>(Range), &VFuncs); + FindVirtualFunctionsInVTable(M, GV.getInitializer(), List, &VFuncs); VFESafeVTablesAndFns[&GV] = VFuncs; } } Index: llvm/lib/Transforms/IPO/GlobalSplit.cpp =================================================================== --- llvm/lib/Transforms/IPO/GlobalSplit.cpp +++ llvm/lib/Transforms/IPO/GlobalSplit.cpp @@ -112,8 +112,21 @@ Type->getOperand(1)})); } - if (GV.hasMetadata(LLVMContext::MD_vcall_visibility)) - SplitGV->setVCallVisibilityMetadata(GV.getVCallVisibility()); + if (GV.hasMetadata(LLVMContext::MD_vcall_visibility)) { + GlobalObject::VCallVisibilityList List = GV.getVCallVisibility(); + + // Find the maximum visibility from all !vcall_visibility + GlobalObject::VCallVisibility TypeVis = + GlobalObject::VCallVisibilityTranslationUnit; + for (auto Entry : List) { + TypeVis = std::min(TypeVis, Entry.Visibility); + } + if (List.empty()) + TypeVis = GlobalObject::VCallVisibilityPublic; + + SplitGV->setVCallVisibility( + {{TypeVis, 0, std::numeric_limits::max()}}); + } } for (User *U : GV.users()) { Index: llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp =================================================================== --- llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -796,16 +796,21 @@ const DenseSet &DynamicExportSymbols) { if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) return; - for (GlobalVariable &GV : M.globals()) + for (GlobalVariable &GV : M.globals()) { // Add linkage unit visibility to any variable with type metadata, which are // the vtable definitions. We won't have an existing vcall_visibility // metadata on vtable definitions with public visibility. - if (GV.hasMetadata(LLVMContext::MD_type) && - GV.getVCallVisibility() == GlobalObject::VCallVisibilityPublic && - // Don't upgrade the visibility for symbols exported to the dynamic - // linker, as we have no information on their eventual use. - !DynamicExportSymbols.count(GV.getGUID())) - GV.setVCallVisibilityMetadata(GlobalObject::VCallVisibilityLinkageUnit); + GlobalObject::VCallVisibilityList List = GV.getVCallVisibility(); + if (GV.hasMetadata(LLVMContext::MD_type) && !List.empty()) { + for (auto &Entry : List) { + if (Entry.Visibility == GlobalObject::VCallVisibilityPublic && + !DynamicExportSymbols.count(GV.getGUID())) { + Entry.Visibility = GlobalObject::VCallVisibilityLinkageUnit; + } + } + GV.setVCallVisibility(List); + } + } } /// If whole program visibility asserted, then upgrade all public vcall @@ -974,10 +979,19 @@ if (!TM.Bits->GV->isConstant()) return false; + GlobalObject::VCallVisibilityList List = TM.Bits->GV->getVCallVisibility(); + // Find the maximum visibility from all !vcall_visibility + GlobalObject::VCallVisibility TypeVis = + GlobalObject::VCallVisibilityTranslationUnit; + for (auto Entry : List) { + TypeVis = std::min(TypeVis, Entry.Visibility); + } + if (List.empty()) + TypeVis = GlobalObject::VCallVisibilityPublic; + // We cannot perform whole program devirtualization analysis on a vtable // with public LTO visibility. - if (TM.Bits->GV->getVCallVisibility() == - GlobalObject::VCallVisibilityPublic) + if (TypeVis == GlobalObject::VCallVisibilityPublic) return false; Constant *Ptr = getPointerAtOffset(TM.Bits->GV->getInitializer(), Index: llvm/test/Transforms/GlobalDCE/virtual-functions-ranges.ll =================================================================== --- llvm/test/Transforms/GlobalDCE/virtual-functions-ranges.ll +++ llvm/test/Transforms/GlobalDCE/virtual-functions-ranges.ll @@ -64,6 +64,26 @@ ; CHECK-SAME: i8* bitcast (void ()* @regular_non_virtual_funcD to i8*) ; CHECK-SAME: }, align 8 +; A vtable that contains multiple ranges. +@vtableE = internal unnamed_addr constant { [6 x i8*] } { [6 x i8*] [ + i8* bitcast (void ()* @regular_non_virtual_funcE1 to i8*), + i8* bitcast (void ()* @vfunc1_live to i8*), + i8* bitcast (void ()* @vfunc2_dead to i8*), + i8* bitcast (void ()* @regular_non_virtual_funcE2 to i8*), + i8* bitcast (void ()* @vfunc3_live to i8*), + i8* bitcast (void ()* @vfunc4_dead to i8*) +]}, align 8, !type !{i64 8, !"vfunc1.type"}, !type !{i64 16, !"vfunc2.type"}, !type !{i64 32, !"vfunc3.type"}, !type !{i64 40, !"vfunc4.type"}, + !vcall_visibility !{i64 2, i64 8, i64 24}, !vcall_visibility !{i64 2, i64 32, i64 48} + +; CHECK: @vtableE = internal unnamed_addr constant { [6 x i8*] } { [6 x i8*] [ +; CHECK-SAME: i8* bitcast (void ()* @regular_non_virtual_funcE1 to i8*), +; CHECK-SAME: i8* bitcast (void ()* @vfunc1_live to i8*), +; CHECK-SAME: i8* null, +; CHECK-SAME: i8* bitcast (void ()* @regular_non_virtual_funcE2 to i8*), +; CHECK-SAME: i8* bitcast (void ()* @vfunc3_live to i8*), +; CHECK-SAME: i8* null +; CHECK-SAME: ] }, align 8 + ; (1) vfunc1_live is referenced from @main, stays alive define internal void @vfunc1_live() { ; CHECK: define internal void @vfunc1_live( @@ -76,6 +96,16 @@ ret void } +define internal void @vfunc3_live() { + ; CHECK: define internal void @vfunc3_live( + ret void +} + +define internal void @vfunc4_dead() { + ; CHECK-NOT: define internal void @vfunc4_dead( + ret void +} + ; (3) not using a range in !vcall_visibility, global gets removed define internal void @regular_non_virtual_funcA() { ; CHECK-NOT: define internal void @regular_non_virtual_funcA( @@ -103,12 +133,24 @@ ret void } +define internal void @regular_non_virtual_funcE1() { + ; CHECK: define internal void @regular_non_virtual_funcE1( + ret void +} + +define internal void @regular_non_virtual_funcE2() { + ; CHECK: define internal void @regular_non_virtual_funcE2( + ret void +} + define void @main() { %1 = ptrtoint { [3 x i8*] }* @vtableA to i64 ; to keep @vtableA alive %2 = ptrtoint { [3 x i8*] }* @vtableB to i64 ; to keep @vtableB alive %3 = ptrtoint { [3 x i8*] }* @vtableC to i64 ; to keep @vtableC alive %4 = ptrtoint { i32, i32, i8* }* @vtableD to i64 ; to keep @vtableD alive - %5 = tail call { i8*, i1 } @llvm.type.checked.load(i8* null, i32 0, metadata !"vfunc1.type") + %5 = ptrtoint { [6 x i8*] }* @vtableE to i64 ; to keep @vtableE alive + %6 = tail call { i8*, i1 } @llvm.type.checked.load(i8* null, i32 0, metadata !"vfunc1.type") + %7 = tail call { i8*, i1 } @llvm.type.checked.load(i8* null, i32 0, metadata !"vfunc3.type") ret void } Index: llvm/test/Transforms/GlobalSplit/basic.ll =================================================================== --- llvm/test/Transforms/GlobalSplit/basic.ll +++ llvm/test/Transforms/GlobalSplit/basic.ll @@ -54,7 +54,7 @@ ; CHECK: [[T1]] = !{i32 0, !"foo"} ; CHECK: [[T2]] = !{i32 15, !"bar"} ; CHECK: [[T3]] = !{i32 16, !"a"} -; CHECK: [[VIS]] = !{i64 2} +; CHECK: [[VIS]] = !{i64 2, i64 0, i64 -1} ; CHECK: [[T4]] = !{i32 1, !"b"} ; CHECK: [[T5]] = !{i32 8, !"c"} !0 = !{i32 0, !"foo"}