diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp --- a/clang/lib/CodeGen/CGVTables.cpp +++ b/clang/lib/CodeGen/CGVTables.cpp @@ -1126,9 +1126,10 @@ } } - if (getCodeGenOpts().VirtualFunctionElimination) { + if (getCodeGenOpts().VirtualFunctionElimination || + getCodeGenOpts().WholeProgramVTables) { llvm::GlobalObject::VCallVisibility TypeVis = GetVCallVisibilityLevel(RD); if (TypeVis != llvm::GlobalObject::VCallVisibilityPublic) - VTable->addVCallVisibilityMetadata(TypeVis); + VTable->setVCallVisibilityMetadata(TypeVis); } } diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -542,6 +542,14 @@ getModule().addModuleFlag(llvm::Module::Override, "Cross-DSO CFI", 1); } + if (CodeGenOpts.WholeProgramVTables) { + // Indicate whether VFE was enabled for this module, so that the + // vcall_visibility metadata added under whole program vtables is handled + // appropriately in the optimizer. + getModule().addModuleFlag(llvm::Module::Error, "Virtual Function Elim", + CodeGenOpts.VirtualFunctionElimination); + } + if (LangOpts.Sanitize.has(SanitizerKind::CFIICall)) { getModule().addModuleFlag(llvm::Module::Override, "CFI Canonical Jump Tables", diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -670,6 +670,10 @@ CGM.HasHiddenLTOVisibility(RD); bool ShouldEmitVFEInfo = CGM.getCodeGenOpts().VirtualFunctionElimination && CGM.HasHiddenLTOVisibility(RD); + // Emit type test when WPD enabled, as we need to ignore vcall_visibility + // used without type checked loads when attempting VFE. + bool ShouldEmitWPDInfo = CGM.getCodeGenOpts().WholeProgramVTables && + CGM.HasHiddenLTOVisibility(RD); llvm::Value *VirtualFn = nullptr; { @@ -677,8 +681,9 @@ llvm::Value *TypeId = nullptr; llvm::Value *CheckResult = nullptr; - if (ShouldEmitCFICheck || ShouldEmitVFEInfo) { - // If doing CFI or VFE, we will need the metadata node to check against. + if (ShouldEmitCFICheck || ShouldEmitVFEInfo || ShouldEmitWPDInfo) { + // If doing CFI, VFE or WPD, we will need the metadata node to check + // against. llvm::Metadata *MD = CGM.CreateMetadataIdentifierForVirtualMemPtrType(QualType(MPT, 0)); TypeId = llvm::MetadataAsValue::get(CGF.getLLVMContext(), MD); @@ -702,7 +707,7 @@ } else { // When not doing VFE, emit a normal load, as it allows more // optimisations than type.checked.load. - if (ShouldEmitCFICheck) { + if (ShouldEmitCFICheck || ShouldEmitWPDInfo) { CheckResult = Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::type_test), {Builder.CreateBitCast(VFPAddr, CGF.Int8PtrTy), TypeId}); @@ -713,7 +718,8 @@ "memptr.virtualfn"); } assert(VirtualFn && "Virtual fuction pointer not created!"); - assert((!ShouldEmitCFICheck || !ShouldEmitVFEInfo || CheckResult) && + assert((!ShouldEmitCFICheck || !ShouldEmitVFEInfo || !ShouldEmitWPDInfo || + CheckResult) && "Check result required but not created!"); if (ShouldEmitCFICheck) { diff --git a/clang/test/CodeGenCXX/vcall-visibility-metadata.cpp b/clang/test/CodeGenCXX/vcall-visibility-metadata.cpp --- a/clang/test/CodeGenCXX/vcall-visibility-metadata.cpp +++ b/clang/test/CodeGenCXX/vcall-visibility-metadata.cpp @@ -1,4 +1,5 @@ -// RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -emit-llvm -fvirtual-function-elimination -fwhole-program-vtables -o - %s | FileCheck %s +// RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -emit-llvm -fvirtual-function-elimination -fwhole-program-vtables -o - %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-VFE +// RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -emit-llvm -fwhole-program-vtables -o - %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOVFE // Anonymous namespace. @@ -83,6 +84,7 @@ return new G(); } - // CHECK-DAG: [[VIS_DSO]] = !{i64 1} // CHECK-DAG: [[VIS_TU]] = !{i64 2} +// CHECK-VFE-DAG: !{i32 1, !"Virtual Function Elim", i32 1} +// CHECK-NOVFE-DAG: !{i32 1, !"Virtual Function Elim", i32 0} diff --git a/llvm/include/llvm/IR/GlobalObject.h b/llvm/include/llvm/IR/GlobalObject.h --- a/llvm/include/llvm/IR/GlobalObject.h +++ b/llvm/include/llvm/IR/GlobalObject.h @@ -178,7 +178,7 @@ void copyMetadata(const GlobalObject *Src, unsigned Offset); void addTypeMetadata(unsigned Offset, Metadata *TypeID); - void addVCallVisibilityMetadata(VCallVisibility Visibility); + void setVCallVisibilityMetadata(VCallVisibility Visibility); VCallVisibility getVCallVisibility() const; protected: diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -1499,7 +1499,10 @@ TypeID})); } -void GlobalObject::addVCallVisibilityMetadata(VCallVisibility Visibility) { +void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) { + // Remove any existing vcall visibility metadata first in case we are + // updating. + eraseMetadata(LLVMContext::MD_vcall_visibility); addMetadata(LLVMContext::MD_vcall_visibility, *MDNode::get(getContext(), {ConstantAsMetadata::get(ConstantInt::get( diff --git a/llvm/lib/Transforms/IPO/GlobalDCE.cpp b/llvm/lib/Transforms/IPO/GlobalDCE.cpp --- a/llvm/lib/Transforms/IPO/GlobalDCE.cpp +++ b/llvm/lib/Transforms/IPO/GlobalDCE.cpp @@ -263,6 +263,15 @@ if (!ClEnableVFE) return; + // If the Virtual Function Elim module flag is present and set to zero, then + // the vcall_visibility metadata was inserted for another optimization (WPD) + // and we may not have type checked loads on all accesses to the vtable. + // Don't attempt VFE in that case. + auto *Val = mdconst::dyn_extract_or_null( + M.getModuleFlag("Virtual Function Elim")); + if (!Val || Val->getZExtValue() == 0) + return; + ScanVTables(M); if (VFESafeVTables.empty()) diff --git a/llvm/lib/Transforms/IPO/GlobalSplit.cpp b/llvm/lib/Transforms/IPO/GlobalSplit.cpp --- a/llvm/lib/Transforms/IPO/GlobalSplit.cpp +++ b/llvm/lib/Transforms/IPO/GlobalSplit.cpp @@ -111,6 +111,9 @@ ConstantInt::get(Int32Ty, ByteOffset - SplitBegin)), Type->getOperand(1)})); } + + if (GV.hasMetadata(LLVMContext::MD_vcall_visibility)) + SplitGV->setVCallVisibilityMetadata(GV.getVCallVisibility()); } for (User *U : GV.users()) { diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll @@ -70,9 +70,12 @@ declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) #2 +!llvm.module.flags = !{!5} + !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFivE.virtual"} !2 = !{i64 2} !3 = !{i64 16, !"_ZTS1B"} !4 = !{i64 16, !"_ZTSM1BFivE.virtual"} +!5 = !{i32 1, !"Virtual Function Elim", i32 1} !10 = !{} diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll @@ -108,6 +108,8 @@ declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) +!llvm.module.flags = !{!7} + !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFiiE.virtual"} !2 = !{i64 24, !"_ZTSM1AFifE.virtual"} @@ -115,4 +117,5 @@ !4 = !{i64 16, !"_ZTS1B"} !5 = !{i64 16, !"_ZTSM1BFiiE.virtual"} !6 = !{i64 24, !"_ZTSM1BFifE.virtual"} +!7 = !{i32 1, !"Virtual Function Elim", i32 1} !12 = !{} diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll @@ -70,9 +70,12 @@ declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) #2 +!llvm.module.flags = !{!5} + !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFivE.virtual"} !2 = !{i64 2} !3 = !{i64 16, !"_ZTS1B"} !4 = !{i64 16, !"_ZTSM1BFivE.virtual"} +!5 = !{i32 1, !"Virtual Function Elim", i32 1} !10 = !{} diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll @@ -110,6 +110,8 @@ declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) +!llvm.module.flags = !{!7} + !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFiiE.virtual"} !2 = !{i64 24, !"_ZTSM1AFifE.virtual"} @@ -117,4 +119,5 @@ !4 = !{i64 16, !"_ZTS1B"} !5 = !{i64 16, !"_ZTSM1BFiiE.virtual"} !6 = !{i64 24, !"_ZTSM1BFifE.virtual"} +!7 = !{i32 1, !"Virtual Function Elim", i32 1} !12 = !{} diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-novfe.ll copy from llvm/test/Transforms/GlobalDCE/virtual-functions.ll copy to llvm/test/Transforms/GlobalDCE/virtual-functions-novfe.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-novfe.ll @@ -1,3 +1,5 @@ +; Tests that VFE is not performed when the Virtual Function Elim metadata set +; to 0. This is the same as virtual-functions.ll otherwise. ; RUN: opt < %s -globaldce -S | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -11,14 +13,12 @@ ; intrinsic. Function test_A makes a call to A::foo, but there is no call to ; A::bar anywhere, so A::bar can be deleted, and its vtable slot replaced with ; null. +; However, with the metadata set to 0 we should not perform this VFE. %struct.A = type { i32 (...)** } -; The pointer to A::bar in the vtable can be removed, because it will never be -; loaded. We replace it with null to keep the layout the same. Because it is at -; the end of the vtable we could potentially shrink the vtable, but don't -; currently do that. -; CHECK: @_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* null] } +; We should retain @_ZN1A3barEv in the vtable. +; CHECK: @_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3barEv to i8*)] } @_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3barEv to i8*)] }, align 8, !type !0, !type !1, !type !2, !vcall_visibility !3 ; A::foo is called, so must be retained. @@ -28,8 +28,9 @@ ret i32 42 } -; A::bar is not used, so can be deleted. -; CHECK-NOT: define internal i32 @_ZN1A3barEv( +; A::bar is not used, so can be deleted with VFE, however, we should not be +; performing that elimination here. +; CHECK: define internal i32 @_ZN1A3barEv( define internal i32 @_ZN1A3barEv(%struct.A* nocapture readnone %this) { entry: ret i32 1337 @@ -48,8 +49,11 @@ ret i32 %call1 } +!llvm.module.flags = !{!4} + !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFivE.virtual"} !2 = !{i64 24, !"_ZTSM1AFivE.virtual"} !3 = !{i64 2} +!4 = !{i32 1, !"Virtual Function Elim", i32 0} !9 = !{} diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll @@ -85,7 +85,7 @@ declare dso_local noalias nonnull i8* @_Znwm(i64) -!llvm.module.flags = !{!5} +!llvm.module.flags = !{!5, !6} !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFvvE.virtual"} @@ -93,3 +93,4 @@ !3 = !{i64 1} ; linkage-unit vcall visibility !4 = !{i64 2} ; translation-unit vcall visibility !5 = !{i32 1, !"LTOPostLink", i32 1} +!6 = !{i32 1, !"Virtual Function Elim", i32 1} diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll @@ -85,10 +85,11 @@ declare dso_local noalias nonnull i8* @_Znwm(i64) -!llvm.module.flags = !{} +!llvm.module.flags = !{!5} !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFvvE.virtual"} !2 = !{i64 0} ; public vcall visibility !3 = !{i64 1} ; linkage-unit vcall visibility !4 = !{i64 2} ; translation-unit vcall visibility +!5 = !{i32 1, !"Virtual Function Elim", i32 1} diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions.ll --- a/llvm/test/Transforms/GlobalDCE/virtual-functions.ll +++ b/llvm/test/Transforms/GlobalDCE/virtual-functions.ll @@ -48,8 +48,11 @@ ret i32 %call1 } +!llvm.module.flags = !{!4} + !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFivE.virtual"} !2 = !{i64 24, !"_ZTSM1AFivE.virtual"} !3 = !{i64 2} +!4 = !{i32 1, !"Virtual Function Elim", i32 1} !9 = !{} diff --git a/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll b/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll --- a/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll +++ b/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll @@ -39,9 +39,10 @@ declare dso_local noalias nonnull i8* @_Znwm(i64) @_ZTVN10__cxxabiv117__class_type_infoE = external dso_local global i8* -!llvm.module.flags = !{!3} +!llvm.module.flags = !{!3, !4} !0 = !{i64 16, !"_ZTS1A"} !1 = !{i64 16, !"_ZTSM1AFvvE.virtual"} !2 = !{i64 2} ; translation-unit vcall visibility !3 = !{i32 1, !"LTOPostLink", i32 1} +!4 = !{i32 1, !"Virtual Function Elim", i32 1}