diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -2698,15 +2698,21 @@ // Don't insert type test assumes if we are forcing public // visibility. !CGM.AlwaysHasLTOVisibilityPublic(RD)) { - llvm::Metadata *MD = - CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); + QualType Ty = QualType(RD->getTypeForDecl(), 0); + llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(Ty); llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD); llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy); + // If we already know that the call has hidden LTO visibility, emit + // @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD + // will convert to @llvm.type.test() if we assert at link time that we have + // whole program visibility. + llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD) + ? llvm::Intrinsic::type_test + : llvm::Intrinsic::public_type_test; llvm::Value *TypeTest = - Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test), - {CastedVTable, TypeId}); + Builder.CreateCall(CGM.getIntrinsic(IID), {CastedVTable, TypeId}); Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest); } } diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -707,8 +707,12 @@ if (ShouldEmitCFICheck || ShouldEmitWPDInfo) { llvm::Value *VFPAddr = Builder.CreateGEP(CGF.Int8Ty, VTable, VTableOffset); + llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD) + ? llvm::Intrinsic::type_test + : llvm::Intrinsic::public_type_test; + CheckResult = Builder.CreateCall( - CGM.getIntrinsic(llvm::Intrinsic::type_test), + CGM.getIntrinsic(IID), {Builder.CreateBitCast(VFPAddr, CGF.Int8PtrTy), TypeId}); } diff --git a/clang/test/CodeGenCXX/cfi-mfcall.cpp b/clang/test/CodeGenCXX/cfi-mfcall.cpp --- a/clang/test/CodeGenCXX/cfi-mfcall.cpp +++ b/clang/test/CodeGenCXX/cfi-mfcall.cpp @@ -10,11 +10,12 @@ struct S : B1, B3 {}; // DEFAULT-NOT: llvm.type.test +// DEFAULT-NOT: llvm.public.type.test void f(S *s, void (S::*p)()) { // WPV: [[OFFSET:%.*]] = sub i64 {{.*}}, 1 // WPV: [[VFPTR:%.*]] = getelementptr i8, i8* %{{.*}}, i64 [[OFFSET]] - // WPV: [[TT:%.*]] = call i1 @llvm.type.test(i8* [[VFPTR]], metadata !"_ZTSM1SFvvE.virtual") + // WPV: [[TT:%.*]] = call i1 @llvm.public.type.test(i8* [[VFPTR]], metadata !"_ZTSM1SFvvE.virtual") // CHECK: [[OFFSET:%.*]] = sub i64 {{.*}}, 1 // CHECK: [[VFPTR:%.*]] = getelementptr i8, i8* %{{.*}}, i64 [[OFFSET]] // CHECK: [[TT:%.*]] = call i1 @llvm.type.test(i8* [[VFPTR]], metadata !"_ZTSM1SFvvE.virtual") diff --git a/clang/test/CodeGenCXX/thinlto-distributed-type-metadata.cpp b/clang/test/CodeGenCXX/thinlto-distributed-type-metadata.cpp --- a/clang/test/CodeGenCXX/thinlto-distributed-type-metadata.cpp +++ b/clang/test/CodeGenCXX/thinlto-distributed-type-metadata.cpp @@ -13,13 +13,13 @@ // The pre-link bitcode produced by clang should contain a type test assume // sequence. -// TT: [[TTREG:%[0-9]+]] = call i1 @llvm.type.test({{.*}}, metadata !"_ZTS1A") +// TT: [[TTREG:%[0-9]+]] = call i1 @llvm.public.type.test({{.*}}, metadata !"_ZTS1A") // TT: void @llvm.assume(i1 [[TTREG]]) // The ThinLTO backend optimized bitcode should not have any type test assume // sequences. // OPT-NOT: @llvm.type.test -// OPT-NOT: call void @llvm.assume +// OPT-NOT: @llvm.public.type.test // We should have only one @llvm.assume call, the one that was expanded // from the builtin in the IR below, not the one fed by the type test. // OPT: %cmp = icmp ne %struct.A* %{{.*}}, null @@ -27,7 +27,7 @@ // Check after the builtin assume again that we don't have a type test assume // sequence. // OPT-NOT: @llvm.type.test -// OPT-NOT: call void @llvm.assume +// OPT-NOT: @llvm.public.type.test // NM: T _Z2afP1A diff --git a/clang/test/CodeGenCXX/thinlto_public_type_test_distributed.ll b/clang/test/CodeGenCXX/thinlto_public_type_test_distributed.ll new file mode 100644 --- /dev/null +++ b/clang/test/CodeGenCXX/thinlto_public_type_test_distributed.ll @@ -0,0 +1,43 @@ +; REQUIRES: x86-registered-target + +; Check that we properly update @llvm.public.type.test with distributed ThinLTO. + +; RUN: opt -thinlto-bc -o %t.o %s + +; RUN: llvm-lto2 run -thinlto-distributed-indexes %t.o \ +; RUN: -o %t2.index \ +; RUN: -r=%t.o,f,px + +; RUN: %clang_cc1 -triple x86_64-grtev4-linux-gnu \ +; RUN: -emit-obj -fthinlto-index=%t.o.thinlto.bc \ +; RUN: -o %t.native.o -x ir %t.o --save-temps=obj +; RUN: llvm-dis %t.native.o.3.import.bc -o - | FileCheck %s --check-prefix=PUBLIC + +; RUN: llvm-lto2 run -thinlto-distributed-indexes %t.o --whole-program-visibility-conf \ +; RUN: -o %t2.index \ +; RUN: -r=%t.o,f,px + +; RUN: %clang_cc1 -triple x86_64-grtev4-linux-gnu \ +; RUN: -emit-obj -fthinlto-index=%t.o.thinlto.bc \ +; RUN: -o %t.native.o -x ir %t.o --save-temps=obj +; RUN: llvm-dis %t.native.o.1.promote.bc -o - | FileCheck %s --check-prefix=HIDDEN + +; PUBLIC-NOT: call {{.*}}@llvm.public.type.test +; PUBLIC-NOT: call {{.*}}@llvm.type.test +; PUBLIC: call void @llvm.assume(i1 true) + +; HIDDEN-NOT: call {{.*}}@llvm.public.type.test +; HIDDEN: call {{.*}}@llvm.type.test + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-grtev4-linux-gnu" + +define i32 @f(ptr %vtable) { +entry: + %p = call i1 @llvm.public.type.test(ptr %vtable, metadata !"_ZTS1A") + call void @llvm.assume(i1 %p) + ret i32 0 +} + +declare void @llvm.assume(i1) +declare i1 @llvm.public.type.test(ptr, metadata) diff --git a/clang/test/CodeGenCXX/type-metadata.cpp b/clang/test/CodeGenCXX/type-metadata.cpp --- a/clang/test/CodeGenCXX/type-metadata.cpp +++ b/clang/test/CodeGenCXX/type-metadata.cpp @@ -1,18 +1,18 @@ // Tests for the cfi-vcall feature: -// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM --check-prefix=NDIAG %s -// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-ABORT %s -// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-recover=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-RECOVER %s +// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM-HIDDEN --check-prefix=NDIAG %s +// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM-HIDDEN --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-ABORT %s +// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-recover=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM-HIDDEN --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-RECOVER %s // RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-pc-windows-msvc -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=MS --check-prefix=TT-MS --check-prefix=NDIAG %s // Tests for the whole-program-vtables feature: -// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM %s -// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=ITANIUM-DEFAULTVIS --check-prefix=TT-ITANIUM %s +// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM-HIDDEN %s +// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=ITANIUM-DEFAULTVIS --check-prefix=TT-ITANIUM-DEFAULT %s // RUN: %clang_cc1 -no-opaque-pointers -O2 -flto -flto-unit -triple x86_64-unknown-linux -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=ITANIUM-OPT %s // RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-pc-windows-msvc -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=MS --check-prefix=TT-MS %s // Tests for cfi + whole-program-vtables: -// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=ITANIUM --check-prefix=TC-ITANIUM %s -// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-pc-windows-msvc -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=MS --check-prefix=TC-MS %s +// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=CFI-VT-ITANIUM --check-prefix=ITANIUM --check-prefix=TC-ITANIUM %s +// RUN: %clang_cc1 -no-opaque-pointers -flto -flto-unit -triple x86_64-pc-windows-msvc -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=CFI-VT-MS --check-prefix=MS --check-prefix=TC-MS %s // ITANIUM: @_ZTV1A = {{[^!]*}}, !type [[A16:![0-9]+]] // ITANIUM-DIAG-SAME: !type [[ALL16:![0-9]+]] @@ -141,7 +141,8 @@ // ITANIUM-DEFAULTVIS: define{{.*}} void @_Z2afP1A // MS: define dso_local void @"?af@@YAXPEAUA@@@Z" void af(A *a) { - // TT-ITANIUM: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"_ZTS1A") + // TT-ITANIUM-HIDDEN: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"_ZTS1A") + // TT-ITANIUM-DEFAULT: [[P:%[^ ]*]] = call i1 @llvm.public.type.test(i8* [[VT:%[^ ]*]], metadata !"_ZTS1A") // TT-MS: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"?AUA@@") // TC-ITANIUM: [[PAIR:%[^ ]*]] = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"_ZTS1A") // TC-MS: [[PAIR:%[^ ]*]] = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"?AUA@@") @@ -173,7 +174,8 @@ // ITANIUM: define internal void @_Z3df1PN12_GLOBAL__N_11DE // MS: define internal void @"?df1@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" void df1(D *d) { - // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) + // TT-ITANIUM-HIDDEN: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) + // TT-ITANIUM-DEFAULT: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUA@@") // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata ![[DTYPE:[0-9]+]]) // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"?AUA@@") @@ -183,7 +185,8 @@ // ITANIUM: define internal void @_Z3dg1PN12_GLOBAL__N_11DE // MS: define internal void @"?dg1@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" void dg1(D *d) { - // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTS1B") + // TT-ITANIUM-HIDDEN: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTS1B") + // TT-ITANIUM-DEFAULT: {{%[^ ]*}} = call i1 @llvm.public.type.test(i8* {{%[^ ]*}}, metadata !"_ZTS1B") // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUB@@") // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 8, metadata !"_ZTS1B") // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"?AUB@@") @@ -193,7 +196,8 @@ // ITANIUM: define internal void @_Z3dh1PN12_GLOBAL__N_11DE // MS: define internal void @"?dh1@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" void dh1(D *d) { - // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE]]) + // TT-ITANIUM-HIDDEN: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE]]) + // TT-ITANIUM-DEFAULT: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE]]) // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 16, metadata ![[DTYPE]]) // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 8, metadata ![[DTYPE:[0-9]+]]) @@ -205,7 +209,8 @@ __attribute__((no_sanitize("cfi"))) void df2(D *d) { // CFI-NVT-NOT: call i1 @llvm.type.test - // CFI-VT: [[P:%[^ ]*]] = call i1 @llvm.type.test + // CFI-VT-ITANIUM: [[P:%[^ ]*]] = call i1 @llvm.type.test + // CFI-VT-MS: [[P:%[^ ]*]] = call i1 @llvm.type.test // CFI-VT: call void @llvm.assume(i1 [[P]]) d->f(); } @@ -215,7 +220,8 @@ __attribute__((no_sanitize("address"))) __attribute__((no_sanitize("cfi-vcall"))) void df3(D *d) { // CFI-NVT-NOT: call i1 @llvm.type.test - // CFI-VT: [[P:%[^ ]*]] = call i1 @llvm.type.test + // CFI-VT-ITANIUM: [[P:%[^ ]*]] = call i1 @llvm.type.test + // CFI-VT-MS: [[P:%[^ ]*]] = call i1 @llvm.type.test // CFI-VT: call void @llvm.assume(i1 [[P]]) d->f(); } @@ -252,7 +258,8 @@ // ITANIUM-DEFAULTVIS: define{{.*}} void @_ZN5test21fEPNS_1DE // MS: define dso_local void @"?f@test2@@YAXPEAUD@1@@Z" void f(D *d) { - // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTSN5test21DE") + // TT-ITANIUM-HIDDEN: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTSN5test21DE") + // TT-ITANIUM-DEFAULT: {{%[^ ]*}} = call i1 @llvm.public.type.test(i8* {{%[^ ]*}}, metadata !"_ZTSN5test21DE") // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUA@test2@@") // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 8, metadata !"_ZTSN5test21DE") // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"?AUA@test2@@") diff --git a/lld/test/ELF/lto/update_public_type_test.ll b/lld/test/ELF/lto/update_public_type_test.ll new file mode 100644 --- /dev/null +++ b/lld/test/ELF/lto/update_public_type_test.ll @@ -0,0 +1,27 @@ +; REQUIRES: x86 + +; Check that we RAUW llvm.public.type.test with either llvm.type.test when --lto-whole-program-visibility is specified, or with true otherwise. + +; RUN: opt --thinlto-bc -o %t.o %s +; RUN: ld.lld %t.o -o %t2.o --save-temps +; RUN: llvm-dis %t.o.1.promote.bc -o - | FileCheck %s --check-prefix=PUB +; RUN: ld.lld %t.o -o %t3.o --save-temps --lto-whole-program-visibility +; RUN: llvm-dis %t.o.1.promote.bc -o - | FileCheck %s --check-prefix=WPV + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-grtev4-linux-gnu" + +declare i1 @llvm.public.type.test(ptr, metadata) +declare void @llvm.assume(i1) + +; PUB-NOT: call {{.*}}@llvm.public.type.test +; PUB-NOT: call {{.*}}@llvm.type.test +; PUB: call {{.*}}@llvm.assume(i1 true) +; WPV: call {{.*}}@llvm.type.test +; WPV: call {{.*}}@llvm.assume + +define void @f(ptr %a) { + %i = call i1 @llvm.public.type.test(ptr %a, metadata !"_ZTS1A") + call void @llvm.assume(i1 %i) + ret void +} diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1757,6 +1757,12 @@ [llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty], [IntrNoMem, IntrWillReturn]>; +// Test whether a pointer is associated with a type metadata identifier. Used +// for public visibility classes that may later be refined to private +// visibility. +def int_public_type_test : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_metadata_ty], + [IntrNoMem, IntrWillReturn, IntrSpeculatable]>; + // Create a branch funnel that implements an indirect call to a limited set of // callees. This needs to be a musttail call. def int_icall_branch_funnel : DefaultAttrsIntrinsic<[], [llvm_vararg_ty], []>; diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -1122,6 +1122,9 @@ /// every summary of a GV is synchronized. bool WithDSOLocalPropagation = false; + /// Indicates that we have whole program visibility. + bool WithWholeProgramVisibility = false; + /// Indicates that summary-based synthetic entry count propagation has run bool HasSyntheticEntryCounts = false; @@ -1280,6 +1283,9 @@ bool withDSOLocalPropagation() const { return WithDSOLocalPropagation; } void setWithDSOLocalPropagation() { WithDSOLocalPropagation = true; } + bool withWholeProgramVisibility() const { return WithWholeProgramVisibility; } + void setWithWholeProgramVisibility() { WithWholeProgramVisibility = true; } + bool isReadOnly(const GlobalVarSummary *GVS) const { return WithAttributePropagation && GVS->maybeReadOnly(); } diff --git a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h --- a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h +++ b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h @@ -240,6 +240,8 @@ uint64_t ByteOffset; }; +void updatePublicTypeTestCalls(Module &M, + bool WholeProgramVisibilityEnabledInLTO); void updateVCallVisibilityInModule( Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet &DynamicExportSymbols); diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -164,11 +164,15 @@ SetVector &TypeCheckedLoadConstVCalls, DominatorTree &DT) { switch (CI->getCalledFunction()->getIntrinsicID()) { - case Intrinsic::type_test: { + case Intrinsic::type_test: + case Intrinsic::public_type_test: { + // errs() << "HIHI " << CI->getParent()->getParent()->getName() <<"\n"; + // CI->dump(); auto *TypeMDVal = cast(CI->getArgOperand(1)); auto *TypeId = dyn_cast(TypeMDVal->getMetadata()); if (!TypeId) break; + // errs() << "WOW\n"; GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString()); // Produce a summary from type.test intrinsics. We only summarize type.test @@ -251,6 +255,8 @@ bool HasLocalsInUsedOrAsm, DenseSet &CantBePromoted, bool IsThinLTO, std::function GetSSICallback) { + // errs() << __FUNCTION__ << " " << F.getName() << "\n"; + // M.dump(); // Summary not currently supported for anonymous functions, they should // have been named. assert(F.hasName()); diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp --- a/llvm/lib/Analysis/TypeMetadataUtils.cpp +++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp @@ -75,7 +75,9 @@ SmallVectorImpl &DevirtCalls, SmallVectorImpl &Assumes, const CallInst *CI, DominatorTree &DT) { - assert(CI->getCalledFunction()->getIntrinsicID() == Intrinsic::type_test); + assert(CI->getCalledFunction()->getIntrinsicID() == Intrinsic::type_test || + CI->getCalledFunction()->getIntrinsicID() == + Intrinsic::public_type_test); const Module *M = CI->getParent()->getParent()->getParent(); diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp --- a/llvm/lib/IR/ModuleSummaryIndex.cpp +++ b/llvm/lib/IR/ModuleSummaryIndex.cpp @@ -105,11 +105,13 @@ Flags |= 0x20; if (withDSOLocalPropagation()) Flags |= 0x40; + if (withWholeProgramVisibility()) + Flags |= 0x80; return Flags; } void ModuleSummaryIndex::setFlags(uint64_t Flags) { - assert(Flags <= 0x7f && "Unexpected bits in flag"); + assert(Flags <= 0xff && "Unexpected bits in flag"); // 1 bit: WithGlobalValueDeadStripping flag. // Set on combined index only. if (Flags & 0x1) @@ -139,6 +141,10 @@ // Set on combined index only. if (Flags & 0x40) setWithDSOLocalPropagation(); + // 1 bit: WithWholeProgramVisibility flag. + // Set on combined index only. + if (Flags & 0x80) + setWithWholeProgramVisibility(); } // Collect for the given module the list of function it defines diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1028,6 +1028,8 @@ }; computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols, isPrevailing, Conf.OptLevel > 0); + if (Conf.HasWholeProgramVisibility) + ThinLTO.CombinedIndex.setWithWholeProgramVisibility(); // Setup output file to emit statistics. auto StatsFileOrErr = setupStatsFile(Conf.StatsFile); @@ -1103,6 +1105,8 @@ updateVCallVisibilityInModule(*RegularLTO.CombinedModule, Conf.HasWholeProgramVisibility, DynamicExportSymbols); + updatePublicTypeTestCalls(*RegularLTO.CombinedModule, + Conf.HasWholeProgramVisibility); if (Conf.PreOptModuleHook && !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule)) diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -40,6 +40,7 @@ #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/Transforms/Utils/SplitModule.h" @@ -560,6 +561,10 @@ // the module, if applicable. Mod.setPartialSampleProfileRatio(CombinedIndex); + updatePublicTypeTestCalls(Mod, + Conf.HasWholeProgramVisibility || + CombinedIndex.withWholeProgramVisibility()); + if (Conf.CodeGenOnly) { codegen(Conf, TM.get(), AddStream, Task, Mod, CombinedIndex); return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile)); diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp --- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -132,6 +132,14 @@ } } + if (Function *PublicTypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::public_type_test))) { + for (const Use &U : PublicTypeTestFunc->uses()) { + auto CI = cast(U.getUser()); + ExternalizeTypeId(CI, 1); + } + } + if (Function *TypeCheckedLoadFunc = M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load))) { for (const Use &U : TypeCheckedLoadFunc->uses()) { diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -790,7 +790,7 @@ const DenseSet &DynamicExportSymbols) { if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) return; - for (GlobalVariable &GV : M.globals()) + for (GlobalVariable &GV : M.globals()) { // Add linkage unit visibility to any variable with type metadata, which are // the vtable definitions. We won't have an existing vcall_visibility // metadata on vtable definitions with public visibility. @@ -800,6 +800,34 @@ // linker, as we have no information on their eventual use. !DynamicExportSymbols.count(GV.getGUID())) GV.setVCallVisibilityMetadata(GlobalObject::VCallVisibilityLinkageUnit); + } +} + +void updatePublicTypeTestCalls(Module &M, + bool WholeProgramVisibilityEnabledInLTO) { + Function *PublicTypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::public_type_test)); + if (!PublicTypeTestFunc) + return; + if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) { + Function *TypeTestFunc = + Intrinsic::getDeclaration(&M, Intrinsic::type_test); + for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) { + auto *CI = cast(U.getUser()); + auto *NewCI = CallInst::Create( + TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)}, None, "", + CI); + CI->replaceAllUsesWith(NewCI); + CI->eraseFromParent(); + } + } else { + auto *True = ConstantInt::getTrue(M.getContext()); + for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) { + auto *CI = cast(U.getUser()); + CI->replaceAllUsesWith(True); + CI->eraseFromParent(); + } + } } /// If whole program visibility asserted, then upgrade all public vcall diff --git a/llvm/test/ThinLTO/X86/devirt_vcall_vis_public.ll b/llvm/test/ThinLTO/X86/devirt_vcall_vis_public.ll --- a/llvm/test/ThinLTO/X86/devirt_vcall_vis_public.ll +++ b/llvm/test/ThinLTO/X86/devirt_vcall_vis_public.ll @@ -10,6 +10,7 @@ ; RUN: -whole-program-visibility \ ; RUN: -o %t3 \ ; RUN: -r=%t2.o,test,px \ +; RUN: -r=%t2.o,test_public,px \ ; RUN: -r=%t2.o,_ZN1A1nEi,p \ ; RUN: -r=%t2.o,_ZN1B1fEi,p \ ; RUN: -r=%t2.o,_ZN1C1fEi,p \ @@ -28,6 +29,7 @@ ; RUN: -verify-machineinstrs=0 \ ; RUN: -o %t3 \ ; RUN: -r=%t.o,test,px \ +; RUN: -r=%t.o,test_public,px \ ; RUN: -r=%t.o,_ZN1A1nEi,p \ ; RUN: -r=%t.o,_ZN1B1fEi,p \ ; RUN: -r=%t.o,_ZN1C1fEi,p \ @@ -50,6 +52,7 @@ ; RUN: -whole-program-visibility \ ; RUN: -o %t5 \ ; RUN: -r=%t4.o,test,px \ +; RUN: -r=%t4.o,test_public,px \ ; RUN: -r=%t4.o,_ZN1A1nEi,p \ ; RUN: -r=%t4.o,_ZN1B1fEi,p \ ; RUN: -r=%t4.o,_ZN1C1fEi,p \ @@ -59,6 +62,8 @@ ; RUN: -r=%t4.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t5.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR +; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi +; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi ; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi ; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi @@ -69,6 +74,7 @@ ; RUN: llvm-lto2 run %t2.o -save-temps -pass-remarks=. \ ; RUN: -o %t3 \ ; RUN: -r=%t2.o,test,px \ +; RUN: -r=%t2.o,test_public,px \ ; RUN: -r=%t2.o,_ZN1A1nEi,p \ ; RUN: -r=%t2.o,_ZN1B1fEi,p \ ; RUN: -r=%t2.o,_ZN1C1fEi,p \ @@ -83,6 +89,7 @@ ; RUN: -verify-machineinstrs=0 \ ; RUN: -o %t3 \ ; RUN: -r=%t.o,test,px \ +; RUN: -r=%t.o,test_public,px \ ; RUN: -r=%t.o,_ZN1A1nEi,p \ ; RUN: -r=%t.o,_ZN1B1fEi,p \ ; RUN: -r=%t.o,_ZN1C1fEi,p \ @@ -103,6 +110,7 @@ ; RUN: llvm-lto2 run %t4.o -save-temps -pass-remarks=. \ ; RUN: -o %t5 \ ; RUN: -r=%t4.o,test,px \ +; RUN: -r=%t4.o,test_public,px \ ; RUN: -r=%t4.o,_ZN1A1nEi,p \ ; RUN: -r=%t4.o,_ZN1B1fEi,p \ ; RUN: -r=%t4.o,_ZN1C1fEi,p \ @@ -120,6 +128,7 @@ ; RUN: -disable-whole-program-visibility \ ; RUN: -o %t3 \ ; RUN: -r=%t2.o,test,px \ +; RUN: -r=%t2.o,test_public,px \ ; RUN: -r=%t2.o,_ZN1A1nEi,p \ ; RUN: -r=%t2.o,_ZN1B1fEi,p \ ; RUN: -r=%t2.o,_ZN1C1fEi,p \ @@ -178,7 +187,44 @@ ; CHECK-IR-LABEL: ret i32 ; CHECK-IR-LABEL: } +; CHECK-IR-LABEL: define i32 @test_public +define i32 @test_public(ptr %obj, ptr %obj2, i32 %a) { +entry: + %vtable = load ptr, ptr %obj + %p = call i1 @llvm.public.type.test(ptr %vtable, metadata !"_ZTS1A") + call void @llvm.assume(i1 %p) + %fptrptr = getelementptr ptr, ptr %vtable, i32 1 + %fptr1 = load ptr, ptr %fptrptr, align 8 + + ; Check that the call was devirtualized. + ; CHECK-IR: %call = tail call i32 @_ZN1A1nEi + ; CHECK-NODEVIRT-IR: %call = tail call i32 %fptr1 + %call = tail call i32 %fptr1(ptr nonnull %obj, i32 %a) + + %fptr22 = load ptr, ptr %vtable, align 8 + + ; We still have to call it as virtual. + ; CHECK-IR: %call3 = tail call i32 %fptr22 + ; CHECK-NODEVIRT-IR: %call3 = tail call i32 %fptr22 + %call3 = tail call i32 %fptr22(ptr nonnull %obj, i32 %call) + + %vtable2 = load ptr, ptr %obj2 + %p2 = call i1 @llvm.public.type.test(ptr %vtable2, metadata !4) + call void @llvm.assume(i1 %p2) + + %fptr33 = load ptr, ptr %vtable2, align 8 + + ; Check that the call was devirtualized. + ; CHECK-IR: %call4 = tail call i32 @_ZN1D1mEi + ; CHECK-NODEVIRT-IR: %call4 = tail call i32 %fptr33 + %call4 = tail call i32 %fptr33(ptr nonnull %obj2, i32 %call3) + ret i32 %call4 +} +; CHECK-IR-LABEL: ret i32 +; CHECK-IR-LABEL: } + declare i1 @llvm.type.test(ptr, metadata) +declare i1 @llvm.public.type.test(ptr, metadata) declare void @llvm.assume(i1) define i32 @_ZN1B1fEi(ptr %this, i32 %a) #0 { diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -65,6 +65,10 @@ cl::desc("Alias Analysis Pipeline"), cl::value_desc("aapipeline")); +static cl::opt WholeProgramVisibilityConf( + "whole-program-visibility-conf", + cl::desc("Set whole program visibility in LTO configuration")); + static cl::opt SaveTemps("save-temps", cl::desc("Save temporary files")); static cl::list SelectSaveTemps( @@ -302,6 +306,7 @@ // Run a custom pipeline, if asked for. Conf.OptPipeline = OptPipeline; Conf.AAPipeline = AAPipeline; + Conf.HasWholeProgramVisibility = WholeProgramVisibilityConf; Conf.OptLevel = OptLevel - '0'; Conf.Freestanding = EnableFreestanding;