Index: cfe/trunk/lib/CodeGen/CGClass.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGClass.cpp +++ cfe/trunk/lib/CodeGen/CGClass.cpp @@ -2645,6 +2645,42 @@ {CastedVTable, ValidVtable}); } +bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) { + if (!CGM.getCodeGenOpts().WholeProgramVTables || + !SanOpts.has(SanitizerKind::CFIVCall) || + !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall) || + !CGM.HasHiddenLTOVisibility(RD)) + return false; + + std::string TypeName = RD->getQualifiedNameAsString(); + return !getContext().getSanitizerBlacklist().isBlacklistedType(TypeName); +} + +llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad( + const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) { + SanitizerScope SanScope(this); + + EmitSanitizerStatReport(llvm::SanStat_CFI_VCall); + + llvm::Metadata *MD = + CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); + llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD); + + llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy); + llvm::Value *CheckedLoad = Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::type_checked_load), + {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset), + TypeId}); + llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1); + + EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall), + "cfi_check_fail", nullptr, nullptr); + + return Builder.CreateBitCast( + Builder.CreateExtractValue(CheckedLoad, 0), + cast(VTable->getType())->getElementType()); +} + // FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do // quite what we want. static const Expr *skipNoOpCastsAndParens(const Expr *E) { Index: cfe/trunk/lib/CodeGen/CodeGenFunction.h =================================================================== --- cfe/trunk/lib/CodeGen/CodeGenFunction.h +++ cfe/trunk/lib/CodeGen/CodeGenFunction.h @@ -1423,6 +1423,15 @@ void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD, llvm::Value *VTable, SourceLocation Loc); + /// Returns whether we should perform a type checked load when loading a + /// virtual function for virtual calls to members of RD. This is generally + /// true when both vcall CFI and whole-program-vtables are enabled. + bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD); + + /// Emit a type checked load from the given vtable. + llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD, llvm::Value *VTable, + uint64_t VTableByteOffset); + /// CanDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given /// expr can be devirtualized. bool CanDevirtualizeMemberFunctionCall(const Expr *Base, Index: cfe/trunk/lib/CodeGen/ItaniumCXXABI.cpp =================================================================== --- cfe/trunk/lib/CodeGen/ItaniumCXXABI.cpp +++ cfe/trunk/lib/CodeGen/ItaniumCXXABI.cpp @@ -1595,12 +1595,18 @@ auto *MethodDecl = cast(GD.getDecl()); llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent()); - CGF.EmitTypeMetadataCodeForVCall(MethodDecl->getParent(), VTable, Loc); - uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); - llvm::Value *VFuncPtr = - CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); - return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); + if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { + return CGF.EmitVTableTypeCheckedLoad( + MethodDecl->getParent(), VTable, + VTableIndex * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); + } else { + CGF.EmitTypeMetadataCodeForVCall(MethodDecl->getParent(), VTable, Loc); + + llvm::Value *VFuncPtr = + CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); + return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); + } } llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( Index: cfe/trunk/lib/CodeGen/MicrosoftCXXABI.cpp =================================================================== --- cfe/trunk/lib/CodeGen/MicrosoftCXXABI.cpp +++ cfe/trunk/lib/CodeGen/MicrosoftCXXABI.cpp @@ -1815,13 +1815,20 @@ MicrosoftVTableContext::MethodVFTableLocation ML = CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD); - if (CGM.getCodeGenOpts().PrepareForLTO) - CGF.EmitTypeMetadataCodeForVCall( - getClassAtVTableLocation(getContext(), GD, ML), VTable, Loc); - llvm::Value *VFuncPtr = - Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); - return Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); + if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { + return CGF.EmitVTableTypeCheckedLoad( + getClassAtVTableLocation(getContext(), GD, ML), VTable, + ML.Index * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); + } else { + if (CGM.getCodeGenOpts().PrepareForLTO) + CGF.EmitTypeMetadataCodeForVCall( + getClassAtVTableLocation(getContext(), GD, ML), VTable, Loc); + + llvm::Value *VFuncPtr = + Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); + return Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); + } } llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( Index: cfe/trunk/test/CodeGenCXX/cfi-stats.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/cfi-stats.cpp +++ cfe/trunk/test/CodeGenCXX/cfi-stats.cpp @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall,cfi-nvcall,cfi-derived-cast,cfi-unrelated-cast,cfi-icall -fsanitize-stats -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall,cfi-nvcall,cfi-derived-cast,cfi-unrelated-cast,cfi-icall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -fsanitize-stats -emit-llvm -o - %s | FileCheck %s // CHECK: [[STATS:@[^ ]*]] = internal global { i8*, i32, [5 x [2 x i8*]] } { i8* null, i32 5, [5 x [2 x i8*]] // CHECK: {{\[\[}}2 x i8*] zeroinitializer, Index: cfe/trunk/test/CodeGenCXX/type-metadata.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/type-metadata.cpp +++ cfe/trunk/test/CodeGenCXX/type-metadata.cpp @@ -1,12 +1,16 @@ // Tests for the cfi-vcall feature: -// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=ITANIUM --check-prefix=ITANIUM-NDIAG --check-prefix=NDIAG %s -// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=ITANIUM --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-ABORT %s -// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-recover=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=ITANIUM --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-RECOVER %s -// RUN: %clang_cc1 -flto -triple x86_64-pc-windows-msvc -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=MS --check-prefix=NDIAG %s +// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM --check-prefix=NDIAG %s +// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-ABORT %s +// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-recover=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM --check-prefix=ITANIUM-DIAG --check-prefix=DIAG --check-prefix=DIAG-RECOVER %s +// RUN: %clang_cc1 -flto -triple x86_64-pc-windows-msvc -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=MS --check-prefix=TT-MS --check-prefix=NDIAG %s // Tests for the whole-program-vtables feature: -// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=ITANIUM %s -// RUN: %clang_cc1 -flto -triple x86_64-pc-windows-msvc -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=MS %s +// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=ITANIUM --check-prefix=TT-ITANIUM %s +// RUN: %clang_cc1 -flto -triple x86_64-pc-windows-msvc -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=VTABLE-OPT --check-prefix=MS --check-prefix=TT-MS %s + +// Tests for cfi + whole-program-vtables: +// RUN: %clang_cc1 -flto -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=ITANIUM --check-prefix=TC-ITANIUM %s +// RUN: %clang_cc1 -flto -triple x86_64-pc-windows-msvc -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=MS --check-prefix=TC-MS %s // ITANIUM: @_ZTV1A = {{[^!]*}}, !type [[A16:![0-9]+]] // ITANIUM-DIAG-SAME: !type [[ALL16:![0-9]+]] @@ -102,8 +106,11 @@ // ITANIUM: define hidden void @_Z2afP1A // MS: define void @"\01?af@@YAXPEAUA@@@Z" void af(A *a) { - // ITANIUM: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"_ZTS1A") - // MS: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"?AUA@@") + // TT-ITANIUM: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"_ZTS1A") + // TT-MS: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"?AUA@@") + // TC-ITANIUM: [[PAIR:%[^ ]*]] = call { i8*, i1 } @llvm.type.checked.load(i8* %2, i32 0, metadata !"_ZTS1A") + // TC-MS: [[PAIR:%[^ ]*]] = call { i8*, i1 } @llvm.type.checked.load(i8* %2, i32 0, metadata !"?AUA@@") + // CFI-VT: [[P:%[^ ]*]] = extractvalue { i8*, i1 } [[PAIR]], 1 // DIAG-NEXT: [[VTVALID0:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT]], metadata !"all-vtables") // VTABLE-OPT: call void @llvm.assume(i1 [[P]]) // CFI-NEXT: br i1 [[P]], label %[[CONTBB:[^ ,]*]], label %[[TRAPBB:[^ ,]*]] @@ -120,7 +127,10 @@ // DIAG-RECOVER-NEXT: br label %[[CONTBB]] // CFI: [[CONTBB]] - // CFI: call void % + // CFI-NVT: [[PTR:%[^ ]*]] = load + // CFI-VT: [[PTRI8:%[^ ]*]] = extractvalue { i8*, i1 } [[PAIR]], 0 + // CFI-VT: [[PTR:%[^ ]*]] = bitcast i8* [[PTRI8]] to + // CFI: call void [[PTR]] #line 123 a->f(); } @@ -128,24 +138,30 @@ // ITANIUM: define internal void @_Z3df1PN12_GLOBAL__N_11DE // MS: define internal void @"\01?df1@@YAXPEAUD@?A@@@Z" void df1(D *d) { - // ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) - // MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUA@@") + // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) + // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUA@@") + // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata ![[DTYPE:[0-9]+]]) + // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"?AUA@@") d->f(); } // ITANIUM: define internal void @_Z3dg1PN12_GLOBAL__N_11DE // MS: define internal void @"\01?dg1@@YAXPEAUD@?A@@@Z" void dg1(D *d) { - // ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTS1B") - // MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUB@@") + // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTS1B") + // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUB@@") + // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 8, metadata !"_ZTS1B") + // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"?AUB@@") d->g(); } // ITANIUM: define internal void @_Z3dh1PN12_GLOBAL__N_11DE // MS: define internal void @"\01?dh1@@YAXPEAUD@?A@@@Z" void dh1(D *d) { - // ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE]]) - // MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) + // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE]]) + // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) + // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 16, metadata ![[DTYPE]]) + // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 8, metadata ![[DTYPE:[0-9]+]]) d->h(); } @@ -153,7 +169,9 @@ // MS: define internal void @"\01?df2@@YAXPEAUD@?A@@@Z" __attribute__((no_sanitize("cfi"))) void df2(D *d) { - // CFI-NOT: call i1 @llvm.type.test + // CFI-NVT-NOT: call i1 @llvm.type.test + // CFI-VT: [[P:%[^ ]*]] = call i1 @llvm.type.test + // CFI-VT: call void @llvm.assume(i1 [[P]]) d->f(); } @@ -161,7 +179,9 @@ // MS: define internal void @"\01?df3@@YAXPEAUD@?A@@@Z" __attribute__((no_sanitize("address"))) __attribute__((no_sanitize("cfi-vcall"))) void df3(D *d) { - // CFI-NOT: call i1 @llvm.type.test + // CFI-NVT-NOT: call i1 @llvm.type.test + // CFI-VT: [[P:%[^ ]*]] = call i1 @llvm.type.test + // CFI-VT: call void @llvm.assume(i1 [[P]]) d->f(); } @@ -196,8 +216,10 @@ // ITANIUM: define hidden void @_ZN5test21fEPNS_1DE // MS: define void @"\01?f@test2@@YAXPEAUD@1@@Z" void f(D *d) { - // ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTSN5test21DE") - // MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUA@test2@@") + // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTSN5test21DE") + // TT-MS: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"?AUA@test2@@") + // TC-ITANIUM: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 8, metadata !"_ZTSN5test21DE") + // TC-MS: {{%[^ ]*}} = call { i8*, i1 } @llvm.type.checked.load(i8* {{%[^ ]*}}, i32 0, metadata !"?AUA@test2@@") d->m_fn1(); }