Index: lib/CodeGen/CGAtomic.cpp =================================================================== --- lib/CodeGen/CGAtomic.cpp +++ lib/CodeGen/CGAtomic.cpp @@ -1251,7 +1251,7 @@ if (IsVolatile) Load->setVolatile(true); if (LVal.getTBAAInfo()) - CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo()); + CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); return Load; } @@ -1780,7 +1780,7 @@ if (IsVolatile) store->setVolatile(true); if (dest.getTBAAInfo()) - CGM.DecorateInstruction(store, dest.getTBAAInfo()); + CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); return; } Index: lib/CodeGen/CGClass.cpp =================================================================== --- lib/CodeGen/CGClass.cpp +++ lib/CodeGen/CGClass.cpp @@ -25,6 +25,7 @@ #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Metadata.h" using namespace clang; using namespace CodeGen; @@ -1850,7 +1851,8 @@ This = ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr); - llvm::Value *VPtrValue = GetVTablePtr(This, VTableGlobal->getType()); + llvm::Value *VPtrValue = + GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass); llvm::Value *Cmp = Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables"); Builder.CreateAssumption(Cmp); @@ -2072,7 +2074,8 @@ VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy); llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField); - CGM.DecorateInstruction(Store, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithTBAA(Store, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass); } CodeGenFunction::VPtrsVector @@ -2162,10 +2165,14 @@ } llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This, - llvm::Type *Ty) { - llvm::Value *VTablePtrSrc = Builder.CreateBitCast(This, Ty->getPointerTo()); + llvm::Type *VTableTy, + const CXXRecordDecl *RD) { + llvm::Value *VTablePtrSrc = + Builder.CreateBitCast(This, VTableTy->getPointerTo()); llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable"); - CGM.DecorateInstruction(VTable, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithTBAA(VTable, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithInvariantGroup(VTable, RD); + return VTable; } @@ -2249,7 +2256,7 @@ EmitBlock(CheckBlock); } - llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy); + llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy, ClassDecl); EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); if (MayBeNull) { Index: lib/CodeGen/CGExpr.cpp =================================================================== --- lib/CodeGen/CGExpr.cpp +++ lib/CodeGen/CGExpr.cpp @@ -1172,7 +1172,8 @@ llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo, TBAAOffset); if (TBAAPath) - CGM.DecorateInstruction(Load, TBAAPath, false/*ConvertTypeToTag*/); + CGM.DecorateInstructionWithTBAA(Load, TBAAPath, + false /*ConvertTypeToTag*/); } bool NeedsBoolCheck = @@ -1284,7 +1285,8 @@ llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo, TBAAOffset); if (TBAAPath) - CGM.DecorateInstruction(Store, TBAAPath, false/*ConvertTypeToTag*/); + CGM.DecorateInstructionWithTBAA(Store, TBAAPath, + false /*ConvertTypeToTag*/); } } @@ -2716,7 +2718,7 @@ else tbaa = CGM.getTBAAInfo(type); if (tbaa) - CGM.DecorateInstruction(load, tbaa); + CGM.DecorateInstructionWithTBAA(load, tbaa); } addr = load; Index: lib/CodeGen/CGExprCXX.cpp =================================================================== --- lib/CodeGen/CGExprCXX.cpp +++ lib/CodeGen/CGExprCXX.cpp @@ -259,7 +259,7 @@ } else { if (SanOpts.has(SanitizerKind::CFINVCall) && MD->getParent()->isDynamicClass()) { - llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy); + llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy, MD->getParent()); EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart()); } Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -1335,7 +1335,8 @@ /// GetVTablePtr - Return the Value of the vtable pointer member pointed /// to by This. - llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty); + llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *VTableTy, + const CXXRecordDecl *RD); enum CFITypeCheckKind { CFITCK_VCall, Index: lib/CodeGen/CodeGenModule.h =================================================================== --- lib/CodeGen/CodeGenModule.h +++ lib/CodeGen/CodeGenModule.h @@ -490,6 +490,27 @@ llvm::DenseMap DeferredEmptyCoverageMappingDecls; std::unique_ptr CoverageMapping; + + /// Helper class for generating invariant.group metadata. + /// For externally visible classes it generates node like this + /// !{ "%mangledName%.%TAG%.invariant.group" } + /// and for internally visible classes it generates empty distinct nodes. + class InvariantGroupMetadataFactory { + CodeGenModule &CGM; + const std::string tag; + llvm::SmallDenseMap distinctMDNodes; + + std::string getFullName(const CXXRecordDecl *ClassDecl) const; + llvm::MDNode *getInternallyVisibleMD(const CXXRecordDecl *ClassDecl); + llvm::MDNode *getExternallyVisibleMD(const CXXRecordDecl *ClassDecl) const; + + public: + InvariantGroupMetadataFactory(CodeGenModule &CGM, std::string tag); + llvm::MDNode *getInvariantMetadata(const CXXRecordDecl *ClassDecl); + }; + + InvariantGroupMetadataFactory vptrInvariantGroupMetadataFactory; + public: CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts, const PreprocessorOptions &ppopts, @@ -659,9 +680,13 @@ /// is the same as the type. For struct-path aware TBAA, the tag /// is different from the type: base type, access type and offset. /// When ConvertTypeToTag is true, we create a tag based on the scalar type. - void DecorateInstruction(llvm::Instruction *Inst, - llvm::MDNode *TBAAInfo, - bool ConvertTypeToTag = true); + void DecorateInstructionWithTBAA(llvm::Instruction *Inst, + llvm::MDNode *TBAAInfo, + bool ConvertTypeToTag = true); + + /// Adds !invariant.barrier !tag to instruction + void DecorateInstructionWithInvariantGroup(llvm::Instruction *I, + const CXXRecordDecl *RD); /// Emit the given number of characters as a value of type size_t. llvm::ConstantInt *getSize(CharUnits numChars); Index: lib/CodeGen/CodeGenModule.cpp =================================================================== --- lib/CodeGen/CodeGenModule.cpp +++ lib/CodeGen/CodeGenModule.cpp @@ -92,7 +92,8 @@ NSConcreteStackBlock(nullptr), BlockObjectAssign(nullptr), BlockObjectDispose(nullptr), BlockDescriptorType(nullptr), GenericBlockLiteralType(nullptr), LifetimeStartFn(nullptr), - LifetimeEndFn(nullptr), SanitizerMD(new SanitizerMetadata(*this)) { + LifetimeEndFn(nullptr), SanitizerMD(new SanitizerMetadata(*this)), + vptrInvariantGroupMetadataFactory(*this, "vptr") { // Initialize the type cache. llvm::LLVMContext &LLVMContext = M.getContext(); @@ -472,9 +473,9 @@ /// and struct-path aware TBAA, the tag has the same format: /// base type, access type and offset. /// When ConvertTypeToTag is true, we create a tag based on the scalar type. -void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst, - llvm::MDNode *TBAAInfo, - bool ConvertTypeToTag) { +void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst, + llvm::MDNode *TBAAInfo, + bool ConvertTypeToTag) { if (ConvertTypeToTag && TBAA) Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAA->getTBAAScalarTagInfo(TBAAInfo)); @@ -482,6 +483,13 @@ Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo); } +void CodeGenModule::DecorateInstructionWithInvariantGroup( + llvm::Instruction *I, const CXXRecordDecl *RD) { + auto *MetaData = + vptrInvariantGroupMetadataFactory.getInvariantMetadata(RD); + I->setMetadata("invariant.group", MetaData); +} + void CodeGenModule::Error(SourceLocation loc, StringRef message) { unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0"); getDiags().Report(Context.getFullLoc(loc), diagID) << message; @@ -1486,6 +1494,47 @@ return !isTriviallyRecursive(F); } +CodeGenModule::InvariantGroupMetadataFactory::InvariantGroupMetadataFactory( + CodeGenModule &CGM_, std::string tag_) + : CGM(CGM_), tag(std::move(tag_)) {} + +std::string CodeGenModule::InvariantGroupMetadataFactory::getFullName( + const CXXRecordDecl *ClassDecl) const { + std::string MangledName; + llvm::raw_string_ostream MangledNameStream(MangledName); + CGM.getCXXABI().getMangleContext().mangleTypeName( + QualType(ClassDecl->getTypeForDecl(), 0), MangledNameStream); + + MangledNameStream << "." << tag << ".invariant.group"; + MangledNameStream.flush(); + return MangledName; +} + +llvm::MDNode * +CodeGenModule::InvariantGroupMetadataFactory::getInternallyVisibleMD( + const CXXRecordDecl *ClassDecl) { + assert(ClassDecl != nullptr); + llvm::MDNode *&MetaData = distinctMDNodes[ClassDecl]; + if (!MetaData) + MetaData = llvm::MDNode::getDistinct(CGM.getLLVMContext(), {}); + return MetaData; +} + +llvm::MDNode * +CodeGenModule::InvariantGroupMetadataFactory::getExternallyVisibleMD( + const CXXRecordDecl *ClassDecl) const { + auto *MDString = llvm::MDString::get(CGM.getLLVMContext(), getFullName(ClassDecl)); + return llvm::MDNode::get(CGM.getLLVMContext(), MDString); +} + +llvm::MDNode * +CodeGenModule::InvariantGroupMetadataFactory::getInvariantMetadata( + const CXXRecordDecl *ClassDecl) { + if (ClassDecl->isExternallyVisible()) + return getExternallyVisibleMD(ClassDecl); + return getInternallyVisibleMD(ClassDecl); +} + /// If the type for the method's class was generated by /// CGDebugInfo::createContextChain(), the cache contains only a /// limited DIType without any declarations. Since EmitFunctionStart() Index: lib/CodeGen/ItaniumCXXABI.cpp =================================================================== --- lib/CodeGen/ItaniumCXXABI.cpp +++ lib/CodeGen/ItaniumCXXABI.cpp @@ -493,7 +493,7 @@ // Cast the adjusted this to a pointer to vtable pointer and load. llvm::Type *VTableTy = Builder.getInt8PtrTy(); - llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy); + llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy, RD); // Apply the offset. llvm::Value *VTableOffset = FnAsInt; @@ -919,7 +919,10 @@ // to pass to the deallocation function. // Grab the vtable pointer as an intptr_t*. - llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo()); + auto *ClassDecl = + cast(ElementType->getAs()->getDecl()); + llvm::Value *VTable = + CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl); // Track back to entry -2 and pull out the offset there. llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64( @@ -1116,8 +1119,10 @@ QualType SrcRecordTy, llvm::Value *ThisPtr, llvm::Type *StdTypeInfoPtrTy) { + auto *ClassDecl = + cast(SrcRecordTy->getAs()->getDecl()); llvm::Value *Value = - CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo()); + CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl); // Load the type info. Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL); @@ -1179,8 +1184,11 @@ CGF.ConvertType(CGF.getContext().getPointerDiffType()); llvm::Type *DestLTy = CGF.ConvertType(DestTy); + auto *ClassDecl = + cast(SrcRecordTy->getAs()->getDecl()); // Get the vtable pointer. - llvm::Value *VTable = CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo()); + llvm::Value *VTable = + CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo(), ClassDecl); // Get the offset-to-top from the vtable. llvm::Value *OffsetToTop = @@ -1206,7 +1214,7 @@ llvm::Value *This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) { - llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy); + llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy, ClassDecl); CharUnits VBaseOffsetOffset = CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl); @@ -1492,10 +1500,11 @@ SourceLocation Loc) { GD = GD.getCanonicalDecl(); Ty = Ty->getPointerTo()->getPointerTo(); - llvm::Value *VTable = CGF.GetVTablePtr(This, Ty); + auto *MethodDecl = cast(GD.getDecl()); + llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent()); if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) - CGF.EmitVTablePtrCheckForCall(cast(GD.getDecl()), VTable, + CGF.EmitVTablePtrCheckForCall(MethodDecl, VTable, CodeGenFunction::CFITCK_VCall, Loc); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); Index: lib/CodeGen/MicrosoftCXXABI.cpp =================================================================== --- lib/CodeGen/MicrosoftCXXABI.cpp +++ lib/CodeGen/MicrosoftCXXABI.cpp @@ -1800,7 +1800,9 @@ Ty = Ty->getPointerTo()->getPointerTo(); llvm::Value *VPtr = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty); + + auto *MethodDecl = cast(GD.getDecl()); + llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent()); MicrosoftVTableContext::MethodVFTableLocation ML = CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD); @@ -1927,7 +1929,8 @@ // Load the vfptr and then callee from the vftable. The callee should have // adjusted 'this' so that the vfptr is at offset zero. llvm::Value *VTable = CGF.GetVTablePtr( - getThisValue(CGF), ThunkTy->getPointerTo()->getPointerTo()); + getThisValue(CGF), ThunkTy->getPointerTo()->getPointerTo(), + MD->getParent()); llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); llvm::Value *Callee = CGF.Builder.CreateLoad(VFuncPtr); Index: test/CodeGenCXX/invariat.barrier-for-vptrs.cpp =================================================================== --- /dev/null +++ test/CodeGenCXX/invariat.barrier-for-vptrs.cpp @@ -0,0 +1,55 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm %s -o - | FileCheck %s + +struct A { + virtual void foo(); +}; + +// CHECK-LABEL: define void @_Z21testExternallyVisiblev() +// CHECK: %vtable = load {{.*}} !invariant.group !1 +// CHECK-LABEL: } + +void testExternallyVisible() { + A *a = new A; + a->foo(); +} + +namespace { + +struct B { + virtual void bar(); +}; + +struct C : B { + void bar(); +}; + +} + +B *get(bool p) { + return p ? new B : new C; +} + +// CHECK-LABEL: define void @_Z21testInternallyVisibleb( +// CHECK: %vtable = load {{.*}}, !invariant.group !2 + +void testInternallyVisible(bool p) { + B *b = get(p); + b->bar(); +} + +// Checking A::A() +// CHECK-LABEL: define linkonce_odr void @_ZN1AC2Ev(%struct.A* %this) +// CHECK: store {{.*}}, !invariant.group !1 +// CHECK-LABEL: } + +// Checking C::C() +// CHECK-LABEL: define internal void @_ZN12_GLOBAL__N_11BC2Ev(%"struct.(anonymous namespace)::B"* +// CHECK: store {{.*}}, !invariant.group !2 + +// Checking B::B() +// CHECK-LABEL: define internal void @_ZN12_GLOBAL__N_11CC2Ev(%"struct.(anonymous namespace)::C"* +// CHECK: store {{.*}}, !invariant.group !3 + +// CHECK: !1 = !{!"_ZTS1A.vptr.invariant.group"} +// CHECK: !2 = distinct !{} +// CHECK: !3 = distinct !{}