Decorating vptr load & stores with !invariant.group

[Prazek]

Summary

Adding !invariant.group to vptr load/stores for devirtualization purposes.

Rebased after Peter commit and on top of strict-vtable-pointers

For more goto:
http://lists.llvm.org/pipermail/cfe-dev/2015-July/044227.htm

[pcc]

lib/CodeGen/CodeGenModule.cpp

1497

This doesn't seem like it needs to be a separate class. See CreateBitSetIdentifierForType/BitSetIdMap in D11857. I'll factor out that functionality into a separate patch to avoid blocking either review on each other.

[pcc]

lib/CodeGen/CodeGenModule.cpp

1497

D12038

[rsmith]

lib/CodeGen/CodeGenModule.cpp

3727–3728

I think this may break the MSVC build, and it looks unrelated to the other changes. Revert this bit, or use llvm::None. In the latter case, commit this separately.

[Prazek]

lib/CodeGen/CodeGenModule.cpp

3727–3728

ok

lib/CodeGen/CGAtomic.cpp

llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
   Load->setAtomic(AO);
 
   // Other decoration.
   Load->setAlignment(getAtomicAlignment().getQuantity());
   if (IsVolatile)
     Load->setVolatile(true);
   if (LVal.getTBAAInfo())
-    CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
+    CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
   return Load;
 }
 
 /// An LValue is a candidate for having its loads and stores be made atomic if
 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
 /// performing such an operation can be performed without a libcall.
 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
   AtomicInfo AI(*this, LV);
if (LVal.isSimple()) {
     if (!isInit)
       store->setAtomic(AO);
 
     // Other decoration.
     store->setAlignment(dest.getAlignment().getQuantity());
     if (IsVolatile)
       store->setVolatile(true);
     if (dest.getTBAAInfo())
-      CGM.DecorateInstruction(store, dest.getTBAAInfo());
+      CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
     return;
   }
 
   // Emit simple atomic update operation.
   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
 }
 
 /// Emit a compare-and-exchange op for atomic type.

lib/CodeGen/CGClass.cpp

 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/EvaluatedExprVisitor.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/CodeGen/CGFunctionInfo.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Metadata.h"
 
 using namespace clang;
 using namespace CodeGen;
 
 CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
     const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
     CastExpr::path_const_iterator End) {
   CharUnits Offset = CharUnits::Zero();
void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr,
 
   // We can just use the base offset in the complete class.
   CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
 
   if (!NonVirtualOffset.isZero())
     This =
         ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr);
 
-  llvm::Value *VPtrValue = GetVTablePtr(This, VTableGlobal->getType());
+  llvm::Value *VPtrValue =
+      GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
   llvm::Value *Cmp =
       Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
   Builder.CreateAssumption(Cmp);
 }
 
 void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
                                                 llvm::Value *This) {
   for (const VPtr &vptr : getVTablePointers(ClassDecl))
void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
   llvm::Type *VTablePtrTy =
       llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
           ->getPointerTo()
           ->getPointerTo();
   VTableField = Builder.CreateBitCast(VTableField, VTablePtrTy->getPointerTo());
   VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
 
   llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
-  CGM.DecorateInstruction(Store, CGM.getTBAAInfoForVTablePtr());
+  CGM.DecorateInstructionWithTBAA(Store, CGM.getTBAAInfoForVTablePtr());
+  CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
 }
 
 CodeGenFunction::VPtrsVector
 CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
   CodeGenFunction::VPtrsVector VPtrsResult;
   VisitedVirtualBasesSetTy VBases;
   getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
                     /*NearestVBase=*/nullptr,
if (CGM.getCXXABI().canInitializeVPtr(Vptr.VTableClass, Vptr.Base.getBase(),
                                           Vptr.NearestVBase))
       InitializeVTablePointer(Vptr);
 
   if (RD->getNumVBases())
     CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
 }
 
 llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This,
-                                           llvm::Type *Ty) {
-  llvm::Value *VTablePtrSrc = Builder.CreateBitCast(This, Ty->getPointerTo());
+                                           llvm::Type *VTableTy,
+                                           const CXXRecordDecl *RD) {
+  llvm::Value *VTablePtrSrc =
+      Builder.CreateBitCast(This, VTableTy->getPointerTo());
   llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
-  CGM.DecorateInstruction(VTable, CGM.getTBAAInfoForVTablePtr());
+  CGM.DecorateInstructionWithTBAA(VTable, CGM.getTBAAInfoForVTablePtr());
+  CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
+
   return VTable;
 }
 
 // If a class has a single non-virtual base and does not introduce or override
 // virtual member functions or fields, it will have the same layout as its base.
 // This function returns the least derived such class.
 //
 // Casting an instance of a base class to such a derived class is technically
if (MayBeNull) {
     llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
     ContBlock = createBasicBlock("cast.cont");
 
     Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
 
     EmitBlock(CheckBlock);
   }
 
-  llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy);
+  llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy, ClassDecl);
   EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
 
   if (MayBeNull) {
     Builder.CreateBr(ContBlock);
     EmitBlock(ContBlock);
   }
 }
 

lib/CodeGen/CGExpr.cpp

llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
   if (Volatile)
     Load->setVolatile(true);
   if (Alignment)
     Load->setAlignment(Alignment);
   if (TBAAInfo) {
     llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo,
                                                       TBAAOffset);
     if (TBAAPath)
-      CGM.DecorateInstruction(Load, TBAAPath, false/*ConvertTypeToTag*/);
+      CGM.DecorateInstructionWithTBAA(Load, TBAAPath,
+                                      false /*ConvertTypeToTag*/);
   }
 
   bool NeedsBoolCheck =
       SanOpts.has(SanitizerKind::Bool) && hasBooleanRepresentation(Ty);
   bool NeedsEnumCheck =
       SanOpts.has(SanitizerKind::Enum) && Ty->getAs<EnumType>();
   if (NeedsBoolCheck || NeedsEnumCheck) {
     SanitizerScope SanScope(this);
void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
 
   llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
   if (Alignment)
     Store->setAlignment(Alignment);
   if (TBAAInfo) {
     llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo,
                                                       TBAAOffset);
     if (TBAAPath)
-      CGM.DecorateInstruction(Store, TBAAPath, false/*ConvertTypeToTag*/);
+      CGM.DecorateInstructionWithTBAA(Store, TBAAPath,
+                                      false /*ConvertTypeToTag*/);
   }
 }
 
 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
                                         bool isInit) {
   EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
                     lvalue.getAlignment().getQuantity(), lvalue.getType(),
                     lvalue.getTBAAInfo(), isInit, lvalue.getTBAABaseType(),
if (const ReferenceType *refType = type->getAs<ReferenceType>()) {
       TBAAPath = false;
       if (CGM.shouldUseTBAA()) {
         llvm::MDNode *tbaa;
         if (mayAlias)
           tbaa = CGM.getTBAAInfo(getContext().CharTy);
         else
           tbaa = CGM.getTBAAInfo(type);
         if (tbaa)
-          CGM.DecorateInstruction(load, tbaa);
+          CGM.DecorateInstructionWithTBAA(load, tbaa);
       }
 
       addr = load;
       mayAlias = false;
       type = refType->getPointeeType();
       if (type->isIncompleteType())
         alignment = CharUnits();
       else

lib/CodeGen/CGExprCXX.cpp

RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
   if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(MD)) {
     Callee = CGM.GetAddrOfFunction(GlobalDecl(Ctor, Ctor_Complete), Ty);
   } else if (UseVirtualCall) {
     Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty,
                                                        CE->getLocStart());
   } else {
     if (SanOpts.has(SanitizerKind::CFINVCall) &&
         MD->getParent()->isDynamicClass()) {
-      llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy);
+      llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy, MD->getParent());
       EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart());
     }
 
     if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier)
       Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty);
     else if (!DevirtualizedMethod)
       Callee = CGM.GetAddrOfFunction(MD, Ty);
     else {

lib/CodeGen/CodeGenFunction.h

void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
                          bool BaseIsNonVirtualPrimaryBase,
                          const CXXRecordDecl *VTableClass,
                          VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
 
   void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
 
   /// GetVTablePtr - Return the Value of the vtable pointer member pointed
   /// to by This.
-  llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty);
+  llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *VTableTy,
+                            const CXXRecordDecl *RD);
 
   enum CFITypeCheckKind {
     CFITCK_VCall,
     CFITCK_NVCall,
     CFITCK_DerivedCast,
     CFITCK_UnrelatedCast,
   };
 

lib/CodeGen/CodeGenModule.h

private:
 
   std::unique_ptr<SanitizerMetadata> SanitizerMD;
 
   /// @}
 
   llvm::DenseMap<const Decl *, bool> DeferredEmptyCoverageMappingDecls;
 
   std::unique_ptr<CoverageMappingModuleGen> CoverageMapping;
+
+  /// Helper class for generating invariant.group metadata.
+  /// For externally visible classes it generates node like this
+  /// !{ "%mangledName%.%TAG%.invariant.group" }
+  /// and for internally visible classes it generates empty distinct nodes.
+  class InvariantGroupMetadataFactory {
+    CodeGenModule &CGM;
+    const std::string tag;
+    llvm::SmallDenseMap<const CXXRecordDecl *, llvm::MDNode *> distinctMDNodes;
+
+    std::string getFullName(const CXXRecordDecl *ClassDecl) const;
+    llvm::MDNode *getInternallyVisibleMD(const CXXRecordDecl *ClassDecl);
+    llvm::MDNode *getExternallyVisibleMD(const CXXRecordDecl *ClassDecl) const;
+
+  public:
+    InvariantGroupMetadataFactory(CodeGenModule &CGM, std::string tag);
+    llvm::MDNode *getInvariantMetadata(const CXXRecordDecl *ClassDecl);
+  };
+
+  InvariantGroupMetadataFactory vptrInvariantGroupMetadataFactory;
+
 public:
   CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts,
                 const PreprocessorOptions &ppopts,
                 const CodeGenOptions &CodeGenOpts, llvm::Module &M,
                 DiagnosticsEngine &Diags,
                 CoverageSourceInfo *CoverageInfo = nullptr);
 
   ~CodeGenModule();
public:
 
   bool isPaddedAtomicType(QualType type);
   bool isPaddedAtomicType(const AtomicType *type);
 
   /// Decorate the instruction with a TBAA tag. For scalar TBAA, the tag
   /// is the same as the type. For struct-path aware TBAA, the tag
   /// is different from the type: base type, access type and offset.
   /// When ConvertTypeToTag is true, we create a tag based on the scalar type.
-  void DecorateInstruction(llvm::Instruction *Inst,
+  void DecorateInstructionWithTBAA(llvm::Instruction *Inst,
                                    llvm::MDNode *TBAAInfo,
                                    bool ConvertTypeToTag = true);
 
+  /// Adds !invariant.barrier !tag to instruction
+  void DecorateInstructionWithInvariantGroup(llvm::Instruction *I,
+                                             const CXXRecordDecl *RD);
+
   /// Emit the given number of characters as a value of type size_t.
   llvm::ConstantInt *getSize(CharUnits numChars);
 
   /// Set the visibility for the given LLVM GlobalValue.
   void setGlobalVisibility(llvm::GlobalValue *GV, const NamedDecl *D) const;
 
   /// Set the TLS mode for the given LLVM GlobalValue for the thread-local
   /// variable declaration D.

lib/CodeGen/CodeGenModule.cpp

: Context(C), LangOpts(C.getLangOpts()), HeaderSearchOpts(HSO),
       OpenCLRuntime(nullptr), OpenMPRuntime(nullptr), CUDARuntime(nullptr),
       DebugInfo(nullptr), ARCData(nullptr),
       NoObjCARCExceptionsMetadata(nullptr), RRData(nullptr), PGOReader(nullptr),
       CFConstantStringClassRef(nullptr), ConstantStringClassRef(nullptr),
       NSConstantStringType(nullptr), NSConcreteGlobalBlock(nullptr),
       NSConcreteStackBlock(nullptr), BlockObjectAssign(nullptr),
       BlockObjectDispose(nullptr), BlockDescriptorType(nullptr),
       GenericBlockLiteralType(nullptr), LifetimeStartFn(nullptr),
-      LifetimeEndFn(nullptr), SanitizerMD(new SanitizerMetadata(*this)) {
+      LifetimeEndFn(nullptr), SanitizerMD(new SanitizerMetadata(*this)),
+      vptrInvariantGroupMetadataFactory(*this, "vptr") {
 
   // Initialize the type cache.
   llvm::LLVMContext &LLVMContext = M.getContext();
   VoidTy = llvm::Type::getVoidTy(LLVMContext);
   Int8Ty = llvm::Type::getInt8Ty(LLVMContext);
   Int16Ty = llvm::Type::getInt16Ty(LLVMContext);
   Int32Ty = llvm::Type::getInt32Ty(LLVMContext);
   Int64Ty = llvm::Type::getInt64Ty(LLVMContext);
if (!TBAA)
     return nullptr;
   return TBAA->getTBAAStructTagInfo(BaseTy, AccessN, O);
 }
 
 /// Decorate the instruction with a TBAA tag. For both scalar TBAA
 /// and struct-path aware TBAA, the tag has the same format:
 /// base type, access type and offset.
 /// When ConvertTypeToTag is true, we create a tag based on the scalar type.
-void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst,
+void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst,
                                                 llvm::MDNode *TBAAInfo,
                                                 bool ConvertTypeToTag) {
   if (ConvertTypeToTag && TBAA)
     Inst->setMetadata(llvm::LLVMContext::MD_tbaa,
                       TBAA->getTBAAScalarTagInfo(TBAAInfo));
   else
     Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo);
 }
 
+void CodeGenModule::DecorateInstructionWithInvariantGroup(
+    llvm::Instruction *I, const CXXRecordDecl *RD) {
+  auto *MetaData =
+      vptrInvariantGroupMetadataFactory.getInvariantMetadata(RD);
+  I->setMetadata("invariant.group", MetaData);
+}
+
 void CodeGenModule::Error(SourceLocation loc, StringRef message) {
   unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0");
   getDiags().Report(Context.getFullLoc(loc), diagID) << message;
 }
 
 /// ErrorUnsupported - Print out an error that codegen doesn't support the
 /// specified stmt yet.
 void CodeGenModule::ErrorUnsupported(const Stmt *S, const char *Type) {
CodeGenModule::shouldEmitFunction(GlobalDecl GD) {
   // PR9614. Avoid cases where the source code is lying to us. An available
   // externally function should have an equivalent function somewhere else,
   // but a function that calls itself is clearly not equivalent to the real
   // implementation.
   // This happens in glibc's btowc and in some configure checks.
   return !isTriviallyRecursive(F);
 }
 
+CodeGenModule::InvariantGroupMetadataFactory::InvariantGroupMetadataFactory(
+    CodeGenModule &CGM_, std::string tag_)
+    : CGM(CGM_), tag(std::move(tag_)) {}
+
+std::string CodeGenModule::InvariantGroupMetadataFactory::getFullName(
+    const CXXRecordDecl *ClassDecl) const {
+  std::string MangledName;
+  llvm::raw_string_ostream MangledNameStream(MangledName);
+  CGM.getCXXABI().getMangleContext().mangleTypeName(
+      QualType(ClassDecl->getTypeForDecl(), 0), MangledNameStream);
+
+  MangledNameStream << "." << tag << ".invariant.group";
+  MangledNameStream.flush();
+  return MangledName;
+}
+
+llvm::MDNode *
+CodeGenModule::InvariantGroupMetadataFactory::getInternallyVisibleMD(
+    const CXXRecordDecl *ClassDecl) {
+  assert(ClassDecl != nullptr);
+  llvm::MDNode *&MetaData = distinctMDNodes[ClassDecl];
+  if (!MetaData)
+    MetaData = llvm::MDNode::getDistinct(CGM.getLLVMContext(), {});
+  return MetaData;
+}
+
+llvm::MDNode *
+CodeGenModule::InvariantGroupMetadataFactory::getExternallyVisibleMD(
+    const CXXRecordDecl *ClassDecl) const {
+  auto *MDString = llvm::MDString::get(CGM.getLLVMContext(), getFullName(ClassDecl));
+  return llvm::MDNode::get(CGM.getLLVMContext(), MDString);
+}
+
+llvm::MDNode *
+CodeGenModule::InvariantGroupMetadataFactory::getInvariantMetadata(
+    const CXXRecordDecl *ClassDecl) {
+  if (ClassDecl->isExternallyVisible())
+    return getExternallyVisibleMD(ClassDecl);
+  return getInternallyVisibleMD(ClassDecl);
+}
+
 /// If the type for the method's class was generated by
 /// CGDebugInfo::createContextChain(), the cache contains only a
 /// limited DIType without any declarations. Since EmitFunctionStart()
 /// needs to find the canonical declaration for each method, we need
 /// to construct the complete type prior to emitting the method.
 void CodeGenModule::CompleteDIClassType(const CXXMethodDecl* D) {
   if (!D->isInstance())
     return;
bool PerformInit =
         VD->getAnyInitializer() &&
         !VD->getAnyInitializer()->isConstantInitializer(getContext(),
                                                         /*ForRef=*/false);
     if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition(
             VD, GetAddrOfGlobalVar(VD), RefExpr->getLocStart(), PerformInit))
       CXXGlobalInits.push_back(InitFunction);
   }
 }
 
 llvm::MDTuple *CodeGenModule::CreateVTableBitSetEntry(
     llvm::GlobalVariable *VTable, CharUnits Offset, const CXXRecordDecl *RD) {
   std::string OutName;
   llvm::raw_string_ostream Out(OutName);
   getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out);
 
   llvm::Metadata *BitsetOps[] = {
       llvm::MDString::get(getLLVMContext(), Out.str()),
       llvm::ConstantAsMetadata::get(VTable),
       llvm::ConstantAsMetadata::get(
           llvm::ConstantInt::get(Int64Ty, Offset.getQuantity()))};
   return llvm::MDTuple::get(getLLVMContext(), BitsetOps);
 }

lib/CodeGen/ItaniumCXXABI.cpp

llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
 
   // In the virtual path, the adjustment left 'This' pointing to the
   // vtable of the correct base subobject.  The "function pointer" is an
   // offset within the vtable (+1 for the virtual flag on non-ARM).
   CGF.EmitBlock(FnVirtual);
 
   // Cast the adjusted this to a pointer to vtable pointer and load.
   llvm::Type *VTableTy = Builder.getInt8PtrTy();
-  llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy);
+  llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy, RD);
 
   // Apply the offset.
   llvm::Value *VTableOffset = FnAsInt;
   if (!UseARMMethodPtrABI)
     VTableOffset = Builder.CreateSub(VTableOffset, ptrdiff_1);
   VTable = Builder.CreateGEP(VTable, VTableOffset);
 
   // Load the virtual function to call.
void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
                                             QualType ElementType,
                                             const CXXDestructorDecl *Dtor) {
   bool UseGlobalDelete = DE->isGlobalDelete();
   if (UseGlobalDelete) {
     // Derive the complete-object pointer, which is what we need
     // to pass to the deallocation function.
 
     // Grab the vtable pointer as an intptr_t*.
-    llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo());
+    auto *ClassDecl =
+        cast<CXXRecordDecl>(ElementType->getAs<RecordType>()->getDecl());
+    llvm::Value *VTable =
+        CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl);
 
     // Track back to entry -2 and pull out the offset there.
     llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
         VTable, -2, "complete-offset.ptr");
     llvm::LoadInst *Offset = CGF.Builder.CreateLoad(OffsetPtr);
     Offset->setAlignment(CGF.PointerAlignInBytes);
 
     // Apply the offset.
void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
   CGF.EmitRuntimeCallOrInvoke(Fn).setDoesNotReturn();
   CGF.Builder.CreateUnreachable();
 }
 
 llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
                                        QualType SrcRecordTy,
                                        llvm::Value *ThisPtr,
                                        llvm::Type *StdTypeInfoPtrTy) {
+  auto *ClassDecl =
+      cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
   llvm::Value *Value =
-      CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo());
+      CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl);
 
   // Load the type info.
   Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL);
   return CGF.Builder.CreateLoad(Value);
 }
 
 bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
                                                        QualType SrcRecordTy) {

 llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF,
                                                   llvm::Value *Value,
                                                   QualType SrcRecordTy,
                                                   QualType DestTy) {
   llvm::Type *PtrDiffLTy =
       CGF.ConvertType(CGF.getContext().getPointerDiffType());
   llvm::Type *DestLTy = CGF.ConvertType(DestTy);
 
+  auto *ClassDecl =
+      cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
   // Get the vtable pointer.
-  llvm::Value *VTable = CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo());
+  llvm::Value *VTable =
+      CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo(), ClassDecl);
 
   // Get the offset-to-top from the vtable.
   llvm::Value *OffsetToTop =
       CGF.Builder.CreateConstInBoundsGEP1_64(VTable, -2ULL);
   OffsetToTop = CGF.Builder.CreateLoad(OffsetToTop, "offset.to.top");
 
   // Finally, add the offset to the pointer.
   Value = CGF.EmitCastToVoidPtr(Value);
bool ItaniumCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
   return true;
 }
 
 llvm::Value *
 ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
                                          llvm::Value *This,
                                          const CXXRecordDecl *ClassDecl,
                                          const CXXRecordDecl *BaseClassDecl) {
-  llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy);
+  llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy, ClassDecl);
   CharUnits VBaseOffsetOffset =
       CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl,
                                                                BaseClassDecl);
 
   llvm::Value *VBaseOffsetPtr =
     CGF.Builder.CreateConstGEP1_64(VTablePtr, VBaseOffsetOffset.getQuantity(),
                                    "vbase.offset.ptr");
   VBaseOffsetPtr = CGF.Builder.CreateBitCast(VBaseOffsetPtr,

 
 llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
                                                       GlobalDecl GD,
                                                       llvm::Value *This,
                                                       llvm::Type *Ty,
                                                       SourceLocation Loc) {
   GD = GD.getCanonicalDecl();
   Ty = Ty->getPointerTo()->getPointerTo();
-  llvm::Value *VTable = CGF.GetVTablePtr(This, Ty);
+  auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl());
+  llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent());
 
   if (CGF.SanOpts.has(SanitizerKind::CFIVCall))
-    CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable,
+    CGF.EmitVTablePtrCheckForCall(MethodDecl, VTable,
                                   CodeGenFunction::CFITCK_VCall, Loc);
 
   uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
   llvm::Value *VFuncPtr =
       CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn");
   return CGF.Builder.CreateLoad(VFuncPtr);
 }
 

lib/CodeGen/MicrosoftCXXABI.cpp

llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
                                                         llvm::Type *Ty,
                                                         SourceLocation Loc) {
   GD = GD.getCanonicalDecl();
   CGBuilderTy &Builder = CGF.Builder;
 
   Ty = Ty->getPointerTo()->getPointerTo();
   llvm::Value *VPtr =
       adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true);
-  llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty);
+
+  auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl());
+  llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent());
 
   MicrosoftVTableContext::MethodVFTableLocation ML =
       CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD);
   if (CGF.SanOpts.has(SanitizerKind::CFIVCall))
     CGF.EmitVTablePtrCheck(getClassAtVTableLocation(getContext(), GD, ML),
                            VTable, CodeGenFunction::CFITCK_VCall, Loc);
 
   llvm::Value *VFuncPtr =
llvm::Function *MicrosoftCXXABI::EmitVirtualMemPtrThunk(
   // Start defining the function.
   CGF.StartFunction(GlobalDecl(), FnInfo.getReturnType(), ThunkFn, FnInfo,
                     FunctionArgs, MD->getLocation(), SourceLocation());
   EmitThisParam(CGF);
 
   // Load the vfptr and then callee from the vftable.  The callee should have
   // adjusted 'this' so that the vfptr is at offset zero.
   llvm::Value *VTable = CGF.GetVTablePtr(
-      getThisValue(CGF), ThunkTy->getPointerTo()->getPointerTo());
+      getThisValue(CGF), ThunkTy->getPointerTo()->getPointerTo(),
+      MD->getParent());
   llvm::Value *VFuncPtr =
       CGF.Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn");
   llvm::Value *Callee = CGF.Builder.CreateLoad(VFuncPtr);
 
   CGF.EmitMustTailThunk(MD, getThisValue(CGF), Callee);
 
   return ThunkFn;
 }

test/CodeGenCXX/invariat.barrier-for-vptrs.cpp

+// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm %s -o - | FileCheck %s
+
+struct A {
+  virtual void foo();
+};
+
+// CHECK-LABEL: define void @_Z21testExternallyVisiblev()
+// CHECK: %vtable = load {{.*}} !invariant.group !1
+// CHECK-LABEL: }
+
+void testExternallyVisible() {
+  A *a = new A;
+  a->foo();
+}
+
+namespace {
+
+struct B {
+  virtual void bar();
+};
+
+struct C : B {
+  void bar();
+};
+
+}
+
+B *get(bool p) {
+  return p ? new B : new C;
+}
+
+// CHECK-LABEL: define void @_Z21testInternallyVisibleb(
+// CHECK:  %vtable = load {{.*}}, !invariant.group !2
+
+void testInternallyVisible(bool p) {
+  B *b = get(p);
+  b->bar();
+}
+
+// Checking A::A()
+// CHECK-LABEL: define linkonce_odr void @_ZN1AC2Ev(%struct.A* %this)
+// CHECK: store {{.*}}, !invariant.group !1
+// CHECK-LABEL: }
+
+// Checking C::C()
+// CHECK-LABEL: define internal void @_ZN12_GLOBAL__N_11BC2Ev(%"struct.(anonymous namespace)::B"*
+// CHECK:  store {{.*}}, !invariant.group !2
+
+// Checking B::B()
+// CHECK-LABEL: define internal void @_ZN12_GLOBAL__N_11CC2Ev(%"struct.(anonymous namespace)::C"*
+// CHECK:  store {{.*}}, !invariant.group !3
+
+// CHECK: !1 = !{!"_ZTS1A.vptr.invariant.group"}
+// CHECK: !2 = distinct !{}
+// CHECK: !3 = distinct !{}