[Introduction] C++ Const object optimization in LLVM

[lvoufo]

Summary

Based on the design doc at
https://docs.google.com/document/d/112O-Q_XrbrU1I4P-oiLCN9u86Qg_BYBdcDsmh7Pn9Nw/edit?usp=sharing,
it handles non-member const objects.
Extensions are discussed in the design doc and will be submitted separately.

[lvoufo]

Clang-formatted.

[lvoufo]

Fix unit test formatting.

[lvoufo]

Fix regression test formatting.

[lvoufo]

All formatting changes should be done by now.

[rsmith]

Your D14419 removes a lot of the code added here. Can you update this patch to not add that code? I'd prefer not to review the portion of this code that you're about to delete.

lib/CodeGen/CGDecl.cpp

369

The call to AddInitializerToStaticVarDecl above indirectly calls EmitCXXGlobalVarDeclInit, which already emits an llvm.invariant.start call when appropriate. Please extend the code in EmitCXXGlobalVarDeclInit to handle these new cases (constant types with non-trivial destructors) rather adding another codepath that creates an llvm.invariant.start here.

877–879

I don't think this check is necessary; if we know some region of memory is locally constant, I don't think it matters whether we can trace it back to an alloca.

884

Do we need to add QualType::isWriteOnce for this? CodeGenModule::isTypeConstant seems to be doing mostly the right set of checks. I suggest you add a new flag to it to indicate whether it should check for trivial destructibility, and then use it here instead of using isWriteOnce.

902

The use of RAII here seems unnecessary. You should be able to emit the invariant start and the invariant end cleanup at the same time, by moving this after the call to EmitAutoVarCleanups.

952–958

This conditional should not be necessary: IRBuilder::CreateBitCast should choose between a bitcast constant and a bitcast instruction for you. Just use the non-constant branch in all cases.

959

Not all of our supported versions of MSVC accept this syntax. Create a separate array of llvm::Value * and pass it in here.

lib/CodeGen/CGDeclCXX.cpp

478

This should be unnecessary if you let EmitCXXGlobalVarDeclInit emit the invariant start.

test/CodeGenCXX/init-invariant.cpp

47–49

Please also check that @llvm.invariant.end is called later in the same global initialization function.

52–53

Likewise for this case.

include/clang/AST/DeclCXX.h

struct DefinitionData {
     bool HasPrivateFields : 1;
 
     /// \brief True when there are protected non-static data members.
     bool HasProtectedFields : 1;
 
     /// \brief True when there are private non-static data members.
     bool HasPublicFields : 1;
 
+    /// \brief True is this class has already been checked as candidate
+    /// for writeonce semantics.
+    bool IsWriteOnceChecked : 1;
+
+    /// \brief True if instances of this class are candidates
+    /// for writeonce semantics.
+    bool IsWriteOnceCandidate : 1;
+
     /// \brief True if this class (or any subobject) has mutable fields.
     bool HasMutableFields : 1;
 
     /// \brief True if this class (or any nested anonymous struct or union)
     /// has variant members.
     bool HasVariantMembers : 1;
 
     /// \brief True if there no non-field members declared by the user.
public:
   method_iterator method_begin() const {
     return method_iterator(decls_begin());
   }
   /// \brief Method past-the-end iterator.
   method_iterator method_end() const {
     return method_iterator(decls_end());
   }
 
+  bool computeWriteOnceCandidacy() const;
+
   /// Iterator access to constructor members.
   typedef specific_decl_iterator<CXXConstructorDecl> ctor_iterator;
   typedef llvm::iterator_range<specific_decl_iterator<CXXConstructorDecl>>
     ctor_range;
 
   ctor_range ctors() const { return ctor_range(ctor_begin(), ctor_end()); }
 
   ctor_iterator ctor_begin() const {

include/clang/AST/Type.h

public:
   /// \brief Retrieve the set of CVR (const-volatile-restrict) qualifiers
   /// applied to this type.
   unsigned getCVRQualifiers() const;
 
   bool isConstant(ASTContext& Ctx) const {
     return QualType::isConstant(*this, Ctx);
   }
 
+  /// \brief Determine if this satisfies 'writeonce' properties.
+  bool isWriteOnce(ASTContext &Ctx);
+
   /// \brief Determine whether this is a Plain Old Data (POD) type (C++ 3.9p10).
   bool isPODType(ASTContext &Context) const;
 
   /// Return true if this is a POD type according to the rules of the C++98
   /// standard, regardless of the current compilation's language.
   bool isCXX98PODType(ASTContext &Context) const;
 
   /// Return true if this is a POD type according to the more relaxed rules

lib/AST/DeclCXX.cpp

void LazyASTUnresolvedSet::getFromExternalSource(ASTContext &C) const {
 
   for (ASTUnresolvedSet::iterator I = Impl.begin(); I != Impl.end(); ++I)
     I.setDecl(cast<NamedDecl>(Source->GetExternalDecl(
         reinterpret_cast<uintptr_t>(I.getDecl()) >> 2)));
   Impl.Decls.setLazy(false);
 }
 
 CXXRecordDecl::DefinitionData::DefinitionData(CXXRecordDecl *D)
-  : UserDeclaredConstructor(false), UserDeclaredSpecialMembers(0),
-    Aggregate(true), PlainOldData(true), Empty(true), Polymorphic(false),
-    Abstract(false), IsStandardLayout(true), HasNoNonEmptyBases(true),
-    HasPrivateFields(false), HasProtectedFields(false), HasPublicFields(false),
-    HasMutableFields(false), HasVariantMembers(false), HasOnlyCMembers(true),
-    HasInClassInitializer(false), HasUninitializedReferenceMember(false),
+    : UserDeclaredConstructor(false),
+      UserDeclaredSpecialMembers(0),
+      Aggregate(true),
+      PlainOldData(true),
+      Empty(true),
+      Polymorphic(false),
+      Abstract(false),
+      IsStandardLayout(true),
+      HasNoNonEmptyBases(true),
+      HasPrivateFields(false),
+      HasProtectedFields(false),
+      HasPublicFields(false),
+      IsWriteOnceChecked(false),
+      IsWriteOnceCandidate(false),
+      HasMutableFields(false),
+      HasVariantMembers(false),
+      HasOnlyCMembers(true),
+      HasInClassInitializer(false),
+      HasUninitializedReferenceMember(false),
       NeedOverloadResolutionForMoveConstructor(false),
       NeedOverloadResolutionForMoveAssignment(false),
       NeedOverloadResolutionForDestructor(false),
       DefaultedMoveConstructorIsDeleted(false),
       DefaultedMoveAssignmentIsDeleted(false),
       DefaultedDestructorIsDeleted(false),
       HasTrivialSpecialMembers(SMF_All),
       DeclaredNonTrivialSpecialMembers(0),
       HasIrrelevantDestructor(true),
       HasConstexprNonCopyMoveConstructor(false),
       DefaultedDefaultConstructorIsConstexpr(true),
       HasConstexprDefaultConstructor(false),
-    HasNonLiteralTypeFieldsOrBases(false), ComputedVisibleConversions(false),
-    UserProvidedDefaultConstructor(false), DeclaredSpecialMembers(0),
+      HasNonLiteralTypeFieldsOrBases(false),
+      ComputedVisibleConversions(false),
+      UserProvidedDefaultConstructor(false),
+      DeclaredSpecialMembers(0),
       ImplicitCopyConstructorHasConstParam(true),
       ImplicitCopyAssignmentHasConstParam(true),
       HasDeclaredCopyConstructorWithConstParam(false),
       HasDeclaredCopyAssignmentWithConstParam(false),
-    IsLambda(false), IsParsingBaseSpecifiers(false), NumBases(0), NumVBases(0),
-    Bases(), VBases(),
-    Definition(D), FirstFriend() {
-}
+      IsLambda(false),
+      IsParsingBaseSpecifiers(false),
+      NumBases(0),
+      NumVBases(0),
+      Bases(),
+      VBases(),
+      Definition(D),
+      FirstFriend() {}
 
 CXXBaseSpecifier *CXXRecordDecl::DefinitionData::getBasesSlowCase() const {
   return Bases.get(Definition->getASTContext().getExternalSource());
 }
 
 CXXBaseSpecifier *CXXRecordDecl::DefinitionData::getVBasesSlowCase() const {
   return VBases.get(Definition->getASTContext().getExternalSource());
 }
if (const CXXRecordDecl *RD =
             Field->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl())
       if (RD->isAnyDestructorNoReturn())
         return true;
 
   // All destructors are not noreturn.
   return false;
 }
 
+/// \brief A class is a candidate for 'writeonce' semantics if
+/// none of its methods writes to memory.
+bool CXXRecordDecl::computeWriteOnceCandidacy() const {
+  // The class must be defined and concrete.
+  if (!isCompleteDefinition() || isDependentType()) return false;
+
+  // If this has already been checked, then return the stored value.
+  if (data().IsWriteOnceChecked) return data().IsWriteOnceCandidate;
+
+  // Otherwise, mark that this is checked.
+  data().IsWriteOnceChecked = true;
+  assert(!data().IsWriteOnceCandidate &&
+         "The candidacy should not have been set yet.");
+
+  // Check each method, excluding constructors and destructors.
+  // NOTE: The invariant intrinsic calls are generated right after the
+  //       construction of a given const object (which must be initialized)
+  //       and right before its destruction at the end of its lifetime.
+  //       Both constructors and destructors modify the allocated memory and
+  //       and will not be called in the middle of an invariant_start/end
+  //       pair, thus may not affect the reduction of loads. So, it does not
+  //       help to handle constructors here as well.
+  for (auto *M : methods()) {
+    if (isa<CXXConstructorDecl>(M) || isa<CXXDestructorDecl>(M)) continue;
+
+    // If we already know that this method does not write to memory, skip it.
+    // NOTE: CodeGenModule::ConstructAttributeList() marks declarations with
+    //       ConstAttr and PureAttr attributes, respectively, as 'readnone' and
+    //       'readonly' (for LLVM).
+    if (M->hasAttr<ConstAttr>() || M->hasAttr<PureAttr>()) continue;
+
+    const FunctionDecl *Def = nullptr;
+    bool IsDefined = M->isDefined(Def);
+
+    // If this is not defined and not virtual, then it may write to memory.
+    // So it's not a candidate for 'writeonce' semantics.
+    // If it is purely virtual then it must be overriden and its overriding
+    // method (in this context) must be a candidate for 'writeonce' semantics.
+    // Skip it.
+    if (!IsDefined) {
+      if (M->isPure()) {
+        assert(M->isVirtual() && "Not defined and pure implies virtual.");
+        continue;
+      }
+      return false;
+    }
+
+    // If this is trivial, skip it.
+    if (Def->hasTrivialBody()) continue;
+
+    // TODO: Any other case?
+  }
+
+  // Check bases
+  for (const auto &I : bases()) {
+    const RecordType *Ty = I.getType()->getAs<RecordType>();
+    assert(Ty && "No type?");
+    CXXRecordDecl *Base =
+        cast_or_null<CXXRecordDecl>(Ty->getDecl()->getDefinition());
+
+    if (!Base || !Base->computeWriteOnceCandidacy()) return false;
+  }
+
+  data().IsWriteOnceCandidate = true;
+  return true;
+}
+
 void CXXRecordDecl::completeDefinition() {
   completeDefinition(nullptr);
 }
 
 void CXXRecordDecl::completeDefinition(CXXFinalOverriderMap *FinalOverriders) {
   RecordDecl::completeDefinition();
   
   // If the class may be abstract (but hasn't been marked as such), check for

lib/AST/Type.cpp

ObjCInterfaceDecl *Interface
       = cast<ObjCInterfaceType>(CanonicalType)->getDecl();
     if (Def)
       *Def = Interface;
     return !Interface->hasDefinition();
   }
   }
 }
 
+bool QualType::isWriteOnce(ASTContext &Context) {
+  // TODO: Include C objects as well?
+  if (Context.getLangOpts().CPlusPlus && isConstant(Context) &&
+      !getTypePtr()->isReferenceType()) {
+    if (const CXXRecordDecl *Record =
+            Context.getBaseElementType(*this)->getAsCXXRecordDecl())
+      return Record->computeWriteOnceCandidacy();
+    return true;
+  }
+  return false;
+}
+
 bool QualType::isPODType(ASTContext &Context) const {
   // C++11 has a more relaxed definition of POD.
   if (Context.getLangOpts().CPlusPlus11)
     return isCXX11PODType(Context);
 
   return isCXX98PODType(Context);
 }
 

lib/CodeGen/CGDecl.cpp

void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
   llvm::Type *expectedType = addr->getType();
 
   llvm::GlobalVariable *var =
     cast<llvm::GlobalVariable>(addr->stripPointerCasts());
   // If this value has an initializer, emit it.
   if (D.getInit())
     var = AddInitializerToStaticVarDecl(D, var);
 
+  MarkWriteOnceWrittenRAII MWO(*this, &D, var);
+
   var->setAlignment(alignment.getQuantity());
 
   if (D.hasAttr<AnnotateAttr>())
     CGM.AddGlobalAnnotations(&D, var);
 
   if (const SectionAttr *SA = D.getAttr<SectionAttr>())
     var->setSection(SA->getName());
 
namespace {
   public:
     CallLifetimeEnd(Address addr, llvm::Value *size)
       : Addr(addr.getPointer()), Size(size) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitLifetimeEnd(Size, Addr);
     }
   };
+
+  /// A cleanup to call @llvm.invariant.end.
+  class CallInvariantEnd final : public EHScopeStack::Cleanup {
+    llvm::CallInst *StartInst;
+    llvm::Value *Addr;
+    llvm::Value *Size;  // TODO: Is this even necessary?
+   public:
+    CallInvariantEnd(llvm::CallInst *C, llvm::Value *addr, llvm::Value *size)
+        : StartInst(C), Addr(addr), Size(size) {}
+
+    void Emit(CodeGenFunction &CGF, Flags flags) override {
+      CGF.EmitInvariantEnd(StartInst, Size, Addr);
+    }
+  };
 }
 
 /// EmitAutoVarWithLifetime - Does the setup required for an automatic
 /// variable with lifetime.
 static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var,
                                     Address addr,
                                     Qualifiers::ObjCLifetime lifetime) {
   switch (lifetime) {
static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init,
   // plopping in more stores.
   unsigned StoreBudget = 6;
   uint64_t SizeLimit = 32;
 
   return GlobalSize > SizeLimit &&
          canEmitInitWithFewStoresAfterMemset(Init, StoreBudget);
 }
 
+/// Emit the code necessary to initialize the given global variable.
+/// TODO: Use Address for Addr (to match lifetime intrisics).
+MarkWriteOnceWrittenRAII::MarkWriteOnceWrittenRAII(CodeGenFunction &CGF,
+                                                   const VarDecl *D,
+                                                   llvm::GlobalVariable *GVAddr)
+    : CGF(CGF), GV(GVAddr), AI(nullptr) {
+  assert(D && "Invalid declaration.");
+
+  llvm::Value *AddrPtr = nullptr;
+  if (GVAddr) {
+    assert(GV && GV == GVAddr && "Caller provides address for globals");
+    AddrPtr = GVAddr;
+  } else {
+    CodeGen::Address Addr = CGF.GetAddrOfLocalVar(D);
+    AddrPtr = Addr.getPointer()->stripPointerCasts();
+    AI = dyn_cast_or_null<llvm::AllocaInst>(AddrPtr);
+  }
+
+  // Only GlobalVariable's and AllocaInst's can be writeonce.
+  // Exit if the given address is none of these.
+  if (!GV && !AI) return;
+
+  // If the address is writeonce, then emit @llvm.invariant.start() intrinsic.
+  // Then, for non-global variables, push the emission of the
+  // @llvm.invariant.end() intrinsic onto the cleanup stack.
+  if ((AI || !GV->isConstant()) && D->getType().isWriteOnce(CGF.getContext()))
+    Args = CGF.EmitInvariantStart(*D, AddrPtr);
+}
+
+MarkWriteOnceWrittenRAII::~MarkWriteOnceWrittenRAII() {
+  if (AI && Args.StartInst) {
+    assert(!GV && "Can't have it both ways!");
+    CGF.EHStack.pushCleanup<CallInvariantEnd>(NormalCleanup, Args.StartInst,
+                                              Args.Addr, Args.Size);
+  }
+}
+
 /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
 /// variable declaration with auto, register, or no storage class specifier.
 /// These turn into simple stack objects, or GlobalValues depending on target.
 void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) {
   AutoVarEmission emission = EmitAutoVarAlloca(D);
   EmitAutoVarInit(emission);
+  MarkWriteOnceWrittenRAII MWO(*this, &D);
   EmitAutoVarCleanups(emission);
 }
 
 /// Emit a lifetime.begin marker if some criteria are satisfied.
 /// \return a pointer to the temporary size Value if a marker was emitted, null
 /// otherwise
 llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size,
                                                 llvm::Value *Addr) {

 
 void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
   Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
   llvm::CallInst *C =
       Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
   C->setDoesNotThrow();
 }
 
+/// Emit code to cause the variable at the given address to be considered as
+/// constant from this point onwards.
+CodeGenModule::InvariantArgs CodeGenFunction::EmitInvariantStart(
+    const VarDecl &D, llvm::Value *Addr) {
+  // Don't emit the intrinsic if we're not optimizing.
+  if (!CGM.getCodeGenOpts().OptimizationLevel)
+    return CodeGenModule::InvariantArgs{};
+
+  // Grab the llvm.invariant.start intrinsic.
+  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
+  llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID);
+
+  // Emit a call with the size in bytes of the object.
+  CharUnits WidthChars = getContext().getTypeSizeInChars(D.getType());
+  uint64_t Width = WidthChars.getQuantity();
+
+  llvm::Value *Size;
+  if (llvm::Constant *CAddr = dyn_cast<llvm::Constant>(Addr)) {
+    Size = llvm::ConstantInt::getSigned(Int64Ty, Width);
+    Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy);
+  } else {
+    Size = llvm::ConstantInt::get(Int64Ty, Width);
+    Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
+  }
+  llvm::CallInst *C = Builder.CreateCall(InvariantStart, {Size, Addr});
+  C->setDoesNotThrow();
+
+  return CodeGenModule::InvariantArgs{C, Size, Addr};
+}
+
+void CodeGenFunction::EmitInvariantEnd(llvm::CallInst *Start, llvm::Value *Size,
+                                       llvm::Value *Addr) {
+  // Grab the llvm.invariant.end intrinsic.
+  llvm::Intrinsic::ID InvEndID = llvm::Intrinsic::invariant_end;
+  llvm::Constant *InvariantEnd = CGM.getIntrinsic(InvEndID);
+
+  // Emit a call with the size in bytes of the object.
+  llvm::CallInst *C = Builder.CreateCall(InvariantEnd, {Start, Size, Addr});
+  C->setDoesNotThrow();
+}
+
 /// EmitAutoVarAlloca - Emit the alloca and debug information for a
 /// local variable.  Does not emit initialization or destruction.
 CodeGenFunction::AutoVarEmission
 CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   QualType Ty = D.getType();
 
   AutoVarEmission emission(D);
 

lib/CodeGen/CGDeclCXX.cpp

function = CodeGenFunction(CGM)
         .generateDestroyHelper(addr, type, CGF.getDestroyer(dtorKind),
                                CGF.needsEHCleanup(dtorKind), &D);
     argument = llvm::Constant::getNullValue(CGF.Int8PtrTy);
   }
 
   CGM.getCXXABI().registerGlobalDtor(CGF, D, function, argument);
 }
 
-/// Emit code to cause the variable at the given address to be considered as
-/// constant from this point onwards.
-static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
-                              llvm::Constant *Addr) {
-  // Don't emit the intrinsic if we're not optimizing.
-  if (!CGF.CGM.getCodeGenOpts().OptimizationLevel)
-    return;
-
-  // Grab the llvm.invariant.start intrinsic.
-  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
-  llvm::Constant *InvariantStart = CGF.CGM.getIntrinsic(InvStartID);
-
-  // Emit a call with the size in bytes of the object.
-  CharUnits WidthChars = CGF.getContext().getTypeSizeInChars(D.getType());
-  uint64_t Width = WidthChars.getQuantity();
-  llvm::Value *Args[2] = { llvm::ConstantInt::getSigned(CGF.Int64Ty, Width),
-                           llvm::ConstantExpr::getBitCast(Addr, CGF.Int8PtrTy)};
-  CGF.Builder.CreateCall(InvariantStart, Args);
-}
-
 void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
                                                llvm::Constant *DeclPtr,
                                                bool PerformInit) {
 
   const Expr *Init = D.getInit();
   QualType T = D.getType();
 
   // The address space of a static local variable (DeclPtr) may be different
void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
 
   if (!T->isReferenceType()) {
     if (getLangOpts().OpenMP && D.hasAttr<OMPThreadPrivateDeclAttr>())
       (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
           &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
           PerformInit, this);
     if (PerformInit)
       EmitDeclInit(*this, D, DeclAddr);
-    if (CGM.isTypeConstant(D.getType(), true))
-      EmitDeclInvariant(*this, D, DeclPtr);
-    else
+    if (CGM.isTypeConstant(D.getType(), true)) {
+      // Generate the @llvm.invariant.start intrinsic call if DeclPtr is
+      // not 'writeonce'. We'll do the same for 'writeonce' addresses later.
+      llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(DeclPtr);
+      assert(GV && "Must point to a global variable.");
+      if (GV->isConstant() && !D.getType().isWriteOnce(getContext()))
+        (void)EmitInvariantStart(D, DeclPtr);
+    } else {
       EmitDeclDestroy(*this, D, DeclAddr);
+    }
     return;
   }
 
   assert(PerformInit && "cannot have constant initializer which needs "
          "destruction for reference");
   RValue RV = EmitReferenceBindingToExpr(Init);
   EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T);
 }
void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
   // Use guarded initialization if the global variable is weak. This
   // occurs for, e.g., instantiated static data members and
   // definitions explicitly marked weak.
   if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage()) {
     EmitCXXGuardedInit(*D, Addr, PerformInit);
   } else {
     EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit);
   }
+  MarkWriteOnceWrittenRAII MWO(*this, D, Addr);
 
   FinishFunction();
 }
 
 void
 CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
                                            ArrayRef<llvm::Function *> Decls,
                                            Address Guard) {

lib/CodeGen/CGExprAgg.cpp

 void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
   assert(E && hasAggregateEvaluationKind(E->getType()) &&
          "Invalid aggregate expression to emit");
   assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
          "slot has bits but no address");
 
   // Optimize the slot if possible.
   CheckAggExprForMemSetUse(Slot, E, *this);
- 
   AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E));
 }
 
 LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
   assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
   Address Temp = CreateMemTemp(E->getType());
   LValue LV = MakeAddrLValue(Temp, E->getType());
   EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed,

lib/CodeGen/CodeGenFunction.h

void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
                                llvm::Value *NumElements,
                                llvm::Value *AllocSizeWithoutCookie);
 
   void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
                         Address Ptr);
 
   llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr);
   void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
+  CodeGenModule::InvariantArgs EmitInvariantStart(const VarDecl &D,
+                                                  llvm::Value *Addr);
+  void EmitInvariantEnd(llvm::CallInst *C, llvm::Value *Size,
+                        llvm::Value *Addr);
 
   llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
   void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
 
   void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
                       QualType DeleteTy);
 
   RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
private:
   BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
                                   const AutoVarEmission &emission);
 
   void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
 
   llvm::Value *GetValueForARMHint(unsigned BuiltinID);
 };
 
+class MarkWriteOnceWrittenRAII {
+  CodeGenFunction &CGF;
+  llvm::GlobalVariable *GV;
+  llvm::AllocaInst *AI;
+  CodeGenModule::InvariantArgs Args;
+
+ public:
+  MarkWriteOnceWrittenRAII(CodeGenFunction &CGF, const VarDecl *D,
+                           llvm::GlobalVariable *GVAddr = nullptr);
+  ~MarkWriteOnceWrittenRAII();
+};
+
 /// Helper class with most of the code for saving a value for a
 /// conditional expression cleanup.
 struct DominatingLLVMValue {
   typedef llvm::PointerIntPair<llvm::Value*, 1, bool> saved_type;
 
   /// Answer whether the given value needs extra work to be saved.
   static bool needsSaving(llvm::Value *value) {
     // If it's not an instruction, we don't need to save.

lib/CodeGen/CodeGenModule.h

Structor(int Priority, llvm::Constant *Initializer,
           AssociatedData(AssociatedData) {}
     int Priority;
     llvm::Constant *Initializer;
     llvm::Constant *AssociatedData;
   };
 
   typedef std::vector<Structor> CtorList;
 
+  struct InvariantArgs {
+    llvm::CallInst *StartInst;
+    llvm::Value *Size;
+    llvm::Value *Addr;
+    InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {}
+    InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A)
+        : StartInst(C), Size(S), Addr(A) {}
+  };
+
 private:
   ASTContext &Context;
   const LangOptions &LangOpts;
   const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info.
   const PreprocessorOptions &PreprocessorOpts; // Only used for debug info.
   const CodeGenOptions &CodeGenOpts;
   llvm::Module &TheModule;
   DiagnosticsEngine &Diags;

test/CodeGenCXX/const-invariant.cpp

+// *** const objects.
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O0 -o - -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-O0
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-L-CO --check-prefix=CHECK-L-CO-OBJ --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-GP-CO --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-OBJ --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-G-CO --check-prefix=CHECK-G-CO-OBJ --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-OBJ
+
+// *** const variables of builtin type.
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O0 -o - -DCONST -DINT | FileCheck %s --check-prefix=CHECK-O0
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DCONST -DINT | FileCheck %s --check-prefix=CHECK-L-CO --check-prefix=CHECK-L-CO-INT --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DCONST -DINT | FileCheck %s --check-prefix=CHECK-GP-CO --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-INT --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DCONST -DINT | FileCheck %s --check-prefix=CHECK-G-CO --check-prefix=CHECK-G-CO-INT --check-prefix=CHECK-NL-CO
+
+// *** non-const objects.
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DOBJ | FileCheck %s --check-prefix=CHECK-NC
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DOBJ | FileCheck %s --check-prefix=CHECK-NC
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DOBJ | FileCheck %s --check-prefix=CHECK-NC
+
+// Check that llvm.invariant.start/end are properly generated on C++ const
+// variables.
+
+// Do not produce markers at -O0.
+// CHECK-O0-NOT: llvm.invariant.start
+// CHECK-O0-NOT: llvm.invariant.end
+
+// Do not produce markers from non-const objects
+// CHECK-NC-NOT: llvm.invariant.start
+// CHECK-NC-NOT: llvm.invariant.end
+
+// const or not?
+#ifdef CONST
+#define Const const
+#else
+#define Const
+#endif
+
+// object or builtin?
+#if defined(OBJ)
+struct A {
+  int a;
+  int b;
+  A(int a) : a(a) {}
+};
+
+// A with explicit destructor
+struct D {
+  int a;
+  int b;
+  D(int a) : a(a) {}
+  ~D();
+};
+
+// A with mutable data field.
+struct M {
+  int a;
+  mutable int b;
+  M(int a) : a(a) {}
+  void f() {}
+};
+
+// A with non-trivial method that may write to memory.
+struct F {
+  int a;
+  F(int a) : a(a) {}
+  void f();
+};
+
+// A non-virtual base class.
+struct Base {
+  int pa;
+  Base(int a);
+};
+struct I : public Base {
+  int a;
+  int b;
+  I(int a);
+};
+
+// A virtual base class, not-writeonce.
+struct VBase {
+  int pa;
+  VBase(int a);
+  virtual void f();
+};
+struct IV : public VBase {
+  int a;
+  int b;
+  IV(int a);
+};
+
+// A virtual base class, writeonce.
+struct CVBase {
+  int pa;
+  CVBase(int a);
+  virtual void f() = 0;
+  virtual void g() {}
+};
+struct CIV : public CVBase {
+  int a;
+  int b;
+  CIV(int a);
+  void f() {}
+};
+
+#define Type A  // C++ class
+#elif defined(INT)
+#define Type int  // Builtin
+#endif
+
+// global, partial global or local?
+#if defined(GLOBAL)
+#undef LOCAL
+#undef LOCAL_PARTIAL
+#elif defined(PART_GLOBAL)
+#undef LOCAL
+#define LOCAL_PARTIAL
+#else
+#define LOCAL
+#define LOCAL_PARTIAL
+#endif
+
+// --------------- START ---------------
+
+const int one();
+
+#ifndef LOCAL
+Const Type i(one());
+#ifdef CONST
+Const Type& i_r = Type(one());
+#endif
+Const Type* i_p = new Type(one());
+Const Type* Const i_pc = new Type(one());
+#endif
+
+#ifndef LOCAL_PARTIAL
+Const Type j = i;
+Type k = i;
+#endif
+
+#if defined(OBJ) && !defined(LOCAL)
+Const D i_d(one());
+Const M i_m(one());
+Const F i_f(one());
+Const I i_i(one());
+Const IV i_iv(one());
+Const CIV i_civ(one());
+#endif
+
+// CHECK-NL-CO: @_ZL1i = internal global
+// CHECK-NL-CO: @i_r = global
+// CHECK-NL-CO: @_ZGR3i_r_ = internal global
+// CHECK-NL-CO: @i_p = global
+// CHECK-NL-CO: @_ZL4i_pc = internal global
+// CHECK-G-CO: @k = global
+// CHECK-NL-CO-OBJ: @_ZL3i_d = internal global
+// CHECK-NL-CO-OBJ: @_ZL3i_m = internal global
+// CHECK-NL-CO-OBJ: @_ZL3i_f = internal global
+// CHECK-NL-CO-OBJ: @_ZL3i_i = internal global
+// CHECK-NL-CO-OBJ: @_ZL4i_iv = internal global
+// CHECK-NL-CO-OBJ: @_ZL5i_civ = internal global
+// CHECK-G-CO: @_ZL1j = internal global
+
+// CHECK-NL-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* @_ZL1i, {{.*}})
+// CHECK-NL-CO-INT: store {{.*}}, {{.*}}* @_ZL1i
+// CHECK-NL-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL1i to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1DC{{[0-9]+}}Ei({{.*}}* @_ZL3i_d, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_d to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1MC{{[0-9]+}}Ei({{.*}}* @_ZL3i_m, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_m to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1FC{{[0-9]+}}Ei({{.*}}* @_ZL3i_f, {{.*}})
+// CHECK-NL-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_f to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1IC{{[0-9]+}}Ei({{.*}}* @_ZL3i_i, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_i to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* @_ZL4i_iv, {{.*}})
+// CHECK-NL-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL4i_iv to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* @_ZL5i_civ, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL5i_civ to i8*))
+
+void foo(const Type* const);
+void bar(Type);
+#if defined(OBJ)
+void foo_d(const D* const);
+void bar_d(D);
+void foo_m(const M* const);
+void bar_m(M);
+void foo_f(const F* const);
+void bar_f(F);
+void foo_i(const I* const);
+void bar_i(I);
+void foo_iv(const IV* const);
+void bar_iv(IV);
+void foo_civ(const CIV* const);
+void bar_civ(CIV);
+#endif
+
+// Example 1: Duplicate loads.
+void ex1() {
+// CHECK: @_Z3ex1v(
+#ifdef LOCAL
+  Const Type i(one());
+#endif
+  // CHECK-L-CO: %i = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(i);
+  foo(&i);  // Does not change i.
+  bar(i);
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+// Example 2: Unnecessary stores and loads.
+void ex2() {
+// CHECK: @_Z3ex2v(
+#ifdef LOCAL
+  Const Type i(one());
+#endif
+#ifdef LOCAL_PARTIAL
+  Const Type j = i;
+#endif
+  // CHECK-L-CO: %i = alloca {{.*}}
+  // CHECK-NG-CO: %j = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  // CHECK-NG-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(i);
+  foo(&i);  // Does not change i, nor j.
+  bar(j);
+  // CHECK-NG-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+// Example 3: Necessary stores and loads.
+void ex3() {
+// CHECK: @_Z3ex3v(
+#ifdef LOCAL
+  Const Type i(one());
+#endif
+#ifdef LOCAL_PARTIAL
+  Type k = i;
+#endif
+  // CHECK-L-CO: %i = alloca {{.*}}
+  // CHECK-NG-CO: %k = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(i);
+  foo(&k);  // Does not change i; May change k.
+  bar(k);
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+#if defined(OBJ)
+
+// Example 1 with explicit destructor:
+//   Call to invariant_end intrinsic must be generated before
+//   call to destructor.
+void ex1_d() {
+// CHECK: @_Z5ex1_dv(
+#ifdef LOCAL
+  Const D i_d(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_d = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1DC{{[0-9]+}}Ei({{.*}}* %i_d,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar_d(i_d);
+  foo_d(&i_d);  // Does not change i_d.
+  bar_d(i_d);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+  // CHECK-L-CO-OBJ: call void @_ZN1DD1Ev(
+}
+
+// Example 1 with mutable data field and a trivial method:
+//   Expect no change from Example 1.
+void ex1_m() {
+// CHECK: @_Z5ex1_mv(
+#ifdef LOCAL
+  Const M i_m(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_m = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1MC{{[0-9]+}}Ei({{.*}}* %i_m,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
+  bar_m(i_m);
+  foo_m(&i_m);  // May change i_m.
+  bar_m(i_m);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 no non-trivial method that may write to memory:
+//   Expect no invariant intrinsic call generated.
+void ex1_f() {
+// CHECK: @_Z5ex1_fv(
+#ifdef LOCAL
+  Const F i_f(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_f = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1FC{{[0-9]+}}Ei({{.*}}* %i_f,
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(
+  bar_f(i_f);
+  foo_f(&i_f);  // May change i_f.
+  bar_f(i_f);
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 with type inheriting non-virtual base:
+//   Expect no change from Example 1.
+void ex1_i() {
+// CHECK: @_Z5ex1_iv(
+#ifdef LOCAL
+  Const I i_i(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_i = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1IC{{[0-9]+}}Ei({{.*}}* %i_i,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
+  bar_i(i_i);
+  foo_i(&i_i);  // Does not change i_i.
+  bar_i(i_i);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 with type inheriting virtual non-writeonce base:
+//   Expect no invariant intrinsics.
+void ex1_iv() {
+// CHECK: @_Z6ex1_ivv(
+#ifdef LOCAL
+  Const IV i_iv(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_iv = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* %i_iv,
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(
+  bar_iv(i_iv);
+  foo_iv(&i_iv);  // Does not change i_iv.
+  bar_iv(i_iv);
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 with type inheriting virtual writeonce base:
+//   Expect no change from Example 1.
+void ex1_civ() {
+// CHECK: @_Z7ex1_civv(
+#ifdef LOCAL
+  Const CIV i_civ(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_civ = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* %i_civ,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
+  bar_civ(i_civ);
+  foo_civ(&i_civ);  // Does not change i_civ.
+  bar_civ(i_civ);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
+}
+
+#endif  // #if defined(OBJ)
+
+// Example 1 with references and pointers:
+//   Expect no invariant intrinsic call generated on non const pointers.
+void ex1_p() {
+// CHECK: @_Z5ex1_pv(
+#ifdef LOCAL
+#ifdef CONST
+  Const Type& i_r = Type(one());
+#endif
+  Const Type* i_p = new Type(one());
+  Const Type* Const i_pc = new Type(one());
+#endif
+
+#ifdef CONST
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %{{.*.tmp}},
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %{{.*.tmp}}
+  // CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.start(
+  bar(i_r);
+  foo(&i_r);  // May change i_r.
+  bar(i_r);
+// CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.end(
+#endif
+
+  // CHECK-L-CO: call {{.*}} i8* @_Znwm(
+  // CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.start(
+  bar(*i_p);
+  foo(i_p);  // May change *i_p.
+  bar(*i_p);
+  // CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.end(
+
+  // CHECK-L-CO: call {{.*}} i8* @_Znwm(
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(*i_pc);
+  foo(i_pc);  // May change *i_p.
+  bar(*i_pc);
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+// CHECK-G-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL1j to i8*))
+// CHECK-G-CO-INT: store {{.*}}, {{.*}}* @_ZL1j

test/CodeGenCXX/init-invariant.cpp

 
 void e() {
   static const A a = A();
 }
 
 // CHECK: call void @_ZN1AC1Ev({{.*}}* nonnull @a)
 // CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @a to i8*))
 
 // CHECK: call void @_ZN1BC1Ev({{.*}}* nonnull @b)
-// CHECK-NOT: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @b to i8*))
+// CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @b to
+// i8*))
 
 // CHECK: call void @_ZN1CC1Ev({{.*}}* nonnull @c)
-// CHECK-NOT: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @c to i8*))
+// CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @c to
+// i8*))
 
 // CHECK: call i32 @_Z1fv(
 // CHECK: store {{.*}}, i32* @d
 // CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @d to i8*))
 
 // CHECK-LABEL: define void @_Z1ev(
 // CHECK: call void @_ZN1AC1Ev(%struct.A* nonnull @_ZZ1evE1a)
 // CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* nonnull bitcast ({{.*}} @_ZZ1evE1a to i8*))
 // CHECK-NOT: llvm.invariant.end