[Extension] Optimizing passed by-value const objects.

[lvoufo]

Summary

Continued from D13603, handling functions that are defined with const parameters.

[lvoufo]

Update patch (Wrong patch submitted earlier).

[rsmith]

This looks fine (subject to the observation in D13603 that MarkWriteOnceWrittenRAII can just be a function rather than an RAII class type).

include/clang/AST/DeclCXX.h

struct DefinitionData {
     bool HasPrivateFields : 1;
 
     /// \brief True when there are protected non-static data members.
     bool HasProtectedFields : 1;
 
     /// \brief True when there are private non-static data members.
     bool HasPublicFields : 1;
 
+    /// \brief True is this class has already been checked as candidate
+    /// for writeonce semantics.
+    bool IsWriteOnceChecked : 1;
+
+    /// \brief True if instances of this class are candidates
+    /// for writeonce semantics.
+    bool IsWriteOnceCandidate : 1;
+
     /// \brief True if this class (or any subobject) has mutable fields.
     bool HasMutableFields : 1;
 
     /// \brief True if this class (or any nested anonymous struct or union)
     /// has variant members.
     bool HasVariantMembers : 1;
 
     /// \brief True if there no non-field members declared by the user.
public:
   method_iterator method_begin() const {
     return method_iterator(decls_begin());
   }
   /// \brief Method past-the-end iterator.
   method_iterator method_end() const {
     return method_iterator(decls_end());
   }
 
+  bool computeWriteOnceCandidacy() const;
+
   /// Iterator access to constructor members.
   typedef specific_decl_iterator<CXXConstructorDecl> ctor_iterator;
   typedef llvm::iterator_range<specific_decl_iterator<CXXConstructorDecl>>
     ctor_range;
 
   ctor_range ctors() const { return ctor_range(ctor_begin(), ctor_end()); }
 
   ctor_iterator ctor_begin() const {

include/clang/AST/Type.h

public:
   /// \brief Retrieve the set of CVR (const-volatile-restrict) qualifiers
   /// applied to this type.
   unsigned getCVRQualifiers() const;
 
   bool isConstant(ASTContext& Ctx) const {
     return QualType::isConstant(*this, Ctx);
   }
 
+  /// \brief Determine if this satisfies 'writeonce' properties.
+  bool isWriteOnce(ASTContext &Ctx);
+
   /// \brief Determine whether this is a Plain Old Data (POD) type (C++ 3.9p10).
   bool isPODType(ASTContext &Context) const;
 
   /// Return true if this is a POD type according to the rules of the C++98
   /// standard, regardless of the current compilation's language.
   bool isCXX98PODType(ASTContext &Context) const;
 
   /// Return true if this is a POD type according to the more relaxed rules

lib/AST/DeclCXX.cpp

void LazyASTUnresolvedSet::getFromExternalSource(ASTContext &C) const {
 
   for (ASTUnresolvedSet::iterator I = Impl.begin(); I != Impl.end(); ++I)
     I.setDecl(cast<NamedDecl>(Source->GetExternalDecl(
         reinterpret_cast<uintptr_t>(I.getDecl()) >> 2)));
   Impl.Decls.setLazy(false);
 }
 
 CXXRecordDecl::DefinitionData::DefinitionData(CXXRecordDecl *D)
-  : UserDeclaredConstructor(false), UserDeclaredSpecialMembers(0),
-    Aggregate(true), PlainOldData(true), Empty(true), Polymorphic(false),
-    Abstract(false), IsStandardLayout(true), HasNoNonEmptyBases(true),
-    HasPrivateFields(false), HasProtectedFields(false), HasPublicFields(false),
-    HasMutableFields(false), HasVariantMembers(false), HasOnlyCMembers(true),
-    HasInClassInitializer(false), HasUninitializedReferenceMember(false),
+    : UserDeclaredConstructor(false),
+      UserDeclaredSpecialMembers(0),
+      Aggregate(true),
+      PlainOldData(true),
+      Empty(true),
+      Polymorphic(false),
+      Abstract(false),
+      IsStandardLayout(true),
+      HasNoNonEmptyBases(true),
+      HasPrivateFields(false),
+      HasProtectedFields(false),
+      HasPublicFields(false),
+      IsWriteOnceChecked(false),
+      IsWriteOnceCandidate(false),
+      HasMutableFields(false),
+      HasVariantMembers(false),
+      HasOnlyCMembers(true),
+      HasInClassInitializer(false),
+      HasUninitializedReferenceMember(false),
       NeedOverloadResolutionForMoveConstructor(false),
       NeedOverloadResolutionForMoveAssignment(false),
       NeedOverloadResolutionForDestructor(false),
       DefaultedMoveConstructorIsDeleted(false),
       DefaultedMoveAssignmentIsDeleted(false),
       DefaultedDestructorIsDeleted(false),
       HasTrivialSpecialMembers(SMF_All),
       DeclaredNonTrivialSpecialMembers(0),
       HasIrrelevantDestructor(true),
       HasConstexprNonCopyMoveConstructor(false),
       DefaultedDefaultConstructorIsConstexpr(true),
       HasConstexprDefaultConstructor(false),
-    HasNonLiteralTypeFieldsOrBases(false), ComputedVisibleConversions(false),
-    UserProvidedDefaultConstructor(false), DeclaredSpecialMembers(0),
+      HasNonLiteralTypeFieldsOrBases(false),
+      ComputedVisibleConversions(false),
+      UserProvidedDefaultConstructor(false),
+      DeclaredSpecialMembers(0),
       ImplicitCopyConstructorHasConstParam(true),
       ImplicitCopyAssignmentHasConstParam(true),
       HasDeclaredCopyConstructorWithConstParam(false),
       HasDeclaredCopyAssignmentWithConstParam(false),
-    IsLambda(false), IsParsingBaseSpecifiers(false), NumBases(0), NumVBases(0),
-    Bases(), VBases(),
-    Definition(D), FirstFriend() {
-}
+      IsLambda(false),
+      IsParsingBaseSpecifiers(false),
+      NumBases(0),
+      NumVBases(0),
+      Bases(),
+      VBases(),
+      Definition(D),
+      FirstFriend() {}
 
 CXXBaseSpecifier *CXXRecordDecl::DefinitionData::getBasesSlowCase() const {
   return Bases.get(Definition->getASTContext().getExternalSource());
 }
 
 CXXBaseSpecifier *CXXRecordDecl::DefinitionData::getVBasesSlowCase() const {
   return VBases.get(Definition->getASTContext().getExternalSource());
 }
if (const CXXRecordDecl *RD =
             Field->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl())
       if (RD->isAnyDestructorNoReturn())
         return true;
 
   // All destructors are not noreturn.
   return false;
 }
 
+/// \brief A class is a candidate for 'writeonce' semantics if
+/// none of its methods writes to memory.
+bool CXXRecordDecl::computeWriteOnceCandidacy() const {
+  // The class must be defined and concrete.
+  if (!isCompleteDefinition() || isDependentType()) return false;
+
+  // If this has already been checked, then return the stored value.
+  if (data().IsWriteOnceChecked) return data().IsWriteOnceCandidate;
+
+  // Otherwise, mark that this is checked.
+  data().IsWriteOnceChecked = true;
+  assert(!data().IsWriteOnceCandidate &&
+         "The candidacy should not have been set yet.");
+
+  // Check each method, excluding constructors and destructors.
+  // NOTE: The invariant intrinsic calls are generated right after the
+  //       construction of a given const object (which must be initialized)
+  //       and right before its destruction at the end of its lifetime.
+  //       Both constructors and destructors modify the allocated memory and
+  //       and will not be called in the middle of an invariant_start/end
+  //       pair, thus may not affect the reduction of loads. So, it does not
+  //       help to handle constructors here as well.
+  for (auto *M : methods()) {
+    if (isa<CXXConstructorDecl>(M) || isa<CXXDestructorDecl>(M)) continue;
+
+    // If we already know that this method does not write to memory, skip it.
+    // NOTE: CodeGenModule::ConstructAttributeList() marks declarations with
+    //       ConstAttr and PureAttr attributes, respectively, as 'readnone' and
+    //       'readonly' (for LLVM).
+    if (M->hasAttr<ConstAttr>() || M->hasAttr<PureAttr>()) continue;
+
+    const FunctionDecl *Def = nullptr;
+    bool IsDefined = M->isDefined(Def);
+
+    // If this is not defined and not virtual, then it may write to memory.
+    // So it's not a candidate for 'writeonce' semantics.
+    // If it is purely virtual then it must be overriden and its overriding
+    // method (in this context) must be a candidate for 'writeonce' semantics.
+    // Skip it.
+    if (!IsDefined) {
+      if (M->isPure()) {
+        assert(M->isVirtual() && "Not defined and pure implies virtual.");
+        continue;
+      }
+      return false;
+    }
+
+    // If this is trivial, skip it.
+    if (Def->hasTrivialBody()) continue;
+
+    // TODO: Any other case?
+  }
+
+  // Check bases
+  for (const auto &I : bases()) {
+    const RecordType *Ty = I.getType()->getAs<RecordType>();
+    assert(Ty && "No type?");
+    CXXRecordDecl *Base =
+        cast_or_null<CXXRecordDecl>(Ty->getDecl()->getDefinition());
+
+    if (!Base || !Base->computeWriteOnceCandidacy()) return false;
+  }
+
+  data().IsWriteOnceCandidate = true;
+  return true;
+}
+
 void CXXRecordDecl::completeDefinition() {
   completeDefinition(nullptr);
 }
 
 void CXXRecordDecl::completeDefinition(CXXFinalOverriderMap *FinalOverriders) {
   RecordDecl::completeDefinition();
   
   // If the class may be abstract (but hasn't been marked as such), check for

lib/AST/Type.cpp

ObjCInterfaceDecl *Interface
       = cast<ObjCInterfaceType>(CanonicalType)->getDecl();
     if (Def)
       *Def = Interface;
     return !Interface->hasDefinition();
   }
   }
 }
 
+bool QualType::isWriteOnce(ASTContext &Context) {
+  // TODO: Include C objects as well?
+  if (Context.getLangOpts().CPlusPlus && isConstant(Context) &&
+      !getTypePtr()->isReferenceType()) {
+    if (const CXXRecordDecl *Record =
+            Context.getBaseElementType(*this)->getAsCXXRecordDecl())
+      return Record->computeWriteOnceCandidacy();
+    return true;
+  }
+  return false;
+}
+
 bool QualType::isPODType(ASTContext &Context) const {
   // C++11 has a more relaxed definition of POD.
   if (Context.getLangOpts().CPlusPlus11)
     return isCXX11PODType(Context);
 
   return isCXX98PODType(Context);
 }
 

lib/CodeGen/CGDecl.cpp

void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
   llvm::Type *expectedType = addr->getType();
 
   llvm::GlobalVariable *var =
     cast<llvm::GlobalVariable>(addr->stripPointerCasts());
   // If this value has an initializer, emit it.
   if (D.getInit())
     var = AddInitializerToStaticVarDecl(D, var);
 
+  MarkWriteOnceWrittenRAII MWO(*this, &D, var);
+
   var->setAlignment(alignment.getQuantity());
 
   if (D.hasAttr<AnnotateAttr>())
     CGM.AddGlobalAnnotations(&D, var);
 
   if (const SectionAttr *SA = D.getAttr<SectionAttr>())
     var->setSection(SA->getName());
 
namespace {
   public:
     CallLifetimeEnd(Address addr, llvm::Value *size)
       : Addr(addr.getPointer()), Size(size) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitLifetimeEnd(Size, Addr);
     }
   };
+
+  /// A cleanup to call @llvm.invariant.end.
+  class CallInvariantEnd final : public EHScopeStack::Cleanup {
+    llvm::CallInst *StartInst;
+    llvm::Value *Addr;
+    llvm::Value *Size;  // TODO: Is this even necessary?
+   public:
+    CallInvariantEnd(llvm::CallInst *C, llvm::Value *addr, llvm::Value *size)
+        : StartInst(C), Addr(addr), Size(size) {}
+
+    void Emit(CodeGenFunction &CGF, Flags flags) override {
+      CGF.EmitInvariantEnd(StartInst, Size, Addr);
+    }
+  };
 }
 
 /// EmitAutoVarWithLifetime - Does the setup required for an automatic
 /// variable with lifetime.
 static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var,
                                     Address addr,
                                     Qualifiers::ObjCLifetime lifetime) {
   switch (lifetime) {
static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init,
   // plopping in more stores.
   unsigned StoreBudget = 6;
   uint64_t SizeLimit = 32;
 
   return GlobalSize > SizeLimit &&
          canEmitInitWithFewStoresAfterMemset(Init, StoreBudget);
 }
 
+/// Emit the code necessary to initialize the given global variable.
+/// TODO: Use Address for Addr (to match lifetime intrisics).
+MarkWriteOnceWrittenRAII::MarkWriteOnceWrittenRAII(CodeGenFunction &CGF,
+                                                   const VarDecl *D,
+                                                   llvm::GlobalVariable *GVAddr)
+    : CGF(CGF), GV(GVAddr), AI(nullptr) {
+  assert(D && "Invalid declaration.");
+
+  llvm::Value *AddrPtr = nullptr;
+  if (GVAddr) {
+    assert(GV && GV == GVAddr && "Caller provides address for globals");
+    AddrPtr = GVAddr;
+  } else {
+    CodeGen::Address Addr = CGF.GetAddrOfLocalVar(D);
+    AddrPtr = Addr.getPointer()->stripPointerCasts();
+    AI = dyn_cast_or_null<llvm::AllocaInst>(AddrPtr);
+  }
+
+  // Only GlobalVariable's and AllocaInst's can be writeonce.
+  // Exit if the given address is none of these.
+  if (!GV && !AI) return;
+
+  // If the address is writeonce, then emit @llvm.invariant.start() intrinsic.
+  // Then, for non-global variables, push the emission of the
+  // @llvm.invariant.end() intrinsic onto the cleanup stack.
+  if ((AI || !GV->isConstant()) && D->getType().isWriteOnce(CGF.getContext()))
+    Args = CGF.EmitInvariantStart(*D, AddrPtr);
+}
+
+MarkWriteOnceWrittenRAII::~MarkWriteOnceWrittenRAII() {
+  if (AI && Args.StartInst) {
+    assert(!GV && "Can't have it both ways!");
+    CGF.EHStack.pushCleanup<CallInvariantEnd>(NormalCleanup, Args.StartInst,
+                                              Args.Addr, Args.Size);
+  }
+}
+
 /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
 /// variable declaration with auto, register, or no storage class specifier.
 /// These turn into simple stack objects, or GlobalValues depending on target.
 void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) {
   AutoVarEmission emission = EmitAutoVarAlloca(D);
   EmitAutoVarInit(emission);
+  MarkWriteOnceWrittenRAII MWO(*this, &D);
   EmitAutoVarCleanups(emission);
 }
 
 /// Emit a lifetime.begin marker if some criteria are satisfied.
 /// \return a pointer to the temporary size Value if a marker was emitted, null
 /// otherwise
 llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size,
                                                 llvm::Value *Addr) {

 
 void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
   Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
   llvm::CallInst *C =
       Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
   C->setDoesNotThrow();
 }
 
+/// Emit code to cause the variable at the given address to be considered as
+/// constant from this point onwards.
+CodeGenModule::InvariantArgs CodeGenFunction::EmitInvariantStart(
+    const VarDecl &D, llvm::Value *Addr) {
+  // Don't emit the intrinsic if we're not optimizing.
+  if (!CGM.getCodeGenOpts().OptimizationLevel)
+    return CodeGenModule::InvariantArgs{};
+
+  // Grab the llvm.invariant.start intrinsic.
+  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
+  llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID);
+
+  // Emit a call with the size in bytes of the object.
+  CharUnits WidthChars = getContext().getTypeSizeInChars(D.getType());
+  uint64_t Width = WidthChars.getQuantity();
+
+  llvm::Value *Size;
+  if (llvm::Constant *CAddr = dyn_cast<llvm::Constant>(Addr)) {
+    Size = llvm::ConstantInt::getSigned(Int64Ty, Width);
+    Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy);
+  } else {
+    Size = llvm::ConstantInt::get(Int64Ty, Width);
+    Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
+  }
+  llvm::CallInst *C = Builder.CreateCall(InvariantStart, {Size, Addr});
+  C->setDoesNotThrow();
+
+  return CodeGenModule::InvariantArgs{C, Size, Addr};
+}
+
+void CodeGenFunction::EmitInvariantEnd(llvm::CallInst *Start, llvm::Value *Size,
+                                       llvm::Value *Addr) {
+  // Grab the llvm.invariant.end intrinsic.
+  llvm::Intrinsic::ID InvEndID = llvm::Intrinsic::invariant_end;
+  llvm::Constant *InvariantEnd = CGM.getIntrinsic(InvEndID);
+
+  // Emit a call with the size in bytes of the object.
+  llvm::CallInst *C = Builder.CreateCall(InvariantEnd, {Start, Size, Addr});
+  C->setDoesNotThrow();
+}
+
 /// EmitAutoVarAlloca - Emit the alloca and debug information for a
 /// local variable.  Does not emit initialization or destruction.
 CodeGenFunction::AutoVarEmission
 CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   QualType Ty = D.getType();
 
   AutoVarEmission emission(D);
 

lib/CodeGen/CGDeclCXX.cpp

function = CodeGenFunction(CGM)
         .generateDestroyHelper(addr, type, CGF.getDestroyer(dtorKind),
                                CGF.needsEHCleanup(dtorKind), &D);
     argument = llvm::Constant::getNullValue(CGF.Int8PtrTy);
   }
 
   CGM.getCXXABI().registerGlobalDtor(CGF, D, function, argument);
 }
 
-/// Emit code to cause the variable at the given address to be considered as
-/// constant from this point onwards.
-static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
-                              llvm::Constant *Addr) {
-  // Don't emit the intrinsic if we're not optimizing.
-  if (!CGF.CGM.getCodeGenOpts().OptimizationLevel)
-    return;
-
-  // Grab the llvm.invariant.start intrinsic.
-  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
-  llvm::Constant *InvariantStart = CGF.CGM.getIntrinsic(InvStartID);
-
-  // Emit a call with the size in bytes of the object.
-  CharUnits WidthChars = CGF.getContext().getTypeSizeInChars(D.getType());
-  uint64_t Width = WidthChars.getQuantity();
-  llvm::Value *Args[2] = { llvm::ConstantInt::getSigned(CGF.Int64Ty, Width),
-                           llvm::ConstantExpr::getBitCast(Addr, CGF.Int8PtrTy)};
-  CGF.Builder.CreateCall(InvariantStart, Args);
-}
-
 void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
                                                llvm::Constant *DeclPtr,
                                                bool PerformInit) {
 
   const Expr *Init = D.getInit();
   QualType T = D.getType();
 
   // The address space of a static local variable (DeclPtr) may be different
void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
 
   if (!T->isReferenceType()) {
     if (getLangOpts().OpenMP && D.hasAttr<OMPThreadPrivateDeclAttr>())
       (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
           &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
           PerformInit, this);
     if (PerformInit)
       EmitDeclInit(*this, D, DeclAddr);
-    if (CGM.isTypeConstant(D.getType(), true))
-      EmitDeclInvariant(*this, D, DeclPtr);
-    else
+    if (CGM.isTypeConstant(D.getType(), true)) {
+      // Generate the @llvm.invariant.start intrinsic call if DeclPtr is
+      // not 'writeonce'. We'll do the same for 'writeonce' addresses later.
+      llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(DeclPtr);
+      assert(GV && "Must point to a global variable.");
+      if (GV->isConstant() && !D.getType().isWriteOnce(getContext()))
+        (void)EmitInvariantStart(D, DeclPtr);
+    } else {
       EmitDeclDestroy(*this, D, DeclAddr);
+    }
     return;
   }
 
   assert(PerformInit && "cannot have constant initializer which needs "
          "destruction for reference");
   RValue RV = EmitReferenceBindingToExpr(Init);
   EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T);
 }
void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
   // Use guarded initialization if the global variable is weak. This
   // occurs for, e.g., instantiated static data members and
   // definitions explicitly marked weak.
   if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage()) {
     EmitCXXGuardedInit(*D, Addr, PerformInit);
   } else {
     EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit);
   }
+  MarkWriteOnceWrittenRAII MWO(*this, D, Addr);
 
   FinishFunction();
 }
 
 void
 CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
                                            ArrayRef<llvm::Function *> Decls,
                                            Address Guard) {

lib/CodeGen/CGExprAgg.cpp

 void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
   assert(E && hasAggregateEvaluationKind(E->getType()) &&
          "Invalid aggregate expression to emit");
   assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
          "slot has bits but no address");
 
   // Optimize the slot if possible.
   CheckAggExprForMemSetUse(Slot, E, *this);
- 
   AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E));
 }
 
 LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
   assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
   Address Temp = CreateMemTemp(E->getType());
   LValue LV = MakeAddrLValue(Temp, E->getType());
   EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed,

lib/CodeGen/CodeGenFunction.h

void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
                                llvm::Value *NumElements,
                                llvm::Value *AllocSizeWithoutCookie);
 
   void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
                         Address Ptr);
 
   llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr);
   void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
+  CodeGenModule::InvariantArgs EmitInvariantStart(const VarDecl &D,
+                                                  llvm::Value *Addr);
+  void EmitInvariantEnd(llvm::CallInst *C, llvm::Value *Size,
+                        llvm::Value *Addr);
 
   llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
   void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
 
   void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
                       QualType DeleteTy);
 
   RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
private:
   BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
                                   const AutoVarEmission &emission);
 
   void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
 
   llvm::Value *GetValueForARMHint(unsigned BuiltinID);
 };
 
+class MarkWriteOnceWrittenRAII {
+  CodeGenFunction &CGF;
+  llvm::GlobalVariable *GV;
+  llvm::AllocaInst *AI;
+  CodeGenModule::InvariantArgs Args;
+
+ public:
+  MarkWriteOnceWrittenRAII(CodeGenFunction &CGF, const VarDecl *D,
+                           llvm::GlobalVariable *GVAddr = nullptr);
+  ~MarkWriteOnceWrittenRAII();
+};
+
 /// Helper class with most of the code for saving a value for a
 /// conditional expression cleanup.
 struct DominatingLLVMValue {
   typedef llvm::PointerIntPair<llvm::Value*, 1, bool> saved_type;
 
   /// Answer whether the given value needs extra work to be saved.
   static bool needsSaving(llvm::Value *value) {
     // If it's not an instruction, we don't need to save.

lib/CodeGen/CodeGenModule.h

Structor(int Priority, llvm::Constant *Initializer,
           AssociatedData(AssociatedData) {}
     int Priority;
     llvm::Constant *Initializer;
     llvm::Constant *AssociatedData;
   };
 
   typedef std::vector<Structor> CtorList;
 
+  struct InvariantArgs {
+    llvm::CallInst *StartInst;
+    llvm::Value *Size;
+    llvm::Value *Addr;
+    InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {}
+    InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A)
+        : StartInst(C), Size(S), Addr(A) {}
+  };
+
 private:
   ASTContext &Context;
   const LangOptions &LangOpts;
   const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info.
   const PreprocessorOptions &PreprocessorOpts; // Only used for debug info.
   const CodeGenOptions &CodeGenOpts;
   llvm::Module &TheModule;
   DiagnosticsEngine &Diags;

test/CodeGenCXX/const-invariant.cpp

+// *** const objects.
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O0 -o - -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-O0
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-L-CO --check-prefix=CHECK-L-CO-OBJ --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-GP-CO --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-OBJ --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-G-CO --check-prefix=CHECK-G-CO-OBJ --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-OBJ
+
+// *** const variables of builtin type.
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O0 -o - -DCONST -DINT | FileCheck %s --check-prefix=CHECK-O0
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DCONST -DINT | FileCheck %s --check-prefix=CHECK-L-CO --check-prefix=CHECK-L-CO-INT --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DCONST -DINT | FileCheck %s --check-prefix=CHECK-GP-CO --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-INT --check-prefix=CHECK-NG-CO
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DCONST -DINT | FileCheck %s --check-prefix=CHECK-G-CO --check-prefix=CHECK-G-CO-INT --check-prefix=CHECK-NL-CO
+
+// *** non-const objects.
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DOBJ | FileCheck %s --check-prefix=CHECK-NC
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DOBJ | FileCheck %s --check-prefix=CHECK-NC
+// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DOBJ | FileCheck %s --check-prefix=CHECK-NC
+
+// Check that llvm.invariant.start/end are properly generated on C++ const
+// variables.
+
+// Do not produce markers at -O0.
+// CHECK-O0-NOT: llvm.invariant.start
+// CHECK-O0-NOT: llvm.invariant.end
+
+// Do not produce markers from non-const objects
+// CHECK-NC-NOT: llvm.invariant.start
+// CHECK-NC-NOT: llvm.invariant.end
+
+// const or not?
+#ifdef CONST
+#define Const const
+#else
+#define Const
+#endif
+
+// object or builtin?
+#if defined(OBJ)
+struct A {
+  int a;
+  int b;
+  A(int a) : a(a) {}
+};
+
+// A with explicit destructor
+struct D {
+  int a;
+  int b;
+  D(int a) : a(a) {}
+  ~D();
+};
+
+// A with mutable data field.
+struct M {
+  int a;
+  mutable int b;
+  M(int a) : a(a) {}
+  void f() {}
+};
+
+// A with non-trivial method that may write to memory.
+struct F {
+  int a;
+  F(int a) : a(a) {}
+  void f();
+};
+
+// A non-virtual base class.
+struct Base {
+  int pa;
+  Base(int a);
+};
+struct I : public Base {
+  int a;
+  int b;
+  I(int a);
+};
+
+// A virtual base class, not-writeonce.
+struct VBase {
+  int pa;
+  VBase(int a);
+  virtual void f();
+};
+struct IV : public VBase {
+  int a;
+  int b;
+  IV(int a);
+};
+
+// A virtual base class, writeonce.
+struct CVBase {
+  int pa;
+  CVBase(int a);
+  virtual void f() = 0;
+  virtual void g() {}
+};
+struct CIV : public CVBase {
+  int a;
+  int b;
+  CIV(int a);
+  void f() {}
+};
+
+#define Type A  // C++ class
+#elif defined(INT)
+#define Type int  // Builtin
+#endif
+
+// global, partial global or local?
+#if defined(GLOBAL)
+#undef LOCAL
+#undef LOCAL_PARTIAL
+#elif defined(PART_GLOBAL)
+#undef LOCAL
+#define LOCAL_PARTIAL
+#else
+#define LOCAL
+#define LOCAL_PARTIAL
+#endif
+
+// --------------- START ---------------
+
+const int one();
+
+#ifndef LOCAL
+Const Type i(one());
+#ifdef CONST
+Const Type& i_r = Type(one());
+#endif
+Const Type* i_p = new Type(one());
+Const Type* Const i_pc = new Type(one());
+#endif
+
+#ifndef LOCAL_PARTIAL
+Const Type j = i;
+Type k = i;
+#endif
+
+#if defined(OBJ) && !defined(LOCAL)
+Const D i_d(one());
+Const M i_m(one());
+Const F i_f(one());
+Const I i_i(one());
+Const IV i_iv(one());
+Const CIV i_civ(one());
+#endif
+
+// CHECK-NL-CO: @_ZL1i = internal global
+// CHECK-NL-CO: @i_r = global
+// CHECK-NL-CO: @_ZGR3i_r_ = internal global
+// CHECK-NL-CO: @i_p = global
+// CHECK-NL-CO: @_ZL4i_pc = internal global
+// CHECK-G-CO: @k = global
+// CHECK-NL-CO-OBJ: @_ZL3i_d = internal global
+// CHECK-NL-CO-OBJ: @_ZL3i_m = internal global
+// CHECK-NL-CO-OBJ: @_ZL3i_f = internal global
+// CHECK-NL-CO-OBJ: @_ZL3i_i = internal global
+// CHECK-NL-CO-OBJ: @_ZL4i_iv = internal global
+// CHECK-NL-CO-OBJ: @_ZL5i_civ = internal global
+// CHECK-G-CO: @_ZL1j = internal global
+
+// CHECK-NL-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* @_ZL1i, {{.*}})
+// CHECK-NL-CO-INT: store {{.*}}, {{.*}}* @_ZL1i
+// CHECK-NL-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL1i to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1DC{{[0-9]+}}Ei({{.*}}* @_ZL3i_d, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_d to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1MC{{[0-9]+}}Ei({{.*}}* @_ZL3i_m, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_m to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1FC{{[0-9]+}}Ei({{.*}}* @_ZL3i_f, {{.*}})
+// CHECK-NL-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_f to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN1IC{{[0-9]+}}Ei({{.*}}* @_ZL3i_i, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_i to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* @_ZL4i_iv, {{.*}})
+// CHECK-NL-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL4i_iv to i8*))
+// CHECK-NL-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* @_ZL5i_civ, {{.*}})
+// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL5i_civ to i8*))
+
+void foo(const Type* const);
+void bar(Type);
+#if defined(OBJ)
+void foo_d(const D* const);
+void bar_d(D);
+void foo_m(const M* const);
+void bar_m(M);
+void foo_f(const F* const);
+void bar_f(F);
+void foo_i(const I* const);
+void bar_i(I);
+void foo_iv(const IV* const);
+void bar_iv(IV);
+void foo_civ(const CIV* const);
+void bar_civ(CIV);
+#endif
+
+// Example 1: Duplicate loads.
+void ex1() {
+// CHECK: @_Z3ex1v(
+#ifdef LOCAL
+  Const Type i(one());
+#endif
+  // CHECK-L-CO: %i = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(i);
+  foo(&i);  // Does not change i.
+  bar(i);
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+// Example 2: Unnecessary stores and loads.
+void ex2() {
+// CHECK: @_Z3ex2v(
+#ifdef LOCAL
+  Const Type i(one());
+#endif
+#ifdef LOCAL_PARTIAL
+  Const Type j = i;
+#endif
+  // CHECK-L-CO: %i = alloca {{.*}}
+  // CHECK-NG-CO: %j = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  // CHECK-NG-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(i);
+  foo(&i);  // Does not change i, nor j.
+  bar(j);
+  // CHECK-NG-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+// Example 3: Necessary stores and loads.
+void ex3() {
+// CHECK: @_Z3ex3v(
+#ifdef LOCAL
+  Const Type i(one());
+#endif
+#ifdef LOCAL_PARTIAL
+  Type k = i;
+#endif
+  // CHECK-L-CO: %i = alloca {{.*}}
+  // CHECK-NG-CO: %k = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(i);
+  foo(&k);  // Does not change i; May change k.
+  bar(k);
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+#if defined(OBJ)
+
+// Example 1 with explicit destructor:
+//   Call to invariant_end intrinsic must be generated before
+//   call to destructor.
+void ex1_d() {
+// CHECK: @_Z5ex1_dv(
+#ifdef LOCAL
+  Const D i_d(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_d = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1DC{{[0-9]+}}Ei({{.*}}* %i_d,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar_d(i_d);
+  foo_d(&i_d);  // Does not change i_d.
+  bar_d(i_d);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+  // CHECK-L-CO-OBJ: call void @_ZN1DD1Ev(
+}
+
+// Example 1 with mutable data field and a trivial method:
+//   Expect no change from Example 1.
+void ex1_m() {
+// CHECK: @_Z5ex1_mv(
+#ifdef LOCAL
+  Const M i_m(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_m = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1MC{{[0-9]+}}Ei({{.*}}* %i_m,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
+  bar_m(i_m);
+  foo_m(&i_m);  // May change i_m.
+  bar_m(i_m);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 no non-trivial method that may write to memory:
+//   Expect no invariant intrinsic call generated.
+void ex1_f() {
+// CHECK: @_Z5ex1_fv(
+#ifdef LOCAL
+  Const F i_f(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_f = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1FC{{[0-9]+}}Ei({{.*}}* %i_f,
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(
+  bar_f(i_f);
+  foo_f(&i_f);  // May change i_f.
+  bar_f(i_f);
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 with type inheriting non-virtual base:
+//   Expect no change from Example 1.
+void ex1_i() {
+// CHECK: @_Z5ex1_iv(
+#ifdef LOCAL
+  Const I i_i(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_i = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN1IC{{[0-9]+}}Ei({{.*}}* %i_i,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
+  bar_i(i_i);
+  foo_i(&i_i);  // Does not change i_i.
+  bar_i(i_i);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 with type inheriting virtual non-writeonce base:
+//   Expect no invariant intrinsics.
+void ex1_iv() {
+// CHECK: @_Z6ex1_ivv(
+#ifdef LOCAL
+  Const IV i_iv(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_iv = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* %i_iv,
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(
+  bar_iv(i_iv);
+  foo_iv(&i_iv);  // Does not change i_iv.
+  bar_iv(i_iv);
+  // CHECK-L-CO-OBJ-NOT: call {{.*}}@llvm.invariant.end(
+}
+
+// Example 1 with type inheriting virtual writeonce base:
+//   Expect no change from Example 1.
+void ex1_civ() {
+// CHECK: @_Z7ex1_civv(
+#ifdef LOCAL
+  Const CIV i_civ(one());
+#endif
+  // CHECK-L-CO-OBJ: %i_civ = alloca {{.*}}
+
+  // CHECK-L-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* %i_civ,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
+  bar_civ(i_civ);
+  foo_civ(&i_civ);  // Does not change i_civ.
+  bar_civ(i_civ);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
+}
+
+#endif  // #if defined(OBJ)
+
+// Example 1 with references and pointers:
+//   Expect no invariant intrinsic call generated on non const pointers.
+void ex1_p() {
+// CHECK: @_Z5ex1_pv(
+#ifdef LOCAL
+#ifdef CONST
+  Const Type& i_r = Type(one());
+#endif
+  Const Type* i_p = new Type(one());
+  Const Type* Const i_pc = new Type(one());
+#endif
+
+#ifdef CONST
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %{{.*.tmp}},
+  // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %{{.*.tmp}}
+  // CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.start(
+  bar(i_r);
+  foo(&i_r);  // May change i_r.
+  bar(i_r);
+// CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.end(
+#endif
+
+  // CHECK-L-CO: call {{.*}} i8* @_Znwm(
+  // CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.start(
+  bar(*i_p);
+  foo(i_p);  // May change *i_p.
+  bar(*i_p);
+  // CHECK-L-CO-NOT: call {{.*}}@llvm.invariant.end(
+
+  // CHECK-L-CO: call {{.*}} i8* @_Znwm(
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8*
+  bar(*i_pc);
+  foo(i_pc);  // May change *i_p.
+  bar(*i_pc);
+  // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
+// CHECK-G-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL1j to i8*))
+// CHECK-G-CO-INT: store {{.*}}, {{.*}}* @_ZL1j

test/CodeGenCXX/init-invariant.cpp

 void e() {
   static const A a = A();
 }
 
 // CHECK: call void @_ZN1AC1Ev({{.*}}* nonnull @a)
 // CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @a to i8*))
 
 // CHECK: call void @_ZN1BC1Ev({{.*}}* nonnull @b)
-// CHECK-NOT: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @b to i8*))
+// CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @b to i8*))
 
 // CHECK: call void @_ZN1CC1Ev({{.*}}* nonnull @c)
-// CHECK-NOT: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @c to i8*))
+// CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @c to i8*))
 
 // CHECK: call i32 @_Z1fv(
 // CHECK: store {{.*}}, i32* @d
 // CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @d to i8*))
 
 // CHECK-LABEL: define void @_Z1ev(
 // CHECK: call void @_ZN1AC1Ev(%struct.A* nonnull @_ZZ1evE1a)
 // CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* nonnull bitcast ({{.*}} @_ZZ1evE1a to i8*))
 // CHECK-NOT: llvm.invariant.end