Extend const optimization logic from D13031 to member objects

lib/CodeGen/CGDecl.cpp

public:
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitLifetimeEnd(Size, Addr);
     }
   };
 
   /// A cleanup to call @llvm.invariant.end.
   class CallInvariantEnd final : public EHScopeStack::Cleanup {
-    llvm::CallInst *StartInst;
-    llvm::Value *Addr;
-    llvm::Value *Size;    // TODO: Is this even necessary?
+    CodeGenModule::InvariantArgs Args;
   public:
-    CallInvariantEnd(llvm::CallInst *C, llvm::Value *addr, llvm::Value *size)
-      : StartInst(C), Addr(addr), Size(size) {}
+    CallInvariantEnd(CodeGenModule::InvariantArgs &args) : Args(args) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
-      CGF.EmitInvariantEnd(StartInst, Size, Addr);
+      CGF.EmitInvariantEnd(Args);
     }
   };
 }
 
 /// EmitAutoVarWithLifetime - Does the setup required for an automatic
 /// variable with lifetime.
 static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var,
                                     Address addr,
static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init,
 
   return GlobalSize > SizeLimit &&
          canEmitInitWithFewStoresAfterMemset(Init, StoreBudget);
 }
 
 /// Emit the code necessary to initialize the given global variable.
 /// TODO: Use Address for Addr (to match lifetime intrisics).
 MarkWriteOnceWritten::MarkWriteOnceWritten(CodeGenFunction &CGF,
-                                           const VarDecl *D,
+                                           const VarDecl* D,
                                            llvm::GlobalVariable *GVAddr)
 : CGF(CGF), GV(GVAddr), AI(nullptr) {
   assert(D && "Invalid declaration.");
 
   llvm::Value *AddrPtr = nullptr;
   if (GVAddr) {
     assert(GV && GV == GVAddr && "Caller provides address for globals");
     AddrPtr = GVAddr;
   } else {
     CodeGen::Address Addr = CGF.GetAddrOfLocalVar(D);
     AddrPtr = Addr.getPointer()->stripPointerCasts();
     AI = dyn_cast_or_null<llvm::AllocaInst>(AddrPtr);
   }
 
-  // Only GlobalVariable's and AllocaInst's can be writeonce.
-  // Exit if the given address is none of these.
-  if (!GV && !AI) return;
+  // Only GlobalVariable's and AllocaInst's can be writeonce.
+  // Exit if the given address is none of these.
+  if (!GV && !AI) return;
 
-  // If the address is writeonce, then emit @llvm.invariant.start() intrinsic.
-  // Then, for non-global variables, push the emission of the
-  // @llvm.invariant.end() intrinsic onto the cleanup stack.
-  if ((AI || !GV->isConstant()) && D->getType().isWriteOnce(CGF.getContext()))
-    Args = CGF.EmitInvariantStart(*D, AddrPtr);
+  // If the address is writeonce, then emit @llvm.invariant.start() intrinsic.
+  // Then, for non-global variables, push the emission of the
+  // @llvm.invariant.end() intrinsic onto the cleanup stack.
+  if (AI || !GV->isConstant())
+    Args = CGF.EmitInvariantStart(*D, AddrPtr, /*IsGlobalConstant =*/false);
 }
 
 MarkWriteOnceWritten::~MarkWriteOnceWritten() {
-  if (AI && Args.StartInst) {
-    assert(!GV && "Can't have it both ways!");
-    CGF.EHStack.pushCleanup<CallInvariantEnd>(NormalCleanup, Args.StartInst,
-                                              Args.Addr, Args.Size);
-  }
+  if (Args.Addr &&
+      dyn_cast<llvm::AllocaInst>(Args.Addr->stripPointerCasts()) &&
+      Args.StartInst)
+    CGF.EHStack.pushCleanup<CallInvariantEnd>(NormalCleanup, Args);
 }
 
 /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
 /// variable declaration with auto, register, or no storage class specifier.
 /// These turn into simple stack objects, or GlobalValues depending on target.
 void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) {
   AutoVarEmission emission = EmitAutoVarAlloca(D);
   EmitAutoVarInit(emission);

 
 void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
   Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
   llvm::CallInst *C =
       Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
   C->setDoesNotThrow();
 }
 
+/// Collect offsets in bits.
+static bool getInvariantOffsets(const CodeGenFunction &CGF, QualType Ty,
+                                llvm::SmallVectorImpl<llvm::Value*> &Args) {
+  ASTContext &Ctx = CGF.getContext();
+  bool FoundWriteOnce = false;
+  if (const CXXRecordDecl *Record =
+      Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) {
+    for (const auto *Field : Record->fields()) {
+      assert(dyn_cast<FieldDecl>(Field) && "Field decls only.");
+      if (Field->getType().isWriteOnce(Ctx)) {
+        FoundWriteOnce = true;
+        CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty);
+        uint64_t Width = WidthChars.getQuantity();
+        Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Width));   // Size
+
+        uint64_t Offset = Ctx.getFieldOffset(Field);
+        Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Offset));  // Offset
+      } else {
+        FoundWriteOnce |= getInvariantOffsets(CGF, Field->getType(), Args);
+      }
+    }
+  }
+  return FoundWriteOnce;
+}
+
 /// Emit code to cause the variable at the given address to be considered as
 /// constant from this point onwards.
 CodeGenModule::InvariantArgs
-CodeGenFunction::EmitInvariantStart(const VarDecl &D, llvm::Value *Addr) {
+CodeGenFunction::EmitInvariantStart(const VarDecl &D, llvm::Value *Addr,
+                                    bool IsGlobalConstant) {
   // Don't emit the intrinsic if we're not optimizing.
   if (!CGM.getCodeGenOpts().OptimizationLevel)
     return CodeGenModule::InvariantArgs{};
 
-  // Grab the llvm.invariant.start intrinsic.
-  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
-  llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID);
+  assert(Addr && "Cannot emit on non-null address.");
 
-  // Emit a call with the size in bytes of the object.
-  CharUnits WidthChars = getContext().getTypeSizeInChars(D.getType());
+  // Collect arguments
+  ASTContext &Ctx = getContext();
+  QualType Ty = D.getType();
+  CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty);
   uint64_t Width = WidthChars.getQuantity();
-
-  llvm::Value *Size;
-  if (llvm::Constant* CAddr = dyn_cast<llvm::Constant>(Addr)) {
-    Size = llvm::ConstantInt::getSigned(Int64Ty, Width);
-    Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy);
-  }
-  else {
-    //Width = CGM.getDataLayout().getTypeAllocSize(LTy);
-    Size = llvm::ConstantInt::get(Int64Ty, Width);
-    Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
+  CodeGenModule::InvariantArgs Args;
+  llvm::Constant* CAddr = dyn_cast<llvm::Constant>(Addr);
+  if (CAddr && IsGlobalConstant) {
+    Args.Size = llvm::ConstantInt::getSigned(Int64Ty, Width);
+    Args.Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy);
+  } else if (Ty.isWriteOnce(Ctx) ||
+             getInvariantOffsets(*this, D.getType(), Args.Offsets)) {
+    Args.Size = llvm::ConstantInt::get(Int64Ty, Width);
+    Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
   }
-  llvm::CallInst *C = Builder.CreateCall(InvariantStart, {Size, Addr});
+
+  if (!Args.Addr)
+    return CodeGenModule::InvariantArgs{};
+
+  // Generate llvm.invariant.start intrinsic call.
+  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
+  llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID);
+  llvm::SmallVector<llvm::Value*, 8> CallArgs;
+  CallArgs.push_back(Args.Size);
+  CallArgs.push_back(Args.Addr);
+  CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end());
+  llvm::CallInst *C = Builder.CreateCall(InvariantStart, CallArgs);
+  Args.StartInst = C;
   C->setDoesNotThrow();
 
-  return CodeGenModule::InvariantArgs{C, Size, Addr};
+  return Args;
 }
 
-void CodeGenFunction::EmitInvariantEnd(llvm::CallInst *Start,
-                                       llvm::Value *Size, llvm::Value *Addr) {
-  // Grab the llvm.invariant.end intrinsic.
+void CodeGenFunction::EmitInvariantEnd(CodeGenModule::InvariantArgs &Args) {
+  assert(Args.Addr && "Emit on non-null address.");
+
+  // Generate the llvm.invariant.end intrinsic call.
   llvm::Intrinsic::ID InvEndID = llvm::Intrinsic::invariant_end;
   llvm::Constant *InvariantEnd = CGM.getIntrinsic(InvEndID);
-
-  // Emit a call with the size in bytes of the object.
-  llvm::CallInst *C = Builder.CreateCall(InvariantEnd, {Start, Size, Addr});
+  llvm::SmallVector<llvm::Value*, 8> CallArgs;
+  CallArgs.push_back(Args.StartInst);
+  CallArgs.push_back(Args.Size);
+  CallArgs.push_back(Args.Addr);
+  //CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end());
+  llvm::CallInst *C = Builder.CreateCall(InvariantEnd, CallArgs);
   C->setDoesNotThrow();
 }
 
 /// EmitAutoVarAlloca - Emit the alloca and debug information for a
 /// local variable.  Does not emit initialization or destruction.
 CodeGenFunction::AutoVarEmission
 CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   QualType Ty = D.getType();

lib/CodeGen/CodeGenFunction.h

void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
                                llvm::Value *AllocSizeWithoutCookie);
 
   void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
                         Address Ptr);
 
   llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr);
   void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
   CodeGenModule::InvariantArgs
-  EmitInvariantStart(const VarDecl &D, llvm::Value *Addr);
-  void EmitInvariantEnd(llvm::CallInst *C,
-                        llvm::Value *Size, llvm::Value *Addr);
+      EmitInvariantStart(const VarDecl &D, llvm::Value *Addr,
+                         bool IsGlobalConstant = true);
+  void EmitInvariantEnd(CodeGenModule::InvariantArgs &Args);
 
   llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
   void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
 
   void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
                       QualType DeleteTy);
 
   RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,

lib/CodeGen/CodeGenModule.h

struct Structor {
     int Priority;
     llvm::Constant *Initializer;
     llvm::Constant *AssociatedData;
   };
 
   typedef std::vector<Structor> CtorList;
 
   struct InvariantArgs {
-    llvm::CallInst *StartInst;
-    llvm::Value *Size;
+    llvm::CallInst *StartInst;  // TODO: Is this necessary?
+    llvm::Value *Size;          // TODO: Is this necessary?
     llvm::Value *Addr;
-    InvariantArgs()
-    : StartInst(nullptr), Size(nullptr), Addr(nullptr) {}
-    InvariantArgs(llvm::CallInst *C, llvm::Value *size, llvm::Value *addr)
-    : StartInst(C), Size(size), Addr(addr) {}
+    llvm::SmallVector<llvm::Value*, 8> Offsets;
+
+    InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {}
+    InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A)
+    : StartInst(C), Size(S), Addr(A) {}
   };
 
 private:
   ASTContext &Context;
   const LangOptions &LangOpts;
   const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info.
   const PreprocessorOptions &PreprocessorOpts; // Only used for debug info.
   const CodeGenOptions &CodeGenOpts;

test/CodeGenCXX/const-invariant.cpp

 #else
  #define Const 
 #endif
 
 // object or builtin?
 #if defined(OBJ)
   struct A {
     int a;
-    int b;
-    A(int a) : a(a) { }
+    Const int b;
+    A(int a) : a(a), b(a) { }
   };
 
   // A with explicit destructor
   struct D {
     int a;
     int b;
     D(int a) : a(a) { }
     ~D();

 // CHECK-NL-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_f to i8*))
 // CHECK-NL-CO-OBJ: call void @_ZN1IC{{[0-9]+}}Ei({{.*}}* @_ZL3i_i, {{.*}})
 // CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_i to i8*))
 // CHECK-NL-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* @_ZL4i_iv, {{.*}})
 // CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL4i_iv to i8*))
 
 void foo(const Type* const);
 void bar(Type);
+void foo(const int* const);
+void bar(int);
 #if defined(OBJ)
 void foo_d(const D* const);
 void bar_d(D);
 void foo_m(const M* const);
 void bar_m(M);
 void foo_f(const F* const);
 void bar_f(F);
 void foo_i(const I* const);
#endif
   // CHECK-L-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* %i_iv,
   // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
   bar_iv(i_iv);
   foo_iv(&i_iv);          // Does not change i_iv.
   bar_iv(i_iv);
   // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
 }
 
+// Example 1 with const member of non-const object:
+// collects offsets in invariant_start call.
+void ex1_cm() {
+// CHECK: @_Z6ex1_cmv(
+#ifdef LOCAL
+  Type i(one());
+#endif
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* {{.*}},
+  bar(i.b);
+  foo(&i.b);           // Does not change i.b.
+  bar(i.b);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
 #endif // #if defined(OBJ)
 
 // Example 1 with references and pointers:
 //   Expect no invariant intrinsic call generated on non const pointers.
 void ex1_p() {
 // CHECK: @_Z5ex1_pv(
 #ifdef LOCAL
 #ifdef CONST