[Extension] Optimizing const member objects.

[lvoufo]

Summary

Continued from D13606...

[lvoufo]

Updated from D14003 (D14008 + D14009), D14004, D14005, D14006 and D14007.

[lvoufo]

Update documentation of invariant_start intrinsic call.

[lvoufo]

Fix diff (wrong one submitted last time).

lib/CodeGen/CGDecl.cpp

public:
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitLifetimeEnd(Size, Addr);
     }
   };
 
   /// A cleanup to call @llvm.invariant.end.
   class CallInvariantEnd final : public EHScopeStack::Cleanup {
-    CodeGenModule::InvariantArgs Args;
+    CodeGenFunction::InvariantArgs Args;
    public:
-    CallInvariantEnd(CodeGenModule::InvariantArgs &args) : Args(args) {}
+    CallInvariantEnd(CodeGenFunction::InvariantArgs &args) : Args(args) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitInvariantEnd(Args);
     }
   };
 }
 
 /// EmitAutoVarWithLifetime - Does the setup required for an automatic

 
 void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
   Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
   llvm::CallInst *C =
       Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
   C->setDoesNotThrow();
 }
 
-/// Collect offsets in bits.
-static bool getInvariantOffsets(const CodeGenFunction &CGF, QualType Ty,
-                                llvm::SmallVectorImpl<llvm::Value *> &Args) {
-  ASTContext &Ctx = CGF.getContext();
-  bool FoundWriteOnce = false;
-  if (const CXXRecordDecl *Record =
-          Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) {
-    for (const auto *Field : Record->fields()) {
-      assert(dyn_cast<FieldDecl>(Field) && "Field decls only.");
-      if (Field->getType().isWriteOnce(Ctx)) {
-        FoundWriteOnce = true;
-        CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty);
-        uint64_t Width = WidthChars.getQuantity();
-        Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Width));  // Size
+CodeGenFunction::OffsetsType&
+CodeGenFunction::ComputeInvariantOffsets(const CXXRecordDecl *Record) {
+  ASTContext &Ctx = getContext();
+  auto &OffsetsInfo = InvariantOffsets.FindAndConstruct(Record).second;
+  OffsetsType &Args = OffsetsInfo.Offsets;
+
+  // If this has already been computed, then return the stored value.
+  if (OffsetsInfo.Computed) return Args;
+
+  // Otherwise, mark that this is computed.
+  OffsetsInfo.Computed = true;
+  assert(Args.empty() && "There should be no offset specified yet.");
+
+  // Trace through fields collecting offsets of writeonce candidates.
+    QualType FieldType = Field->getType();
+    if (FieldType.isWriteOnce(Ctx)) {
+      CharUnits WidthChars = Ctx.getTypeSizeInChars(FieldType);
+      Args.push_back(llvm::ConstantInt::get(Int64Ty, Width));  // Size
 
-        uint64_t Offset = Ctx.getFieldOffset(Field);
-        Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Offset));  // Offset
-      } else {
-        FoundWriteOnce |= getInvariantOffsets(CGF, Field->getType(), Args);
-      }
-    }
-  }
-  return FoundWriteOnce;
+      Args.push_back(llvm::ConstantInt::get(Int64Ty, Offset));  // Offset
+    } else if (const CXXRecordDecl *RecField =
+               Ctx.getBaseElementType(FieldType)->getAsCXXRecordDecl()) {
+      auto &FieldArgs = ComputeInvariantOffsets(RecField);
+      Args.insert(Args.end(), FieldArgs.begin(), FieldArgs.end());
+
+  return Args;
 }
 
 /// Emit code to cause the variable at the given address to be considered as
 /// constant from this point onwards.
-CodeGenModule::InvariantArgs CodeGenFunction::EmitInvariantStart(
+CodeGenFunction::InvariantArgs CodeGenFunction::EmitInvariantStart(
     const VarDecl &D, llvm::Value *Addr, bool IsGlobalConstant) {
   // Don't emit the intrinsic if we're not optimizing.
   if (!CGM.getCodeGenOpts().OptimizationLevel)
-    return CodeGenModule::InvariantArgs{};
+    return InvariantArgs{};
 
   assert(Addr && "Cannot emit on non-null address.");
 
   // Collect arguments
   ASTContext &Ctx = getContext();
   QualType Ty = D.getType();
   CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty);
   uint64_t Width = WidthChars.getQuantity();
-  CodeGenModule::InvariantArgs Args;
+  InvariantArgs Args;
+  OffsetsType Offsets;
   llvm::Constant *CAddr = dyn_cast<llvm::Constant>(Addr);
   if (CAddr && IsGlobalConstant) {
     Args.Size = llvm::ConstantInt::getSigned(Int64Ty, Width);
     Args.Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy);
-  } else if (Ty.isWriteOnce(Ctx) ||
-             getInvariantOffsets(*this, D.getType(), Args.Offsets)) {
+  } else if (Ty.isWriteOnce(Ctx)) {
     Args.Size = llvm::ConstantInt::get(Int64Ty, Width);
     Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
+  } else if (const CXXRecordDecl *Record =
+              Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) {
+    Offsets = ComputeInvariantOffsets(Record);
+    // If there are invariant offsets in this non-writeonce record,
+    // then emit the intrinsic call with the offsets. Otherwise,
+    // do not emit anything.
+    if (!Offsets.empty()) {
+      Args.Size = llvm::ConstantInt::get(Int64Ty, Width);
+      Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
+    }
   }
 
-  if (!Args.Addr) return CodeGenModule::InvariantArgs{};
+  if (!Args.Addr) return InvariantArgs{};
 
   // Generate llvm.invariant.start intrinsic call.
   llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
   llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID);
   llvm::SmallVector<llvm::Value *, 8> CallArgs;
   CallArgs.push_back(Args.Size);
   CallArgs.push_back(Args.Addr);
-  CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end());
+  CallArgs.insert(CallArgs.end(), Offsets.begin(), Offsets.end());
   llvm::CallInst *C = Builder.CreateCall(InvariantStart, CallArgs);
   Args.StartInst = C;
   C->setDoesNotThrow();
 
   return Args;
 }
 
-void CodeGenFunction::EmitInvariantEnd(CodeGenModule::InvariantArgs &Args) {
+void CodeGenFunction::EmitInvariantEnd(CodeGenFunction::InvariantArgs &Args) {
   assert(Args.Addr && "Emit on non-null address.");
 
   // Generate the llvm.invariant.end intrinsic call.
   llvm::Intrinsic::ID InvEndID = llvm::Intrinsic::invariant_end;
   llvm::Constant *InvariantEnd = CGM.getIntrinsic(InvEndID);
   llvm::SmallVector<llvm::Value *, 8> CallArgs;
   CallArgs.push_back(Args.StartInst);
   CallArgs.push_back(Args.Size);
   CallArgs.push_back(Args.Addr);
-  // CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end());
   llvm::CallInst *C = Builder.CreateCall(InvariantEnd, CallArgs);
   C->setDoesNotThrow();
 }
 
 /// EmitAutoVarAlloca - Emit the alloca and debug information for a
 /// local variable.  Does not emit initialization or destruction.
 CodeGenFunction::AutoVarEmission
 CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {

lib/CodeGen/CodeGenFunction.h

void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
                                llvm::Value *NumElements,
                                llvm::Value *AllocSizeWithoutCookie);
 
   void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
                         Address Ptr);
 
   llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr);
   void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
-  CodeGenModule::InvariantArgs EmitInvariantStart(const VarDecl &D,
-                                                  llvm::Value *Addr,
-                                                  bool IsGlobalConstant = true);
-  void EmitInvariantEnd(CodeGenModule::InvariantArgs &Args);
+
+  /// \brief Specifes arguments to invariant_start/end intrinsic calls.
+  struct InvariantArgs {
+    llvm::CallInst *StartInst;  // Contains invariant offsets.
+    llvm::Value *Size;          // TODO: Is this necessary?
+    llvm::Value *Addr;
+
+    InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {}
+    InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A)
+        : StartInst(C), Size(S), Addr(A) {}
+  };
+
+  /// \brief Specifies type of invariant offsets in a given record.
+  typedef llvm::SmallVector<llvm::Value *, 8> OffsetsType;
+
+private:
+
+  /// \brief Specifies offsets and whether they have already been computed.
+  /// Note: empty offsets may or may not have been computed.
+  struct OffsetsInfoType {
+    bool Computed;
+    OffsetsType Offsets;
+    OffsetsInfoType() : Computed(false) { }
+  };
+
+  /// \brief A collection of invariant offsets per given record.
+  llvm::DenseMap<const CXXRecordDecl *, OffsetsInfoType> InvariantOffsets;
+
+  /// \brief Compute the invariant offsets of a given Record.
+  OffsetsType& ComputeInvariantOffsets(const CXXRecordDecl *Record);
+
+public:
+  InvariantArgs EmitInvariantStart(const VarDecl &D, llvm::Value *Addr,
+  void EmitInvariantEnd(InvariantArgs &Args);
 
   llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
   void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
 
   void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
                       QualType DeleteTy);
 
   RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
private:
 
   llvm::Value *GetValueForARMHint(unsigned BuiltinID);
 };
 
 class MarkWriteOnceWrittenRAII {
   CodeGenFunction &CGF;
   llvm::GlobalVariable *GV;
   llvm::AllocaInst *AI;
-  CodeGenModule::InvariantArgs Args;
+  CodeGenFunction::InvariantArgs Args;
 
  public:
   MarkWriteOnceWrittenRAII(CodeGenFunction &CGF, const VarDecl *D,
                            llvm::GlobalVariable *GVAddr = nullptr);
   ~MarkWriteOnceWrittenRAII();
 };
 
 /// Helper class with most of the code for saving a value for a

test/CodeGenCXX/const-invariant.cpp

 // CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_i to i8*))
 // CHECK-NL-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* @_ZL4i_iv, {{.*}})
 // CHECK-NL-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL4i_iv to i8*))
 // CHECK-NL-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* @_ZL5i_civ, {{.*}})
 // CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL5i_civ to i8*))
 
 void foo(const Type* const);
 void bar(Type);
+void foo(const int* const);
+void bar(int);
 #if defined(OBJ)
 void foo_d(const D* const);
 void bar_d(D);
 void foo_m(const M* const);
 void bar_m(M);
 void foo_f(const F* const);
 void bar_f(F);
 void foo_i(const I* const);
#endif
   // CHECK-L-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* %i_civ,
   // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
   bar_civ(i_civ);
   foo_civ(&i_civ);  // Does not change i_civ.
   bar_civ(i_civ);
   // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
 }
 
+// Example 1 with const member of non-const object:
+// collects offsets in invariant_start call.
+void ex1_cm() {
+// CHECK: @_Z6ex1_cmv(
+#ifdef LOCAL
+  Type i(one());
+#endif
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* {{.*}},
+  bar(i.b);
+  foo(&i.b);  // Does not change i.b.
+  bar(i.b);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
 #endif  // #if defined(OBJ)
 
 // Example 1 with references and pointers:
 //   Expect no invariant intrinsic call generated on non const pointers.
 void ex1_p() {
 // CHECK: @_Z5ex1_pv(
 #ifdef LOCAL
 #ifdef CONST