[Extension] Optimizing const member objects.

[lvoufo]

Summary

Continued from D13603...

When generating invariant_start intrinsic calls for any given (instantiated) object, instead of simply recording information about whether the object is 'const', record the offsets of sub-fields of the objects that are 'const'. This turns the definition of the invariant_start instruction into a variadic function of at least two arguments, where the first two arguments are information about the allocated memory.

If the invariant_start call instruction only has two arguments, then the object is 'writeonce'. Otherwise, it is not 'writeonce' but has 'writeonce' sub-fields at the indicated offsets.

[lvoufo]

Remove commented out line -- Offsets not needed on invariant_end calls after all.

[lvoufo]

Reify computed invariant offsets, and while we're at it, cleanup InvariantArgs: move it out of CodeGenModule.h and remove Offsets member field since offsets are already in the start instructions.

[rsmith]

lib/CodeGen/CGDecl.cpp

933

Instead of tracking a separate Computed flag, use InvariantOffsets.count(Record) here, and insert the value below.

943–959

It'd be good to also handle the case of a const field of class type that contains a mutable member -- in that case, all members other than the mutable member can be marked as invariant.

956–957

The recursive call to ComputeInvariatOffsets here invalidates your Args reference (DenseMap insertions invalidate all iterators and references to map elements).

lib/CodeGen/CGDecl.cpp

public:
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitLifetimeEnd(Size, Addr);
     }
   };
 
   /// A cleanup to call @llvm.invariant.end.
   class CallInvariantEnd final : public EHScopeStack::Cleanup {
-    CodeGenModule::InvariantArgs Args;
+    CodeGenFunction::InvariantArgs Args;
    public:
-    CallInvariantEnd(CodeGenModule::InvariantArgs &args) : Args(args) {}
+    CallInvariantEnd(CodeGenFunction::InvariantArgs &args) : Args(args) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitInvariantEnd(Args);
     }
   };
 }
 
 /// EmitAutoVarWithLifetime - Does the setup required for an automatic

 
 void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
   Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
   llvm::CallInst *C =
       Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
   C->setDoesNotThrow();
 }
 
-/// Collect offsets in bits.
-static bool getInvariantOffsets(const CodeGenFunction &CGF, QualType Ty,
-                                llvm::SmallVectorImpl<llvm::Value *> &Args) {
-  ASTContext &Ctx = CGF.getContext();
-  bool FoundWriteOnce = false;
-  if (const CXXRecordDecl *Record =
-          Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) {
-    for (const auto *Field : Record->fields()) {
-      assert(dyn_cast<FieldDecl>(Field) && "Field decls only.");
-      if (Field->getType().isWriteOnce(Ctx)) {
-        FoundWriteOnce = true;
-        CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty);
-        uint64_t Width = WidthChars.getQuantity();
-        Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Width));  // Size
+CodeGenFunction::OffsetsType&
+CodeGenFunction::ComputeInvariantOffsets(const CXXRecordDecl *Record) {
+  ASTContext &Ctx = getContext();
+  auto &OffsetsInfo = InvariantOffsets.FindAndConstruct(Record).second;
+  OffsetsType &Args = OffsetsInfo.Offsets;
+
+  // If this has already been computed, then return the stored value.
+  if (OffsetsInfo.Computed) return Args;
+
+  // Otherwise, mark that this is computed.
+  OffsetsInfo.Computed = true;
+  assert(Args.empty() && "There should be no offset specified yet.");
+
+  // Trace through fields collecting offsets of writeonce candidates.
+    QualType FieldType = Field->getType();
+    if (FieldType.isWriteOnce(Ctx)) {
+      CharUnits WidthChars = Ctx.getTypeSizeInChars(FieldType);
+      Args.push_back(llvm::ConstantInt::get(Int64Ty, Width));  // Size
 
-        uint64_t Offset = Ctx.getFieldOffset(Field);
-        Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Offset));  // Offset
-      } else {
-        FoundWriteOnce |= getInvariantOffsets(CGF, Field->getType(), Args);
-      }
-    }
-  }
-  return FoundWriteOnce;
+      Args.push_back(llvm::ConstantInt::get(Int64Ty, Offset));  // Offset
+    } else if (const CXXRecordDecl *RecField =
+               Ctx.getBaseElementType(FieldType)->getAsCXXRecordDecl()) {
+      auto &FieldArgs = ComputeInvariantOffsets(RecField);
+      Args.insert(Args.end(), FieldArgs.begin(), FieldArgs.end());
+
+  return Args;
 }
 
 /// Emit code to cause the variable at the given address to be considered as
 /// constant from this point onwards.
-CodeGenModule::InvariantArgs CodeGenFunction::EmitInvariantStart(
+CodeGenFunction::InvariantArgs CodeGenFunction::EmitInvariantStart(
     const VarDecl &D, llvm::Value *Addr, bool IsGlobalConstant) {
   // Don't emit the intrinsic if we're not optimizing.
   if (!CGM.getCodeGenOpts().OptimizationLevel)
-    return CodeGenModule::InvariantArgs{};
+    return InvariantArgs{};
 
   assert(Addr && "Cannot emit on non-null address.");
 
   // Collect arguments
   ASTContext &Ctx = getContext();
   QualType Ty = D.getType();
   CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty);
   uint64_t Width = WidthChars.getQuantity();
-  CodeGenModule::InvariantArgs Args;
+  InvariantArgs Args;
+  OffsetsType Offsets;
   llvm::Constant *CAddr = dyn_cast<llvm::Constant>(Addr);
   if (CAddr && IsGlobalConstant) {
     Args.Size = llvm::ConstantInt::getSigned(Int64Ty, Width);
     Args.Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy);
-  } else if (Ty.isWriteOnce(Ctx) ||
-             getInvariantOffsets(*this, D.getType(), Args.Offsets)) {
+  } else if (Ty.isWriteOnce(Ctx)) {
     Args.Size = llvm::ConstantInt::get(Int64Ty, Width);
     Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
+  } else if (const CXXRecordDecl *Record =
+              Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) {
+    Offsets = ComputeInvariantOffsets(Record);
+    // If there are invariant offsets in this non-writeonce record,
+    // then emit the intrinsic call with the offsets. Otherwise,
+    // do not emit anything.
+    if (!Offsets.empty()) {
+      Args.Size = llvm::ConstantInt::get(Int64Ty, Width);
+      Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy);
+    }
   }
 
-  if (!Args.Addr) return CodeGenModule::InvariantArgs{};
+  if (!Args.Addr) return InvariantArgs{};
 
   // Generate llvm.invariant.start intrinsic call.
   llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
   llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID);
   llvm::SmallVector<llvm::Value *, 8> CallArgs;
   CallArgs.push_back(Args.Size);
   CallArgs.push_back(Args.Addr);
-  CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end());
+  CallArgs.insert(CallArgs.end(), Offsets.begin(), Offsets.end());
   llvm::CallInst *C = Builder.CreateCall(InvariantStart, CallArgs);
   Args.StartInst = C;
   C->setDoesNotThrow();
 
   return Args;
 }
 
-void CodeGenFunction::EmitInvariantEnd(CodeGenModule::InvariantArgs &Args) {
+void CodeGenFunction::EmitInvariantEnd(CodeGenFunction::InvariantArgs &Args) {
   assert(Args.Addr && "Emit on non-null address.");
 
   // Generate the llvm.invariant.end intrinsic call.
   llvm::Intrinsic::ID InvEndID = llvm::Intrinsic::invariant_end;
   llvm::Constant *InvariantEnd = CGM.getIntrinsic(InvEndID);
   llvm::SmallVector<llvm::Value *, 8> CallArgs;
   CallArgs.push_back(Args.StartInst);
   CallArgs.push_back(Args.Size);
   CallArgs.push_back(Args.Addr);
-  // CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end());
   llvm::CallInst *C = Builder.CreateCall(InvariantEnd, CallArgs);
   C->setDoesNotThrow();
 }
 
 /// EmitAutoVarAlloca - Emit the alloca and debug information for a
 /// local variable.  Does not emit initialization or destruction.
 CodeGenFunction::AutoVarEmission
 CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {

lib/CodeGen/CodeGenFunction.h

void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
                                llvm::Value *NumElements,
                                llvm::Value *AllocSizeWithoutCookie);
 
   void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
                         Address Ptr);
 
   llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr);
   void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
-  CodeGenModule::InvariantArgs EmitInvariantStart(const VarDecl &D,
-                                                  llvm::Value *Addr,
-                                                  bool IsGlobalConstant = true);
-  void EmitInvariantEnd(CodeGenModule::InvariantArgs &Args);
+
+  /// \brief Specifes arguments to invariant_start/end intrinsic calls.
+  struct InvariantArgs {
+    llvm::CallInst *StartInst;  // Contains invariant offsets.
+    llvm::Value *Size;          // TODO: Is this necessary?
+    llvm::Value *Addr;
+
+    InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {}
+    InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A)
+        : StartInst(C), Size(S), Addr(A) {}
+  };
+
+  /// \brief Specifies type of invariant offsets in a given record.
+  typedef llvm::SmallVector<llvm::Value *, 8> OffsetsType;
+
+private:
+
+  /// \brief Specifies offsets and whether they have already been computed.
+  /// Note: empty offsets may or may not have been computed.
+  struct OffsetsInfoType {
+    bool Computed;
+    OffsetsType Offsets;
+    OffsetsInfoType() : Computed(false) { }
+  };
+
+  /// \brief A collection of invariant offsets per given record.
+  llvm::DenseMap<const CXXRecordDecl *, OffsetsInfoType> InvariantOffsets;
+
+  /// \brief Compute the invariant offsets of a given Record.
+  OffsetsType& ComputeInvariantOffsets(const CXXRecordDecl *Record);
+
+public:
+  InvariantArgs EmitInvariantStart(const VarDecl &D, llvm::Value *Addr,
+  void EmitInvariantEnd(InvariantArgs &Args);
 
   llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
   void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
 
   void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
                       QualType DeleteTy);
 
   RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
private:
 
   llvm::Value *GetValueForARMHint(unsigned BuiltinID);
 };
 
 class MarkWriteOnceWrittenRAII {
   CodeGenFunction &CGF;
   llvm::GlobalVariable *GV;
   llvm::AllocaInst *AI;
-  CodeGenModule::InvariantArgs Args;
+  CodeGenFunction::InvariantArgs Args;
 
  public:
   MarkWriteOnceWrittenRAII(CodeGenFunction &CGF, const VarDecl *D,
                            llvm::GlobalVariable *GVAddr = nullptr);
   ~MarkWriteOnceWrittenRAII();
 };
 
 /// Helper class with most of the code for saving a value for a

lib/CodeGen/CodeGenModule.h

Structor(int Priority, llvm::Constant *Initializer,
           AssociatedData(AssociatedData) {}
     int Priority;
     llvm::Constant *Initializer;
     llvm::Constant *AssociatedData;
   };
 
   typedef std::vector<Structor> CtorList;
 
-  struct InvariantArgs {
-    llvm::CallInst *StartInst;  // TODO: Is this necessary?
-    llvm::Value *Size;          // TODO: Is this necessary?
-    llvm::Value *Addr;
-    llvm::SmallVector<llvm::Value *, 8> Offsets;
-
-    InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {}
-    InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A)
-        : StartInst(C), Size(S), Addr(A) {}
-  };
-
 private:
   ASTContext &Context;
   const LangOptions &LangOpts;
   const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info.
   const PreprocessorOptions &PreprocessorOpts; // Only used for debug info.
   const CodeGenOptions &CodeGenOpts;
   llvm::Module &TheModule;
   DiagnosticsEngine &Diags;

test/CodeGenCXX/const-invariant.cpp

 // CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_i to i8*))
 // CHECK-NL-CO-OBJ: call void @_ZN2IVC{{[0-9]+}}Ei({{.*}}* @_ZL4i_iv, {{.*}})
 // CHECK-NL-CO-OBJ-NOT: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL4i_iv to i8*))
 // CHECK-NL-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* @_ZL5i_civ, {{.*}})
 // CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL5i_civ to i8*))
 
 void foo(const Type* const);
 void bar(Type);
+void foo(const int* const);
+void bar(int);
 #if defined(OBJ)
 void foo_d(const D* const);
 void bar_d(D);
 void foo_m(const M* const);
 void bar_m(M);
 void foo_f(const F* const);
 void bar_f(F);
 void foo_i(const I* const);
#endif
   // CHECK-L-CO-OBJ: call void @_ZN3CIVC{{[0-9]+}}Ei({{.*}}* %i_civ,
   // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(
   bar_civ(i_civ);
   foo_civ(&i_civ);  // Does not change i_civ.
   bar_civ(i_civ);
   // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end(
 }
 
+// Example 1 with const member of non-const object:
+// collects offsets in invariant_start call.
+void ex1_cm() {
+// CHECK: @_Z6ex1_cmv(
+#ifdef LOCAL
+  Type i(one());
+#endif
+
+  // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i,
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* {{.*}},
+  bar(i.b);
+  foo(&i.b);  // Does not change i.b.
+  bar(i.b);
+  // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8*
+}
+
 #endif  // #if defined(OBJ)
 
 // Example 1 with references and pointers:
 //   Expect no invariant intrinsic call generated on non const pointers.
 void ex1_p() {
 // CHECK: @_Z5ex1_pv(
 #ifdef LOCAL
 #ifdef CONST