Index: lib/CodeGen/CGDecl.cpp =================================================================== --- lib/CodeGen/CGDecl.cpp +++ lib/CodeGen/CGDecl.cpp @@ -530,15 +530,12 @@ /// A cleanup to call @llvm.invariant.end. class CallInvariantEnd final : public EHScopeStack::Cleanup { - llvm::CallInst *StartInst; - llvm::Value *Addr; - llvm::Value *Size; // TODO: Is this even necessary? + CodeGenModule::InvariantArgs Args; public: - CallInvariantEnd(llvm::CallInst *C, llvm::Value *addr, llvm::Value *size) - : StartInst(C), Addr(addr), Size(size) {} + CallInvariantEnd(CodeGenModule::InvariantArgs &args) : Args(args) {} void Emit(CodeGenFunction &CGF, Flags flags) override { - CGF.EmitInvariantEnd(StartInst, Size, Addr); + CGF.EmitInvariantEnd(Args); } }; } @@ -859,7 +856,7 @@ /// Emit the code necessary to initialize the given global variable. /// TODO: Use Address for Addr (to match lifetime intrisics). MarkWriteOnceWritten::MarkWriteOnceWritten(CodeGenFunction &CGF, - const VarDecl *D, + const VarDecl* D, llvm::GlobalVariable *GVAddr) : CGF(CGF), GV(GVAddr), AI(nullptr) { assert(D && "Invalid declaration."); @@ -874,23 +871,22 @@ AI = dyn_cast_or_null(AddrPtr); } - // Only GlobalVariable's and AllocaInst's can be writeonce. - // Exit if the given address is none of these. - if (!GV && !AI) return; + // Only GlobalVariable's and AllocaInst's can be writeonce. + // Exit if the given address is none of these. + if (!GV && !AI) return; - // If the address is writeonce, then emit @llvm.invariant.start() intrinsic. - // Then, for non-global variables, push the emission of the - // @llvm.invariant.end() intrinsic onto the cleanup stack. - if ((AI || !GV->isConstant()) && D->getType().isWriteOnce(CGF.getContext())) - Args = CGF.EmitInvariantStart(*D, AddrPtr); + // If the address is writeonce, then emit @llvm.invariant.start() intrinsic. + // Then, for non-global variables, push the emission of the + // @llvm.invariant.end() intrinsic onto the cleanup stack. + if (AI || !GV->isConstant()) + Args = CGF.EmitInvariantStart(*D, AddrPtr, /*IsGlobalConstant =*/false); } MarkWriteOnceWritten::~MarkWriteOnceWritten() { - if (AI && Args.StartInst) { - assert(!GV && "Can't have it both ways!"); - CGF.EHStack.pushCleanup(NormalCleanup, Args.StartInst, - Args.Addr, Args.Size); - } + if (Args.Addr && + dyn_cast(Args.Addr->stripPointerCasts()) && + Args.StartInst) + CGF.EHStack.pushCleanup(NormalCleanup, Args); } /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a @@ -932,46 +928,87 @@ C->setDoesNotThrow(); } +/// Collect offsets in bits. +static bool getInvariantOffsets(const CodeGenFunction &CGF, QualType Ty, + llvm::SmallVectorImpl &Args) { + ASTContext &Ctx = CGF.getContext(); + bool FoundWriteOnce = false; + if (const CXXRecordDecl *Record = + Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) { + for (const auto *Field : Record->fields()) { + assert(dyn_cast(Field) && "Field decls only."); + if (Field->getType().isWriteOnce(Ctx)) { + FoundWriteOnce = true; + CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty); + uint64_t Width = WidthChars.getQuantity(); + Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Width)); // Size + + uint64_t Offset = Ctx.getFieldOffset(Field); + Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Offset)); // Offset + } else { + FoundWriteOnce |= getInvariantOffsets(CGF, Field->getType(), Args); + } + } + } + return FoundWriteOnce; +} + /// Emit code to cause the variable at the given address to be considered as /// constant from this point onwards. CodeGenModule::InvariantArgs -CodeGenFunction::EmitInvariantStart(const VarDecl &D, llvm::Value *Addr) { +CodeGenFunction::EmitInvariantStart(const VarDecl &D, llvm::Value *Addr, + bool IsGlobalConstant) { // Don't emit the intrinsic if we're not optimizing. if (!CGM.getCodeGenOpts().OptimizationLevel) return CodeGenModule::InvariantArgs{}; - // Grab the llvm.invariant.start intrinsic. - llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start; - llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID); + assert(Addr && "Cannot emit on non-null address."); - // Emit a call with the size in bytes of the object. - CharUnits WidthChars = getContext().getTypeSizeInChars(D.getType()); + // Collect arguments + ASTContext &Ctx = getContext(); + QualType Ty = D.getType(); + CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty); uint64_t Width = WidthChars.getQuantity(); - - llvm::Value *Size; - if (llvm::Constant* CAddr = dyn_cast(Addr)) { - Size = llvm::ConstantInt::getSigned(Int64Ty, Width); - Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy); - } - else { - //Width = CGM.getDataLayout().getTypeAllocSize(LTy); - Size = llvm::ConstantInt::get(Int64Ty, Width); - Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + CodeGenModule::InvariantArgs Args; + llvm::Constant* CAddr = dyn_cast(Addr); + if (CAddr && IsGlobalConstant) { + Args.Size = llvm::ConstantInt::getSigned(Int64Ty, Width); + Args.Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy); + } else if (Ty.isWriteOnce(Ctx) || + getInvariantOffsets(*this, D.getType(), Args.Offsets)) { + Args.Size = llvm::ConstantInt::get(Int64Ty, Width); + Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy); } - llvm::CallInst *C = Builder.CreateCall(InvariantStart, {Size, Addr}); + + if (!Args.Addr) + return CodeGenModule::InvariantArgs{}; + + // Generate llvm.invariant.start intrinsic call. + llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start; + llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID); + llvm::SmallVector CallArgs; + CallArgs.push_back(Args.Size); + CallArgs.push_back(Args.Addr); + CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end()); + llvm::CallInst *C = Builder.CreateCall(InvariantStart, CallArgs); + Args.StartInst = C; C->setDoesNotThrow(); - return CodeGenModule::InvariantArgs{C, Size, Addr}; + return Args; } -void CodeGenFunction::EmitInvariantEnd(llvm::CallInst *Start, - llvm::Value *Size, llvm::Value *Addr) { - // Grab the llvm.invariant.end intrinsic. +void CodeGenFunction::EmitInvariantEnd(CodeGenModule::InvariantArgs &Args) { + assert(Args.Addr && "Emit on non-null address."); + + // Generate the llvm.invariant.end intrinsic call. llvm::Intrinsic::ID InvEndID = llvm::Intrinsic::invariant_end; llvm::Constant *InvariantEnd = CGM.getIntrinsic(InvEndID); - - // Emit a call with the size in bytes of the object. - llvm::CallInst *C = Builder.CreateCall(InvariantEnd, {Start, Size, Addr}); + llvm::SmallVector CallArgs; + CallArgs.push_back(Args.StartInst); + CallArgs.push_back(Args.Size); + CallArgs.push_back(Args.Addr); + //CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end()); + llvm::CallInst *C = Builder.CreateCall(InvariantEnd, CallArgs); C->setDoesNotThrow(); } Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -1905,9 +1905,9 @@ llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr); void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr); CodeGenModule::InvariantArgs - EmitInvariantStart(const VarDecl &D, llvm::Value *Addr); - void EmitInvariantEnd(llvm::CallInst *C, - llvm::Value *Size, llvm::Value *Addr); + EmitInvariantStart(const VarDecl &D, llvm::Value *Addr, + bool IsGlobalConstant = true); + void EmitInvariantEnd(CodeGenModule::InvariantArgs &Args); llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E); void EmitCXXDeleteExpr(const CXXDeleteExpr *E); Index: lib/CodeGen/CodeGenModule.h =================================================================== --- lib/CodeGen/CodeGenModule.h +++ lib/CodeGen/CodeGenModule.h @@ -260,13 +260,14 @@ typedef std::vector CtorList; struct InvariantArgs { - llvm::CallInst *StartInst; - llvm::Value *Size; + llvm::CallInst *StartInst; // TODO: Is this necessary? + llvm::Value *Size; // TODO: Is this necessary? llvm::Value *Addr; - InvariantArgs() - : StartInst(nullptr), Size(nullptr), Addr(nullptr) {} - InvariantArgs(llvm::CallInst *C, llvm::Value *size, llvm::Value *addr) - : StartInst(C), Size(size), Addr(addr) {} + llvm::SmallVector Offsets; + + InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {} + InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A) + : StartInst(C), Size(S), Addr(A) {} }; private: Index: test/CodeGenCXX/const-invariant.cpp =================================================================== --- test/CodeGenCXX/const-invariant.cpp +++ test/CodeGenCXX/const-invariant.cpp @@ -36,8 +36,8 @@ #if defined(OBJ) struct A { int a; - int b; - A(int a) : a(a) { } + Const int b; + A(int a) : a(a), b(a) { } }; // A with explicit destructor @@ -159,6 +159,8 @@ void foo(const Type* const); void bar(Type); +void foo(const int* const); +void bar(int); #if defined(OBJ) void foo_d(const D* const); void bar_d(D); @@ -322,6 +324,22 @@ // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end( } +// Example 1 with const member of non-const object: +// collects offsets in invariant_start call. +void ex1_cm() { +// CHECK: @_Z6ex1_cmv( +#ifdef LOCAL + Type i(one()); +#endif + + // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i, + // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* {{.*}}, + bar(i.b); + foo(&i.b); // Does not change i.b. + bar(i.b); + // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8* +} + #endif // #if defined(OBJ) // Example 1 with references and pointers: