Index: include/clang/AST/Type.h =================================================================== --- include/clang/AST/Type.h +++ include/clang/AST/Type.h @@ -709,6 +709,8 @@ return QualType::isConstant(*this, Ctx); } + bool isWriteOnce(ASTContext &Ctx); + /// \brief Determine whether this is a Plain Old Data (POD) type (C++ 3.9p10). bool isPODType(ASTContext &Context) const; Index: lib/AST/Type.cpp =================================================================== --- lib/AST/Type.cpp +++ lib/AST/Type.cpp @@ -1936,6 +1936,19 @@ } } +bool QualType::isWriteOnce(ASTContext &Context) { + + if (!Context.getLangOpts().CPlusPlus || + !isConstant(Context) || getTypePtr()->isReferenceType()) + return false; + + if (const CXXRecordDecl *Record + = Context.getBaseElementType(*this)->getAsCXXRecordDecl()) + return !Record->hasMutableFields(); + + return true; +} + bool QualType::isPODType(ASTContext &Context) const { // C++11 has a more relaxed definition of POD. if (Context.getLangOpts().CPlusPlus11) Index: lib/CodeGen/CGDecl.cpp =================================================================== --- lib/CodeGen/CGDecl.cpp +++ lib/CodeGen/CGDecl.cpp @@ -366,6 +366,8 @@ if (D.getInit()) var = AddInitializerToStaticVarDecl(D, var); + MarkWriteOnceWritten MWO(*this, D, var); + var->setAlignment(alignment.getQuantity()); if (D.hasAttr()) @@ -525,6 +527,20 @@ CGF.EmitLifetimeEnd(Size, Addr); } }; + + /// A cleanup to call @llvm.invariant.end. + class CallInvariantEnd final : public EHScopeStack::Cleanup { + llvm::CallInst *StartInst; + llvm::Value *Addr; + llvm::Value *Size; // TODO: Is this even necessary? + public: + CallInvariantEnd(llvm::CallInst *C, llvm::Value *addr, llvm::Value *size) + : StartInst(C), Addr(addr), Size(size) {} + + void Emit(CodeGenFunction &CGF, Flags flags) override { + CGF.EmitInvariantEnd(StartInst, Size, Addr); + } + }; } /// EmitAutoVarWithLifetime - Does the setup required for an automatic @@ -840,12 +856,42 @@ canEmitInitWithFewStoresAfterMemset(Init, StoreBudget); } +/// Emit the code necessary to initialize the given global variable. +/// TODO: Use Address for Addr (to match lifetime intrisics). +MarkWriteOnceWritten::MarkWriteOnceWritten(CodeGenFunction &CGF, + const VarDecl& D, llvm::Value *Addr) +: GV(dyn_cast_or_null(Addr)), + AI(dyn_cast_or_null(Addr)), + Args{nullptr, 0, nullptr}, + CGF(CGF) { + + // Only GlobalVariable's and AllocaInst's can be writeonce. + // Exit if the given address is none of these. + if (!GV && !AI) return; + + // If the address is writeonce, then emit @llvm.invariant.start() intrinsic. + // Then, for non-global variables, push the emission of the + // @llvm.invariant.end() intrinsic onto the cleanup stack. + if ((AI || !GV->isConstant()) && D.getType().isWriteOnce(CGF.getContext())) + Args = CGF.EmitInvariantStart(D, Addr); +} + +MarkWriteOnceWritten::~MarkWriteOnceWritten() { + if (AI && Args.StartInst) { + assert(!GV && "Can't have it both ways!"); + CGF.EHStack.pushCleanup(NormalCleanup, Args.StartInst, + Args.Addr, Args.Size); + } +} + /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a /// variable declaration with auto, register, or no storage class specifier. /// These turn into simple stack objects, or GlobalValues depending on target. void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) { AutoVarEmission emission = EmitAutoVarAlloca(D); EmitAutoVarInit(emission); + MarkWriteOnceWritten MWO(*this, D, emission.Addr.isValid() ? + emission.Addr.getPointer() : nullptr); EmitAutoVarCleanups(emission); } @@ -878,6 +924,49 @@ C->setDoesNotThrow(); } +/// Emit code to cause the variable at the given address to be considered as +/// constant from this point onwards. +CodeGenModule::InvariantArgs +CodeGenFunction::EmitInvariantStart(const VarDecl &D, llvm::Value *Addr) { + // Don't emit the intrinsic if we're not optimizing. + if (!CGM.getCodeGenOpts().OptimizationLevel) + return CodeGenModule::InvariantArgs{nullptr, 0, nullptr}; + + // Grab the llvm.invariant.start intrinsic. + llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start; + llvm::Constant *InvariantStart = CGM.getIntrinsic(InvStartID); + + // Emit a call with the size in bytes of the object. + CharUnits WidthChars = getContext().getTypeSizeInChars(D.getType()); + uint64_t Width = WidthChars.getQuantity(); + + llvm::Value *Size; + if (llvm::Constant* CAddr = dyn_cast(Addr)) { + Size = llvm::ConstantInt::getSigned(Int64Ty, Width); + Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy); + } + else { + //Width = CGM.getDataLayout().getTypeAllocSize(LTy); + Size = llvm::ConstantInt::get(Int64Ty, Width); + Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + } + llvm::CallInst *C = Builder.CreateCall(InvariantStart, {Size, Addr}); + C->setDoesNotThrow(); + + return CodeGenModule::InvariantArgs{C, Size, Addr}; +} + +void CodeGenFunction::EmitInvariantEnd(llvm::CallInst *Start, + llvm::Value *Size, llvm::Value *Addr) { + // Grab the llvm.invariant.end intrinsic. + llvm::Intrinsic::ID InvEndID = llvm::Intrinsic::invariant_end; + llvm::Constant *InvariantEnd = CGM.getIntrinsic(InvEndID); + + // Emit a call with the size in bytes of the object. + llvm::CallInst *C = Builder.CreateCall(InvariantEnd, {Start, Size, Addr}); + C->setDoesNotThrow(); +} + /// EmitAutoVarAlloca - Emit the alloca and debug information for a /// local variable. Does not emit initialization or destruction. CodeGenFunction::AutoVarEmission Index: lib/CodeGen/CGDeclCXX.cpp =================================================================== --- lib/CodeGen/CGDeclCXX.cpp +++ lib/CodeGen/CGDeclCXX.cpp @@ -109,26 +109,6 @@ CGM.getCXXABI().registerGlobalDtor(CGF, D, function, argument); } -/// Emit code to cause the variable at the given address to be considered as -/// constant from this point onwards. -static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D, - llvm::Constant *Addr) { - // Don't emit the intrinsic if we're not optimizing. - if (!CGF.CGM.getCodeGenOpts().OptimizationLevel) - return; - - // Grab the llvm.invariant.start intrinsic. - llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start; - llvm::Constant *InvariantStart = CGF.CGM.getIntrinsic(InvStartID); - - // Emit a call with the size in bytes of the object. - CharUnits WidthChars = CGF.getContext().getTypeSizeInChars(D.getType()); - uint64_t Width = WidthChars.getQuantity(); - llvm::Value *Args[2] = { llvm::ConstantInt::getSigned(CGF.Int64Ty, Width), - llvm::ConstantExpr::getBitCast(Addr, CGF.Int8PtrTy)}; - CGF.Builder.CreateCall(InvariantStart, Args); -} - void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::Constant *DeclPtr, bool PerformInit) { @@ -168,9 +148,13 @@ PerformInit, this); if (PerformInit) EmitDeclInit(*this, D, DeclAddr); - if (CGM.isTypeConstant(D.getType(), true)) - EmitDeclInvariant(*this, D, DeclPtr); - else + if (CGM.isTypeConstant(D.getType(), true)) { + // Generate the @llvm.invariant.start intrinsic call if DeclPtr is + // not 'writeonce'. We'll do the same for 'writeonce' addresses later. + llvm::GlobalVariable* GV = dyn_cast(DeclPtr); + if (GV && (GV->isConstant() || !D.getType().isWriteOnce(getContext()))) + (void) EmitInvariantStart(D, DeclPtr); + } else EmitDeclDestroy(*this, D, DeclAddr); return; } @@ -489,6 +473,7 @@ } else { EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit); } + MarkWriteOnceWritten MWO(*this, *D, Addr); FinishFunction(); } Index: lib/CodeGen/CGExprAgg.cpp =================================================================== --- lib/CodeGen/CGExprAgg.cpp +++ lib/CodeGen/CGExprAgg.cpp @@ -1402,7 +1402,7 @@ // Optimize the slot if possible. CheckAggExprForMemSetUse(Slot, E, *this); - + AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast(E)); } Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -1904,6 +1904,10 @@ llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr); void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr); + CodeGenModule::InvariantArgs EmitInvariantStart(const VarDecl &D, + llvm::Value *Addr); + void EmitInvariantEnd(llvm::CallInst *C, + llvm::Value *Size, llvm::Value *Addr); llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E); void EmitCXXDeleteExpr(const CXXDeleteExpr *E); @@ -3215,6 +3219,17 @@ llvm::Value *GetValueForARMHint(unsigned BuiltinID); }; +class MarkWriteOnceWritten { + llvm::GlobalVariable *GV; + llvm::AllocaInst *AI; + CodeGenModule::InvariantArgs Args; + CodeGenFunction &CGF; +public: + MarkWriteOnceWritten(CodeGenFunction &CGF, + const VarDecl& D, llvm::Value *Addr); + ~MarkWriteOnceWritten(); +}; + /// Helper class with most of the code for saving a value for a /// conditional expression cleanup. struct DominatingLLVMValue { Index: lib/CodeGen/CodeGenModule.h =================================================================== --- lib/CodeGen/CodeGenModule.h +++ lib/CodeGen/CodeGenModule.h @@ -259,6 +259,14 @@ typedef std::vector CtorList; + struct InvariantArgs { + llvm::CallInst *StartInst; + llvm::Value *Size; + llvm::Value *Addr; + InvariantArgs(llvm::CallInst *C, llvm::Value *size, llvm::Value *addr) + : StartInst(C), Size(size), Addr(addr) {} + }; + private: ASTContext &Context; const LangOptions &LangOpts; Index: test/CodeGenCXX/const-invariant.cpp =================================================================== --- /dev/null +++ test/CodeGenCXX/const-invariant.cpp @@ -0,0 +1,180 @@ +// *** const objects. +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O0 -o - -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-O0 +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-L-CO --check-prefix=CHECK-L-CO-OBJ --check-prefix=CHECK-NG-CO +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-GP-CO --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-OBJ --check-prefix=CHECK-NG-CO +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DCONST -DOBJ | FileCheck %s --check-prefix=CHECK-G-CO --check-prefix=CHECK-G-CO-OBJ --check-prefix=CHECK-NL-CO + +// *** const variables of builtin type. +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O0 -o - -DCONST -DINT | FileCheck %s --check-prefix=CHECK-O0 +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DCONST -DINT | FileCheck %s --check-prefix=CHECK-L-CO --check-prefix=CHECK-L-CO-INT --check-prefix=CHECK-NG-CO +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DCONST -DINT | FileCheck %s --check-prefix=CHECK-GP-CO --check-prefix=CHECK-NL-CO --check-prefix=CHECK-NL-CO-INT --check-prefix=CHECK-NG-CO +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DCONST -DINT | FileCheck %s --check-prefix=CHECK-G-CO --check-prefix=CHECK-G-CO-INT --check-prefix=CHECK-NL-CO + +// *** non-const objects. +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DOBJ | FileCheck %s --check-prefix=CHECK-NC +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DPART_GLOBAL -DOBJ | FileCheck %s --check-prefix=CHECK-NC +// RUN: %clang -x c++ -emit-llvm -target x86_64-linux-gnu -S %s -O2 -mllvm -disable-llvm-optzns -o - -DGLOBAL -DOBJ | FileCheck %s --check-prefix=CHECK-NC + +// Check that llvm.invariant.start/end are properly generated on C++ const variables. + +// Do not produce markers at -O0. +// CHECK-O0-NOT: llvm.invariant.start +// CHECK-O0-NOT: llvm.invariant.end + +// Do not produce markers from non-const objects +// CHECK-NC-NOT: llvm.invariant.start +// CHECK-NC-NOT: llvm.invariant.end + +// const or not? +#ifdef CONST + #define Const const +#else + #define Const +#endif + +// object or builtin? +#if defined(OBJ) + struct A { + int a; + int b; + A(int a) : a(a) { } + }; + + struct A_d { + int a; + int b; + A_d(int a) : a(a) { } + ~A_d(); + }; + + #define Type A // C++ class +#elif defined(INT) + #define Type int // Builtin +#endif + +// global, partial global or local? +#if defined(GLOBAL) + #undef LOCAL + #undef LOCAL_PARTIAL +#elif defined(PART_GLOBAL) + #undef LOCAL + #define LOCAL_PARTIAL +#else + #define LOCAL + #define LOCAL_PARTIAL +#endif + +// --------------- START --------------- + +const int one(); + +#ifndef LOCAL +Const Type i(one()); +#endif + +#ifndef LOCAL_PARTIAL +Const Type j = i; +Type k = i; +#endif + +#if defined(OBJ) && !defined(LOCAL) +Const A_d i_d(one()); +#endif + + +// CHECK-NL-CO: @_ZL1i = internal global +// CHECK-G-CO: @k = global +// CHECK-G-CO: @_ZL1j = internal global +// CHECK-NL-CO-OBJ: @_ZL3i_d = internal global + +// CHECK-NL-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* @_ZL1i, {{.*}}) +// CHECK-NL-CO-INT: store {{.*}}, {{.*}}* @_ZL1i +// CHECK-NL-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL1i to i8*)) +// CHECK-NL-CO-OBJ: call void @_ZN3A_dC{{[0-9]+}}Ei({{.*}}* @_ZL3i_d, {{.*}}) +// CHECK-NL-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL3i_d to i8*)) + + +void foo(const Type* const); +void bar(Type); +#if defined(OBJ) +void foo_d(const A_d* const); +void bar_d(A_d); +#endif + + +void ex1() { +#ifdef LOCAL + Const Type i(one()); +#endif + // CHECK-L-CO: %i = alloca {{.*}} + + // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i, + // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i + // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* + bar(i); + foo(&i); // Does not change i. + bar(i); + // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8* +} + +void ex2() { +#ifdef LOCAL + Const Type i(one()); +#endif +#ifdef LOCAL_PARTIAL + Const Type j = i; +#endif + // CHECK-L-CO: %i = alloca {{.*}} + // CHECK-NG-CO: %j = alloca {{.*}} + + // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i, + // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i + // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* + // CHECK-NG-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* + bar(i); + foo(&i); // Does not change i, nor j. + bar(j); + // CHECK-NG-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8* + // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8* +} + +void ex3() { +#ifdef LOCAL + Const Type i(one()); +#endif +#ifdef LOCAL_PARTIAL + Type k = i; +#endif + // CHECK-L-CO: %i = alloca {{.*}} + // CHECK-NG-CO: %k = alloca {{.*}} + + // CHECK-L-CO-OBJ: call void @_ZN1AC{{[0-9]+}}Ei({{.*}}* %i, + // CHECK-L-CO-INT: store {{.*}}, {{.*}}* %i + // CHECK-L-CO: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* + bar(i); + foo(&k); // Does not change i; May change k. + bar(k); + // CHECK-L-CO: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8* +} + +#if defined(OBJ) +void ex1_d() { +#ifdef LOCAL + Const A_d i_d(one()); +#endif + // CHECK-L-CO-OBJ: %i_d = alloca {{.*}} + + // CHECK-L-CO-OBJ: call void @_ZN3A_dC{{[0-9]+}}Ei({{.*}}* %i_d, + // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* + bar_d(i_d); + foo_d(&i_d); // Does not change i_d. + bar_d(i_d); + // CHECK-L-CO-OBJ: call {{.*}}@llvm.invariant.end({{.*}}, i64 {{[0-9]+}}, i8* + // CHECK-L-CO-OBJ: call void @_ZN3A_dD1Ev( +} +#endif + +// CHECK-G-CO-OBJ: call {{.*}}@llvm.invariant.start(i64 {{[0-9]+}}, i8* bitcast ({{.*}} @_ZL1j to i8*)) +// CHECK-G-CO-INT: store {{.*}}, {{.*}}* @_ZL1j + + Index: test/CodeGenCXX/init-invariant.cpp =================================================================== --- test/CodeGenCXX/init-invariant.cpp +++ test/CodeGenCXX/init-invariant.cpp @@ -48,7 +48,7 @@ // CHECK-NOT: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @b to i8*)) // CHECK: call void @_ZN1CC1Ev({{.*}}* nonnull @c) -// CHECK-NOT: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @c to i8*)) +// CHECK: call {{.*}}@llvm.invariant.start(i64 4, i8* bitcast ({{.*}} @c to i8*)) // CHECK: call i32 @_Z1fv( // CHECK: store {{.*}}, i32* @d