Index: lib/CodeGen/CGDecl.cpp =================================================================== --- lib/CodeGen/CGDecl.cpp +++ lib/CodeGen/CGDecl.cpp @@ -530,9 +530,9 @@ /// A cleanup to call @llvm.invariant.end. class CallInvariantEnd final : public EHScopeStack::Cleanup { - CodeGenModule::InvariantArgs Args; + CodeGenFunction::InvariantArgs Args; public: - CallInvariantEnd(CodeGenModule::InvariantArgs &args) : Args(args) {} + CallInvariantEnd(CodeGenFunction::InvariantArgs &args) : Args(args) {} void Emit(CodeGenFunction &CGF, Flags flags) override { CGF.EmitInvariantEnd(Args); @@ -927,38 +927,47 @@ C->setDoesNotThrow(); } -/// Collect offsets in bits. -static bool getInvariantOffsets(const CodeGenFunction &CGF, QualType Ty, - llvm::SmallVectorImpl &Args) { - ASTContext &Ctx = CGF.getContext(); - bool FoundWriteOnce = false; - if (const CXXRecordDecl *Record = - Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) { - for (const auto *Field : Record->fields()) { - assert(dyn_cast(Field) && "Field decls only."); - if (Field->getType().isWriteOnce(Ctx)) { - FoundWriteOnce = true; - CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty); - uint64_t Width = WidthChars.getQuantity(); - Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Width)); // Size - - uint64_t Offset = Ctx.getFieldOffset(Field); - Args.push_back(llvm::ConstantInt::get(CGF.Int64Ty, Offset)); // Offset - } else { - FoundWriteOnce |= getInvariantOffsets(CGF, Field->getType(), Args); - } +CodeGenFunction::OffsetsType& +CodeGenFunction::ComputeInvariantOffsets(const CXXRecordDecl *Record) { + ASTContext &Ctx = getContext(); + auto &OffsetsInfo = InvariantOffsets.FindAndConstruct(Record).second; + OffsetsType &Args = OffsetsInfo.Offsets; + + // If this has already been computed, then return the stored value. + if (OffsetsInfo.Computed) return Args; + + // Otherwise, mark that this is computed. + OffsetsInfo.Computed = true; + assert(Args.empty() && "There should be no offset specified yet."); + + // Trace through fields collecting offsets of writeonce candidates. + for (const auto *Field : Record->fields()) { + assert(dyn_cast(Field) && "Field decls only."); + QualType FieldType = Field->getType(); + if (FieldType.isWriteOnce(Ctx)) { + CharUnits WidthChars = Ctx.getTypeSizeInChars(FieldType); + uint64_t Width = WidthChars.getQuantity(); + Args.push_back(llvm::ConstantInt::get(Int64Ty, Width)); // Size + + uint64_t Offset = Ctx.getFieldOffset(Field); + Args.push_back(llvm::ConstantInt::get(Int64Ty, Offset)); // Offset + } else if (const CXXRecordDecl *RecField = + Ctx.getBaseElementType(FieldType)->getAsCXXRecordDecl()) { + auto &FieldArgs = ComputeInvariantOffsets(RecField); + Args.insert(Args.end(), FieldArgs.begin(), FieldArgs.end()); } } - return FoundWriteOnce; + + return Args; } /// Emit code to cause the variable at the given address to be considered as /// constant from this point onwards. -CodeGenModule::InvariantArgs CodeGenFunction::EmitInvariantStart( +CodeGenFunction::InvariantArgs CodeGenFunction::EmitInvariantStart( const VarDecl &D, llvm::Value *Addr, bool IsGlobalConstant) { // Don't emit the intrinsic if we're not optimizing. if (!CGM.getCodeGenOpts().OptimizationLevel) - return CodeGenModule::InvariantArgs{}; + return InvariantArgs{}; assert(Addr && "Cannot emit on non-null address."); @@ -967,18 +976,28 @@ QualType Ty = D.getType(); CharUnits WidthChars = Ctx.getTypeSizeInChars(Ty); uint64_t Width = WidthChars.getQuantity(); - CodeGenModule::InvariantArgs Args; + InvariantArgs Args; + OffsetsType Offsets; llvm::Constant *CAddr = dyn_cast(Addr); if (CAddr && IsGlobalConstant) { Args.Size = llvm::ConstantInt::getSigned(Int64Ty, Width); Args.Addr = llvm::ConstantExpr::getBitCast(CAddr, Int8PtrTy); - } else if (Ty.isWriteOnce(Ctx) || - getInvariantOffsets(*this, D.getType(), Args.Offsets)) { + } else if (Ty.isWriteOnce(Ctx)) { Args.Size = llvm::ConstantInt::get(Int64Ty, Width); Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + } else if (const CXXRecordDecl *Record = + Ctx.getBaseElementType(Ty)->getAsCXXRecordDecl()) { + Offsets = ComputeInvariantOffsets(Record); + // If there are invariant offsets in this non-writeonce record, + // then emit the intrinsic call with the offsets. Otherwise, + // do not emit anything. + if (!Offsets.empty()) { + Args.Size = llvm::ConstantInt::get(Int64Ty, Width); + Args.Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + } } - if (!Args.Addr) return CodeGenModule::InvariantArgs{}; + if (!Args.Addr) return InvariantArgs{}; // Generate llvm.invariant.start intrinsic call. llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start; @@ -986,7 +1005,7 @@ llvm::SmallVector CallArgs; CallArgs.push_back(Args.Size); CallArgs.push_back(Args.Addr); - CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end()); + CallArgs.insert(CallArgs.end(), Offsets.begin(), Offsets.end()); llvm::CallInst *C = Builder.CreateCall(InvariantStart, CallArgs); Args.StartInst = C; C->setDoesNotThrow(); @@ -994,7 +1013,7 @@ return Args; } -void CodeGenFunction::EmitInvariantEnd(CodeGenModule::InvariantArgs &Args) { +void CodeGenFunction::EmitInvariantEnd(CodeGenFunction::InvariantArgs &Args) { assert(Args.Addr && "Emit on non-null address."); // Generate the llvm.invariant.end intrinsic call. @@ -1004,7 +1023,6 @@ CallArgs.push_back(Args.StartInst); CallArgs.push_back(Args.Size); CallArgs.push_back(Args.Addr); - // CallArgs.insert(CallArgs.end(), Args.Offsets.begin(), Args.Offsets.end()); llvm::CallInst *C = Builder.CreateCall(InvariantEnd, CallArgs); C->setDoesNotThrow(); } Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -1904,10 +1904,41 @@ llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr); void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr); - CodeGenModule::InvariantArgs EmitInvariantStart(const VarDecl &D, - llvm::Value *Addr, - bool IsGlobalConstant = true); - void EmitInvariantEnd(CodeGenModule::InvariantArgs &Args); + + /// \brief Specifes arguments to invariant_start/end intrinsic calls. + struct InvariantArgs { + llvm::CallInst *StartInst; // Contains invariant offsets. + llvm::Value *Size; // TODO: Is this necessary? + llvm::Value *Addr; + + InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {} + InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A) + : StartInst(C), Size(S), Addr(A) {} + }; + + /// \brief Specifies type of invariant offsets in a given record. + typedef llvm::SmallVector OffsetsType; + +private: + + /// \brief Specifies offsets and whether they have already been computed. + /// Note: empty offsets may or may not have been computed. + struct OffsetsInfoType { + bool Computed; + OffsetsType Offsets; + OffsetsInfoType() : Computed(false) { } + }; + + /// \brief A collection of invariant offsets per given record. + llvm::DenseMap InvariantOffsets; + + /// \brief Compute the invariant offsets of a given Record. + OffsetsType& ComputeInvariantOffsets(const CXXRecordDecl *Record); + +public: + InvariantArgs EmitInvariantStart(const VarDecl &D, llvm::Value *Addr, + bool IsGlobalConstant = true); + void EmitInvariantEnd(InvariantArgs &Args); llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E); void EmitCXXDeleteExpr(const CXXDeleteExpr *E); @@ -3226,7 +3257,7 @@ CodeGenFunction &CGF; llvm::GlobalVariable *GV; llvm::AllocaInst *AI; - CodeGenModule::InvariantArgs Args; + CodeGenFunction::InvariantArgs Args; public: MarkWriteOnceWrittenRAII(CodeGenFunction &CGF, const VarDecl *D, Index: lib/CodeGen/CodeGenModule.h =================================================================== --- lib/CodeGen/CodeGenModule.h +++ lib/CodeGen/CodeGenModule.h @@ -259,17 +259,6 @@ typedef std::vector CtorList; - struct InvariantArgs { - llvm::CallInst *StartInst; // TODO: Is this necessary? - llvm::Value *Size; // TODO: Is this necessary? - llvm::Value *Addr; - llvm::SmallVector Offsets; - - InvariantArgs() : StartInst(nullptr), Size(nullptr), Addr(nullptr) {} - InvariantArgs(llvm::CallInst *C, llvm::Value *S, llvm::Value *A) - : StartInst(C), Size(S), Addr(A) {} - }; - private: ASTContext &Context; const LangOptions &LangOpts;