diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -112,19 +112,6 @@ return CGBuilderBaseTy::CreateStore(Val, Addr, IsVolatile); } - /// Emit a load from an i1 flag variable. - llvm::LoadInst *CreateFlagLoad(llvm::Value *Addr, - const llvm::Twine &Name = "") { - assert(Addr->getType()->getPointerElementType() == getInt1Ty()); - return CreateAlignedLoad(getInt1Ty(), Addr, CharUnits::One(), Name); - } - - /// Emit a store to an i1 flag variable. - llvm::StoreInst *CreateFlagStore(bool Value, llvm::Value *Addr) { - assert(Addr->getType()->getPointerElementType() == getInt1Ty()); - return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One()); - } - // Temporarily use old signature; clang will be updated to an Address overload // in a subsequent patch. llvm::AtomicCmpXchgInst * diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -4678,19 +4678,20 @@ if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { const llvm::DataLayout &DL = CGM.getDataLayout(); llvm::Instruction *IP = CallArgs.getStackBase(); - llvm::AllocaInst *AI; + auto Align = CallInfo.getArgStructAlignment(); if (IP) { IP = IP->getNextNode(); - AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), - "argmem", IP); + unsigned AS = DL.getAllocaAddrSpace(); + llvm::AllocaInst *AI = new llvm::AllocaInst(ArgStruct, AS, "argmem", IP); + AI->setAlignment(Align.getAsAlign()); + ArgMemory = Address(AI, Align); } else { - AI = CreateTempAlloca(ArgStruct, "argmem"); + LangAS AS = getASTAllocaAddressSpace(); + ArgMemory = CreateTempAllocaInAS(ArgStruct, Align, AS, "argmem"); } - auto Align = CallInfo.getArgStructAlignment(); - AI->setAlignment(Align.getAsAlign()); + auto *AI = cast(ArgMemory.getPointer()); AI->setUsedWithInAlloca(true); assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); - ArgMemory = Address(AI, Align); } ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp --- a/clang/lib/CodeGen/CGCleanup.cpp +++ b/clang/lib/CodeGen/CGCleanup.cpp @@ -41,8 +41,9 @@ return saved_type(V, ScalarLiteral); // Everything else needs an alloca. - Address addr = - CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue"); + Address addr = CGF.CreateTempAllocaInAS( + V->getType(), CGF.PreferredAlignmentForIRType(V->getType()), + CGF.getASTAllocaAddressSpace(), "saved-rvalue"); CGF.Builder.CreateStore(V, addr); return saved_type(addr.getPointer(), ScalarAddress); } @@ -51,7 +52,9 @@ CodeGenFunction::ComplexPairTy V = rv.getComplexVal(); llvm::Type *ComplexTy = llvm::StructType::get(V.first->getType(), V.second->getType()); - Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex"); + Address addr = CGF.CreateTempAllocaInAS( + ComplexTy, CGF.PreferredAlignmentForIRType(ComplexTy), + CGF.getASTAllocaAddressSpace(), "saved-complex"); CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0)); CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1)); return saved_type(addr.getPointer(), ComplexAddress); @@ -279,8 +282,9 @@ Address CodeGenFunction::createCleanupActiveFlag() { // Create a variable to decide whether the cleanup needs to be run. - Address active = CreateTempAllocaWithoutCast( - Builder.getInt1Ty(), CharUnits::One(), "cleanup.cond"); + LangAS AS = getASTAllocaAddressSpace(); + Address active = CreateTempAllocaInAS(Builder.getInt1Ty(), CharUnits::One(), + AS, "cleanup.cond"); // Initialize it to false at a site that's guaranteed to be run // before each evaluation. @@ -455,8 +459,9 @@ if (AI && AI->isStaticAlloca()) continue; - Address Tmp = - CreateDefaultAlignTempAlloca(Inst->getType(), "tmp.exprcleanup"); + Address Tmp = CreateTempAllocaInAS( + Inst->getType(), PreferredAlignmentForIRType(Inst->getType()), + getASTAllocaAddressSpace(), "tmp.exprcleanup"); // Find an insertion point after Inst and spill it to the temporary. llvm::BasicBlock::iterator InsertBefore; @@ -1299,9 +1304,13 @@ } Address CodeGenFunction::getNormalCleanupDestSlot() { - if (!NormalCleanupDest.isValid()) + if (!NormalCleanupDest.isValid()) { + llvm::Type *Ty = Builder.getInt32Ty(); + CharUnits Align = PreferredAlignmentForIRType(Ty); + LangAS AS = getASTAllocaAddressSpace(); NormalCleanupDest = - CreateDefaultAlignTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot"); + CreateTempAllocaInAS(Ty, Align, AS, "cleanup.dest.slot"); + } return NormalCleanupDest; } diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -50,7 +50,7 @@ // exception. If it did, 'true' is stored in this variable, and the coroutine // body must be skipped. If the promise type does not define an exception // handler, this is null. - llvm::Value *ResumeEHVar = nullptr; + Address ResumeEHVar = Address::invalid(); // Stores the jump destination just before the coroutine memory is freed. // This is the destination that every suspend point jumps to for the cleanup @@ -231,9 +231,11 @@ CXXTryStmt *TryStmt = nullptr; if (Coro.ExceptionHandler && Kind == AwaitKind::Init && memberCallExpressionCanThrow(S.getResumeExpr())) { - Coro.ResumeEHVar = - CGF.CreateTempAlloca(Builder.getInt1Ty(), Prefix + Twine("resume.eh")); - Builder.CreateFlagStore(true, Coro.ResumeEHVar); + llvm::Type *Ty = Builder.getInt1Ty(); + CharUnits Align = CGF.PreferredAlignmentForIRType(Ty); + LangAS AS = CGF.getASTAllocaAddressSpace(); + Coro.ResumeEHVar = CGF.CreateTempAllocaInAS(Ty, Align, AS, "resume.eh"); + Builder.CreateStore(Builder.getTrue(), Coro.ResumeEHVar); auto Loc = S.getResumeExpr()->getExprLoc(); auto *Catch = new (CGF.getContext()) @@ -251,7 +253,7 @@ Res.RV = CGF.EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult); if (TryStmt) { - Builder.CreateFlagStore(false, Coro.ResumeEHVar); + Builder.CreateStore(Builder.getFalse(), Coro.ResumeEHVar); CGF.ExitCXXTryStmt(*TryStmt); } @@ -656,11 +658,11 @@ // If we didn't generate the IR (perhaps because 'await_resume' was marked // as 'noexcept'), then we skip this check. BasicBlock *ContBB = nullptr; - if (CurCoro.Data->ResumeEHVar) { + if (CurCoro.Data->ResumeEHVar.isValid()) { BasicBlock *BodyBB = createBasicBlock("coro.resumed.body"); ContBB = createBasicBlock("coro.resumed.cont"); - Value *SkipBody = Builder.CreateFlagLoad(CurCoro.Data->ResumeEHVar, - "coro.resumed.eh"); + Value *SkipBody = + Builder.CreateLoad(CurCoro.Data->ResumeEHVar, "coro.resumed.eh"); Builder.CreateCondBr(SkipBody, ContBB, BodyBB); EmitBlock(BodyBB); } diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -496,24 +496,23 @@ template struct DestroyNRVOVariable : EHScopeStack::Cleanup { - DestroyNRVOVariable(Address addr, QualType type, llvm::Value *NRVOFlag) + DestroyNRVOVariable(Address addr, QualType type, Address NRVOFlag) : NRVOFlag(NRVOFlag), Loc(addr), Ty(type) {} - llvm::Value *NRVOFlag; + Address NRVOFlag; Address Loc; QualType Ty; void Emit(CodeGenFunction &CGF, Flags flags) override { // Along the exceptions path we always execute the dtor. - bool NRVO = flags.isForNormalCleanup() && NRVOFlag; + bool NRVO = flags.isForNormalCleanup() && NRVOFlag.isValid(); llvm::BasicBlock *SkipDtorBB = nullptr; if (NRVO) { // If we exited via NRVO, we skip the destructor call. llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused"); SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor"); - llvm::Value *DidNRVO = - CGF.Builder.CreateFlagLoad(NRVOFlag, "nrvo.val"); + llvm::Value *DidNRVO = CGF.Builder.CreateLoad(NRVOFlag, "nrvo.val"); CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB); CGF.EmitBlock(RunDtorBB); } @@ -529,7 +528,7 @@ struct DestroyNRVOVariableCXX final : DestroyNRVOVariable { DestroyNRVOVariableCXX(Address addr, QualType type, - const CXXDestructorDecl *Dtor, llvm::Value *NRVOFlag) + const CXXDestructorDecl *Dtor, Address NRVOFlag) : DestroyNRVOVariable(addr, type, NRVOFlag), Dtor(Dtor) {} @@ -544,7 +543,7 @@ struct DestroyNRVOVariableC final : DestroyNRVOVariable { - DestroyNRVOVariableC(Address addr, llvm::Value *NRVOFlag, QualType Ty) + DestroyNRVOVariableC(Address addr, Address NRVOFlag, QualType Ty) : DestroyNRVOVariable(addr, Ty, NRVOFlag) {} void emitDestructorCall(CodeGenFunction &CGF) { @@ -1369,8 +1368,10 @@ StringRef NameRef = Name.toStringRef(Buffer); auto &Ident = getContext().Idents.getOwn(NameRef); VLAExprNames.push_back(&Ident); - auto SizeExprAddr = - CreateDefaultAlignTempAlloca(VlaSize.NumElts->getType(), NameRef); + llvm::Type *Ty = VlaSize.NumElts->getType(); + CharUnits Align = PreferredAlignmentForIRType(Ty); + LangAS AS = getASTAllocaAddressSpace(); + auto SizeExprAddr = CreateTempAllocaInAS(Ty, Align, AS, NameRef); Builder.CreateStore(VlaSize.NumElts, SizeExprAddr); Dimensions.emplace_back(SizeExprAddr.getPointer(), Type1D.getUnqualifiedType()); @@ -1508,8 +1509,8 @@ Builder.CreateStore(Zero, NRVOFlag); // Record the NRVO flag for this variable. - NRVOFlags[&D] = NRVOFlag.getPointer(); - emission.NRVOFlag = NRVOFlag.getPointer(); + NRVOFlags.insert(std::make_pair(&D, NRVOFlag)); + emission.NRVOFlag = NRVOFlag; } } } else { @@ -1983,7 +1984,7 @@ case QualType::DK_cxx_destructor: // If there's an NRVO flag on the emission, we need a different // cleanup. - if (emission.NRVOFlag) { + if (emission.NRVOFlag.isValid()) { assert(!type->isArrayType()); CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor(); EHStack.pushCleanup(cleanupKind, addr, type, dtor, @@ -2009,7 +2010,7 @@ case QualType::DK_nontrivial_c_struct: destroyer = CodeGenFunction::destroyNonTrivialCStruct; - if (emission.NRVOFlag) { + if (emission.NRVOFlag.isValid()) { assert(!type->isArrayType()); EHStack.pushCleanup(cleanupKind, addr, emission.NRVOFlag, type); diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp --- a/clang/lib/CodeGen/CGException.cpp +++ b/clang/lib/CodeGen/CGException.cpp @@ -419,15 +419,22 @@ } Address CodeGenFunction::getExceptionSlot() { - if (!ExceptionSlot) - ExceptionSlot = CreateTempAlloca(Int8PtrTy, "exn.slot"); - return Address(ExceptionSlot, getPointerAlign()); + if (!ExceptionSlot.isValid()) { + CharUnits Align = getPointerAlign(); + LangAS AS = getASTAllocaAddressSpace(); + ExceptionSlot = CreateTempAllocaInAS(Int8PtrTy, Align, AS, "exn.slot"); + } + return ExceptionSlot; } Address CodeGenFunction::getEHSelectorSlot() { - if (!EHSelectorSlot) - EHSelectorSlot = CreateTempAlloca(Int32Ty, "ehselector.slot"); - return Address(EHSelectorSlot, CharUnits::fromQuantity(4)); + if (!EHSelectorSlot.isValid()) { + CharUnits Align = CharUnits::fromQuantity(4); + LangAS AS = getASTAllocaAddressSpace(); + EHSelectorSlot = + CreateTempAllocaInAS(Int32Ty, Align, AS, "ehselector.slot"); + } + return EHSelectorSlot; } llvm::Value *CodeGenFunction::getExceptionFromSlot() { @@ -1313,10 +1320,9 @@ namespace { struct CallEndCatchForFinally final : EHScopeStack::Cleanup { - llvm::Value *ForEHVar; + Address ForEHVar; llvm::FunctionCallee EndCatchFn; - CallEndCatchForFinally(llvm::Value *ForEHVar, - llvm::FunctionCallee EndCatchFn) + CallEndCatchForFinally(Address ForEHVar, llvm::FunctionCallee EndCatchFn) : ForEHVar(ForEHVar), EndCatchFn(EndCatchFn) {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -1325,7 +1331,7 @@ CGF.createBasicBlock("finally.cleanup.cont"); llvm::Value *ShouldEndCatch = - CGF.Builder.CreateFlagLoad(ForEHVar, "finally.endcatch"); + CGF.Builder.CreateLoad(ForEHVar, "finally.endcatch"); CGF.Builder.CreateCondBr(ShouldEndCatch, EndCatchBB, CleanupContBB); CGF.EmitBlock(EndCatchBB); CGF.EmitRuntimeCallOrInvoke(EndCatchFn); // catch-all, so might throw @@ -1335,14 +1341,14 @@ struct PerformFinally final : EHScopeStack::Cleanup { const Stmt *Body; - llvm::Value *ForEHVar; + Address ForEHVar; llvm::FunctionCallee EndCatchFn; llvm::FunctionCallee RethrowFn; - llvm::Value *SavedExnVar; + Address SavedExnVar; - PerformFinally(const Stmt *Body, llvm::Value *ForEHVar, + PerformFinally(const Stmt *Body, Address ForEHVar, llvm::FunctionCallee EndCatchFn, - llvm::FunctionCallee RethrowFn, llvm::Value *SavedExnVar) + llvm::FunctionCallee RethrowFn, Address SavedExnVar) : Body(Body), ForEHVar(ForEHVar), EndCatchFn(EndCatchFn), RethrowFn(RethrowFn), SavedExnVar(SavedExnVar) {} @@ -1368,14 +1374,13 @@ llvm::BasicBlock *ContBB = CGF.createBasicBlock("finally.cont"); llvm::Value *ShouldRethrow = - CGF.Builder.CreateFlagLoad(ForEHVar, "finally.shouldthrow"); + CGF.Builder.CreateLoad(ForEHVar, "finally.shouldthrow"); CGF.Builder.CreateCondBr(ShouldRethrow, RethrowBB, ContBB); CGF.EmitBlock(RethrowBB); - if (SavedExnVar) { + if (SavedExnVar.isValid()) { CGF.EmitRuntimeCallOrInvoke(RethrowFn, - CGF.Builder.CreateAlignedLoad(CGF.Int8PtrTy, SavedExnVar, - CGF.getPointerAlign())); + CGF.Builder.CreateLoad(SavedExnVar)); } else { CGF.EmitRuntimeCallOrInvoke(RethrowFn); } @@ -1424,9 +1429,13 @@ // But we can't use the exception slot because the @finally might // have a landing pad (which would overwrite the exception slot). llvm::FunctionType *rethrowFnTy = rethrowFn.getFunctionType(); - SavedExnVar = nullptr; - if (rethrowFnTy->getNumParams()) - SavedExnVar = CGF.CreateTempAlloca(CGF.Int8PtrTy, "finally.exn"); + SavedExnVar = Address::invalid(); + if (rethrowFnTy->getNumParams()) { + CharUnits Align = CGF.getPointerAlign(); + LangAS AS = CGF.getASTAllocaAddressSpace(); + SavedExnVar = + CGF.CreateTempAllocaInAS(CGF.Int8PtrTy, Align, AS, "finally.exn"); + } // A finally block is a statement which must be executed on any edge // out of a given scope. Unlike a cleanup, the finally block may @@ -1446,8 +1455,11 @@ RethrowDest = CGF.getJumpDestInCurrentScope(CGF.getUnreachableBlock()); // Whether the finally block is being executed for EH purposes. - ForEHVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), "finally.for-eh"); - CGF.Builder.CreateFlagStore(false, ForEHVar); + llvm::Type *FlagTy = CGF.Builder.getInt1Ty(); + ForEHVar = CGF.CreateTempAllocaInAS( + FlagTy, CGF.PreferredAlignmentForIRType(FlagTy), + CGF.getASTAllocaAddressSpace(), "finally.for-eh"); + CGF.Builder.CreateStore(CGF.Builder.getFalse(), ForEHVar); // Enter a normal cleanup which will perform the @finally block. CGF.EHStack.pushCleanup(NormalCleanup, body, @@ -1483,13 +1495,14 @@ } // If we need to remember the exception pointer to rethrow later, do so. - if (SavedExnVar) { - if (!exn) exn = CGF.getExceptionFromSlot(); - CGF.Builder.CreateAlignedStore(exn, SavedExnVar, CGF.getPointerAlign()); + if (SavedExnVar.isValid()) { + if (!exn) + exn = CGF.getExceptionFromSlot(); + CGF.Builder.CreateStore(exn, SavedExnVar); } // Tell the cleanups in the finally block that we're do this for EH. - CGF.Builder.CreateFlagStore(true, ForEHVar); + CGF.Builder.CreateStore(CGF.Builder.getTrue(), ForEHVar); // Thread a jump through the finally cleanup. CGF.EmitBranchThroughCleanup(RethrowDest); diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -62,70 +62,52 @@ return Builder.CreateBitCast(value, destType); } -/// CreateTempAlloca - This creates a alloca and inserts it into the entry -/// block. -Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, - CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize) { - auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); +/// CreateTempAllocaInAS - Create an alloca in \p AddressSpace with alignment \p +/// Align. Leave the result in \p AddressSpace. +Address CodeGenFunction::CreateTempAllocaInAS(llvm::Type *Ty, CharUnits Align, + LangAS AddressSpace, + const Twine &Name, + llvm::Value *ArraySize) { + auto AS = getContext().getTargetAddressSpace(AddressSpace); + llvm::AllocaInst *Alloca = + ArraySize ? Builder.CreateAlloca(Ty, AS, ArraySize, Name) + : new llvm::AllocaInst(Ty, AS, ArraySize, Name, AllocaInsertPt); Alloca->setAlignment(Align.getAsAlign()); return Address(Alloca, Align); } -/// CreateTempAlloca - This creates a alloca and inserts it into the entry -/// block. The alloca is casted to default address space if necessary. +/// CreateTempAlloca - Create an alloca as with CreateTempAllocaInAS, then cast +/// the result to LangAS::Default if necessary. Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, const Twine &Name, llvm::Value *ArraySize, Address *AllocaAddr) { - auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize); + LangAS AddressSpace = getASTAllocaAddressSpace(); + auto Alloca = CreateTempAllocaInAS(Ty, Align, AddressSpace, Name, ArraySize); if (AllocaAddr) *AllocaAddr = Alloca; llvm::Value *V = Alloca.getPointer(); - // Alloca always returns a pointer in alloca address space, which may - // be different from the type defined by the language. For example, - // in C++ the auto variables are in the default address space. Therefore - // cast alloca to the default address space when necessary. - if (getASTAllocaAddressSpace() != LangAS::Default) { - auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default); + + // Alloca returns a pointer in the specified address space, which may be + // different from the type defined by the language. For example, in C++, auto + // variables are in the default address space. Therefore cast alloca to the + // default address space when necessary. + if (AddressSpace != LangAS::Default) { llvm::IRBuilderBase::InsertPointGuard IPG(Builder); // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt, // otherwise alloca is inserted at the current insertion point of the // builder. if (!ArraySize) Builder.SetInsertPoint(AllocaInsertPt); + llvm::Type *DestTy = + Ty->getPointerTo(getContext().getTargetAddressSpace(LangAS::Default)); V = getTargetHooks().performAddrSpaceCast( - *this, V, getASTAllocaAddressSpace(), LangAS::Default, - Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); + *this, V, AddressSpace, LangAS::Default, DestTy, /*non-null*/ true); } return Address(V, Align); } -/// CreateTempAlloca - This creates an alloca and inserts it into the entry -/// block if \p ArraySize is nullptr, otherwise inserts it at the current -/// insertion point of the builder. -llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, - const Twine &Name, - llvm::Value *ArraySize) { - if (ArraySize) - return Builder.CreateAlloca(Ty, ArraySize, Name); - return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(), - ArraySize, Name, AllocaInsertPt); -} - -/// CreateDefaultAlignTempAlloca - This creates an alloca with the -/// default alignment of the corresponding LLVM type, which is *not* -/// guaranteed to be related in any way to the expected alignment of -/// an AST type that might have been lowered to Ty. -Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name) { - CharUnits Align = - CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlignment(Ty)); - return CreateTempAlloca(Ty, Align, Name); -} - void CodeGenFunction::InitTempAlloca(Address Var, llvm::Value *Init) { auto *Alloca = Var.getPointer(); assert(isa(Alloca) || @@ -169,7 +151,8 @@ Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align, const Twine &Name) { - return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name); + LangAS AS = getASTAllocaAddressSpace(); + return CreateTempAllocaInAS(ConvertTypeForMem(Ty), Align, AS, Name); } Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, @@ -3046,7 +3029,9 @@ // Pointers are passed directly, everything else is passed by address. if (!V->getType()->isPointerTy()) { - Address Ptr = CreateDefaultAlignTempAlloca(V->getType()); + auto Align = PreferredAlignmentForIRType(V->getType()); + LangAS AS = getASTAllocaAddressSpace(); + Address Ptr = CreateTempAllocaInAS(V->getType(), Align, AS); Builder.CreateStore(V, Ptr); V = Ptr.getPointer(); } diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -1792,14 +1792,15 @@ DeleteArgs.add(RValue::get(DeletePtr), ArgTy); // Pass the std::destroying_delete tag if present. - llvm::AllocaInst *DestroyingDeleteTag = nullptr; + Address DestroyingDeleteTag = Address::invalid(); if (Params.DestroyingDelete) { QualType DDTag = *ParamTypeIt++; llvm::Type *Ty = getTypes().ConvertType(DDTag); CharUnits Align = CGM.getNaturalTypeAlignment(DDTag); - DestroyingDeleteTag = CreateTempAlloca(Ty, "destroying.delete.tag"); - DestroyingDeleteTag->setAlignment(Align.getAsAlign()); - DeleteArgs.add(RValue::getAggregate(Address(DestroyingDeleteTag, Align)), DDTag); + LangAS AS = getASTAllocaAddressSpace(); + DestroyingDeleteTag = + CreateTempAllocaInAS(Ty, Align, AS, "destroying.delete.tag"); + DeleteArgs.add(RValue::getAggregate(DestroyingDeleteTag), DDTag); } // Pass the size if the delete function has a size_t parameter. @@ -1840,8 +1841,11 @@ // If call argument lowering didn't use the destroying_delete_t alloca, // remove it again. - if (DestroyingDeleteTag && DestroyingDeleteTag->use_empty()) - DestroyingDeleteTag->eraseFromParent(); + if (DestroyingDeleteTag.isValid()) { + auto *Inst = cast(DestroyingDeleteTag.getPointer()); + if (Inst->use_empty()) + Inst->eraseFromParent(); + } } namespace { diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -2117,7 +2117,9 @@ isa(DstTy)) || (isa(SrcTy) && isa(DstTy))) { - Address Addr = CGF.CreateDefaultAlignTempAlloca(SrcTy, "saved-value"); + CharUnits Align = CGF.PreferredAlignmentForIRType(SrcTy); + LangAS AS = CGF.getASTAllocaAddressSpace(); + Address Addr = CGF.CreateTempAllocaInAS(SrcTy, Align, AS, "saved-value"); LValue LV = CGF.MakeAddrLValue(Addr, E->getType()); CGF.EmitStoreOfScalar(Src, LV); Addr = Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(DestTy), diff --git a/clang/lib/CodeGen/CGGPUBuiltin.cpp b/clang/lib/CodeGen/CGGPUBuiltin.cpp --- a/clang/lib/CodeGen/CGGPUBuiltin.cpp +++ b/clang/lib/CodeGen/CGGPUBuiltin.cpp @@ -106,14 +106,23 @@ // that the alignment of the llvm type was the same as the alignment of the // clang type. llvm::Type *AllocaTy = llvm::StructType::create(ArgTypes, "printf_args"); - llvm::Value *Alloca = CreateTempAlloca(AllocaTy); + CharUnits Align = PreferredAlignmentForIRType(AllocaTy); + LangAS AS = getASTAllocaAddressSpace(); + Address Alloca = CreateTempAllocaInAS(AllocaTy, Align, AS); for (unsigned I = 1, NumArgs = Args.size(); I < NumArgs; ++I) { - llvm::Value *P = Builder.CreateStructGEP(AllocaTy, Alloca, I - 1); + Address P = Builder.CreateStructGEP(Alloca, I - 1); llvm::Value *Arg = Args[I].getRValue(*this).getScalarVal(); - Builder.CreateAlignedStore(Arg, P, DL.getPrefTypeAlign(Arg->getType())); + // FIXME: Changing the following line to Builder.CreateStore(Arg, P) + // results in a test failure in OpenMP/nvptx_target_printf_codegen, in + // that a store of an i32 is expected to have alignment 4 on a 64-bit + // target, but using the alignment from P results in a store with + // alignment 8. Could this actually be correct? + Builder.CreateAlignedStore(Arg, P.getPointer(), + DL.getPrefTypeAlign(Arg->getType())); } - BufferPtr = Builder.CreatePointerCast(Alloca, llvm::Type::getInt8PtrTy(Ctx)); + BufferPtr = Builder.CreatePointerCast(Alloca.getPointer(), + llvm::Type::getInt8PtrTy(Ctx)); } // Invoke vprintf and return. diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -2109,9 +2109,10 @@ // OutlinedFn(>id, &zero_bound, CapturedStruct); Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); - Address ZeroAddrBound = - CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, - /*Name=*/".bound.zero.addr"); + CharUnits Align = CGF.PreferredAlignmentForIRType(CGF.Int32Ty); + LangAS AS = CGF.getASTAllocaAddressSpace(); + Address ZeroAddrBound = CGF.CreateTempAllocaInAS( + CGF.Int32Ty, Align, AS, /*Name=*/".bound.zero.addr"); CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0)); llvm::SmallVector OutlinedFnArgs; // ThreadId for serialized parallels is 0. diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -1485,8 +1485,10 @@ if (!CGF.HaveInsertPoint()) return; - Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, - /*Name=*/".zero.addr"); + Address ZeroAddr = CGF.CreateTempAllocaInAS( + CGF.Int32Ty, CGF.PreferredAlignmentForIRType(CGF.Int32Ty), + CGF.getASTAllocaAddressSpace(), + /*Name=*/".zero.addr"); CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); llvm::SmallVector OutlinedFnArgs; OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); @@ -1517,9 +1519,11 @@ // TODO: Is that needed? CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); - Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca( - llvm::ArrayType::get(CGM.VoidPtrTy, CapturedVars.size()), - "captured_vars_addrs"); + llvm::Type *VarsTy = + llvm::ArrayType::get(CGM.VoidPtrTy, CapturedVars.size()); + Address CapturedVarsAddrs = CGF.CreateTempAllocaInAS( + VarsTy, CGF.PreferredAlignmentForIRType(VarsTy), + CGF.getASTAllocaAddressSpace(), "captured_vars_addrs"); // There's something to share. if (!CapturedVars.empty()) { // Prepare for parallel region. Indicate the outlined function. @@ -3473,8 +3477,10 @@ const auto *RD = CS.getCapturedRecordDecl(); auto CurField = RD->field_begin(); - Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, - /*Name=*/".zero.addr"); + Address ZeroAddr = CGF.CreateTempAllocaInAS( + CGF.Int32Ty, CGF.PreferredAlignmentForIRType(CGF.Int32Ty), + CGF.getASTAllocaAddressSpace(), + /*Name=*/".zero.addr"); CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); // Get the array of arguments. SmallVector Args; @@ -3487,8 +3493,9 @@ // Use global memory for data sharing. // Handle passing of global args to workers. - Address GlobalArgs = - CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); + Address GlobalArgs = CGF.CreateTempAllocaInAS( + CGF.VoidPtrPtrTy, CGF.PreferredAlignmentForIRType(CGF.VoidPtrPtrTy), + CGF.getASTAllocaAddressSpace(), "global_args"); llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer(); llvm::Value *DataSharingArgs[] = {GlobalArgsPtr}; CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -1261,8 +1261,11 @@ // If there is an NRVO flag for this variable, set it to 1 into indicate // that the cleanup code should not destroy the variable. - if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()]) - Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag); + const auto I = NRVOFlags.find(S.getNRVOCandidate()); + if (I != NRVOFlags.end()) { + Address NRVOFlag = I->second; + Builder.CreateStore(Builder.getTrue(), NRVOFlag); + } } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) { // Make sure not to return anything, but evaluate the expression // for side effects. diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -576,7 +576,7 @@ /// A mapping from NRVO variables to the flags used to indicate /// when the NRVO has been applied to this variable. - llvm::DenseMap NRVOFlags; + llvm::DenseMap NRVOFlags; EHScopeStack EHStack; llvm::SmallVector LifetimeExtendedCleanupStack; @@ -623,11 +623,11 @@ /// The exception slot. All landing pads write the current exception pointer /// into this alloca. - llvm::Value *ExceptionSlot = nullptr; + Address ExceptionSlot = Address::invalid(); /// The selector slot. Under the MandatoryCleanup model, all landing pads /// write the current selector value into this alloca. - llvm::AllocaInst *EHSelectorSlot = nullptr; + Address EHSelectorSlot = Address::invalid(); /// A stack of exception code slots. Entering an __except block pushes a slot /// on the stack and leaving pops one. The __exception_code() intrinsic loads @@ -704,11 +704,11 @@ /// An i1 variable indicating whether or not the @finally is /// running for an exception. - llvm::AllocaInst *ForEHVar; + Address ForEHVar = Address::invalid(); /// An i8* variable into which the exception pointer to rethrow /// has been saved. - llvm::AllocaInst *SavedExnVar; + Address SavedExnVar = Address::invalid(); public: void enter(CodeGenFunction &CGF, const Stmt *Finally, @@ -2474,54 +2474,58 @@ TBAAAccessInfo *TBAAInfo = nullptr); LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy); - /// CreateTempAlloca - This creates an alloca and inserts it into the entry - /// block if \p ArraySize is nullptr, otherwise inserts it at the current - /// insertion point of the builder. The caller is responsible for setting an - /// appropriate alignment on - /// the alloca. + /// CreateTempAllocaInAS - Create an alloca in \p AddressSpace with alignment + /// \p Align. /// /// \p ArraySize is the number of array elements to be allocated if it - /// is not nullptr. + /// is not nullptr. + /// + /// The alloca will be inserted into the entry block if \p ArraySize is + /// nullptr. Otherwise it is inserted at the current insertion point of the + /// builder. + Address CreateTempAllocaInAS(llvm::Type *Ty, CharUnits Align, + LangAS AddrSpace, const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr); + + /// CreateTempAlloca - Create an alloca with CreateTempAllocaInAS, then cast + /// the result to LangAS::Default if necessary. /// /// LangAS::Default is the address space of pointers to local variables and - /// temporaries, as exposed in the source language. In certain - /// configurations, this is not the same as the alloca address space, and a - /// cast is needed to lift the pointer from the alloca AS into - /// LangAS::Default. This can happen when the target uses a restricted - /// address space for the stack but the source language requires - /// LangAS::Default to be a generic address space. The latter condition is - /// common for most programming languages; OpenCL is an exception in that - /// LangAS::Default is the private address space, which naturally maps - /// to the stack. + /// temporaries, as exposed in the source language. In certain configurations, + /// this is not the same as the alloca address space, and a cast is needed to + /// lift the pointer from the alloca AS into LangAS::Default. This can happen + /// when the target uses a restricted address space for the stack but the + /// source language requires LangAS::Default to be a generic address + /// space. The latter condition is common for most programming languages; + /// OpenCL is an exception in that LangAS::Default is the private address + /// space, which naturally maps to the stack. /// /// Because the address of a temporary is often exposed to the program in /// various ways, this function will perform the cast. The original alloca /// instruction is returned through \p Alloca if it is not nullptr. /// - /// The cast is not performaed in CreateTempAllocaWithoutCast. This is - /// more efficient if the caller knows that the address will not be exposed. - llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr); - Address CreateTempAlloca(llvm::Type *Ty, CharUnits align, + /// If the caller knows that the address will not be exposed, it is more + /// efficient to use CreateTempAllocaInAS instead, to avoid any unneeded + /// addrspace casts. + Address CreateTempAlloca(llvm::Type *Ty, CharUnits Align, const Twine &Name = "tmp", llvm::Value *ArraySize = nullptr, Address *Alloca = nullptr); - Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr); - /// CreateDefaultAlignedTempAlloca - This creates an alloca with the - /// default ABI alignment of the given LLVM type. + /// PreferredAlignmentForIRType - Return the preferred alignment for the IR + /// type \p Ty. /// - /// IMPORTANT NOTE: This is *not* generally the right alignment for - /// any given AST type that happens to have been lowered to the - /// given IR type. This should only ever be used for function-local, - /// IR-driven manipulations like saving and restoring a value. Do - /// not hand this address off to arbitrary IRGen routines, and especially - /// do not pass it as an argument to a function that might expect a - /// properly ABI-aligned value. - Address CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name = "tmp"); + /// IMPORTANT NOTE: This is *not* generally the right alignment for any given + /// AST type that happens to have been lowered to the given IR type. This + /// should only ever be used for allocating function-local values used in + /// IR-driven manipulations like saving and restoring a value. Do not use + /// this alignment for allocating arbitrary IRGen routines, and especially do + /// not use it to allocate values that might be passed to functions that + /// expect a properly ABI-aligned value. + CharUnits PreferredAlignmentForIRType(llvm::Type *Ty) { + return CharUnits::fromQuantity( + CGM.getDataLayout().getPrefTypeAlignment(Ty)); + } /// InitTempAlloca - Provide an initial value for the given alloca which /// will be observable at all locations in the function. @@ -3008,7 +3012,7 @@ /// as a global constant. Address Addr; - llvm::Value *NRVOFlag; + Address NRVOFlag; /// True if the variable is a __block variable that is captured by an /// escaping block. @@ -3028,12 +3032,13 @@ struct Invalid {}; AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()), - AllocaAddr(Address::invalid()) {} + NRVOFlag(Address::invalid()), AllocaAddr(Address::invalid()) {} AutoVarEmission(const VarDecl &variable) - : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr), - IsEscapingByRef(false), IsConstantAggregate(false), - SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {} + : Variable(&variable), Addr(Address::invalid()), + NRVOFlag(Address::invalid()), IsEscapingByRef(false), + IsConstantAggregate(false), SizeForLifetimeMarkers(nullptr), + AllocaAddr(Address::invalid()) {} bool wasEmittedAsGlobal() const { return !Addr.isValid(); } diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -979,7 +979,10 @@ // If we're checking the return value, allocate space for a pointer to a // precise source location of the checked return statement. if (requiresReturnValueCheck()) { - ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr"); + CharUnits Align = PreferredAlignmentForIRType(Int8PtrTy); + LangAS AS = getASTAllocaAddressSpace(); + ReturnLocation = + CreateTempAllocaInAS(Int8PtrTy, Align, AS, "return.sloc.ptr"); InitTempAlloca(ReturnLocation, llvm::ConstantPointerNull::get(Int8PtrTy)); } @@ -1068,8 +1071,10 @@ ++AI; ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign()); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { + CharUnits Align = PreferredAlignmentForIRType(Int8PtrTy); + LangAS AS = getASTAllocaAddressSpace(); ReturnValuePointer = - CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr"); + CreateTempAllocaInAS(Int8PtrTy, Align, AS, "result.ptr"); Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast( ReturnValue.getPointer(), Int8PtrTy), ReturnValuePointer);