Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -1812,6 +1812,9 @@ must be a multiple of 8-bits. If omitted, the natural stack alignment defaults to "unspecified", which does not prevent any alignment promotions. +``A
`` + Specifies the address space of objects created by '``alloca``'. + Defaults to the default address space of 0. ``p[n]:::`` This specifies the *size* of a pointer and its ```` and ````\erred alignments for address space ``n``. All sizes are in @@ -7188,7 +7191,7 @@ :: - = alloca [inalloca] [, ] [, align ] ; yields type*:result + = alloca [inalloca] [, ] [, align ] [, addrspace()] ; yields type addrspace(num)*:result Overview: """"""""" @@ -7196,7 +7199,7 @@ The '``alloca``' instruction allocates memory on the stack frame of the currently executing function, to be automatically released when this function returns to its caller. The object is always allocated in the -generic address space (address space zero). +address space for allocas indicated in the datalayout. Arguments: """""""""" Index: include/llvm/IR/DataLayout.h =================================================================== --- include/llvm/IR/DataLayout.h +++ include/llvm/IR/DataLayout.h @@ -104,6 +104,7 @@ /// Defaults to false. bool BigEndian; + unsigned AllocaAddrSpace; unsigned StackNaturalAlign; enum ManglingModeT { @@ -186,6 +187,7 @@ clear(); StringRepresentation = DL.StringRepresentation; BigEndian = DL.isBigEndian(); + AllocaAddrSpace = DL.AllocaAddrSpace; StackNaturalAlign = DL.StackNaturalAlign; ManglingMode = DL.ManglingMode; LegalIntWidths = DL.LegalIntWidths; @@ -241,6 +243,7 @@ } unsigned getStackAlignment() const { return StackNaturalAlign; } + unsigned getAllocaAddrSpace() const { return AllocaAddrSpace; } bool hasMicrosoftFastStdCallMangling() const { return ManglingMode == MM_WinCOFFX86; Index: include/llvm/IR/IRBuilder.h =================================================================== --- include/llvm/IR/IRBuilder.h +++ include/llvm/IR/IRBuilder.h @@ -33,6 +33,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" @@ -1089,9 +1090,15 @@ // Instruction creation methods: Memory Instructions //===--------------------------------------------------------------------===// + AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace, + Value *ArraySize = nullptr, const Twine &Name = "") { + return Insert(new AllocaInst(Ty, AddrSpace, ArraySize), Name); + } + AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr, const Twine &Name = "") { - return Insert(new AllocaInst(Ty, ArraySize), Name); + const DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + return Insert(new AllocaInst(Ty, DL.getAllocaAddrSpace(), ArraySize), Name); } // \brief Provided to resolve 'CreateLoad(Ptr, "...")' correctly, instead of // converting the string to 'bool' for the isVolatile parameter. Index: include/llvm/IR/Instructions.h =================================================================== --- include/llvm/IR/Instructions.h +++ include/llvm/IR/Instructions.h @@ -67,18 +67,21 @@ AllocaInst *cloneImpl() const; public: - explicit AllocaInst(Type *Ty, Value *ArraySize = nullptr, + explicit AllocaInst(Type *Ty, unsigned AddrSpace, + Value *ArraySize = nullptr, const Twine &Name = "", Instruction *InsertBefore = nullptr); - AllocaInst(Type *Ty, Value *ArraySize, + AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, const Twine &Name, BasicBlock *InsertAtEnd); - AllocaInst(Type *Ty, const Twine &Name, Instruction *InsertBefore = nullptr); - AllocaInst(Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd); + AllocaInst(Type *Ty, unsigned AddrSpace, + const Twine &Name, Instruction *InsertBefore = nullptr); + AllocaInst(Type *Ty, unsigned AddrSpace, + const Twine &Name, BasicBlock *InsertAtEnd); - AllocaInst(Type *Ty, Value *ArraySize, unsigned Align, + AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, unsigned Align, const Twine &Name = "", Instruction *InsertBefore = nullptr); - AllocaInst(Type *Ty, Value *ArraySize, unsigned Align, + AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, unsigned Align, const Twine &Name, BasicBlock *InsertAtEnd); // Out of line virtual method, so the vtable, etc. has a home. Index: lib/AsmParser/LLParser.h =================================================================== --- lib/AsmParser/LLParser.h +++ lib/AsmParser/LLParser.h @@ -242,6 +242,8 @@ bool ParseOrdering(AtomicOrdering &Ordering); bool ParseOptionalStackAlignment(unsigned &Alignment); bool ParseOptionalCommaAlign(unsigned &Alignment, bool &AteExtraComma); + bool ParseOptionalCommaAddrSpace(unsigned &AddrSpace, LocTy &Loc, + bool &AteExtraComma); bool ParseOptionalCommaInAlloca(bool &IsInAlloca); bool parseAllocSizeArguments(unsigned &ElemSizeArg, Optional &HowManyArg); Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -1852,6 +1852,34 @@ return false; } +/// ParseOptionalCommaAddrSpace +/// ::= +/// ::= ',' addrspace(1) +/// +/// This returns with AteExtraComma set to true if it ate an excess comma at the +/// end. +bool LLParser::ParseOptionalCommaAddrSpace(unsigned &AddrSpace, + LocTy &Loc, + bool &AteExtraComma) { + AteExtraComma = false; + while (EatIfPresent(lltok::comma)) { + // Metadata at the end is an early exit. + if (Lex.getKind() == lltok::MetadataVar) { + AteExtraComma = true; + return false; + } + + Loc = Lex.getLoc(); + if (Lex.getKind() != lltok::kw_addrspace) + return Error(Lex.getLoc(), "expected metadata or 'addrspace'"); + + if (ParseOptionalAddrSpace(AddrSpace)) + return true; + } + + return false; +} + bool LLParser::parseAllocSizeArguments(unsigned &BaseSizeArg, Optional &HowManyArg) { Lex.Lex(); @@ -6030,8 +6058,9 @@ /// (',' 'align' i32)? int LLParser::ParseAlloc(Instruction *&Inst, PerFunctionState &PFS) { Value *Size = nullptr; - LocTy SizeLoc, TyLoc; + LocTy SizeLoc, TyLoc, ASLoc; unsigned Alignment = 0; + unsigned AddrSpace = 0; Type *Ty = nullptr; bool IsInAlloca = EatIfPresent(lltok::kw_inalloca); @@ -6045,12 +6074,21 @@ bool AteExtraComma = false; if (EatIfPresent(lltok::comma)) { if (Lex.getKind() == lltok::kw_align) { - if (ParseOptionalAlignment(Alignment)) return true; + if (ParseOptionalAlignment(Alignment)) + return true; + if (ParseOptionalCommaAddrSpace(AddrSpace, ASLoc, AteExtraComma)) + return true; + } else if (Lex.getKind() == lltok::kw_addrspace) { + ASLoc = Lex.getLoc(); + if (ParseOptionalAddrSpace(AddrSpace)) + return true; } else if (Lex.getKind() == lltok::MetadataVar) { AteExtraComma = true; } else { if (ParseTypeAndValue(Size, SizeLoc, PFS) || - ParseOptionalCommaAlign(Alignment, AteExtraComma)) + ParseOptionalCommaAlign(Alignment, AteExtraComma) || + (!AteExtraComma && + ParseOptionalCommaAddrSpace(AddrSpace, ASLoc, AteExtraComma))) return true; } } @@ -6058,7 +6096,14 @@ if (Size && !Size->getType()->isIntegerTy()) return Error(SizeLoc, "element count must have integer type"); - AllocaInst *AI = new AllocaInst(Ty, Size, Alignment); + const DataLayout &DL = M->getDataLayout(); + unsigned AS = DL.getAllocaAddrSpace(); + if (AS != AddrSpace) { + // TODO: In the future it should be possible to specify addrspace per-alloca. + return Error(ASLoc, "address space must match datalayout"); + } + + AllocaInst *AI = new AllocaInst(Ty, AS, Size, Alignment); AI->setUsedWithInAlloca(IsInAlloca); AI->setSwiftError(IsSwiftError); Inst = AI; Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -4017,7 +4017,12 @@ } if (!Ty || !Size) return error("Invalid record"); - AllocaInst *AI = new AllocaInst(Ty, Size, Align); + + // FIXME: Make this an optional field. + const DataLayout &DL = TheModule->getDataLayout(); + unsigned AS = DL.getAllocaAddrSpace(); + + AllocaInst *AI = new AllocaInst(Ty, AS, Size, Align); AI->setUsedWithInAlloca(InAlloca); AI->setSwiftError(SwiftError); I = AI; Index: lib/CodeGen/SjLjEHPrepare.cpp =================================================================== --- lib/CodeGen/SjLjEHPrepare.cpp +++ lib/CodeGen/SjLjEHPrepare.cpp @@ -175,8 +175,8 @@ // because the value needs to be added to the global context list. auto &DL = F.getParent()->getDataLayout(); unsigned Align = DL.getPrefTypeAlignment(FunctionContextTy); - FuncCtx = new AllocaInst(FunctionContextTy, nullptr, Align, "fn_context", - &EntryBB->front()); + FuncCtx = new AllocaInst(FunctionContextTy, DL.getAllocaAddrSpace(), + nullptr, Align, "fn_context", &EntryBB->front()); // Fill in the function context structure. for (LandingPadInst *LPI : LPads) { Index: lib/CodeGen/WinEHPrepare.cpp =================================================================== --- lib/CodeGen/WinEHPrepare.cpp +++ lib/CodeGen/WinEHPrepare.cpp @@ -86,6 +86,7 @@ // All fields are reset by runOnFunction. EHPersonality Personality = EHPersonality::Unknown; + const DataLayout *DL = nullptr; DenseMap BlockColors; MapVector> FuncletBlocks; }; @@ -111,6 +112,7 @@ if (!isFuncletEHPersonality(Personality)) return false; + DL = &Fn.getParent()->getDataLayout(); return prepareExplicitEH(Fn); } @@ -1070,7 +1072,7 @@ if (!isa(EHPad)) { // If the EHPad isn't a terminator, then we can insert a load in this block // that will dominate all uses. - SpillSlot = new AllocaInst(PN->getType(), nullptr, + SpillSlot = new AllocaInst(PN->getType(), DL->getAllocaAddrSpace(), nullptr, Twine(PN->getName(), ".wineh.spillslot"), &F.getEntryBlock().front()); Value *V = new LoadInst(SpillSlot, Twine(PN->getName(), ".wineh.reload"), @@ -1157,7 +1159,7 @@ Function &F) { // Lazilly create the spill slot. if (!SpillSlot) - SpillSlot = new AllocaInst(V->getType(), nullptr, + SpillSlot = new AllocaInst(V->getType(), DL->getAllocaAddrSpace(), nullptr, Twine(V->getName(), ".wineh.spillslot"), &F.getEntryBlock().front()); Index: lib/IR/AsmWriter.cpp =================================================================== --- lib/IR/AsmWriter.cpp +++ lib/IR/AsmWriter.cpp @@ -3115,6 +3115,12 @@ if (AI->getAlignment()) { Out << ", align " << AI->getAlignment(); } + + unsigned AddrSpace = AI->getType()->getAddressSpace(); + if (AddrSpace != 0) { + Out << ", addrspace(" << AddrSpace << ')'; + } + } else if (isa(I)) { if (Operand) { Out << ' '; Index: lib/IR/DataLayout.cpp =================================================================== --- lib/IR/DataLayout.cpp +++ lib/IR/DataLayout.cpp @@ -174,6 +174,7 @@ LayoutMap = nullptr; BigEndian = false; + AllocaAddrSpace = 0; StackNaturalAlign = 0; ManglingMode = MM_None; NonIntegralAddressSpaces.clear(); @@ -352,6 +353,12 @@ StackNaturalAlign = inBytes(getInt(Tok)); break; } + case 'A': { // Default stack/alloca address space. + AllocaAddrSpace = getInt(Tok); + if (!isUInt<24>(AllocaAddrSpace)) + report_fatal_error("Invalid address space, must be a 24bit integer"); + break; + } case 'm': if (!Tok.empty()) report_fatal_error("Unexpected trailing characters after mangling specifier in datalayout string"); @@ -394,6 +401,7 @@ bool DataLayout::operator==(const DataLayout &Other) const { bool Ret = BigEndian == Other.BigEndian && + AllocaAddrSpace == Other.AllocaAddrSpace && StackNaturalAlign == Other.StackNaturalAlign && ManglingMode == Other.ManglingMode && LegalIntWidths == Other.LegalIntWidths && Index: lib/IR/Instructions.cpp =================================================================== --- lib/IR/Instructions.cpp +++ lib/IR/Instructions.cpp @@ -1199,34 +1199,38 @@ return Amt; } -AllocaInst::AllocaInst(Type *Ty, const Twine &Name, Instruction *InsertBefore) - : AllocaInst(Ty, /*ArraySize=*/nullptr, Name, InsertBefore) {} - -AllocaInst::AllocaInst(Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd) - : AllocaInst(Ty, /*ArraySize=*/nullptr, Name, InsertAtEnd) {} - -AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, const Twine &Name, +AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name, Instruction *InsertBefore) - : AllocaInst(Ty, ArraySize, /*Align=*/0, Name, InsertBefore) {} + : AllocaInst(Ty, AddrSpace, /*ArraySize=*/nullptr, Name, InsertBefore) {} -AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, const Twine &Name, +AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name, BasicBlock *InsertAtEnd) - : AllocaInst(Ty, ArraySize, /*Align=*/0, Name, InsertAtEnd) {} + : AllocaInst(Ty, AddrSpace, /*ArraySize=*/nullptr, Name, InsertAtEnd) {} -AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align, +AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, const Twine &Name, Instruction *InsertBefore) - : UnaryInstruction(PointerType::getUnqual(Ty), Alloca, - getAISize(Ty->getContext(), ArraySize), InsertBefore), - AllocatedType(Ty) { + : AllocaInst(Ty, AddrSpace, ArraySize, /*Align=*/0, Name, InsertBefore) {} + +AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, + const Twine &Name, BasicBlock *InsertAtEnd) + : AllocaInst(Ty, AddrSpace, ArraySize, /*Align=*/0, Name, InsertAtEnd) {} + +AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, + unsigned Align, const Twine &Name, + Instruction *InsertBefore) + : UnaryInstruction(PointerType::get(Ty, AddrSpace), Alloca, + getAISize(Ty->getContext(), ArraySize), InsertBefore), + AllocatedType(Ty) { setAlignment(Align); assert(!Ty->isVoidTy() && "Cannot allocate void!"); setName(Name); } -AllocaInst::AllocaInst(Type *Ty, Value *ArraySize, unsigned Align, - const Twine &Name, BasicBlock *InsertAtEnd) - : UnaryInstruction(PointerType::getUnqual(Ty), Alloca, - getAISize(Ty->getContext(), ArraySize), InsertAtEnd), +AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, + unsigned Align, const Twine &Name, + BasicBlock *InsertAtEnd) + : UnaryInstruction(PointerType::get(Ty, AddrSpace), Alloca, + getAISize(Ty->getContext(), ArraySize), InsertAtEnd), AllocatedType(Ty) { setAlignment(Align); assert(!Ty->isVoidTy() && "Cannot allocate void!"); @@ -3828,6 +3832,7 @@ AllocaInst *AllocaInst::cloneImpl() const { AllocaInst *Result = new AllocaInst(getAllocatedType(), + getType()->getAddressSpace(), (Value *)getOperand(0), getAlignment()); Result->setUsedWithInAlloca(isUsedWithInAlloca()); Result->setSwiftError(isSwiftError()); Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -3169,8 +3169,9 @@ void Verifier::visitAllocaInst(AllocaInst &AI) { SmallPtrSet Visited; PointerType *PTy = AI.getType(); - Assert(PTy->getAddressSpace() == 0, - "Allocation instruction pointer not in the generic address space!", + // TODO: Relax this restriction? + Assert(PTy->getAddressSpace() == DL.getAllocaAddrSpace(), + "Allocation instruction pointer not in the stack address space!", &AI); Assert(AI.getAllocatedType()->isSized(&Visited), "Cannot allocate unsized type", &AI); Index: lib/Target/NVPTX/NVPTXLowerArgs.cpp =================================================================== --- lib/Target/NVPTX/NVPTXLowerArgs.cpp +++ lib/Target/NVPTX/NVPTXLowerArgs.cpp @@ -159,7 +159,8 @@ assert(PType && "Expecting pointer type in handleByValParam"); Type *StructType = PType->getElementType(); - AllocaInst *AllocA = new AllocaInst(StructType, Arg->getName(), FirstInst); + unsigned AS = Func->getParent()->getDataLayout().getAllocaAddrSpace(); + AllocaInst *AllocA = new AllocaInst(StructType, AS, Arg->getName(), FirstInst); // Set the alignment to alignment of the byval parameter. This is because, // later load/stores assume that alignment, and we are going to replace // the use of the byval parameter with this alloca instruction. Index: lib/Transforms/Coroutines/CoroElide.cpp =================================================================== --- lib/Transforms/Coroutines/CoroElide.cpp +++ lib/Transforms/Coroutines/CoroElide.cpp @@ -127,7 +127,8 @@ // is spilled into the coroutine frame and recreate the alignment information // here. Possibly we will need to do a mini SROA here and break the coroutine // frame into individual AllocaInst recreating the original alignment. - auto *Frame = new AllocaInst(FrameTy, "", InsertPt); + const DataLayout &DL = F->getParent()->getDataLayout(); + auto *Frame = new AllocaInst(FrameTy, DL.getAllocaAddrSpace(), "", InsertPt); auto *FrameVoidPtr = new BitCastInst(Frame, Type::getInt8PtrTy(C), "vFrame", InsertPt); Index: lib/Transforms/IPO/ArgumentPromotion.cpp =================================================================== --- lib/Transforms/IPO/ArgumentPromotion.cpp +++ lib/Transforms/IPO/ArgumentPromotion.cpp @@ -349,6 +349,8 @@ Call->eraseFromParent(); } + const DataLayout &DL = F->getParent()->getDataLayout(); + // Since we have now created the new function, splice the body of the old // function right into the new function, leaving the old rotting hulk of the // function empty. @@ -376,7 +378,8 @@ // Just add all the struct element types. Type *AgTy = cast(I->getType())->getElementType(); - Value *TheAlloca = new AllocaInst(AgTy, nullptr, "", InsertPt); + Value *TheAlloca = new AllocaInst(AgTy, DL.getAllocaAddrSpace(), nullptr, + "", InsertPt); StructType *STy = cast(AgTy); Value *Idxs[2] = {ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), nullptr}; Index: lib/Transforms/IPO/GlobalOpt.cpp =================================================================== --- lib/Transforms/IPO/GlobalOpt.cpp +++ lib/Transforms/IPO/GlobalOpt.cpp @@ -1819,12 +1819,14 @@ GS.AccessingFunction->doesNotRecurse() && isPointerValueDeadOnEntryToFunction(GS.AccessingFunction, GV, LookupDomTree)) { + const DataLayout &DL = GV->getParent()->getDataLayout(); + DEBUG(dbgs() << "LOCALIZING GLOBAL: " << *GV << "\n"); Instruction &FirstI = const_cast(*GS.AccessingFunction ->getEntryBlock().begin()); Type *ElemTy = GV->getValueType(); // FIXME: Pass Global's alignment when globals have alignment - AllocaInst *Alloca = new AllocaInst(ElemTy, nullptr, + AllocaInst *Alloca = new AllocaInst(ElemTy, DL.getAllocaAddrSpace(), nullptr, GV->getName(), &FirstI); if (!isa(GV->getInitializer())) new StoreInst(GV->getInitializer(), Alloca, &FirstI); Index: lib/Transforms/Instrumentation/DataFlowSanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -331,6 +331,10 @@ DFSanFunction &DFSF; DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {} + const DataLayout &getDataLayout() const { + return DFSF.F->getParent()->getDataLayout(); + } + void visitOperandShadowInst(Instruction &I); void visitBinaryOperator(BinaryOperator &BO); @@ -1482,7 +1486,8 @@ auto *LabelVATy = ArrayType::get(DFSF.DFS.ShadowTy, CS.arg_size() - FT->getNumParams()); auto *LabelVAAlloca = new AllocaInst( - LabelVATy, "labelva", &DFSF.F->getEntryBlock().front()); + LabelVATy, getDataLayout().getAllocaAddrSpace(), + "labelva", &DFSF.F->getEntryBlock().front()); for (unsigned n = 0; i != CS.arg_end(); ++i, ++n) { auto LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, n); @@ -1495,8 +1500,9 @@ if (!FT->getReturnType()->isVoidTy()) { if (!DFSF.LabelReturnAlloca) { DFSF.LabelReturnAlloca = - new AllocaInst(DFSF.DFS.ShadowTy, "labelreturn", - &DFSF.F->getEntryBlock().front()); + new AllocaInst(DFSF.DFS.ShadowTy, + getDataLayout().getAllocaAddrSpace(), + "labelreturn", &DFSF.F->getEntryBlock().front()); } Args.push_back(DFSF.LabelReturnAlloca); } @@ -1575,7 +1581,8 @@ unsigned VarArgSize = CS.arg_size() - FT->getNumParams(); ArrayType *VarArgArrayTy = ArrayType::get(DFSF.DFS.ShadowTy, VarArgSize); AllocaInst *VarArgShadow = - new AllocaInst(VarArgArrayTy, "", &DFSF.F->getEntryBlock().front()); + new AllocaInst(VarArgArrayTy, getDataLayout().getAllocaAddrSpace(), + "", &DFSF.F->getEntryBlock().front()); Args.push_back(IRB.CreateConstGEP2_32(VarArgArrayTy, VarArgShadow, 0, 0)); for (unsigned n = 0; i != e; ++i, ++n) { IRB.CreateStore( Index: lib/Transforms/Instrumentation/MemorySanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -3062,6 +3062,8 @@ } void finalizeInstrumentation() override { + const DataLayout &DL = F.getParent()->getDataLayout(); + assert(!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); if (!VAStartInstrumentationList.empty()) { @@ -3362,6 +3364,7 @@ assert(!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); if (!VAStartInstrumentationList.empty()) { + const DataLayout &DL = F.getParent()->getDataLayout(); // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); Index: lib/Transforms/Scalar/RewriteStatepointsForGC.cpp =================================================================== --- lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -1619,8 +1619,10 @@ // Emit alloca for "LiveValue" and record it in "allocaMap" and // "PromotableAllocas" + const DataLayout &DL = F.getParent()->getDataLayout(); auto emitAllocaFor = [&](Value *LiveValue) { - AllocaInst *Alloca = new AllocaInst(LiveValue->getType(), "", + AllocaInst *Alloca = new AllocaInst(LiveValue->getType(), + DL.getAllocaAddrSpace(), "", F.getEntryBlock().getFirstNonPHI()); AllocaMap[LiveValue] = Alloca; PromotableAllocas.push_back(Alloca); Index: lib/Transforms/Scalar/SROA.cpp =================================================================== --- lib/Transforms/Scalar/SROA.cpp +++ lib/Transforms/Scalar/SROA.cpp @@ -2295,7 +2295,8 @@ #endif return getAdjustedPtr(IRB, DL, &NewAI, - APInt(DL.getPointerSizeInBits(), Offset), PointerTy, + APInt(DL.getPointerTypeSizeInBits(PointerTy), Offset), + PointerTy, #ifndef NDEBUG Twine(OldName) + "." #else @@ -2370,6 +2371,8 @@ Value *OldOp = LI.getOperand(0); assert(OldOp == OldPtr); + unsigned AS = LI.getPointerAddressSpace(); + Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8) : LI.getType(); const bool IsLoadPastEnd = DL.getTypeStoreSize(TargetTy) > SliceSize; @@ -2406,7 +2409,7 @@ "endian_shift"); } } else { - Type *LTy = TargetTy->getPointerTo(); + Type *LTy = TargetTy->getPointerTo(AS); LoadInst *NewLI = IRB.CreateAlignedLoad(getNewAllocaSlicePtr(IRB, LTy), getSliceAlign(TargetTy), LI.isVolatile(), LI.getName()); @@ -2434,7 +2437,7 @@ // the computed value, and then replace the placeholder with LI, leaving // LI only used for this computation. Value *Placeholder = - new LoadInst(UndefValue::get(LI.getType()->getPointerTo())); + new LoadInst(UndefValue::get(LI.getType()->getPointerTo(AS))); V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset, "insert"); LI.replaceAllUsesWith(V); @@ -2547,7 +2550,8 @@ NewSI = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), SI.isVolatile()); } else { - Value *NewPtr = getNewAllocaSlicePtr(IRB, V->getType()->getPointerTo()); + unsigned AS = SI.getPointerAddressSpace(); + Value *NewPtr = getNewAllocaSlicePtr(IRB, V->getType()->getPointerTo(AS)); NewSI = IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(V->getType()), SI.isVolatile()); } @@ -3862,7 +3866,7 @@ if (Alignment <= DL.getABITypeAlignment(SliceTy)) Alignment = 0; NewAI = new AllocaInst( - SliceTy, nullptr, Alignment, + SliceTy, AI.getType()->getAddressSpace(), nullptr, Alignment, AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI); ++NumNewAllocas; } Index: lib/Transforms/Utils/CodeExtractor.cpp =================================================================== --- lib/Transforms/Utils/CodeExtractor.cpp +++ lib/Transforms/Utils/CodeExtractor.cpp @@ -440,8 +440,10 @@ // Emit a call to the new function, passing in: *pointer to struct (if // aggregating parameters), or plan inputs and allocated memory for outputs std::vector params, StructValues, ReloadOutputs, Reloads; - - LLVMContext &Context = newFunction->getContext(); + + Module *M = newFunction->getParent(); + LLVMContext &Context = M->getContext(); + const DataLayout &DL = M->getDataLayout(); // Add inputs as params, or to be filled into the struct for (Value *input : inputs) @@ -456,8 +458,9 @@ StructValues.push_back(output); } else { AllocaInst *alloca = - new AllocaInst(output->getType(), nullptr, output->getName() + ".loc", - &codeReplacer->getParent()->front().front()); + new AllocaInst(output->getType(), DL.getAllocaAddrSpace(), + nullptr, output->getName() + ".loc", + &codeReplacer->getParent()->front().front()); ReloadOutputs.push_back(alloca); params.push_back(alloca); } @@ -473,7 +476,8 @@ // Allocate a struct at the beginning of this function StructArgTy = StructType::get(newFunction->getContext(), ArgTypes); - Struct = new AllocaInst(StructArgTy, nullptr, "structArg", + Struct = new AllocaInst(StructArgTy, DL.getAllocaAddrSpace(), nullptr, + "structArg", &codeReplacer->getParent()->front().front()); params.push_back(Struct); Index: lib/Transforms/Utils/DemoteRegToStack.cpp =================================================================== --- lib/Transforms/Utils/DemoteRegToStack.cpp +++ lib/Transforms/Utils/DemoteRegToStack.cpp @@ -28,15 +28,17 @@ return nullptr; } + Function *F = I.getParent()->getParent(); + const DataLayout &DL = F->getParent()->getDataLayout(); + // Create a stack slot to hold the value. AllocaInst *Slot; if (AllocaPoint) { - Slot = new AllocaInst(I.getType(), nullptr, + Slot = new AllocaInst(I.getType(), DL.getAllocaAddrSpace(), nullptr, I.getName()+".reg2mem", AllocaPoint); } else { - Function *F = I.getParent()->getParent(); - Slot = new AllocaInst(I.getType(), nullptr, I.getName() + ".reg2mem", - &F->getEntryBlock().front()); + Slot = new AllocaInst(I.getType(), DL.getAllocaAddrSpace(), nullptr, + I.getName() + ".reg2mem", &F->getEntryBlock().front()); } // We cannot demote invoke instructions to the stack if their normal edge @@ -110,14 +112,17 @@ return nullptr; } + const DataLayout &DL = P->getModule()->getDataLayout(); + // Create a stack slot to hold the value. AllocaInst *Slot; if (AllocaPoint) { - Slot = new AllocaInst(P->getType(), nullptr, + Slot = new AllocaInst(P->getType(), DL.getAllocaAddrSpace(), nullptr, P->getName()+".reg2mem", AllocaPoint); } else { Function *F = P->getParent()->getParent(); - Slot = new AllocaInst(P->getType(), nullptr, P->getName() + ".reg2mem", + Slot = new AllocaInst(P->getType(), DL.getAllocaAddrSpace(), nullptr, + P->getName() + ".reg2mem", &F->getEntryBlock().front()); } Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -1225,6 +1225,7 @@ Type *AggTy = ArgTy->getElementType(); Function *Caller = TheCall->getFunction(); + const DataLayout &DL = Caller->getParent()->getDataLayout(); // If the called function is readonly, then it could not mutate the caller's // copy of the byval'd memory. In this case, it is safe to elide the copy and @@ -1238,31 +1239,30 @@ AssumptionCache *AC = IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*Caller) : nullptr; - const DataLayout &DL = Caller->getParent()->getDataLayout(); // If the pointer is already known to be sufficiently aligned, or if we can // round it up to a larger alignment, then we don't need a temporary. if (getOrEnforceKnownAlignment(Arg, ByValAlignment, DL, TheCall, AC) >= ByValAlignment) return Arg; - + // Otherwise, we have to make a memcpy to get a safe alignment. This is bad // for code quality, but rarely happens and is required for correctness. } // Create the alloca. If we have DataLayout, use nice alignment. - unsigned Align = - Caller->getParent()->getDataLayout().getPrefTypeAlignment(AggTy); + unsigned Align = DL.getPrefTypeAlignment(AggTy); // If the byval had an alignment specified, we *must* use at least that // alignment, as it is required by the byval argument (and uses of the // pointer inside the callee). Align = std::max(Align, ByValAlignment); - - Value *NewAlloca = new AllocaInst(AggTy, nullptr, Align, Arg->getName(), + + Value *NewAlloca = new AllocaInst(AggTy, DL.getAllocaAddrSpace(), + nullptr, Align, Arg->getName(), &*Caller->begin()->begin()); IFI.StaticAllocas.push_back(cast(NewAlloca)); - + // Uses of the argument in the function should use our new alloca // instead. return NewAlloca; Index: test/Assembler/alloca-addrspace-parse-error-0.ll =================================================================== --- /dev/null +++ test/Assembler/alloca-addrspace-parse-error-0.ll @@ -0,0 +1,11 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +target datalayout = "A1" + +; CHECK: :8:3: error: expected metadata after comma +define void @use_alloca() { + %alloca = alloca i32, addrspace(1), + ret void +} + +!0 = !{} Index: test/Assembler/alloca-addrspace-parse-error-1.ll =================================================================== --- /dev/null +++ test/Assembler/alloca-addrspace-parse-error-1.ll @@ -0,0 +1,12 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +target datalayout = "A1" + +; addrspace and align in wrong order +; CHECK: :8:39: error: expected metadata after comma +define void @use_alloca() { + %alloca = alloca i32, addrspace(1), align 4 + ret void +} + +!0 = !{} Index: test/Assembler/alloca-addrspace0.ll =================================================================== --- /dev/null +++ test/Assembler/alloca-addrspace0.ll @@ -0,0 +1,24 @@ +; RUN: llvm-as < %s | llvm-dis | FileCheck %s + +target datalayout = "A0" +; CHECK: target datalayout = "A0" + + +; CHECK: %alloca_scalar_no_align = alloca i32 +; CHECK-NEXT: %alloca_scalar_align4 = alloca i32, align 4 +; CHECK-NEXT: %alloca_scalar_no_align_metadata = alloca i32, !foo !0 +; CHECK-NEXT: %alloca_scalar_align4_metadata = alloca i32, align 4, !foo !0 +; CHECK-NEXT: %alloca_inalloca_scalar_no_align = alloca inalloca i32 +; CHECK-NEXT: %alloca_inalloca_scalar_align4_metadata = alloca inalloca i32, align 4, !foo !0 +define void @use_alloca() { + %alloca_scalar_no_align = alloca i32, addrspace(0) + %alloca_scalar_align4 = alloca i32, align 4, addrspace(0) + %alloca_scalar_no_align_metadata = alloca i32, addrspace(0), !foo !0 + %alloca_scalar_align4_metadata = alloca i32, align 4, addrspace(0), !foo !0 + %alloca_inalloca_scalar_no_align = alloca inalloca i32, addrspace(0) + %alloca_inalloca_scalar_align4_metadata = alloca inalloca i32, align 4, addrspace(0), !foo !0 + + ret void +} + +!0 = !{} Index: test/Assembler/datalayout-alloca-addrspace-mismatch-0.ll =================================================================== --- /dev/null +++ test/Assembler/datalayout-alloca-addrspace-mismatch-0.ll @@ -0,0 +1,9 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +target datalayout = "A1" + +; CHECK: :7:41: error: address space must match datalayout +define void @use_alloca() { + %alloca_scalar_no_align = alloca i32, addrspace(2) + ret void +} Index: test/Assembler/datalayout-alloca-addrspace-mismatch-1.ll =================================================================== --- /dev/null +++ test/Assembler/datalayout-alloca-addrspace-mismatch-1.ll @@ -0,0 +1,9 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +target datalayout = "A1" + +; CHECK: :7:50: error: address space must match datalayout +define void @use_alloca() { + %alloca_scalar_no_align = alloca i32, align 4, addrspace(2) + ret void +} Index: test/Assembler/datalayout-alloca-addrspace-mismatch-2.ll =================================================================== --- /dev/null +++ test/Assembler/datalayout-alloca-addrspace-mismatch-2.ll @@ -0,0 +1,11 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +target datalayout = "A1" + +; CHECK: :7:50: error: address space must match datalayout +define void @use_alloca() { + %alloca_scalar_no_align = alloca i32, align 4, addrspace(2), !foo !0 + ret void +} + +!0 = !{} Index: test/Assembler/datalayout-alloca-addrspace.ll =================================================================== --- /dev/null +++ test/Assembler/datalayout-alloca-addrspace.ll @@ -0,0 +1,23 @@ +; RUN: llvm-as < %s | llvm-dis | FileCheck %s + +target datalayout = "A1" +; CHECK: target datalayout = "A1" + +; CHECK: %alloca_scalar_no_align = alloca i32, addrspace(1) +; CHECK-NEXT: %alloca_scalar_align4 = alloca i32, align 4, addrspace(1) +; CHECK-NEXT: %alloca_scalar_no_align_metadata = alloca i32, addrspace(1), !foo !0 +; CHECK-NEXT: %alloca_scalar_align4_metadata = alloca i32, align 4, addrspace(1), !foo !0 +; CHECK-NEXT: %alloca_inalloca_scalar_no_align = alloca inalloca i32, addrspace(1) +; CHECK-NEXT: %alloca_inalloca_scalar_align4_metadata = alloca inalloca i32, align 4, addrspace(1), !foo !0 +define void @use_alloca() { + %alloca_scalar_no_align = alloca i32, addrspace(1) + %alloca_scalar_align4 = alloca i32, align 4, addrspace(1) + %alloca_scalar_no_align_metadata = alloca i32, addrspace(1), !foo !0 + %alloca_scalar_align4_metadata = alloca i32, align 4, addrspace(1), !foo !0 + %alloca_inalloca_scalar_no_align = alloca inalloca i32, addrspace(1) + %alloca_inalloca_scalar_align4_metadata = alloca inalloca i32, align 4, addrspace(1), !foo !0 + + ret void +} + +!0 = !{} Index: test/Assembler/invalid-datalayout-alloca-addrspace.ll =================================================================== --- /dev/null +++ test/Assembler/invalid-datalayout-alloca-addrspace.ll @@ -0,0 +1,4 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +target datalayout = "A16777216" +; CHECK: Invalid address space, must be a 24bit integer Index: test/Transforms/SROA/alloca-address-space.ll =================================================================== --- /dev/null +++ test/Transforms/SROA/alloca-address-space.ll @@ -0,0 +1,84 @@ +; RUN: opt < %s -sroa -S | FileCheck %s +target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64-A2" + +declare void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* nocapture, i8 addrspace(2)* nocapture readonly, i32, i32, i1) +declare void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(2)* nocapture readonly, i32, i32, i1) +declare void @llvm.memcpy.p2i8.p1i8.i32(i8 addrspace(2)* nocapture, i8 addrspace(1)* nocapture readonly, i32, i32, i1) +declare void @llvm.memcpy.p1i8.p1i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture readonly, i32, i32, i1) + + + +; CHECK-LABEL: @test_address_space_1_1( +; CHECK: load <2 x i64>, <2 x i64> addrspace(1)* %a, align 2 +; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(1)* {{.*}}, align 2 +; CHECK: ret void +define void @test_address_space_1_1(<2 x i64> addrspace(1)* %a, i16 addrspace(1)* %b) { + %aa = alloca <2 x i64>, align 16, addrspace(2) + %aptr = bitcast <2 x i64> addrspace(1)* %a to i8 addrspace(1)* + %aaptr = bitcast <2 x i64> addrspace(2)* %aa to i8 addrspace(2)* + call void @llvm.memcpy.p2i8.p1i8.i32(i8 addrspace(2)* %aaptr, i8 addrspace(1)* %aptr, i32 16, i32 2, i1 false) + %bptr = bitcast i16 addrspace(1)* %b to i8 addrspace(1)* + call void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* %bptr, i8 addrspace(2)* %aaptr, i32 16, i32 2, i1 false) + ret void +} + +; CHECK-LABEL: @test_address_space_1_0( +; CHECK: load <2 x i64>, <2 x i64> addrspace(1)* %a, align 2 +; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(2)* {{.*}}, align 2 +; CHECK: ret void +define void @test_address_space_1_0(<2 x i64> addrspace(1)* %a, i16 addrspace(2)* %b) { + %aa = alloca <2 x i64>, align 16, addrspace(2) + %aptr = bitcast <2 x i64> addrspace(1)* %a to i8 addrspace(1)* + %aaptr = bitcast <2 x i64> addrspace(2)* %aa to i8 addrspace(2)* + call void @llvm.memcpy.p2i8.p1i8.i32(i8 addrspace(2)* %aaptr, i8 addrspace(1)* %aptr, i32 16, i32 2, i1 false) + %bptr = bitcast i16 addrspace(2)* %b to i8 addrspace(2)* + call void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* %bptr, i8 addrspace(2)* %aaptr, i32 16, i32 2, i1 false) + ret void +} + +; CHECK-LABEL: @test_address_space_0_1( +; CHECK: load <2 x i64>, <2 x i64> addrspace(2)* %a, align 2 +; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(1)* {{.*}}, align 2 +; CHECK: ret void +define void @test_address_space_0_1(<2 x i64> addrspace(2)* %a, i16 addrspace(1)* %b) { + %aa = alloca <2 x i64>, align 16, addrspace(2) + %aptr = bitcast <2 x i64> addrspace(2)* %a to i8 addrspace(2)* + %aaptr = bitcast <2 x i64> addrspace(2)* %aa to i8 addrspace(2)* + call void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* %aaptr, i8 addrspace(2)* %aptr, i32 16, i32 2, i1 false) + %bptr = bitcast i16 addrspace(1)* %b to i8 addrspace(1)* + call void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* %bptr, i8 addrspace(2)* %aaptr, i32 16, i32 2, i1 false) + ret void +} + +%struct.struct_test_27.0.13 = type { i32, float, i64, i8, [4 x i32] } + +; CHECK-LABEL: @copy_struct( +; CHECK-NOT: memcpy +define void @copy_struct([5 x i64] %in.coerce) { +for.end: + %in = alloca %struct.struct_test_27.0.13, align 8, addrspace(2) + %0 = bitcast %struct.struct_test_27.0.13 addrspace(2)* %in to [5 x i64] addrspace(2)* + store [5 x i64] %in.coerce, [5 x i64] addrspace(2)* %0, align 8 + %scevgep9 = getelementptr %struct.struct_test_27.0.13, %struct.struct_test_27.0.13 addrspace(2)* %in, i32 0, i32 4, i32 0 + %scevgep910 = bitcast i32 addrspace(2)* %scevgep9 to i8 addrspace(2)* + call void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* undef, i8 addrspace(2)* %scevgep910, i32 16, i32 4, i1 false) + ret void +} + +%union.anon = type { i32* } + +@g = common global i32 0, align 4 +@l = common addrspace(3) global i32 0, align 4 + +; Make sure an illegal bitcast isn't introduced +; CHECK-LABEL: @pr27557( +; CHECK: %[[CAST:.*]] = bitcast i32* addrspace(2)* {{.*}} to i32 addrspace(3)* addrspace(2)* +; CHECK: store i32 addrspace(3)* @l, i32 addrspace(3)* addrspace(2)* %[[CAST]] +define void @pr27557() { + %1 = alloca %union.anon, align 8, addrspace(2) + %2 = bitcast %union.anon addrspace(2)* %1 to i32* addrspace(2)* + store i32* @g, i32* addrspace(2)* %2, align 8 + %3 = bitcast %union.anon addrspace(2)* %1 to i32 addrspace(3)* addrspace(2)* + store i32 addrspace(3)* @l, i32 addrspace(3)* addrspace(2)* %3, align 8 + ret void +} Index: tools/llvm-stress/llvm-stress.cpp =================================================================== --- tools/llvm-stress/llvm-stress.cpp +++ tools/llvm-stress/llvm-stress.cpp @@ -424,7 +424,9 @@ void Act() override { Type *Tp = pickType(); - PT->push_back(new AllocaInst(Tp, "A", BB->getFirstNonPHI())); + const DataLayout &DL = BB->getModule()->getDataLayout(); + PT->push_back(new AllocaInst(Tp, DL.getAllocaAddrSpace(), + "A", BB->getFirstNonPHI())); } }; Index: unittests/Analysis/ScalarEvolutionTest.cpp =================================================================== --- unittests/Analysis/ScalarEvolutionTest.cpp +++ unittests/Analysis/ScalarEvolutionTest.cpp @@ -306,9 +306,11 @@ // %bitcast2 = bitcast i8* %select to i32* // br i1 undef, label %loop, label %exit + const DataLayout &DL = F->getParent()->getDataLayout(); BranchInst *Br = BranchInst::Create( LoopBB, ExitBB, UndefValue::get(Type::getInt1Ty(Context)), LoopBB); - AllocaInst *Alloca = new AllocaInst(I32Ty, "alloca", Br); + AllocaInst *Alloca = new AllocaInst(I32Ty, DL.getAllocaAddrSpace(), + "alloca", Br); ConstantInt *Ci32 = ConstantInt::get(Context, APInt(32, 1)); GetElementPtrInst *Gep0 = GetElementPtrInst::Create(I32Ty, Alloca, Ci32, "gep0", Br);