diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -2541,14 +2541,19 @@ LLVMTypeRef *ParamTypes, size_t ParamCount); +LLVM_ATTRIBUTE_C_DEPRECATED(LLVMTypeRef LLVMIntrinsicGetType( + LLVMContextRef Ctx, unsigned ID, + LLVMTypeRef *ParamTypes, size_t ParamCount), + "Use LLVMIntrinsicGetType2 instead to support " + "datalayout-dependent intrinsics"); /** * Retrieves the type of an intrinsic. For overloaded intrinsics, parameter * types must be provided to uniquely identify an overload. * * @see llvm::Intrinsic::getType() */ -LLVMTypeRef LLVMIntrinsicGetType(LLVMContextRef Ctx, unsigned ID, - LLVMTypeRef *ParamTypes, size_t ParamCount); +LLVMTypeRef LLVMIntrinsicGetType2(LLVMModuleRef Mod, unsigned ID, + LLVMTypeRef *ParamTypes, size_t ParamCount); /** * Retrieves the name of an intrinsic. diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -29,6 +29,7 @@ class LLVMContext; class Module; class AttributeList; +class DataLayout; /// This namespace contains an enum with a value for every intrinsic/builtin /// function known by LLVM. The enum values are returned by @@ -74,8 +75,8 @@ std::string getNameNoUnnamedTypes(ID Id, ArrayRef Tys); /// Return the function type for an intrinsic. - FunctionType *getType(LLVMContext &Context, ID id, - ArrayRef Tys = None); + FunctionType *getType(LLVMContext &Context, ID id, const DataLayout &DL, + ArrayRef Tys = None); /// Returns true if the intrinsic can be overloaded. bool isOverloaded(ID id); @@ -143,6 +144,7 @@ AMX, PPCQuad, AnyPtrToElt, + DataLayoutQualPointer, } Kind; union { @@ -152,6 +154,7 @@ unsigned Struct_NumElements; unsigned Argument_Info; ElementCount Vector_Width; + char DataLayout_Pointer_AddressSpace; }; enum ArgKind { @@ -210,6 +213,7 @@ Result.Vector_Width = ElementCount::get(Width, IsScalable); return Result; } + unsigned getPointerAddressSpace(const DataLayout &DL) const; }; /// Return the IIT table descriptor for the specified intrinsic into an array @@ -230,6 +234,7 @@ /// otherwise. MatchIntrinsicTypesResult matchIntrinsicSignature(FunctionType *FTy, ArrayRef &Infos, + const DataLayout &DL, SmallVectorImpl &ArgTys); /// Verify if the intrinsic has variable arguments. This method is intended to @@ -243,7 +248,8 @@ /// AgTys vector. /// /// Returns false if the given function is not a valid intrinsic call. - bool getIntrinsicSignature(Function *F, SmallVectorImpl &ArgTys); + bool getIntrinsicSignature(Function *F, const DataLayout &DL, + SmallVectorImpl &ArgTys); // Checks if the intrinsic name matches with its signature and if not // returns the declaration with the same signature and remangled name. diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -182,6 +182,14 @@ int AddrSpace = addrspace; } +class LLVMDataLayoutPointerType + : LLVMType{ + LLVMType ElTy = elty; + // Must be one of 'A', 'G', 'P' (the DataLayout-defined address spaces) + string AddrSpace = addrspace; +} + + class LLVMPointerType : LLVMQualPointerType; @@ -261,6 +269,9 @@ def llvm_ptr_ty : LLVMPointerType; // i8* def llvm_ptrptr_ty : LLVMPointerType; // i8** def llvm_anyptr_ty : LLVMAnyPointerType; // (space)i8* +def llvm_prog_ptr_ty : LLVMDataLayoutPointerType; // (prog)i8* +def llvm_alloca_ptr_ty : LLVMDataLayoutPointerType; // (alloca)i8* +def llvm_global_ptr_ty : LLVMDataLayoutPointerType; // (globals)i8* def llvm_empty_ty : LLVMType; // { } def llvm_descriptor_ty : LLVMPointerType; // { }* def llvm_metadata_ty : LLVMType; // !{...} diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -2362,7 +2362,17 @@ LLVMTypeRef *ParamTypes, size_t ParamCount) { auto IID = llvm_map_to_intrinsic_id(ID); ArrayRef Tys(unwrap(ParamTypes), ParamCount); - return wrap(llvm::Intrinsic::getType(*unwrap(Ctx), IID, Tys)); + DataLayout DefaultDL(""); + return wrap(llvm::Intrinsic::getType(*unwrap(Ctx), IID, DefaultDL, Tys)); +} + +LLVMTypeRef LLVMIntrinsicGetType2(LLVMModuleRef Mod, unsigned ID, + LLVMTypeRef *ParamTypes, size_t ParamCount) { + auto IID = llvm_map_to_intrinsic_id(ID); + ArrayRef Tys(unwrap(ParamTypes), ParamCount); + Module *M = unwrap(Mod); + return wrap( + llvm::Intrinsic::getType(M->getContext(), IID, M->getDataLayout(), Tys)); } const char *LLVMIntrinsicCopyOverloadedName(unsigned ID, diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -964,9 +964,10 @@ if (HasUnnamedType) { assert(M && "unnamed types need a module"); if (!FT) - FT = Intrinsic::getType(M->getContext(), Id, Tys); + FT = Intrinsic::getType(M->getContext(), Id, M->getDataLayout(), Tys); else - assert((FT == Intrinsic::getType(M->getContext(), Id, Tys)) && + assert((FT == Intrinsic::getType(M->getContext(), Id, M->getDataLayout(), + Tys)) && "Provided FunctionType must match arguments"); return M->getUniqueIntrinsicName(Result, Id, FT); } @@ -1050,6 +1051,7 @@ IIT_ANYPTR_TO_ELT = 56, IIT_I2 = 57, IIT_I4 = 58, + IIT_DATALAYOUTPTR = 59, }; static void DecodeIITType(unsigned &NextElt, ArrayRef Infos, @@ -1189,6 +1191,12 @@ DecodeIITType(NextElt, Infos, Info, OutputTable); return; } + case IIT_DATALAYOUTPTR: { // [DATALAYOUTPTR addrspace-character, subtype] + OutputTable.push_back(IITDescriptor::get( + IITDescriptor::DataLayoutQualPointer, Infos[NextElt++])); + DecodeIITType(NextElt, Infos, Info, OutputTable); + return; + } case IIT_ARG: { unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); OutputTable.push_back(IITDescriptor::get(IITDescriptor::Argument, ArgInfo)); @@ -1296,6 +1304,24 @@ #include "llvm/IR/IntrinsicImpl.inc" #undef GET_INTRINSIC_GENERATOR_GLOBAL +unsigned +Intrinsic::IITDescriptor::getPointerAddressSpace(const DataLayout &DL) const { + if (Kind == DataLayoutQualPointer) { + switch (DataLayout_Pointer_AddressSpace) { + case 'A': + return DL.getAllocaAddrSpace(); + case 'G': + return DL.getDefaultGlobalsAddressSpace(); + case 'P': + return DL.getProgramAddressSpace(); + default: + llvm_unreachable("Invalid pointer address space character"); + } + } + assert(Kind == Pointer); + return Pointer_AddressSpace; +} + void Intrinsic::getIntrinsicInfoTableEntries(ID id, SmallVectorImpl &T){ // Check to see if the intrinsic's type was expressible by the table. @@ -1330,7 +1356,8 @@ } static Type *DecodeFixedType(ArrayRef &Infos, - ArrayRef Tys, LLVMContext &Context) { + ArrayRef Tys, LLVMContext &Context, + const DataLayout &DL) { using namespace Intrinsic; IITDescriptor D = Infos.front(); @@ -1353,15 +1380,17 @@ case IITDescriptor::Integer: return IntegerType::get(Context, D.Integer_Width); case IITDescriptor::Vector: - return VectorType::get(DecodeFixedType(Infos, Tys, Context), + return VectorType::get(DecodeFixedType(Infos, Tys, Context, DL), D.Vector_Width); - case IITDescriptor::Pointer: - return PointerType::get(DecodeFixedType(Infos, Tys, Context), - D.Pointer_AddressSpace); + case IITDescriptor::DataLayoutQualPointer: + case IITDescriptor::Pointer: { + unsigned AS = D.getPointerAddressSpace(DL); + return PointerType::get(DecodeFixedType(Infos, Tys, Context, DL), AS); + } case IITDescriptor::Struct: { SmallVector Elts; for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i) - Elts.push_back(DecodeFixedType(Infos, Tys, Context)); + Elts.push_back(DecodeFixedType(Infos, Tys, Context, DL)); return StructType::get(Context, Elts); } case IITDescriptor::Argument: @@ -1394,7 +1423,7 @@ return VectorType::getHalfElementsVectorType(cast( Tys[D.getArgumentNumber()])); case IITDescriptor::SameVecWidthArgument: { - Type *EltTy = DecodeFixedType(Infos, Tys, Context); + Type *EltTy = DecodeFixedType(Infos, Tys, Context, DL); Type *Ty = Tys[D.getArgumentNumber()]; if (auto *VTy = dyn_cast(Ty)) return VectorType::get(EltTy, VTy->getElementCount()); @@ -1434,17 +1463,17 @@ llvm_unreachable("unhandled"); } -FunctionType *Intrinsic::getType(LLVMContext &Context, - ID id, ArrayRef Tys) { +FunctionType *Intrinsic::getType(LLVMContext &Context, ID id, + const DataLayout &DL, ArrayRef Tys) { SmallVector Table; getIntrinsicInfoTableEntries(id, Table); ArrayRef TableRef = Table; - Type *ResultTy = DecodeFixedType(TableRef, Tys, Context); + Type *ResultTy = DecodeFixedType(TableRef, Tys, Context, DL); SmallVector ArgTys; while (!TableRef.empty()) - ArgTys.push_back(DecodeFixedType(TableRef, Tys, Context)); + ArgTys.push_back(DecodeFixedType(TableRef, Tys, Context, DL)); // DecodeFixedType returns Void for IITDescriptor::Void and IITDescriptor::VarArg // If we see void type as the type of the last argument, it is vararg intrinsic @@ -1481,7 +1510,7 @@ Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef Tys) { // There can never be multiple globals with the same name of different types, // because intrinsics must be a specific type. - auto *FT = getType(M->getContext(), id, Tys); + auto *FT = getType(M->getContext(), id, M->getDataLayout(), Tys); return cast( M->getOrInsertFunction( Tys.empty() ? getName(id) : getName(id, Tys, M, FT), FT) @@ -1501,11 +1530,11 @@ using DeferredIntrinsicMatchPair = std::pair>; -static bool matchIntrinsicType( - Type *Ty, ArrayRef &Infos, - SmallVectorImpl &ArgTys, - SmallVectorImpl &DeferredChecks, - bool IsDeferredCheck) { +static bool +matchIntrinsicType(Type *Ty, ArrayRef &Infos, + const DataLayout &DL, SmallVectorImpl &ArgTys, + SmallVectorImpl &DeferredChecks, + bool IsDeferredCheck) { using namespace Intrinsic; // If we ran out of descriptors, there are too many arguments. @@ -1538,13 +1567,14 @@ case IITDescriptor::Vector: { VectorType *VT = dyn_cast(Ty); return !VT || VT->getElementCount() != D.Vector_Width || - matchIntrinsicType(VT->getElementType(), Infos, ArgTys, + matchIntrinsicType(VT->getElementType(), Infos, DL, ArgTys, DeferredChecks, IsDeferredCheck); } + case IITDescriptor::DataLayoutQualPointer: case IITDescriptor::Pointer: { PointerType *PT = dyn_cast(Ty); - if (!PT || PT->getAddressSpace() != D.Pointer_AddressSpace) - return true; + if (!PT || PT->getAddressSpace() != D.getPointerAddressSpace(DL)) + return true; if (!PT->isOpaque()) { /* Manually consume a pointer to empty struct descriptor, which is * used for externref. We don't want to enforce that the struct is @@ -1556,7 +1586,7 @@ return false; } return matchIntrinsicType(PT->getNonOpaquePointerElementType(), Infos, - ArgTys, DeferredChecks, IsDeferredCheck); + DL, ArgTys, DeferredChecks, IsDeferredCheck); } // Consume IIT descriptors relating to the pointer element type. // FIXME: Intrinsic type matching of nested single value types or even @@ -1579,7 +1609,7 @@ return true; for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i) - if (matchIntrinsicType(ST->getElementType(i), Infos, ArgTys, + if (matchIntrinsicType(ST->getElementType(i), Infos, DL, ArgTys, DeferredChecks, IsDeferredCheck)) return true; return false; @@ -1664,7 +1694,7 @@ return true; EltTy = ThisArgType->getElementType(); } - return matchIntrinsicType(EltTy, Infos, ArgTys, DeferredChecks, + return matchIntrinsicType(EltTy, Infos, DL, ArgTys, DeferredChecks, IsDeferredCheck); } case IITDescriptor::PtrToArgument: { @@ -1776,27 +1806,26 @@ llvm_unreachable("unhandled"); } -Intrinsic::MatchIntrinsicTypesResult -Intrinsic::matchIntrinsicSignature(FunctionType *FTy, - ArrayRef &Infos, - SmallVectorImpl &ArgTys) { +Intrinsic::MatchIntrinsicTypesResult Intrinsic::matchIntrinsicSignature( + FunctionType *FTy, ArrayRef &Infos, + const DataLayout &DL, SmallVectorImpl &ArgTys) { SmallVector DeferredChecks; - if (matchIntrinsicType(FTy->getReturnType(), Infos, ArgTys, DeferredChecks, - false)) - return MatchIntrinsicTypes_NoMatchRet; + if (matchIntrinsicType(FTy->getReturnType(), Infos, DL, ArgTys, + DeferredChecks, false)) + return MatchIntrinsicTypes_NoMatchRet; unsigned NumDeferredReturnChecks = DeferredChecks.size(); for (auto *Ty : FTy->params()) - if (matchIntrinsicType(Ty, Infos, ArgTys, DeferredChecks, false)) - return MatchIntrinsicTypes_NoMatchArg; + if (matchIntrinsicType(Ty, Infos, DL, ArgTys, DeferredChecks, false)) + return MatchIntrinsicTypes_NoMatchArg; for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) { DeferredIntrinsicMatchPair &Check = DeferredChecks[I]; - if (matchIntrinsicType(Check.first, Check.second, ArgTys, DeferredChecks, - true)) - return I < NumDeferredReturnChecks ? MatchIntrinsicTypes_NoMatchRet - : MatchIntrinsicTypes_NoMatchArg; + if (matchIntrinsicType(Check.first, Check.second, DL, ArgTys, + DeferredChecks, true)) + return I < NumDeferredReturnChecks ? MatchIntrinsicTypes_NoMatchRet + : MatchIntrinsicTypes_NoMatchArg; } return MatchIntrinsicTypes_Match; @@ -1822,7 +1851,7 @@ return true; } -bool Intrinsic::getIntrinsicSignature(Function *F, +bool Intrinsic::getIntrinsicSignature(Function *F, const DataLayout &DL, SmallVectorImpl &ArgTys) { Intrinsic::ID ID = F->getIntrinsicID(); if (!ID) @@ -1832,7 +1861,7 @@ getIntrinsicInfoTableEntries(ID, Table); ArrayRef TableRef = Table; - if (Intrinsic::matchIntrinsicSignature(F->getFunctionType(), TableRef, + if (Intrinsic::matchIntrinsicSignature(F->getFunctionType(), TableRef, DL, ArgTys) != Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) { return false; @@ -1845,7 +1874,7 @@ Optional Intrinsic::remangleIntrinsicFunction(Function *F) { SmallVector ArgTys; - if (!getIntrinsicSignature(F, ArgTys)) + if (!getIntrinsicSignature(F, F->getParent()->getDataLayout(), ArgTys)) return None; Intrinsic::ID ID = F->getIntrinsicID(); diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -986,7 +986,7 @@ FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false); SmallVector OverloadTys; Intrinsic::MatchIntrinsicTypesResult Res = - matchIntrinsicSignature(FTy, TableRef, OverloadTys); + matchIntrinsicSignature(FTy, TableRef, M->getDataLayout(), OverloadTys); (void)Res; assert(Res == Intrinsic::MatchIntrinsicTypes_Match && TableRef.empty() && "Wrong types for intrinsic!"); diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -4915,7 +4915,7 @@ // Walk the descriptors to extract overloaded types. SmallVector ArgTys; Intrinsic::MatchIntrinsicTypesResult Res = - Intrinsic::matchIntrinsicSignature(IFTy, TableRef, ArgTys); + Intrinsic::matchIntrinsicSignature(IFTy, TableRef, DL, ArgTys); Check(Res != Intrinsic::MatchIntrinsicTypes_NoMatchRet, "Intrinsic has incorrect return type!", IF); Check(Res != Intrinsic::MatchIntrinsicTypes_NoMatchArg, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -120,7 +120,8 @@ std::function &, SmallVectorImpl &)> Func) { SmallVector ArgTys; - if (!Intrinsic::getIntrinsicSignature(OldIntr.getCalledFunction(), ArgTys)) + if (!Intrinsic::getIntrinsicSignature(OldIntr.getCalledFunction(), + IC.getDataLayout(), ArgTys)) return None; SmallVector Args(OldIntr.args()); @@ -1168,7 +1169,8 @@ // Validate function argument and return types, extracting overloaded types // along the way. SmallVector OverloadTys; - if (!Intrinsic::getIntrinsicSignature(II.getCalledFunction(), OverloadTys)) + if (!Intrinsic::getIntrinsicSignature(II.getCalledFunction(), + IC.getDataLayout(), OverloadTys)) return nullptr; Module *M = II.getParent()->getParent()->getParent(); diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -257,6 +257,7 @@ IIT_ANYPTR_TO_ELT = 56, IIT_I2 = 57, IIT_I4 = 58, + IIT_DATALAYOUTPTR = 59, }; static void EncodeFixedValueType(MVT::SimpleValueType VT, @@ -381,16 +382,28 @@ } case MVT::iPTR: { - unsigned AddrSpace = 0; - if (R->isSubClassOf("LLVMQualPointerType")) { - AddrSpace = R->getValueAsInt("AddrSpace"); - assert(AddrSpace < 256 && "Address space exceeds 255"); - } - if (AddrSpace) { - Sig.push_back(IIT_ANYPTR); - Sig.push_back(AddrSpace); + if (R->isSubClassOf("LLVMDataLayoutPointerType")) { + StringRef AddrSpaceChar = R->getValueAsString("AddrSpace"); + if (AddrSpaceChar.size() != 1 || + !StringRef("AGP").contains(AddrSpaceChar)) + PrintFatalError(R, + "Invalid address space value: '" + AddrSpaceChar + "'"); + Sig.push_back(IIT_DATALAYOUTPTR); + Sig.push_back(AddrSpaceChar[0]); } else { - Sig.push_back(IIT_PTR); + unsigned AddrSpace = 0; + if (R->isSubClassOf("LLVMQualPointerType")) { + AddrSpace = R->getValueAsInt("AddrSpace"); + if (!isUInt<8>(AddrSpace)) + PrintFatalError(R, + "Address space " + Twine(AddrSpace) + " exceeds 255"); + } + if (AddrSpace) { + Sig.push_back(IIT_ANYPTR); + Sig.push_back(AddrSpace); + } else { + Sig.push_back(IIT_PTR); + } } return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, NextArgCode, Sig, Mapping); diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -283,8 +283,8 @@ ArrayRef tableRef = table; SmallVector overloadedArgTys; - if (llvm::Intrinsic::matchIntrinsicSignature(ft, tableRef, - overloadedArgTys) != + if (llvm::Intrinsic::matchIntrinsicSignature( + ft, tableRef, module->getDataLayout(), overloadedArgTys) != llvm::Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) { return op.emitOpError("intrinsic type is not a match"); }