diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -142,6 +142,7 @@ VecOfBitcastsToInt, AMX, PPCQuad, + AnyPtrToElt, } Kind; union { @@ -180,14 +181,15 @@ return (ArgKind)(Argument_Info & 7); } - // VecOfAnyPtrsToElt uses both an overloaded argument (for address space) - // and a reference argument (for matching vector width and element types) + // VecOfAnyPtrsToElt and AnyPtrToElt uses both an overloaded argument (for + // address space) and a reference argument (for matching vector width and + // element types) unsigned getOverloadArgNumber() const { - assert(Kind == VecOfAnyPtrsToElt); + assert(Kind == VecOfAnyPtrsToElt || Kind == AnyPtrToElt); return Argument_Info >> 16; } unsigned getRefArgNumber() const { - assert(Kind == VecOfAnyPtrsToElt); + assert(Kind == VecOfAnyPtrsToElt || Kind == AnyPtrToElt); return Argument_Info & 0xFFFF; } diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -212,6 +212,7 @@ class LLVMPointerTo : LLVMMatchType; class LLVMPointerToElt : LLVMMatchType; +class LLVMAnyPointerToElt : LLVMMatchType; class LLVMVectorOfAnyPointersToElt : LLVMMatchType; class LLVMVectorElementType : LLVMMatchType; @@ -1412,14 +1413,14 @@ // Experimental strided memory accesses def int_experimental_vp_strided_store : DefaultAttrsIntrinsic<[], [ llvm_anyvector_ty, - LLVMPointerToElt<0>, + LLVMAnyPointerToElt<0>, llvm_anyint_ty, // Stride in bytes LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty], [ NoCapture>, IntrNoSync, IntrWriteMem, IntrArgMemOnly, IntrWillReturn ]>; def int_experimental_vp_strided_load : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [ LLVMPointerToElt<0>, + [ LLVMAnyPointerToElt<0>, llvm_anyint_ty, // Stride in bytes LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty], diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -983,7 +983,8 @@ IIT_PPCF128 = 52, IIT_V3 = 53, IIT_EXTERNREF = 54, - IIT_FUNCREF = 55 + IIT_FUNCREF = 55, + IIT_ANYPTR_TO_ELT = 56, }; static void DecodeIITType(unsigned &NextElt, ArrayRef Infos, @@ -1157,6 +1158,13 @@ OutputTable.push_back(IITDescriptor::get(IITDescriptor::PtrToElt, ArgInfo)); return; } + case IIT_ANYPTR_TO_ELT: { + unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + OutputTable.push_back( + IITDescriptor::get(IITDescriptor::AnyPtrToElt, ArgNo, RefNo)); + return; + } case IIT_VEC_OF_ANYPTRS_TO_ELT: { unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); @@ -1348,6 +1356,9 @@ case IITDescriptor::VecOfAnyPtrsToElt: // Return the overloaded type (which determines the pointers address space) return Tys[D.getOverloadArgNumber()]; + case IITDescriptor::AnyPtrToElt: + // Return the overloaded type (which determines the pointers address space) + return Tys[D.getOverloadArgNumber()]; } llvm_unreachable("unhandled"); } @@ -1588,6 +1599,30 @@ return !ThisArgType->isOpaqueOrPointeeTypeMatches( ReferenceType->getElementType()); } + case IITDescriptor::AnyPtrToElt: { + unsigned RefArgNumber = D.getRefArgNumber(); + if (RefArgNumber >= ArgTys.size()) { + if (IsDeferredCheck) + return true; + // If forward referencing, already add the pointer type and + // defer the checks for later. + ArgTys.push_back(Ty); + return DeferCheck(Ty); + } + + if (!IsDeferredCheck) { + assert(D.getOverloadArgNumber() == ArgTys.size() && + "Table consistency error"); + ArgTys.push_back(Ty); + } + + auto *ReferenceType = dyn_cast(ArgTys[RefArgNumber]); + auto *ThisArgType = dyn_cast(Ty); + if (!ThisArgType || !ReferenceType) + return true; + return !ThisArgType->isOpaqueOrPointeeTypeMatches( + ReferenceType->getElementType()); + } case IITDescriptor::VecOfAnyPtrsToElt: { unsigned RefArgNumber = D.getRefArgNumber(); if (RefArgNumber >= ArgTys.size()) { diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -506,8 +506,8 @@ M, VPID, {ReturnType, Params[0]->getType()}); break; case Intrinsic::experimental_vp_strided_load: - VPFunc = - Intrinsic::getDeclaration(M, VPID, {ReturnType, Params[1]->getType()}); + VPFunc = Intrinsic::getDeclaration( + M, VPID, {ReturnType, Params[0]->getType(), Params[1]->getType()}); break; case Intrinsic::vp_gather: VPFunc = Intrinsic::getDeclaration( @@ -519,7 +519,8 @@ break; case Intrinsic::experimental_vp_strided_store: VPFunc = Intrinsic::getDeclaration( - M, VPID, {Params[0]->getType(), Params[2]->getType()}); + M, VPID, + {Params[0]->getType(), Params[1]->getType(), Params[2]->getType()}); break; case Intrinsic::vp_scatter: VPFunc = Intrinsic::getDeclaration( diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp --- a/llvm/unittests/IR/VPIntrinsicTest.cpp +++ b/llvm/unittests/IR/VPIntrinsicTest.cpp @@ -61,6 +61,9 @@ Str << "declare void " "@llvm.experimental.vp.strided.store.v8i32.i32(<8 x i32>, " "i32*, i32, <8 x i1>, i32) "; + Str << "declare void " + "@llvm.experimental.vp.strided.store.v8i32.p1i32.i32(<8 x i32>, " + "i32 addrspace(1)*, i32, <8 x i1>, i32) "; Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x " "i32*>, <8 x i1>, i32) "; Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x " @@ -68,6 +71,9 @@ Str << "declare <8 x i32> " "@llvm.experimental.vp.strided.load.v8i32.i32(i32*, i32, <8 " "x i1>, i32) "; + Str << "declare <8 x i32> " + "@llvm.experimental.vp.strided.load.v8i32.p1i32.i32(i32 " + "addrspace(1)*, i32, <8 x i1>, i32) "; Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x " "i1>, i32) "; diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -252,7 +252,8 @@ IIT_PPCF128 = 52, IIT_V3 = 53, IIT_EXTERNREF = 54, - IIT_FUNCREF = 55 + IIT_FUNCREF = 55, + IIT_ANYPTR_TO_ELT = 56, }; static void EncodeFixedValueType(MVT::SimpleValueType VT, @@ -327,6 +328,13 @@ // Encode LLVMMatchType ArgNo Sig.push_back(Number); return; + } else if (R->isSubClassOf("LLVMAnyPointerToElt")) { + Sig.push_back(IIT_ANYPTR_TO_ELT); + // Encode overloaded ArgNo + Sig.push_back(NextArgCode++); + // Encode LLVMMatchType ArgNo + Sig.push_back(Number); + return; } else if (R->isSubClassOf("LLVMPointerToElt")) Sig.push_back(IIT_PTR_TO_ELT); else if (R->isSubClassOf("LLVMVectorElementType")) @@ -415,6 +423,9 @@ if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) { ArgCodes.push_back(3 /*vAny*/); ++NumInserted; + } else if (R->isSubClassOf("LLVMAnyPointerToElt")) { + ArgCodes.push_back(4 /*iPTRAny*/); + ++NumInserted; } return; }