diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1364,33 +1364,36 @@ static bool classof(const SDNode *N) { // For some targets, we lower some target intrinsics to a MemIntrinsicNode // with either an intrinsic or a target opcode. - return N->getOpcode() == ISD::LOAD || - N->getOpcode() == ISD::STORE || - N->getOpcode() == ISD::PREFETCH || - N->getOpcode() == ISD::ATOMIC_CMP_SWAP || - N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || - N->getOpcode() == ISD::ATOMIC_SWAP || - N->getOpcode() == ISD::ATOMIC_LOAD_ADD || - N->getOpcode() == ISD::ATOMIC_LOAD_SUB || - N->getOpcode() == ISD::ATOMIC_LOAD_AND || - N->getOpcode() == ISD::ATOMIC_LOAD_CLR || - N->getOpcode() == ISD::ATOMIC_LOAD_OR || - N->getOpcode() == ISD::ATOMIC_LOAD_XOR || - N->getOpcode() == ISD::ATOMIC_LOAD_NAND || - N->getOpcode() == ISD::ATOMIC_LOAD_MIN || - N->getOpcode() == ISD::ATOMIC_LOAD_MAX || - N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || - N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || - N->getOpcode() == ISD::ATOMIC_LOAD_FADD || - N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || - N->getOpcode() == ISD::ATOMIC_LOAD || - N->getOpcode() == ISD::ATOMIC_STORE || - N->getOpcode() == ISD::MLOAD || - N->getOpcode() == ISD::MSTORE || - N->getOpcode() == ISD::MGATHER || - N->getOpcode() == ISD::MSCATTER || - N->isMemIntrinsic() || - N->isTargetMemoryOpcode(); + switch (N->getOpcode()) { + case ISD::LOAD: + case ISD::STORE: + case ISD::PREFETCH: + case ISD::ATOMIC_CMP_SWAP: + case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: + case ISD::ATOMIC_SWAP: + case ISD::ATOMIC_LOAD_ADD: + case ISD::ATOMIC_LOAD_SUB: + case ISD::ATOMIC_LOAD_AND: + case ISD::ATOMIC_LOAD_CLR: + case ISD::ATOMIC_LOAD_OR: + case ISD::ATOMIC_LOAD_XOR: + case ISD::ATOMIC_LOAD_NAND: + case ISD::ATOMIC_LOAD_MIN: + case ISD::ATOMIC_LOAD_MAX: + case ISD::ATOMIC_LOAD_UMIN: + case ISD::ATOMIC_LOAD_UMAX: + case ISD::ATOMIC_LOAD_FADD: + case ISD::ATOMIC_LOAD_FSUB: + case ISD::ATOMIC_LOAD: + case ISD::ATOMIC_STORE: + case ISD::MLOAD: + case ISD::MSTORE: + case ISD::MGATHER: + case ISD::MSCATTER: + return true; + default: + return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); + } } }; diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -386,19 +386,32 @@ // Whether \p ID is a VP intrinsic ID. static bool IsVPIntrinsic(Intrinsic::ID); - /// \return the mask parameter or nullptr. + /// \return The mask parameter or nullptr. Value *getMaskParam() const; - /// \return the vector length parameter or nullptr. + /// \return The vector length parameter or nullptr. Value *getVectorLengthParam() const; - /// \return whether the vector length param can be ignored. + /// \return Whether the vector length param can be ignored. bool canIgnoreVectorLengthParam() const; - /// \return the static element count (vector number of elements) the vector + /// \return The static element count (vector number of elements) the vector /// length parameter applies to. ElementCount getStaticVectorLength() const; + /// \return The alignment of the pointer used by this load/store/gather or + /// scatter. + MaybeAlign getPointerAlignment() const; + // MaybeAlign setPointerAlignment(Align NewAlign); // TODO + + /// \return The pointer operand of this load,store, gather or scatter. + Value *getMemoryPointerParam() const; + static Optional getMemoryPointerParamPos(Intrinsic::ID); + + /// \return The data (payload) operand of this store or scatter. + Value *getMemoryDataParam() const; + static Optional getMemoryDataParamPos(Intrinsic::ID); + // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { return IsVPIntrinsic(I->getIntrinsicID()); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1337,6 +1337,32 @@ [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; //===---------------- Vector Predication Intrinsics --------------===// +// Memory Intrinsics +def int_vp_store : DefaultAttrsIntrinsic<[], + [ llvm_anyvector_ty, + LLVMAnyPointerType>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ NoCapture>, IntrNoSync, IntrWriteMem, IntrArgMemOnly, IntrWillReturn ]>; + +def int_vp_load : DefaultAttrsIntrinsic<[ llvm_anyvector_ty], + [ LLVMAnyPointerType>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ NoCapture>, IntrNoSync, IntrReadMem, IntrWillReturn, IntrArgMemOnly ]>; + +def int_vp_gather: DefaultAttrsIntrinsic<[ llvm_anyvector_ty], + [ LLVMVectorOfAnyPointersToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ IntrReadMem, IntrNoSync, IntrWillReturn, IntrArgMemOnly ]>; + +def int_vp_scatter: DefaultAttrsIntrinsic<[], + [ llvm_anyvector_ty, + LLVMVectorOfAnyPointersToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ IntrArgMemOnly, IntrNoSync, IntrWillReturn ]>; // TODO allow IntrNoCapture for vectors of pointers // Speculatable Binary operators let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in { diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def --- a/llvm/include/llvm/IR/VPIntrinsics.def +++ b/llvm/include/llvm/IR/VPIntrinsics.def @@ -88,6 +88,17 @@ #define HANDLE_VP_TO_OPC(OPC) #endif +// Map this VP intrinsic to its canonical functional intrinsic. +#ifndef HANDLE_VP_TO_INTRIN +#define HANDLE_VP_TO_INTRIN(ID) +#endif + +// This VP Intrinsic is a memory operation +// The pointer arg is at POINTERPOS and the data arg is at DATAPOS. +#ifndef HANDLE_VP_IS_MEMOP +#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) +#endif + /// } Property Macros ///// Integer Arithmetic { @@ -146,6 +157,35 @@ ///// } Integer Arithmetic +///// Memory Operations { +// llvm.vp.store(ptr,val,mask,vlen) +BEGIN_REGISTER_VP(vp_store, 2, 3, VP_STORE, 0) +HANDLE_VP_TO_OPC(Store) +HANDLE_VP_TO_INTRIN(masked_store) +HANDLE_VP_IS_MEMOP(vp_store, 1, 0) +END_REGISTER_VP(vp_store, VP_STORE) + +// llvm.vp.scatter(ptr,val,mask,vlen) +BEGIN_REGISTER_VP(vp_scatter, 2, 3, VP_SCATTER, 0) +HANDLE_VP_TO_INTRIN(masked_scatter) +HANDLE_VP_IS_MEMOP(vp_scatter, 1, 0) +END_REGISTER_VP(vp_scatter, VP_SCATTER) + +// llvm.vp.load(ptr,mask,vlen) +BEGIN_REGISTER_VP(vp_load, 1, 2, VP_LOAD, -1) +HANDLE_VP_TO_OPC(Load) +HANDLE_VP_TO_INTRIN(masked_load) +HANDLE_VP_IS_MEMOP(vp_load, 0, None) +END_REGISTER_VP(vp_load, VP_LOAD) + +// llvm.vp.gather(ptr,mask,vlen) +BEGIN_REGISTER_VP(vp_gather, 1, 2, VP_GATHER, -1) +HANDLE_VP_TO_INTRIN(masked_gather) +HANDLE_VP_IS_MEMOP(vp_gather, 0, None) +END_REGISTER_VP(vp_gather, VP_GATHER) + +///// } Memory Operations + #undef BEGIN_REGISTER_VP #undef BEGIN_REGISTER_VP_INTRINSIC @@ -154,3 +194,5 @@ #undef END_REGISTER_VP_INTRINSIC #undef END_REGISTER_VP_SDNODE #undef HANDLE_VP_TO_OPC +#undef HANDLE_VP_TO_INTRIN +#undef HANDLE_VP_IS_MEMOP diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -292,6 +292,53 @@ } } +/// \return the alignment of the pointer used by this load/store/gather or +/// scatter. +MaybeAlign VPIntrinsic::getPointerAlignment() const { + Optional PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID()); + assert(PtrParamOpt.hasValue() && "no pointer argument!"); + return getParamAlign(PtrParamOpt.getValue()); +} + +/// \return The pointer operand of this load,store, gather or scatter. +Value *VPIntrinsic::getMemoryPointerParam() const { + if (auto PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID())) + return getArgOperand(PtrParamOpt.getValue()); + return nullptr; +} + +Optional VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) { + switch (VPID) { + default: + return None; + +#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \ + case Intrinsic::VPID: \ + return POINTERPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + +/// \return The data (payload) operand of this store or scatter. +Value *VPIntrinsic::getMemoryDataParam() const { + auto DataParamOpt = getMemoryDataParamPos(getIntrinsicID()); + if (!DataParamOpt.hasValue()) + return nullptr; + return getArgOperand(DataParamOpt.getValue()); +} + +Optional VPIntrinsic::getMemoryDataParamPos(Intrinsic::ID VPID) { + switch (VPID) { + default: + return None; + +#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \ + case Intrinsic::VPID: \ + return DATAPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + bool VPIntrinsic::IsVPIntrinsic(Intrinsic::ID ID) { switch (ID) { default: diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp --- a/llvm/unittests/IR/VPIntrinsicTest.cpp +++ b/llvm/unittests/IR/VPIntrinsicTest.cpp @@ -44,7 +44,11 @@ " declare <8 x i32> @llvm.vp.or.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) " " declare <8 x i32> @llvm.vp.ashr.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) " " declare <8 x i32> @llvm.vp.lshr.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) " -" declare <8 x i32> @llvm.vp.shl.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) ", +" declare <8 x i32> @llvm.vp.shl.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) " +" declare void @llvm.vp.store.v16i32.p0v16i32(<16 x i32>, <16 x i32>*, <16 x i1>, i32) " +" declare void @llvm.vp.scatter.v16i32.v16p0i32(<16 x i32>, <16 x i32*>, <16 x i1>, i32) " +" declare <16 x i32> @llvm.vp.load.v16i32.p0v16i32(<16 x i32>*, <16 x i1>, i32) " +" declare <16 x i32> @llvm.vp.gather.v16i32.v16p0i32(<16 x i32*>, <16 x i1>, i32) ", Err, C); } };