diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1367,33 +1367,36 @@ static bool classof(const SDNode *N) { // For some targets, we lower some target intrinsics to a MemIntrinsicNode // with either an intrinsic or a target opcode. - return N->getOpcode() == ISD::LOAD || - N->getOpcode() == ISD::STORE || - N->getOpcode() == ISD::PREFETCH || - N->getOpcode() == ISD::ATOMIC_CMP_SWAP || - N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || - N->getOpcode() == ISD::ATOMIC_SWAP || - N->getOpcode() == ISD::ATOMIC_LOAD_ADD || - N->getOpcode() == ISD::ATOMIC_LOAD_SUB || - N->getOpcode() == ISD::ATOMIC_LOAD_AND || - N->getOpcode() == ISD::ATOMIC_LOAD_CLR || - N->getOpcode() == ISD::ATOMIC_LOAD_OR || - N->getOpcode() == ISD::ATOMIC_LOAD_XOR || - N->getOpcode() == ISD::ATOMIC_LOAD_NAND || - N->getOpcode() == ISD::ATOMIC_LOAD_MIN || - N->getOpcode() == ISD::ATOMIC_LOAD_MAX || - N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || - N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || - N->getOpcode() == ISD::ATOMIC_LOAD_FADD || - N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || - N->getOpcode() == ISD::ATOMIC_LOAD || - N->getOpcode() == ISD::ATOMIC_STORE || - N->getOpcode() == ISD::MLOAD || - N->getOpcode() == ISD::MSTORE || - N->getOpcode() == ISD::MGATHER || - N->getOpcode() == ISD::MSCATTER || - N->isMemIntrinsic() || - N->isTargetMemoryOpcode(); + switch (N->getOpcode()) { + case ISD::LOAD: + case ISD::STORE: + case ISD::PREFETCH: + case ISD::ATOMIC_CMP_SWAP: + case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: + case ISD::ATOMIC_SWAP: + case ISD::ATOMIC_LOAD_ADD: + case ISD::ATOMIC_LOAD_SUB: + case ISD::ATOMIC_LOAD_AND: + case ISD::ATOMIC_LOAD_CLR: + case ISD::ATOMIC_LOAD_OR: + case ISD::ATOMIC_LOAD_XOR: + case ISD::ATOMIC_LOAD_NAND: + case ISD::ATOMIC_LOAD_MIN: + case ISD::ATOMIC_LOAD_MAX: + case ISD::ATOMIC_LOAD_UMIN: + case ISD::ATOMIC_LOAD_UMAX: + case ISD::ATOMIC_LOAD_FADD: + case ISD::ATOMIC_LOAD_FSUB: + case ISD::ATOMIC_LOAD: + case ISD::ATOMIC_STORE: + case ISD::MLOAD: + case ISD::MSTORE: + case ISD::MGATHER: + case ISD::MSCATTER: + return true; + default: + return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); + } } }; diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -403,21 +403,34 @@ // Whether \p ID is a VP intrinsic ID. static bool isVPIntrinsic(Intrinsic::ID); - /// \return the mask parameter or nullptr. + /// \return The mask parameter or nullptr. Value *getMaskParam() const; void setMaskParam(Value *); - /// \return the vector length parameter or nullptr. + /// \return The vector length parameter or nullptr. Value *getVectorLengthParam() const; void setVectorLengthParam(Value *); - /// \return whether the vector length param can be ignored. + /// \return Whether the vector length param can be ignored. bool canIgnoreVectorLengthParam() const; - /// \return the static element count (vector number of elements) the vector + /// \return The static element count (vector number of elements) the vector /// length parameter applies to. ElementCount getStaticVectorLength() const; + /// \return The alignment of the pointer used by this load/store/gather or + /// scatter. + MaybeAlign getPointerAlignment() const; + // MaybeAlign setPointerAlignment(Align NewAlign); // TODO + + /// \return The pointer operand of this load,store, gather or scatter. + Value *getMemoryPointerParam() const; + static Optional getMemoryPointerParamPos(Intrinsic::ID); + + /// \return The data (payload) operand of this store or scatter. + Value *getMemoryDataParam() const; + static Optional getMemoryDataParamPos(Intrinsic::ID); + // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { return isVPIntrinsic(I->getIntrinsicID()); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1368,6 +1368,32 @@ [], [IntrNoMem]>; //===---------------- Vector Predication Intrinsics --------------===// +// Memory Intrinsics +def int_vp_store : DefaultAttrsIntrinsic<[], + [ llvm_anyvector_ty, + LLVMAnyPointerType>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ NoCapture>, IntrNoSync, IntrWriteMem, IntrArgMemOnly, IntrWillReturn ]>; + +def int_vp_load : DefaultAttrsIntrinsic<[ llvm_anyvector_ty], + [ LLVMAnyPointerType>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ NoCapture>, IntrNoSync, IntrReadMem, IntrWillReturn, IntrArgMemOnly ]>; + +def int_vp_gather: DefaultAttrsIntrinsic<[ llvm_anyvector_ty], + [ LLVMVectorOfAnyPointersToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ IntrReadMem, IntrNoSync, IntrWillReturn, IntrArgMemOnly ]>; + +def int_vp_scatter: DefaultAttrsIntrinsic<[], + [ llvm_anyvector_ty, + LLVMVectorOfAnyPointersToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_i32_ty], + [ IntrArgMemOnly, IntrNoSync, IntrWillReturn ]>; // TODO allow IntrNoCapture for vectors of pointers // Speculatable Binary operators let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in { diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def --- a/llvm/include/llvm/IR/VPIntrinsics.def +++ b/llvm/include/llvm/IR/VPIntrinsics.def @@ -100,6 +100,17 @@ #define HANDLE_VP_TO_CONSTRAINEDFP(HASROUND, HASEXCEPT, INTRINID) #endif +// Map this VP intrinsic to its canonical functional intrinsic. +#ifndef HANDLE_VP_TO_INTRIN +#define HANDLE_VP_TO_INTRIN(ID) +#endif + +// This VP Intrinsic is a memory operation +// The pointer arg is at POINTERPOS and the data arg is at DATAPOS. +#ifndef HANDLE_VP_IS_MEMOP +#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) +#endif + /// } Property Macros ///// Integer Arithmetic { @@ -191,6 +202,36 @@ ///// } Floating-Point Arithmetic +///// Memory Operations { +// llvm.vp.store(ptr,val,mask,vlen) +BEGIN_REGISTER_VP(vp_store, 2, 3, VP_STORE, 0) +HANDLE_VP_TO_OPC(Store) +HANDLE_VP_TO_INTRIN(masked_store) +HANDLE_VP_IS_MEMOP(vp_store, 1, 0) +END_REGISTER_VP(vp_store, VP_STORE) + +// llvm.vp.scatter(ptr,val,mask,vlen) +BEGIN_REGISTER_VP(vp_scatter, 2, 3, VP_SCATTER, 0) +HANDLE_VP_TO_INTRIN(masked_scatter) +HANDLE_VP_IS_MEMOP(vp_scatter, 1, 0) +END_REGISTER_VP(vp_scatter, VP_SCATTER) + +// llvm.vp.load(ptr,mask,vlen) +BEGIN_REGISTER_VP(vp_load, 1, 2, VP_LOAD, -1) +HANDLE_VP_TO_OPC(Load) +HANDLE_VP_TO_INTRIN(masked_load) +HANDLE_VP_IS_MEMOP(vp_load, 0, None) +END_REGISTER_VP(vp_load, VP_LOAD) + +// llvm.vp.gather(ptr,mask,vlen) +BEGIN_REGISTER_VP(vp_gather, 1, 2, VP_GATHER, -1) +HANDLE_VP_TO_INTRIN(masked_gather) +HANDLE_VP_IS_MEMOP(vp_gather, 0, None) +END_REGISTER_VP(vp_gather, VP_GATHER) + +///// } Memory Operations + + #undef BEGIN_REGISTER_VP #undef BEGIN_REGISTER_VP_INTRINSIC #undef BEGIN_REGISTER_VP_SDNODE @@ -199,3 +240,5 @@ #undef END_REGISTER_VP_SDNODE #undef HANDLE_VP_TO_OPC #undef HANDLE_VP_TO_CONSTRAINEDFP +#undef HANDLE_VP_TO_INTRIN +#undef HANDLE_VP_IS_MEMOP diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -1,4 +1,4 @@ -//===-- InstrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===// +//===-- IntrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -340,6 +340,53 @@ } } +/// \return the alignment of the pointer used by this load/store/gather or +/// scatter. +MaybeAlign VPIntrinsic::getPointerAlignment() const { + Optional PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID()); + assert(PtrParamOpt.hasValue() && "no pointer argument!"); + return getParamAlign(PtrParamOpt.getValue()); +} + +/// \return The pointer operand of this load,store, gather or scatter. +Value *VPIntrinsic::getMemoryPointerParam() const { + if (auto PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID())) + return getArgOperand(PtrParamOpt.getValue()); + return nullptr; +} + +Optional VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) { + switch (VPID) { + default: + return None; + +#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \ + case Intrinsic::VPID: \ + return POINTERPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + +/// \return The data (payload) operand of this store or scatter. +Value *VPIntrinsic::getMemoryDataParam() const { + auto DataParamOpt = getMemoryDataParamPos(getIntrinsicID()); + if (!DataParamOpt.hasValue()) + return nullptr; + return getArgOperand(DataParamOpt.getValue()); +} + +Optional VPIntrinsic::getMemoryDataParamPos(Intrinsic::ID VPID) { + switch (VPID) { + default: + return None; + +#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \ + case Intrinsic::VPID: \ + return DATAPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + bool VPIntrinsic::isVPIntrinsic(Intrinsic::ID ID) { switch (ID) { default: @@ -424,10 +471,35 @@ Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID, ArrayRef Params) { assert(isVPIntrinsic(VPID) && "not a VP intrinsic"); - - // TODO: Extend this for other VP intrinsics as they are upstreamed. This - // works for binary arithmetic VP intrinsics. - auto *VPFunc = Intrinsic::getDeclaration(M, VPID, Params[0]->getType()); + Function *VPFunc; + switch (VPID) { + default: + VPFunc = Intrinsic::getDeclaration(M, VPID, Params[0]->getType()); + break; + case Intrinsic::vp_load: + VPFunc = Intrinsic::getDeclaration( + M, VPID, + {Params[0]->getType()->getPointerElementType(), Params[0]->getType()}); + break; + case Intrinsic::vp_gather: + VPFunc = Intrinsic::getDeclaration( + M, VPID, + {VectorType::get(cast(Params[0]->getType()) + ->getElementType() + ->getPointerElementType(), + cast(Params[0]->getType())), + Params[0]->getType()}); + break; + case Intrinsic::vp_store: + VPFunc = Intrinsic::getDeclaration( + M, VPID, + {Params[1]->getType()->getPointerElementType(), Params[1]->getType()}); + break; + case Intrinsic::vp_scatter: + VPFunc = Intrinsic::getDeclaration( + M, VPID, {Params[0]->getType(), Params[1]->getType()}); + break; + } assert(VPFunc && "Could not declare VP intrinsic"); return VPFunc; } diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp --- a/llvm/unittests/IR/VPIntrinsicTest.cpp +++ b/llvm/unittests/IR/VPIntrinsicTest.cpp @@ -46,6 +46,11 @@ Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode << ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) "; + Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, <8 x i1>, i32) "; + Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x i32*>, <8 x i1>, i32) "; + Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x i1>, i32) "; + Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x i1>, i32) "; + return parseAssemblyString(Str.str(), Err, C); } };