[VP] Preserve address space of pointer for strided load/store intrinsics.

[craig.topper]

Summary

This adds LLVMAnyPointerToElt to use instead of LLVMPointerToElt.
This allows us to preserve the address space as part of the type
overload for the intrinsic, but still require the vector element
type to match the pointer type.

[simoll]

Landing this will break VP strided load/store RVV and VE tests. I'll attach a diff for the VE ones that you can merge into this patch.

[simoll]

Update for vp_strided_load|store.ll tests to be fused into this patch (no attribution necessary):

vp_strided_load_ve.patch16 KBDownload

[craig.topper]

In D122042#3395504, @simoll wrote:

Landing this will break VP strided load/store RVV and VE tests. I'll attach a diff for the VE ones that you can merge into this patch.

The parser barely relies on the type parts of the name so it doesn't fail. But I'll add in your patch. The RVV tests haven't been committed yet

[craig.topper]

Update VE tests

[nikic]

LGTM

llvm/include/llvm/IR/Intrinsics.h

enum IITDescriptorKind {
       PtrToElt,
       VecOfAnyPtrsToElt,
       VecElementArgument,
       Subdivide2Argument,
       Subdivide4Argument,
       VecOfBitcastsToInt,
       AMX,
       PPCQuad,
+      AnyPtrToElt,
     } Kind;
 
     union {
       unsigned Integer_Width;
       unsigned Float_Width;
       unsigned Pointer_AddressSpace;
       unsigned Struct_NumElements;
       unsigned Argument_Info;
ArgKind getArgumentKind() const {
       assert(Kind == Argument || Kind == ExtendArgument ||
              Kind == TruncArgument || Kind == HalfVecArgument ||
              Kind == SameVecWidthArgument || Kind == PtrToArgument ||
              Kind == VecElementArgument || Kind == Subdivide2Argument ||
              Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt);
       return (ArgKind)(Argument_Info & 7);
     }
 
-    // VecOfAnyPtrsToElt uses both an overloaded argument (for address space)
-    // and a reference argument (for matching vector width and element types)
+    // VecOfAnyPtrsToElt and AnyPtrToElt uses both an overloaded argument (for
+    // address space) and a reference argument (for matching vector width and
+    // element types)
     unsigned getOverloadArgNumber() const {
-      assert(Kind == VecOfAnyPtrsToElt);
+      assert(Kind == VecOfAnyPtrsToElt || Kind == AnyPtrToElt);
       return Argument_Info >> 16;
     }
     unsigned getRefArgNumber() const {
-      assert(Kind == VecOfAnyPtrsToElt);
+      assert(Kind == VecOfAnyPtrsToElt || Kind == AnyPtrToElt);
       return Argument_Info & 0xFFFF;
     }
 
     static IITDescriptor get(IITDescriptorKind K, unsigned Field) {
       IITDescriptor Result = { K, { Field } };
       return Result;
     }
 

llvm/include/llvm/IR/Intrinsics.td

 // number of elements.
 class LLVMScalarOrSameVectorWidth<int idx, LLVMType elty>
   : LLVMMatchType<idx> {
   ValueType ElTy = elty.VT;
 }
 
 class LLVMPointerTo<int num> : LLVMMatchType<num>;
 class LLVMPointerToElt<int num> : LLVMMatchType<num>;
+class LLVMAnyPointerToElt<int num> : LLVMMatchType<num>;
 class LLVMVectorOfAnyPointersToElt<int num> : LLVMMatchType<num>;
 class LLVMVectorElementType<int num> : LLVMMatchType<num>;
 
 // Match the type of another intrinsic parameter that is expected to be a
 // vector type, but change the element count to be half as many.
 class LLVMHalfElementsVectorType<int num> : LLVMMatchType<num>;
 
 // Match the type of another intrinsic parameter that is expected to be a
def int_vp_scatter: DefaultAttrsIntrinsic<[],
                                 LLVMVectorOfAnyPointersToElt<0>,
                                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                 llvm_i32_ty],
                               [ IntrArgMemOnly, IntrNoSync, IntrWillReturn ]>; // TODO allow IntrNoCapture for vectors of pointers
 
 // Experimental strided memory accesses
 def int_experimental_vp_strided_store : DefaultAttrsIntrinsic<[],
                              [ llvm_anyvector_ty,
-                               LLVMPointerToElt<0>,
+                               LLVMAnyPointerToElt<0>,
                                llvm_anyint_ty, // Stride in bytes
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty],
                              [ NoCapture<ArgIndex<1>>, IntrNoSync, IntrWriteMem, IntrArgMemOnly, IntrWillReturn ]>;
 
 def int_experimental_vp_strided_load  : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
-                             [ LLVMPointerToElt<0>,
+                             [ LLVMAnyPointerToElt<0>,
                                llvm_anyint_ty, // Stride in bytes
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty],
                              [ NoCapture<ArgIndex<0>>, IntrNoSync, IntrReadMem, IntrWillReturn, IntrArgMemOnly ]>;
 
 // Speculatable Binary operators
 let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in {
   def int_vp_add : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],

llvm/lib/IR/Function.cpp

enum IIT_Info {
   IIT_V128 = 47,
   IIT_BF16 = 48,
   IIT_STRUCT9 = 49,
   IIT_V256 = 50,
   IIT_AMX  = 51,
   IIT_PPCF128 = 52,
   IIT_V3 = 53,
   IIT_EXTERNREF = 54,
-  IIT_FUNCREF = 55
+  IIT_FUNCREF = 55,
+  IIT_ANYPTR_TO_ELT = 56,
 };
 
 static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
                       IIT_Info LastInfo,
                       SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
   using namespace Intrinsic;
 
   bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC);
OutputTable.push_back(IITDescriptor::get(IITDescriptor::PtrToArgument,
                                              ArgInfo));
     return;
   }
   case IIT_PTR_TO_ELT: {
     unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::PtrToElt, ArgInfo));
     return;
   }
+  case IIT_ANYPTR_TO_ELT: {
+    unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(
+        IITDescriptor::get(IITDescriptor::AnyPtrToElt, ArgNo, RefNo));
+    return;
+  }
   case IIT_VEC_OF_ANYPTRS_TO_ELT: {
     unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
     unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
     OutputTable.push_back(
         IITDescriptor::get(IITDescriptor::VecOfAnyPtrsToElt, ArgNo, RefNo));
     return;
   }
   case IIT_EMPTYSTRUCT:
case IITDescriptor::VecOfBitcastsToInt: {
     Type *Ty = Tys[D.getArgumentNumber()];
     VectorType *VTy = dyn_cast<VectorType>(Ty);
     assert(VTy && "Expected an argument of Vector Type");
     return VectorType::getInteger(VTy);
   }
   case IITDescriptor::VecOfAnyPtrsToElt:
     // Return the overloaded type (which determines the pointers address space)
     return Tys[D.getOverloadArgNumber()];
+  case IITDescriptor::AnyPtrToElt:
+    // Return the overloaded type (which determines the pointers address space)
+    return Tys[D.getOverloadArgNumber()];
   }
   llvm_unreachable("unhandled");
 }
 
 FunctionType *Intrinsic::getType(LLVMContext &Context,
                                  ID id, ArrayRef<Type*> Tys) {
   SmallVector<IITDescriptor, 8> Table;
   getIntrinsicInfoTableEntries(id, Table);
case IITDescriptor::PtrToElt: {
         dyn_cast<VectorType> (ArgTys[D.getArgumentNumber()]);
       PointerType *ThisArgType = dyn_cast<PointerType>(Ty);
 
       if (!ThisArgType || !ReferenceType)
         return true;
       return !ThisArgType->isOpaqueOrPointeeTypeMatches(
           ReferenceType->getElementType());
     }
+    case IITDescriptor::AnyPtrToElt: {
+      unsigned RefArgNumber = D.getRefArgNumber();
+      if (RefArgNumber >= ArgTys.size()) {
+        if (IsDeferredCheck)
+          return true;
+        // If forward referencing, already add the pointer type and
+        // defer the checks for later.
+        ArgTys.push_back(Ty);
+        return DeferCheck(Ty);
+      }
+
+      if (!IsDeferredCheck) {
+        assert(D.getOverloadArgNumber() == ArgTys.size() &&
+               "Table consistency error");
+        ArgTys.push_back(Ty);
+      }
+
+      auto *ReferenceType = dyn_cast<VectorType>(ArgTys[RefArgNumber]);
+      auto *ThisArgType = dyn_cast<PointerType>(Ty);
+      if (!ThisArgType || !ReferenceType)
+        return true;
+      return !ThisArgType->isOpaqueOrPointeeTypeMatches(
+          ReferenceType->getElementType());
+    }
     case IITDescriptor::VecOfAnyPtrsToElt: {
       unsigned RefArgNumber = D.getRefArgNumber();
       if (RefArgNumber >= ArgTys.size()) {
         if (IsDeferredCheck)
           return true;
         // If forward referencing, already add the pointer-vector type and
         // defer the checks for later.
         ArgTys.push_back(Ty);

llvm/lib/IR/IntrinsicInst.cpp

Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
   case Intrinsic::vp_select:
     VPFunc = Intrinsic::getDeclaration(M, VPID, {Params[1]->getType()});
     break;
   case Intrinsic::vp_load:
     VPFunc = Intrinsic::getDeclaration(
         M, VPID, {ReturnType, Params[0]->getType()});
     break;
   case Intrinsic::experimental_vp_strided_load:
-    VPFunc =
-        Intrinsic::getDeclaration(M, VPID, {ReturnType, Params[1]->getType()});
+    VPFunc = Intrinsic::getDeclaration(
+        M, VPID, {ReturnType, Params[0]->getType(), Params[1]->getType()});
     break;
   case Intrinsic::vp_gather:
     VPFunc = Intrinsic::getDeclaration(
         M, VPID, {ReturnType, Params[0]->getType()});
     break;
   case Intrinsic::vp_store:
     VPFunc = Intrinsic::getDeclaration(
         M, VPID, {Params[0]->getType(), Params[1]->getType()});
     break;
   case Intrinsic::experimental_vp_strided_store:
     VPFunc = Intrinsic::getDeclaration(
-        M, VPID, {Params[0]->getType(), Params[2]->getType()});
+        M, VPID,
+        {Params[0]->getType(), Params[1]->getType(), Params[2]->getType()});
     break;
   case Intrinsic::vp_scatter:
     VPFunc = Intrinsic::getDeclaration(
         M, VPID, {Params[0]->getType(), Params[1]->getType()});
     break;
   }
   assert(VPFunc && "Could not declare VP intrinsic");
   return VPFunc;

llvm/test/CodeGen/VE/Vector/vp_strided_load.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=ve-unknown-unknown -mattr=+vpu | FileCheck %s
 
-declare <256 x float> @llvm.experimental.vp.strided.load.v256f32.i64(float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare <256 x float> @llvm.experimental.vp.strided.load.v256f32.p0f32.i64(float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc <256 x float> @vp_strided_load_v256f32_rrm(float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256f32_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vseq %v0
 ; CHECK-NEXT:    vmulu.l %v0, %s1, %v0, %vm1
 ; CHECK-NEXT:    vaddu.l %v0, %s0, %v0, %vm1
 ; CHECK-NEXT:    vgtu %v0, %v0, 0, 0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  %r = call <256 x float> @llvm.experimental.vp.strided.load.v256f32.i64(float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  %r = call <256 x float> @llvm.experimental.vp.strided.load.v256f32.p0f32.i64(float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret <256 x float> %r
 }
 
 define fastcc <256 x float> @vp_strided_load_v256f32_rr(float* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256f32_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vldu %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x float> @llvm.experimental.vp.strided.load.v256f32.i64(float* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x float> @llvm.experimental.vp.strided.load.v256f32.p0f32.i64(float* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret <256 x float> %r
 }
 
 define fastcc <256 x float> @vp_strided_load_v256f32_ri(float* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256f32_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vldu %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x float> @llvm.experimental.vp.strided.load.v256f32.i64(float* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x float> @llvm.experimental.vp.strided.load.v256f32.p0f32.i64(float* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret <256 x float> %r
 }
 
-declare <256 x i32> @llvm.experimental.vp.strided.load.v256i32.i64(i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare <256 x i32> @llvm.experimental.vp.strided.load.v256i32.p0i32.i64(i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc <256 x i32> @vp_strided_load_v256i32_rrm(i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256i32_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vseq %v0
 ; CHECK-NEXT:    vmulu.l %v0, %s1, %v0, %vm1
 ; CHECK-NEXT:    vaddu.l %v0, %s0, %v0, %vm1
 ; CHECK-NEXT:    vgtl.zx %v0, %v0, 0, 0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  %r = call <256 x i32> @llvm.experimental.vp.strided.load.v256i32.i64(i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  %r = call <256 x i32> @llvm.experimental.vp.strided.load.v256i32.p0i32.i64(i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret <256 x i32> %r
 }
 
 define fastcc <256 x i32> @vp_strided_load_v256i32_rr(i32* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256i32_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vldl.zx %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x i32> @llvm.experimental.vp.strided.load.v256i32.i64(i32* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x i32> @llvm.experimental.vp.strided.load.v256i32.p0i32.i64(i32* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret <256 x i32> %r
 }
 
 define fastcc <256 x i32> @vp_strided_load_v256i32_ri(i32* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256i32_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vldl.zx %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x i32> @llvm.experimental.vp.strided.load.v256i32.i64(i32* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x i32> @llvm.experimental.vp.strided.load.v256i32.p0i32.i64(i32* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret <256 x i32> %r
 }
 
-declare <256 x double> @llvm.experimental.vp.strided.load.v256f64.i64(double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare <256 x double> @llvm.experimental.vp.strided.load.v256f64.p0f64.i64(double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc <256 x double> @vp_strided_load_v256f64_rrm(double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256f64_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vseq %v0
 ; CHECK-NEXT:    vmulu.l %v0, %s1, %v0, %vm1
 ; CHECK-NEXT:    vaddu.l %v0, %s0, %v0, %vm1
 ; CHECK-NEXT:    vgt %v0, %v0, 0, 0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  %r = call <256 x double> @llvm.experimental.vp.strided.load.v256f64.i64(double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  %r = call <256 x double> @llvm.experimental.vp.strided.load.v256f64.p0f64.i64(double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret <256 x double> %r
 }
 
 define fastcc <256 x double> @vp_strided_load_v256f64_rr(double* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256f64_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x double> @llvm.experimental.vp.strided.load.v256f64.i64(double* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x double> @llvm.experimental.vp.strided.load.v256f64.p0f64.i64(double* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret <256 x double> %r
 }
 
 define fastcc <256 x double> @vp_strided_load_v256f64_ri(double* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256f64_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x double> @llvm.experimental.vp.strided.load.v256f64.i64(double* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x double> @llvm.experimental.vp.strided.load.v256f64.p0f64.i64(double* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret <256 x double> %r
 }
 
-declare <256 x i64> @llvm.experimental.vp.strided.load.v256i64.i64(i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare <256 x i64> @llvm.experimental.vp.strided.load.v256i64.p0i64.i64(i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc <256 x i64> @vp_strided_load_v256i64_rrm(i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256i64_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vseq %v0
 ; CHECK-NEXT:    vmulu.l %v0, %s1, %v0, %vm1
 ; CHECK-NEXT:    vaddu.l %v0, %s0, %v0, %vm1
 ; CHECK-NEXT:    vgt %v0, %v0, 0, 0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  %r = call <256 x i64> @llvm.experimental.vp.strided.load.v256i64.i64(i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  %r = call <256 x i64> @llvm.experimental.vp.strided.load.v256i64.p0i64.i64(i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret <256 x i64> %r
 }
 
 define fastcc <256 x i64> @vp_strided_load_v256i64_rr(i64* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256i64_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x i64> @llvm.experimental.vp.strided.load.v256i64.i64(i64* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x i64> @llvm.experimental.vp.strided.load.v256i64.p0i64.i64(i64* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret <256 x i64> %r
 }
 
 define fastcc <256 x i64> @vp_strided_load_v256i64_ri(i64* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_load_v256i64_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  %r = call <256 x i64> @llvm.experimental.vp.strided.load.v256i64.i64(i64* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  %r = call <256 x i64> @llvm.experimental.vp.strided.load.v256i64.p0i64.i64(i64* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret <256 x i64> %r
 }

llvm/test/CodeGen/VE/Vector/vp_strided_store.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=ve-unknown-unknown -mattr=+vpu | FileCheck %s
 
-declare void @llvm.experimental.vp.strided.store.v256f32.i64(<256 x float> %val, float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare void @llvm.experimental.vp.strided.store.v256f32.p0f32.i64(<256 x float> %val, float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc void @vp_strided_store_v256f32_rrm(<256 x float> %val, float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256f32_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vstu %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  call void @llvm.experimental.vp.strided.store.v256f32.i64(<256 x float> %val, float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256f32.p0f32.i64(<256 x float> %val, float* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256f32_rr(<256 x float> %val, float* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256f32_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vstu %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256f32.i64(<256 x float> %val, float* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256f32.p0f32.i64(<256 x float> %val, float* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256f32_ri(<256 x float> %val, float* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256f32_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vstu %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256f32.i64(<256 x float> %val, float* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256f32.p0f32.i64(<256 x float> %val, float* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret void
 }
 
-declare void @llvm.experimental.vp.strided.store.v256i32.i64(<256 x i32> %val, i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare void @llvm.experimental.vp.strided.store.v256i32.p0i32.i64(<256 x i32> %val, i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc void @vp_strided_store_v256i32_rrm(<256 x i32> %val, i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256i32_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vstl %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  call void @llvm.experimental.vp.strided.store.v256i32.i64(<256 x i32> %val, i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256i32.p0i32.i64(<256 x i32> %val, i32* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256i32_rr(<256 x i32> %val, i32* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256i32_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vstl %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256i32.i64(<256 x i32> %val, i32* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256i32.p0i32.i64(<256 x i32> %val, i32* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256i32_ri(<256 x i32> %val, i32* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256i32_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vstl %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256i32.i64(<256 x i32> %val, i32* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256i32.p0i32.i64(<256 x i32> %val, i32* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret void
 }
 
-declare void @llvm.experimental.vp.strided.store.v256f64.i64(<256 x double> %val, double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare void @llvm.experimental.vp.strided.store.v256f64.p0f64.i64(<256 x double> %val, double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc void @vp_strided_store_v256f64_rrm(<256 x double> %val, double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256f64_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vst %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  call void @llvm.experimental.vp.strided.store.v256f64.i64(<256 x double> %val, double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256f64.p0f64.i64(<256 x double> %val, double* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256f64_rr(<256 x double> %val, double* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256f64_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vst %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256f64.i64(<256 x double> %val, double* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256f64.p0f64.i64(<256 x double> %val, double* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256f64_ri(<256 x double> %val, double* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256f64_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vst %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256f64.i64(<256 x double> %val, double* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256f64.p0f64.i64(<256 x double> %val, double* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret void
 }
 
-declare void @llvm.experimental.vp.strided.store.v256i64.i64(<256 x i64> %val, i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+declare void @llvm.experimental.vp.strided.store.v256i64.p0i64.i64(<256 x i64> %val, i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
 
 define fastcc void @vp_strided_store_v256i64_rrm(<256 x i64> %val, i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256i64_rrm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vst %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
-  call void @llvm.experimental.vp.strided.store.v256i64.i64(<256 x i64> %val, i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256i64.p0i64.i64(<256 x i64> %val, i64* %ptr, i64 %stride, <256 x i1> %mask, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256i64_rr(<256 x i64> %val, i64* %ptr, i64 %stride, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256i64_rr:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vst %v0, %s1, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256i64.i64(<256 x i64> %val, i64* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256i64.p0i64.i64(<256 x i64> %val, i64* %ptr, i64 %stride, <256 x i1> %allones, i32 %evl)
   ret void
 }
 
 define fastcc void @vp_strided_store_v256i64_ri(<256 x i64> %val, i64* %ptr, i32 %evl) {
 ; CHECK-LABEL: vp_strided_store_v256i64_ri:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vst %v0, 24, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
   %one = insertelement <256 x i1> undef, i1 1, i32 0
   %allones = shufflevector <256 x i1> %one, <256 x i1> undef, <256 x i32> zeroinitializer
-  call void @llvm.experimental.vp.strided.store.v256i64.i64(<256 x i64> %val, i64* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
+  call void @llvm.experimental.vp.strided.store.v256i64.p0i64.i64(<256 x i64> %val, i64* %ptr, i64 24, <256 x i1> %allones, i32 %evl)
   ret void
 }

llvm/unittests/IR/VPIntrinsicTest.cpp

std::unique_ptr<Module> createVPDeclarationModule() {
     Str << " declare <8 x float> @llvm.vp.fma.v8f32(<8 x float>, <8 x float>, "
            "<8 x float>, <8 x i1>, i32) ";
 
     Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, "
            "<8 x i1>, i32) ";
     Str << "declare void "
            "@llvm.experimental.vp.strided.store.v8i32.i32(<8 x i32>, "
            "i32*, i32, <8 x i1>, i32) ";
+    Str << "declare void "
+           "@llvm.experimental.vp.strided.store.v8i32.p1i32.i32(<8 x i32>, "
+           "i32 addrspace(1)*, i32, <8 x i1>, i32) ";
     Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x "
            "i32*>, <8 x i1>, i32) ";
     Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x "
            "i1>, i32) ";
     Str << "declare <8 x i32> "
            "@llvm.experimental.vp.strided.load.v8i32.i32(i32*, i32, <8 "
            "x i1>, i32) ";
+    Str << "declare <8 x i32> "
+           "@llvm.experimental.vp.strided.load.v8i32.p1i32.i32(i32 "
+           "addrspace(1)*, i32, <8 x i1>, i32) ";
     Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x "
            "i1>, i32) ";
 
     for (const char *ReductionOpcode : ReductionIntOpcodes)
       Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
           << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
 
     for (const char *ReductionOpcode : ReductionFPOpcodes)

llvm/utils/TableGen/IntrinsicEmitter.cpp

enum IIT_Info {
   IIT_V128 = 47,
   IIT_BF16 = 48,
   IIT_STRUCT9 = 49,
   IIT_V256 = 50,
   IIT_AMX  = 51,
   IIT_PPCF128 = 52,
   IIT_V3 = 53,
   IIT_EXTERNREF = 54,
-  IIT_FUNCREF = 55
+  IIT_FUNCREF = 55,
+  IIT_ANYPTR_TO_ELT = 56,
 };
 
 static void EncodeFixedValueType(MVT::SimpleValueType VT,
                                  std::vector<unsigned char> &Sig) {
   if (MVT(VT).isInteger()) {
     unsigned BitWidth = MVT(VT).getFixedSizeInBits();
     switch (BitWidth) {
     default: PrintFatalError("unhandled integer type width in intrinsic!");
else if (R->isSubClassOf("LLVMPointerTo"))
       Sig.push_back(IIT_PTR_TO_ARG);
     else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
       Sig.push_back(IIT_VEC_OF_ANYPTRS_TO_ELT);
       // Encode overloaded ArgNo
       Sig.push_back(NextArgCode++);
       // Encode LLVMMatchType<Number> ArgNo
       Sig.push_back(Number);
       return;
+    } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
+      Sig.push_back(IIT_ANYPTR_TO_ELT);
+      // Encode overloaded ArgNo
+      Sig.push_back(NextArgCode++);
+      // Encode LLVMMatchType<Number> ArgNo
+      Sig.push_back(Number);
+      return;
     } else if (R->isSubClassOf("LLVMPointerToElt"))
       Sig.push_back(IIT_PTR_TO_ELT);
     else if (R->isSubClassOf("LLVMVectorElementType"))
       Sig.push_back(IIT_VEC_ELEMENT);
     else if (R->isSubClassOf("LLVMSubdivide2VectorType"))
       Sig.push_back(IIT_SUBDIVIDE2_ARG);
     else if (R->isSubClassOf("LLVMSubdivide4VectorType"))
       Sig.push_back(IIT_SUBDIVIDE4_ARG);

 
 static void UpdateArgCodes(Record *R, std::vector<unsigned char> &ArgCodes,
                            unsigned int &NumInserted,
                            SmallVectorImpl<unsigned char> &Mapping) {
   if (R->isSubClassOf("LLVMMatchType")) {
     if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
       ArgCodes.push_back(3 /*vAny*/);
       ++NumInserted;
+    } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
+      ArgCodes.push_back(4 /*iPTRAny*/);
+      ++NumInserted;
     }
     return;
   }
 
   unsigned Tmp = 0;
   switch (getValueType(R->getValueAsDef("VT"))) {
   default: break;
   case MVT::iPTR: