[AArch64][SVE] Change pointer type of nontemporal load/store intrinsics

[sdesmalen]

Summary

This fixes a discrepancy between the non-temporal loads/store
intrinsics and other SVE load intrinsics (such as nf/ff), so
that Clang can use the same code to generate these intrinsics.

[efriedma]

LGTM

[andwar]

LGTM

llvm/include/llvm/IR/IntrinsicsAArch64.td

 def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
 def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
 
 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 
   class AdvSIMD_1Vec_PredLoad_Intrinsic
     : Intrinsic<[llvm_anyvector_ty],
                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
-                 LLVMPointerTo<0>],
-                [IntrReadMem, IntrArgMemOnly]>;
-
-  class AdvSIMD_1Vec_PredFaultingLoad_Intrinsic
-    : Intrinsic<[llvm_anyvector_ty],
-                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  LLVMPointerToElt<0>],
                 [IntrReadMem, IntrArgMemOnly]>;
 
   class AdvSIMD_1Vec_PredStore_Intrinsic
     : Intrinsic<[],
                 [llvm_anyvector_ty,
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
-                 LLVMPointerTo<0>],
+                 LLVMPointerToElt<0>],
                 [IntrArgMemOnly, NoCapture<2>]>;
 
   class AdvSIMD_SVE_Index_Intrinsic
     : Intrinsic<[llvm_anyvector_ty],
                 [LLVMVectorElementType<0>,
                  LLVMVectorElementType<0>],
                 [IntrNoMem]>;
 
: Intrinsic<[],
                 ],
                 [IntrInaccessibleMemOrArgMemOnly, ImmArg<3>]>;
 
 //
 // Loads
 //
 
 def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
-
-def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredFaultingLoad_Intrinsic;
-def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredFaultingLoad_Intrinsic;
+def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
+def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
 
 //
 // Stores
 //
 
 def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
 
 //

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

case Intrinsic::aarch64_stxp:
     Info.ptrVal = I.getArgOperand(2);
     Info.offset = 0;
     Info.align = Align(16);
     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
     return true;
   case Intrinsic::aarch64_sve_ldnt1: {
     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = MVT::getVT(PtrTy->getElementType());
+    Info.memVT = MVT::getVT(I.getType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
     Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MONonTemporal;
     return true;
   }
   case Intrinsic::aarch64_sve_stnt1: {
     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(2)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = MVT::getVT(PtrTy->getElementType());
+    Info.memVT = MVT::getVT(I.getOperand(0)->getType());
     Info.ptrVal = I.getArgOperand(2);
     Info.offset = 0;
     Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MONonTemporal;
     return true;
   }
   default:
     break;

llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll

 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 ;
 ; LDNT1B
 ;
 
-define <vscale x 16 x i8> @ldnt1b_i8(<vscale x 16 x i1> %pred, <vscale x 16 x i8>* %addr) {
+define <vscale x 16 x i8> @ldnt1b_i8(<vscale x 16 x i1> %pred, i8* %addr) {
 ; CHECK-LABEL: ldnt1b_i8:
 ; CHECK: ldnt1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> %pred,
-                                                                 <vscale x 16 x i8>* %addr)
+                                                                 i8* %addr)
   ret <vscale x 16 x i8> %res
 }
 
 ;
 ; LDNT1H
 ;
 
-define <vscale x 8 x i16> @ldnt1h_i16(<vscale x 8 x i1> %pred, <vscale x 8 x i16>* %addr) {
+define <vscale x 8 x i16> @ldnt1h_i16(<vscale x 8 x i1> %pred, i16* %addr) {
 ; CHECK-LABEL: ldnt1h_i16:
 ; CHECK: ldnt1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> %pred,
-                                                                 <vscale x 8 x i16>* %addr)
+                                                                 i16* %addr)
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 8 x half> @ldnt1h_f16(<vscale x 8 x i1> %pred, <vscale x 8 x half>* %addr) {
+define <vscale x 8 x half> @ldnt1h_f16(<vscale x 8 x i1> %pred, half* %addr) {
 ; CHECK-LABEL: ldnt1h_f16:
 ; CHECK: ldnt1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> %pred,
-                                                                  <vscale x 8 x half>* %addr)
+                                                                  half* %addr)
   ret <vscale x 8 x half> %res
 }
 
 ;
 ; LDNT1W
 ;
 
-define <vscale x 4 x i32> @ldnt1w_i32(<vscale x 4 x i1> %pred, <vscale x 4 x i32>* %addr) {
+define <vscale x 4 x i32> @ldnt1w_i32(<vscale x 4 x i1> %pred, i32* %addr) {
 ; CHECK-LABEL: ldnt1w_i32:
 ; CHECK: ldnt1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> %pred,
-                                                                 <vscale x 4 x i32>* %addr)
+                                                                 i32* %addr)
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 4 x float> @ldnt1w_f32(<vscale x 4 x i1> %pred, <vscale x 4 x float>* %addr) {
+define <vscale x 4 x float> @ldnt1w_f32(<vscale x 4 x i1> %pred, float* %addr) {
 ; CHECK-LABEL: ldnt1w_f32:
 ; CHECK: ldnt1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> %pred,
-                                                                   <vscale x 4 x float>* %addr)
+                                                                   float* %addr)
   ret <vscale x 4 x float> %res
 }
 
 ;
 ; LDNT1D
 ;
 
-define <vscale x 2 x i64> @ldnt1d_i64(<vscale x 2 x i1> %pred, <vscale x 2 x i64>* %addr) {
+define <vscale x 2 x i64> @ldnt1d_i64(<vscale x 2 x i1> %pred, i64* %addr) {
 ; CHECK-LABEL: ldnt1d_i64:
 ; CHECK: ldnt1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> %pred,
-                                                                 <vscale x 2 x i64>* %addr)
+                                                                 i64* %addr)
   ret <vscale x 2 x i64> %res
 }
 
-define <vscale x 2 x double> @ldnt1d_f64(<vscale x 2 x i1> %pred, <vscale x 2 x double>* %addr) {
+define <vscale x 2 x double> @ldnt1d_f64(<vscale x 2 x i1> %pred, double* %addr) {
 ; CHECK-LABEL: ldnt1d_f64:
 ; CHECK: ldnt1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> %pred,
-                                                                    <vscale x 2 x double>* %addr)
+                                                                    double* %addr)
   ret <vscale x 2 x double> %res
 }
 
-declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>*)
-declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>*)
-declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>*)
-declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>*)
-declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>*)
-declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>*)
-declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>*)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, float*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, double*)

llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll

 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 ;
 ; STNT1B
 ;
 
-define void @stnt1b_i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pred, <vscale x 16 x i8>* %addr) {
+define void @stnt1b_i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pred, i8* %addr) {
 ; CHECK-LABEL: stnt1b_i8:
 ; CHECK: stnt1b { z0.b }, p0, [x0]
 ; CHECK-NEXT: ret
   call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> %data,
                                             <vscale x 16 x i1> %pred,
-                                            <vscale x 16 x i8>* %addr)
+                                            i8* %addr)
   ret void
 }
 
 ;
 ; STNT1H
 ;
 
-define void @stnt1h_i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, <vscale x 8 x i16>* %addr) {
+define void @stnt1h_i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, i16* %addr) {
 ; CHECK-LABEL: stnt1h_i16:
 ; CHECK: stnt1h { z0.h }, p0, [x0]
 ; CHECK-NEXT: ret
   call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> %data,
                                             <vscale x 8 x i1> %pred,
-                                            <vscale x 8 x i16>* %addr)
+                                            i16* %addr)
   ret void
 }
 
-define void @stnt1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, <vscale x 8 x half>* %addr) {
+define void @stnt1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half* %addr) {
 ; CHECK-LABEL: stnt1h_f16:
 ; CHECK: stnt1h { z0.h }, p0, [x0]
 ; CHECK-NEXT: ret
   call void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half> %data,
                                             <vscale x 8 x i1> %pred,
-                                            <vscale x 8 x half>* %addr)
+                                            half* %addr)
   ret void
 }
 
 ;
 ; STNT1W
 ;
 
-define void @stnt1w_i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, <vscale x 4 x i32>* %addr) {
+define void @stnt1w_i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i32* %addr) {
 ; CHECK-LABEL: stnt1w_i32:
 ; CHECK: stnt1w { z0.s }, p0, [x0]
 ; CHECK-NEXT: ret
   call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> %data,
                                             <vscale x 4 x i1> %pred,
-                                            <vscale x 4 x i32>* %addr)
+                                            i32* %addr)
   ret void
 }
 
-define void @stnt1w_f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pred, <vscale x 4 x float>* %addr) {
+define void @stnt1w_f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pred, float* %addr) {
 ; CHECK-LABEL: stnt1w_f32:
 ; CHECK: stnt1w { z0.s }, p0, [x0]
 ; CHECK-NEXT: ret
   call void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float> %data,
                                             <vscale x 4 x i1> %pred,
-                                            <vscale x 4 x float>* %addr)
+                                            float* %addr)
   ret void
 }
 
 ;
 ; STNT1D
 ;
 
-define void @stnt1d_i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, <vscale x 2 x i64>* %addr) {
+define void @stnt1d_i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i64* %addr) {
 ; CHECK-LABEL: stnt1d_i64:
 ; CHECK: stnt1d { z0.d }, p0, [x0]
 ; CHECK-NEXT: ret
   call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> %data,
                                             <vscale x 2 x i1> %pred,
-                                            <vscale x 2 x i64>* %addr)
+                                            i64* %addr)
   ret void
 }
 
-define void @stnt1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, <vscale x 2 x double>* %addr) {
+define void @stnt1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, double* %addr) {
 ; CHECK-LABEL: stnt1d_f64:
 ; CHECK: stnt1d { z0.d }, p0, [x0]
 ; CHECK-NEXT: ret
   call void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double> %data,
                                             <vscale x 2 x i1> %pred,
-                                            <vscale x 2 x double>* %addr)
+                                            double* %addr)
   ret void
 }
 
-declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>*)
-declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>*)
-declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>*)
-declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>*)
-declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>*)
-declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>*)
-declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>*)
+declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
+declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
+declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
+declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
+declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
+declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)

llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll

 ; CHECK-NEXT: rdvl x8, #8
 ; CHECK-NEXT: add x8, x0, x8
 ; CHECK-NEXT: ldnt1d { z[[DATA:[0-9]+]].d }, p0/z, [x{{[0-9]+}}]
 ; CHECK-NEXT: rdvl x8, #-9
 ; CHECK-NEXT: add x8, x0, x8
 ; CHECK-NEXT: stnt1d { z[[DATA]].d }, p0, [x{{[0-9]+}}]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %base, i64 8
+  %base_load_bc = bitcast <vscale x 2 x i64>* %base_load to i64*
   %data = call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> %mask,
-                                                                  <vscale x 2 x i64>* %base_load)
+                                                                  i64* %base_load_bc)
   %base_store = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64> * %base, i64 -9
+  %base_store_bc = bitcast <vscale x 2 x i64>* %base_store to i64*
   call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> %data,
                                             <vscale x 2 x i1> %mask,
-                                            <vscale x 2 x i64>* %base_store)
+                                            i64* %base_store_bc)
   ret void
 }
 
 ; 2-lane non-temporal load/stores
 
 
 define void @test_masked_ldst_sv2i64(<vscale x 2 x i64> * %base, <vscale x 2 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i64:
 ; CHECK-NEXT: ldnt1d { z[[DATA:[0-9]+]].d }, p0/z, [x0, #-8, mul vl]
 ; CHECK-NEXT: stnt1d { z[[DATA]].d }, p0, [x0, #-7, mul vl]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %base, i64 -8
+  %base_load_bc = bitcast <vscale x 2 x i64>* %base_load to i64*
   %data = call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> %mask,
-                                                                  <vscale x 2 x i64>* %base_load)
+                                                                  i64* %base_load_bc)
   %base_store = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64> * %base, i64 -7
+  %base_store_bc = bitcast <vscale x 2 x i64>* %base_store to i64*
   call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> %data,
                                             <vscale x 2 x i1> %mask,
-                                            <vscale x 2 x i64>* %base_store)
+                                            i64* %base_store_bc)
   ret void
 }
 
 define void @test_masked_ldst_sv2f64(<vscale x 2 x double> * %base, <vscale x 2 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2f64:
 ; CHECK-NEXT: ldnt1d { z[[DATA:[0-9]+]].d }, p0/z, [x0, #-6, mul vl]
 ; CHECK-NEXT: stnt1d { z[[DATA]].d }, p0, [x0, #-5, mul vl]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %base, i64 -6
+  %base_load_bc = bitcast <vscale x 2 x double>* %base_load to double*
   %data = call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> %mask,
-                                                                    <vscale x 2 x double>* %base_load)
+                                                                    double* %base_load_bc)
   %base_store = getelementptr <vscale x 2 x double>, <vscale x 2 x double> * %base, i64 -5
+  %base_store_bc = bitcast <vscale x 2 x double>* %base_store to double*
   call void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double> %data,
                                             <vscale x 2 x i1> %mask,
-                                            <vscale x 2 x double>* %base_store)
+                                            double* %base_store_bc)
   ret void
 }
 
 ; 4-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv4i32(<vscale x 4 x i32> * %base, <vscale x 4 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4i32:
 ; CHECK-NEXT: ldnt1w { z[[DATA:[0-9]+]].s }, p0/z, [x0, #6, mul vl]
 ; CHECK-NEXT: stnt1w { z[[DATA]].s }, p0, [x0, #7, mul vl]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %base, i64 6
+  %base_load_bc = bitcast <vscale x 4 x i32>* %base_load to i32*
   %data = call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> %mask,
-                                                                  <vscale x 4 x i32>* %base_load)
+                                                                  i32* %base_load_bc)
   %base_store = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32> * %base, i64 7
+  %base_store_bc = bitcast <vscale x 4 x i32>* %base_store to i32*
   call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> %data,
                                             <vscale x 4 x i1> %mask,
-                                            <vscale x 4 x i32>* %base_store)
+                                            i32* %base_store_bc)
   ret void
 }
 
 define void @test_masked_ldst_sv4f32(<vscale x 4 x float> * %base, <vscale x 4 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4f32:
 ; CHECK-NEXT: ldnt1w { z[[DATA:[0-9]+]].s }, p0/z, [x0, #-1, mul vl]
 ; CHECK-NEXT: stnt1w { z[[DATA]].s }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %base, i64 -1
+  %base_load_bc = bitcast <vscale x 4 x float>* %base_load to float*
   %data = call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> %mask,
-                                                                    <vscale x 4 x float>* %base_load)
+                                                                    float* %base_load_bc)
   %base_store = getelementptr <vscale x 4 x float>, <vscale x 4 x float> * %base, i64 2
+  %base_store_bc = bitcast <vscale x 4 x float>* %base_store to float*
   call void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float> %data,
                                             <vscale x 4 x i1> %mask,
-                                            <vscale x 4 x float>* %base_store)
+                                            float* %base_store_bc)
   ret void
 }
 
 
 ; 8-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv8i16(<vscale x 8 x i16> * %base, <vscale x 8 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv8i16:
 ; CHECK-NEXT: ldnt1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, #6, mul vl]
 ; CHECK-NEXT: stnt1h { z[[DATA]].h }, p0, [x0, #7, mul vl]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %base, i64 6
+  %base_load_bc = bitcast <vscale x 8 x i16>* %base_load to i16*
   %data = call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> %mask,
-                                                                  <vscale x 8 x i16>* %base_load)
+                                                                  i16* %base_load_bc)
   %base_store = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16> * %base, i64 7
+  %base_store_bc = bitcast <vscale x 8 x i16>* %base_store to i16*
   call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> %data,
                                             <vscale x 8 x i1> %mask,
-                                            <vscale x 8 x i16>* %base_store)
+                                            i16* %base_store_bc)
   ret void
 }
 
 define void @test_masked_ldst_sv8f16(<vscale x 8 x half> * %base, <vscale x 8 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv8f16:
 ; CHECK-NEXT: ldnt1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, #-1, mul vl]
 ; CHECK-NEXT: stnt1h { z[[DATA]].h }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %base, i64 -1
+  %base_load_bc = bitcast <vscale x 8 x half>* %base_load to half*
   %data = call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> %mask,
-                                                                   <vscale x 8 x half>* %base_load)
+                                                                   half* %base_load_bc)
   %base_store = getelementptr <vscale x 8 x half>, <vscale x 8 x half> * %base, i64 2
+  %base_store_bc = bitcast <vscale x 8 x half>* %base_store to half*
   call void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half> %data,
                                             <vscale x 8 x i1> %mask,
-                                            <vscale x 8 x half>* %base_store)
+                                            half* %base_store_bc)
   ret void
 }
 
 ; 16-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv16i8(<vscale x 16 x i8> * %base, <vscale x 16 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv16i8:
 ; CHECK-NEXT: ldnt1b { z[[DATA:[0-9]+]].b }, p0/z, [x0, #6, mul vl]
 ; CHECK-NEXT: stnt1b { z[[DATA]].b }, p0, [x0, #7, mul vl]
 ; CHECK-NEXT: ret
   %base_load = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base, i64 6
+  %base_load_bc = bitcast <vscale x 16 x i8>* %base_load to i8*
   %data = call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> %mask,
-                                                                  <vscale x 16 x i8>* %base_load)
+                                                                  i8* %base_load_bc)
   %base_store = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8> * %base, i64 7
+  %base_store_bc = bitcast <vscale x 16 x i8>* %base_store to i8*
   call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> %data,
                                             <vscale x 16 x i1> %mask,
-                                            <vscale x 16 x i8>* %base_store)
+                                            i8* %base_store_bc)
   ret void
 }
 
 ; 2-element non-temporal loads.
-declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>*)
-declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>*)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, double*)
 
 ; 4-element non-temporal loads.
-declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>*)
-declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>*)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, float*)
 
 ; 8-element non-temporal loads.
-declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>*)
-declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>*)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, half*)
 
 ; 16-element non-temporal loads.
-declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i8*)
 
 ; 2-element non-temporal stores.
-declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>*)
-declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>*)
+declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
+declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
                                                                       
 ; 4-element non-temporal stores.                                        
-declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>*)
-declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>*)
+declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
+declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
                                                                       
 ; 8-element non-temporal stores.                                        
-declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>*)
-declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>*)
+declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
+declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
 
 ; 16-element non-temporal stores.
-declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
 

llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-reg.ll

 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve --asm-verbose=false < %s | FileCheck %s
 
 ; 2-lane non-temporal load/stores
 
 define void @test_masked_ldst_sv2i64(i64* %base, <vscale x 2 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i64:
 ; CHECK-NEXT: ldnt1d { z[[DATA:[0-9]+]].d }, p0/z, [x0, x1, lsl #3]
 ; CHECK-NEXT: stnt1d { z[[DATA]].d }, p0, [x0, x1, lsl #3]
 ; CHECK-NEXT: ret
-  %base_i64 = getelementptr i64, i64* %base, i64 %offset
-  %base_addr = bitcast i64* %base_i64 to <vscale x 2 x i64>*
+  %gep = getelementptr i64, i64* %base, i64 %offset
   %data = call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> %mask,
-                                                                  <vscale x 2 x i64>* %base_addr)
+                                                                  i64* %gep)
   call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> %data,
                                             <vscale x 2 x i1> %mask,
-                                            <vscale x 2 x i64>* %base_addr)
+                                            i64* %gep)
   ret void
 }
 
 define void @test_masked_ldst_sv2f64(double* %base, <vscale x 2 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2f64:
 ; CHECK-NEXT: ldnt1d { z[[DATA:[0-9]+]].d }, p0/z, [x0, x1, lsl #3]
 ; CHECK-NEXT: stnt1d { z[[DATA]].d }, p0, [x0, x1, lsl #3]
 ; CHECK-NEXT: ret
-  %base_double = getelementptr double, double* %base, i64 %offset
-  %base_addr = bitcast double* %base_double to <vscale x 2 x double>*
+  %gep = getelementptr double, double* %base, i64 %offset
   %data = call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> %mask,
-                                                                    <vscale x 2 x double>* %base_addr)
+                                                                    double* %gep)
   call void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double> %data,
                                             <vscale x 2 x i1> %mask,
-                                            <vscale x 2 x double>* %base_addr)
+                                            double* %gep)
   ret void
 }
 
 ; 4-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv4i32(i32* %base, <vscale x 4 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4i32:
 ; CHECK-NEXT: ldnt1w { z[[DATA:[0-9]+]].s }, p0/z, [x0, x1, lsl #2]
 ; CHECK-NEXT: stnt1w { z[[DATA]].s }, p0, [x0, x1, lsl #2]
 ; CHECK-NEXT: ret
-  %base_i32 = getelementptr i32, i32* %base, i64 %offset
-  %base_addr = bitcast i32* %base_i32 to <vscale x 4 x i32>*
+  %gep = getelementptr i32, i32* %base, i64 %offset
   %data = call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> %mask,
-                                                                  <vscale x 4 x i32>* %base_addr)
+                                                                  i32* %gep)
   call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> %data,
                                             <vscale x 4 x i1> %mask,
-                                            <vscale x 4 x i32>* %base_addr)
+                                            i32* %gep)
   ret void
 }
 
 define void @test_masked_ldst_sv4f32(float* %base, <vscale x 4 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4f32:
 ; CHECK-NEXT: ldnt1w { z[[DATA:[0-9]+]].s }, p0/z, [x0, x1, lsl #2]
 ; CHECK-NEXT: stnt1w { z[[DATA]].s }, p0, [x0, x1, lsl #2]
 ; CHECK-NEXT: ret
-  %base_float = getelementptr float, float* %base, i64 %offset
-  %base_addr = bitcast float* %base_float to <vscale x 4 x float>*
+  %gep = getelementptr float, float* %base, i64 %offset
   %data = call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> %mask,
-                                                                    <vscale x 4 x float>* %base_addr)
+                                                                    float* %gep)
   call void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float> %data,
                                             <vscale x 4 x i1> %mask,
-                                            <vscale x 4 x float>* %base_addr)
+                                            float* %gep)
   ret void
 }
 
 
 ; 8-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv8i16(i16* %base, <vscale x 8 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv8i16:
 ; CHECK-NEXT: ldnt1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, x1, lsl #1]
 ; CHECK-NEXT: stnt1h { z[[DATA]].h }, p0, [x0, x1, lsl #1]
 ; CHECK-NEXT: ret
-  %base_i16 = getelementptr i16, i16* %base, i64 %offset
-  %base_addr = bitcast i16* %base_i16 to <vscale x 8 x i16>*
+  %gep = getelementptr i16, i16* %base, i64 %offset
   %data = call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> %mask,
-                                                                  <vscale x 8 x i16>* %base_addr)
+                                                                  i16* %gep)
   call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> %data,
                                             <vscale x 8 x i1> %mask,
-                                            <vscale x 8 x i16>* %base_addr)
+                                            i16* %gep)
   ret void
 }
 
 define void @test_masked_ldst_sv8f16(half* %base, <vscale x 8 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv8f16:
 ; CHECK-NEXT: ldnt1h { z[[DATA:[0-9]+]].h }, p0/z, [x0, x1, lsl #1]
 ; CHECK-NEXT: stnt1h { z[[DATA]].h }, p0, [x0, x1, lsl #1]
 ; CHECK-NEXT: ret
-  %base_half = getelementptr half, half* %base, i64 %offset
-  %base_addr = bitcast half* %base_half to <vscale x 8 x half>*
+  %gep = getelementptr half, half* %base, i64 %offset
   %data = call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> %mask,
-                                                                   <vscale x 8 x half>* %base_addr)
+                                                                   half* %gep)
   call void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half> %data,
                                             <vscale x 8 x i1> %mask,
-                                            <vscale x 8 x half>* %base_addr)
+                                            half* %gep)
   ret void
 }
 
 ; 16-lane non-temporal load/stores.
 
 define void @test_masked_ldst_sv16i8(i8* %base, <vscale x 16 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv16i8:
 ; CHECK-NEXT: ldnt1b { z[[DATA:[0-9]+]].b }, p0/z, [x0, x1]
 ; CHECK-NEXT: stnt1b { z[[DATA]].b }, p0, [x0, x1]
 ; CHECK-NEXT: ret
-  %base_i8 = getelementptr i8, i8* %base, i64 %offset
-  %base_addr = bitcast i8* %base_i8 to <vscale x 16 x i8>*
+  %gep = getelementptr i8, i8* %base, i64 %offset
   %data = call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> %mask,
-                                                                  <vscale x 16 x i8>* %base_addr)
+                                                                  i8* %gep)
   call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> %data,
                                             <vscale x 16 x i1> %mask,
-                                            <vscale x 16 x i8>* %base_addr)
+                                            i8* %gep)
   ret void
 }
 
 ; 2-element non-temporal loads.
-declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>*)
-declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>*)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, double*)
 
 ; 4-element non-temporal loads.
-declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>*)
-declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>*)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, float*)
 
 ; 8-element non-temporal loads.
-declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>*)
-declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>*)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, half*)
 
 ; 16-element non-temporal loads.
-declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i8*)
 
 ; 2-element non-temporal stores.
-declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>*)
-declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>*)
+declare void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
+declare void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
                                                                       
 ; 4-element non-temporal stores.                                        
-declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>*)
-declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>*)
+declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
+declare void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
                                                                       
 ; 8-element non-temporal stores.                                        
-declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>*)
-declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>*)
+declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
+declare void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
 
 ; 16-element non-temporal stores.
-declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)