diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -1363,6 +1363,22 @@ Stride); break; } + case Intrinsic::vp_gather: + case Intrinsic::vp_scatter: { + auto *VPI = cast(CI); + unsigned IID = CI->getIntrinsicID(); + bool IsWrite = IID == Intrinsic::vp_scatter; + if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads) + return; + unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID); + Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType(); + // FIXME: Indeed we could set the alignment of pointer vector of + // vp.gather/scatter. Use the accruate alignment here. + Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Align(1), + VPI->getMaskParam(), + VPI->getVectorLengthParam()); + break; + } default: for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) { if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) || @@ -1515,10 +1531,17 @@ IRB.SetInsertPoint(ThenTerm); } - if (Stride) - Index = IRB.CreateMul(Index, Stride); - - Value *InstrumentedAddress = IRB.CreateGEP(VTy, Addr, {Zero, Index}); + Value *InstrumentedAddress; + if (isa(Addr->getType())) { + assert( + cast(Addr->getType())->getElementType()->isPointerTy() && + "Expected vector of pointer."); + InstrumentedAddress = IRB.CreateExtractElement(Addr, Index); + } else { + if (Stride) + Index = IRB.CreateMul(Index, Stride); + InstrumentedAddress = IRB.CreateGEP(VTy, Addr, {Zero, Index}); + } doInstrumentAddress(Pass, I, &*IRB.GetInsertPoint(), InstrumentedAddress, Alignment, Granularity, ElemTypeSize, IsWrite, SizeArgument, UseCalls, Exp); diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll --- a/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll @@ -405,3 +405,81 @@ tail call void @llvm.experimental.vp.strided.store.nxv4f32.i32( %arg, ptr %p, i32 %stride, %mask, i32 %evl) ret void } + +; Test vp gather and scatter. +declare @llvm.vp.gather.nxv4f32.v4p0(, , i32) +declare void @llvm.vp.scatter.nxv4f32.v4p0(, , , i32) + +define @scalable.gather.nxv4f32( %vp, %mask, i32 %evl) sanitize_address { +; CHECK-LABEL: @scalable.gather.nxv4f32( +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[EVL:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP12:%.*]] +; CHECK: 2: +; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[EVL]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]]) +; CHECK-NEXT: br label [[DOTSPLIT:%.*]] +; CHECK: .split: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP2]] ], [ [[IV_NEXT:%.*]], [[TMP11:%.*]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement [[MASK:%.*]], i64 [[IV]] +; CHECK-NEXT: br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP11]] +; CHECK: 8: +; CHECK-NEXT: [[TMP9:%.*]] = extractelement [[VP:%.*]], i64 [[IV]] +; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP9]] to i64 +; CHECK-NEXT: call void @__asan_loadN(i64 [[TMP10]], i64 4) +; CHECK-NEXT: br label [[TMP11]] +; CHECK: 11: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP6]] +; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]] +; CHECK: .split.split: +; CHECK-NEXT: br label [[TMP12]] +; CHECK: 12: +; CHECK-NEXT: [[RES:%.*]] = tail call @llvm.vp.gather.nxv4f32.nxv4p0( [[VP]], [[MASK]], i32 [[EVL]]) +; CHECK-NEXT: ret [[RES]] +; +; DISABLED-LABEL: @scalable.gather.nxv4f32( +; DISABLED-NEXT: [[RES:%.*]] = tail call @llvm.vp.gather.nxv4f32.nxv4p0( [[VP:%.*]], [[MASK:%.*]], i32 [[EVL:%.*]]) +; DISABLED-NEXT: ret [[RES]] +; + %res = tail call @llvm.vp.gather.nxv4f32.v4p0( %vp, %mask, i32 %evl) + ret %res +} + +define void @scalable.scatter.nxv4f32( %arg, %vp, %mask, i32 %evl) sanitize_address { +; CHECK-LABEL: @scalable.scatter.nxv4f32( +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[EVL:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP12:%.*]] +; CHECK: 2: +; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[EVL]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]]) +; CHECK-NEXT: br label [[DOTSPLIT:%.*]] +; CHECK: .split: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP2]] ], [ [[IV_NEXT:%.*]], [[TMP11:%.*]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement [[MASK:%.*]], i64 [[IV]] +; CHECK-NEXT: br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP11]] +; CHECK: 8: +; CHECK-NEXT: [[TMP9:%.*]] = extractelement [[VP:%.*]], i64 [[IV]] +; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP9]] to i64 +; CHECK-NEXT: call void @__asan_storeN(i64 [[TMP10]], i64 4) +; CHECK-NEXT: br label [[TMP11]] +; CHECK: 11: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[IV_CHECK:%.*]] = icmp eq i64 [[IV_NEXT]], [[TMP6]] +; CHECK-NEXT: br i1 [[IV_CHECK]], label [[DOTSPLIT_SPLIT:%.*]], label [[DOTSPLIT]] +; CHECK: .split.split: +; CHECK-NEXT: br label [[TMP12]] +; CHECK: 12: +; CHECK-NEXT: tail call void @llvm.vp.scatter.nxv4f32.nxv4p0( [[ARG:%.*]], [[VP]], [[MASK]], i32 [[EVL]]) +; CHECK-NEXT: ret void +; +; DISABLED-LABEL: @scalable.scatter.nxv4f32( +; DISABLED-NEXT: tail call void @llvm.vp.scatter.nxv4f32.nxv4p0( [[ARG:%.*]], [[VP:%.*]], [[MASK:%.*]], i32 [[EVL:%.*]]) +; DISABLED-NEXT: ret void +; + tail call void @llvm.vp.scatter.nxv4f32.v4p0( %arg, %vp, %mask, i32 %evl) + ret void +}