Index: llvm/lib/CodeGen/ExpandVectorPredication.cpp =================================================================== --- llvm/lib/CodeGen/ExpandVectorPredication.cpp +++ llvm/lib/CodeGen/ExpandVectorPredication.cpp @@ -427,6 +427,15 @@ break; } + case Intrinsic::vp_scatter: + NewMemoryInst = Builder.CreateMaskedScatter( + DataParam, PtrParam, AlignOpt.valueOrOne(), MaskParam); + break; + case Intrinsic::vp_gather: + NewMemoryInst = Builder.CreateMaskedGather(VPI.getType(), PtrParam, + AlignOpt.valueOrOne(), MaskParam, + nullptr, VPI.getName()); + break; } assert(NewMemoryInst); @@ -515,6 +524,8 @@ break; case Intrinsic::vp_load: case Intrinsic::vp_store: + case Intrinsic::vp_gather: + case Intrinsic::vp_scatter: return expandPredicationInMemoryIntrinsic(Builder, VPI); } Index: llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll @@ -0,0 +1,32 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt --expandvp -S < %s | FileCheck %s + +declare @llvm.vp.gather.nxv1i64.nxv1p0i64(, , i32) + +define @vpgather_nxv1i64( %ptrs, %m, i32 zeroext %evl) { +; CHECK-LABEL: @vpgather_nxv1i64( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv1i1.i32(i32 0, i32 [[EVL:%.*]]) +; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M:%.*]] +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE]], 1 +; CHECK-NEXT: [[V1:%.*]] = call @llvm.masked.gather.nxv1i64.nxv1p0i64( [[PTRS:%.*]], i32 1, [[TMP2]], undef) +; CHECK-NEXT: ret [[V1]] +; + %v = call @llvm.vp.gather.nxv1i64.nxv1p0i64( %ptrs, %m, i32 %evl) + ret %v +} + +declare void @llvm.vp.scatter.nxv1i64.nxv1p0i64(, , , i32) + +define void @vpscatter_nxv1i64( %val, %ptrs, %m, i32 zeroext %evl) { +; CHECK-LABEL: @vpscatter_nxv1i64( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv1i1.i32(i32 0, i32 [[EVL:%.*]]) +; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M:%.*]] +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE]], 1 +; CHECK-NEXT: call void @llvm.masked.scatter.nxv1i64.nxv1p0i64( [[VAL:%.*]], [[PTRS:%.*]], i32 1, [[TMP2]]) +; CHECK-NEXT: ret void +; + call void @llvm.vp.scatter.nxv1i64.nxv1p0i64( %val, %ptrs, %m, i32 %evl) + ret void +}