diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -3833,6 +3833,38 @@ static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT, int Pattern) { + + if (Pattern != AArch64SVEPredPattern::all) { + // For vectors that are exactly getMaxSVEVectorSizeInBits big, we can + // use AArch64SVEPredPattern::all, which can enable the use of unpredicated + // variants of instructions when available. + const auto &Subtarget = + static_cast(DAG.getSubtarget()); + unsigned MinSVESize = Subtarget.getMinSVEVectorSizeInBits(); + unsigned MaxSVESize = Subtarget.getMaxSVEVectorSizeInBits(); + if (MaxSVESize && MinSVESize == MaxSVESize) { + unsigned PredCount = getNumElementsFromSVEPredPattern(Pattern); + unsigned ElementCount = 0; + switch (VT.getSimpleVT().SimpleTy) { + default: + llvm_unreachable("unexpected element type for SVE predicate"); + case MVT::nxv16i1: + ElementCount = MaxSVESize / 8; + break; + case MVT::nxv8i1: + ElementCount = MaxSVESize / 16; + break; + case MVT::nxv4i1: + ElementCount = MaxSVESize / 32; + break; + case MVT::nxv2i1: + ElementCount = MaxSVESize / 64; + break; + } + if (PredCount && ElementCount == PredCount) + Pattern = AArch64SVEPredPattern::all; + } + } return DAG.getNode(AArch64ISD::PTRUE, DL, VT, DAG.getTargetConstant(Pattern, DL, MVT::i32)); } @@ -17975,10 +18007,6 @@ getSVEPredPatternFromNumElements(VT.getVectorNumElements()); assert(PgPattern && "Unexpected element count for SVE predicate"); - // TODO: For vectors that are exactly getMaxSVEVectorSizeInBits big, we can - // use AArch64SVEPredPattern::all, which can enable the use of unpredicated - // variants of instructions when available. - MVT MaskVT; switch (VT.getVectorElementType().getSimpleVT().SimpleTy) { default: diff --git a/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll b/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll --- a/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll +++ b/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll @@ -226,18 +226,17 @@ ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ptrue p1.s, vl8 ; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0] -; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] ; CHECK-NEXT: cntd x8 +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] ; CHECK-NEXT: subs x8, x8, #8 ; CHECK-NEXT: csel x8, xzr, x8, lo ; CHECK-NEXT: mov w9, #8 ; CHECK-NEXT: cmp x8, #8 -; CHECK-NEXT: ptrue p1.d, vl8 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: uunpklo z0.d, z1.s ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1d { z0.d }, p1, [x9, x8, lsl #3] +; CHECK-NEXT: st1d { z0.d }, p0, [x9, x8, lsl #3] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-extract-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-vector.ll --- a/llvm/test/CodeGen/AArch64/sve-extract-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-vector.ll @@ -186,12 +186,11 @@ ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w10, #4 ; CHECK-NEXT: cmp x9, #4 -; CHECK-NEXT: ptrue p1.d, vl4 ; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: csel x9, x9, x10, lo ; CHECK-NEXT: mov x10, sp -; CHECK-NEXT: ld1d { z0.d }, p1/z, [x10, x9, lsl #3] -; CHECK-NEXT: st1d { z0.d }, p1, [x8] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x10, x9, lsl #3] +; CHECK-NEXT: st1d { z0.d }, p0, [x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll --- a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll @@ -329,19 +329,18 @@ ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x8 ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] ; CHECK-NEXT: subs x8, x8, #4 ; CHECK-NEXT: csel x8, xzr, x8, lo ; CHECK-NEXT: mov w9, #4 ; CHECK-NEXT: cmp x8, #4 -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1d { z0.d }, p1, [sp] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: st1d { z1.d }, p0, [x9, x8, lsl #3] -; CHECK-NEXT: ld1d { z0.d }, p1/z, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-vscale-attr.ll b/llvm/test/CodeGen/AArch64/sve-vscale-attr.ll --- a/llvm/test/CodeGen/AArch64/sve-vscale-attr.ll +++ b/llvm/test/CodeGen/AArch64/sve-vscale-attr.ll @@ -63,7 +63,7 @@ ; CHECK-LABEL: func_vscale2_2: ; CHECK: // %bb.0: ; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] @@ -108,7 +108,7 @@ define void @func_vscale4_4(<16 x i32>* %a, <16 x i32>* %b) #4 { ; CHECK-LABEL: func_vscale4_4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: add z0.s, p0/m, z0.s, z1.s