diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -7630,11 +7630,18 @@ SDValue AArch64TargetLowering::LowerVECTOR_SPLICE(SDValue Op, SelectionDAG &DAG) const { - EVT Ty = Op.getValueType(); + auto Op1 = Op.getOperand(0); auto Idx = Op.getConstantOperandAPInt(2); - if (Idx.sge(-1) && Idx.slt(Ty.getVectorMinNumElements())) + unsigned MinSVEBlocks = std::max( + Subtarget->getMinSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock, 1U); + + if (Idx == 0) + return Op1; + + if (Idx.sge(-1) && Idx.slt(Ty.getVectorMinNumElements() * MinSVEBlocks)) return Op; + return SDValue(); } diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -2598,14 +2598,14 @@ } // Splice with lane bigger or equal to 0 - def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 (sve_ext_imm_0_15 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_15:$index)>; - def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 (sve_ext_imm_0_7 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_7:$index)>; - def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 (sve_ext_imm_0_3 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_3:$index)>; - def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 (sve_ext_imm_0_1 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_1:$index)>; + def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 (sve_ext_imm_0_255 i32:$index)))), + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_255:$index)>; + def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 (sve_ext_imm_0_127 i32:$index)))), + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_127:$index)>; + def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 (sve_ext_imm_0_63 i32:$index)))), + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_63:$index)>; + def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 (sve_ext_imm_0_31 i32:$index)))), + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_31:$index)>; } // End HasSVEorStreamingSVE diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -264,10 +264,10 @@ def sve_cnt_shl_imm : ComplexPattern">; -def sve_ext_imm_0_1 : ComplexPattern">; -def sve_ext_imm_0_3 : ComplexPattern">; -def sve_ext_imm_0_7 : ComplexPattern">; -def sve_ext_imm_0_15 : ComplexPattern">; +def sve_ext_imm_0_31 : ComplexPattern">; +def sve_ext_imm_0_63 : ComplexPattern">; +def sve_ext_imm_0_127 : ComplexPattern">; +def sve_ext_imm_0_255 : ComplexPattern">; def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), (int_aarch64_sve_cntp node:$pred, node:$src2), [{ diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll --- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll +++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll @@ -10,7 +10,6 @@ define @splice_nxv16i8_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv16i8_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv16i8( %a, %b, i32 0) ret %res @@ -48,10 +47,41 @@ ret %res } +define @splice_nxv16i8_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv16i8_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #31 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv16i8( %a, %b, i32 31) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv16i8_clamped_idx_vscale( %a, %b) #0 { +; CHECK-LABEL: splice_nxv16i8_clamped_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: rdvl x9, #1 +; CHECK-NEXT: sub x9, x9, #1 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w10, #32 +; CHECK-NEXT: cmp x9, #32 +; CHECK-NEXT: st1b { z0.b }, p0, [sp] +; CHECK-NEXT: st1b { z1.b }, p0, [x8, #1, mul vl] +; CHECK-NEXT: csel x9, x9, x10, lo +; CHECK-NEXT: ld1b { z0.b }, p0/z, [x8, x9] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv16i8( %a, %b, i32 32) + ret %res +} + define @splice_nxv8i16_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv8i16_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv8i16( %a, %b, i32 0) ret %res @@ -89,10 +119,41 @@ ret %res } +define @splice_nxv8i16_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv8i16_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #30 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv8i16( %a, %b, i32 15) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv8i16_clamped_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv8i16_clamped_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cnth x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #16 +; CHECK-NEXT: cmp x10, #16 +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: st1h { z1.h }, p0, [x8, #1, mul vl] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ld1h { z0.h }, p0/z, [x8, x9, lsl #1] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv8i16( %a, %b, i32 16) + ret %res +} + define @splice_nxv4i32_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv4i32_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv4i32( %a, %b, i32 0) ret %res @@ -130,10 +191,41 @@ ret %res } +define @splice_nxv4i32_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv4i32_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #28 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv4i32( %a, %b, i32 7) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv4i32_clamped_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv4i32_clamped_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cntw x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #8 +; CHECK-NEXT: cmp x10, #8 +; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: st1w { z1.s }, p0, [x8, #1, mul vl] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8, x9, lsl #2] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv4i32( %a, %b, i32 8) + ret %res +} + define @splice_nxv2i64_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv2i64_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv2i64( %a, %b, i32 0) ret %res @@ -171,6 +263,38 @@ ret %res } +define @splice_nxv2i64_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2i64_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #24 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2i64( %a, %b, i32 3) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv2i64_clamped_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2i64_clamped_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cntd x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #4 +; CHECK-NEXT: cmp x10, #4 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [x8, #1, mul vl] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8, x9, lsl #3] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2i64( %a, %b, i32 4) + ret %res +} + define @splice_nxv2f16_neg_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv2f16_neg_idx: ; CHECK: // %bb.0: @@ -205,7 +329,6 @@ define @splice_nxv2f16_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv2f16_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv2f16( %a, %b, i32 0) ret %res @@ -245,6 +368,40 @@ ret %res } +define @splice_nxv2f16_1_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2f16_1_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #24 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2f16( %a, %b, i32 3) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv2f16_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2f16_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cntd x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: mov w9, #4 +; CHECK-NEXT: cmp x10, #4 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: st1h { z1.h }, p0, [x8, #1, mul vl] +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x8, x9] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2f16( %a, %b, i32 4) + ret %res +} + define @splice_nxv4f16_neg_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv4f16_neg_idx: ; CHECK: // %bb.0: @@ -279,7 +436,6 @@ define @splice_nxv4f16_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv4f16_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv4f16( %a, %b, i32 0) ret %res @@ -319,11 +475,43 @@ ret %res } +define @splice_nxv4f16_3_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv4f16_3_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #28 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv4f16( %a, %b, i32 7) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv4f16_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv4f16_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cntw x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: mov w9, #8 +; CHECK-NEXT: cmp x10, #8 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: st1h { z1.h }, p0, [x8, #1, mul vl] +; CHECK-NEXT: lsl x9, x9, #2 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x8, x9] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv4f16( %a, %b, i32 8) + ret %res +} define @splice_nxv8f16_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv8f16_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv8f16( %a, %b, i32 0) ret %res @@ -361,6 +549,38 @@ ret %res } +define @splice_nxv8f16_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv8f16_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #30 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv8f16( %a, %b, i32 15) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv8f16_clamped_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv8f16_clamped_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cnth x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #16 +; CHECK-NEXT: cmp x10, #16 +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: st1h { z1.h }, p0, [x8, #1, mul vl] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ld1h { z0.h }, p0/z, [x8, x9, lsl #1] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv8f16( %a, %b, i32 16) + ret %res +} + define @splice_nxv2f32_neg_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv2f32_neg_idx: ; CHECK: // %bb.0: @@ -395,7 +615,6 @@ define @splice_nxv2f32_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv2f32_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv2f32( %a, %b, i32 0) ret %res @@ -435,10 +654,43 @@ ret %res } +define @splice_nxv2f32_1_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2f32_1_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #24 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2f32( %a, %b, i32 3) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv2f32_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2f32_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cntd x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: mov w9, #4 +; CHECK-NEXT: cmp x10, #4 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: st1w { z1.s }, p0, [x8, #1, mul vl] +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x8, x9] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2f32( %a, %b, i32 4) + ret %res +} + define @splice_nxv4f32_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv4f32_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv4f32( %a, %b, i32 0) ret %res @@ -476,10 +728,41 @@ ret %res } +define @splice_nxv4f32_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv4f32_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #28 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv4f32( %a, %b, i32 7) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv4f32_clamped_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv4f32_clamped_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cntw x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #8 +; CHECK-NEXT: cmp x10, #8 +; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: st1w { z1.s }, p0, [x8, #1, mul vl] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8, x9, lsl #2] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv4f32( %a, %b, i32 8) + ret %res +} + define @splice_nxv2f64_first_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv2f64_first_idx: ; CHECK: // %bb.0: -; CHECK-NEXT: ext z0.b, z0.b, z1.b, #0 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.splice.nxv2f64( %a, %b, i32 0) ret %res @@ -517,6 +800,38 @@ ret %res } +define @splice_nxv2f64_last_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2f64_last_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: ext z0.b, z0.b, z1.b, #24 +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2f64( %a, %b, i32 3) + ret %res +} + +; Ensure index is clamped when we cannot prove it's less than VL-1. +define @splice_nxv2f64_clamped_idx_vscale( %a, %b) #1 { +; CHECK-LABEL: splice_nxv2f64_clamped_idx_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: cntd x10 +; CHECK-NEXT: sub x10, x10, #1 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #4 +; CHECK-NEXT: cmp x10, #4 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [x8, #1, mul vl] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8, x9, lsl #3] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %res = call @llvm.experimental.vector.splice.nxv2f64( %a, %b, i32 4) + ret %res +} + ; Ensure predicate based splice is promoted to use ZPRs. define @splice_nxv2i1_idx( %a, %b) #0 { ; CHECK-LABEL: splice_nxv2i1_idx: @@ -1191,3 +1506,4 @@ declare @llvm.experimental.vector.splice.nxv2f64(, , i32) attributes #0 = { nounwind "target-features"="+sve" } +attributes #1 = { nounwind "target-features"="+sve" vscale_range(2,2) }