diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -12469,7 +12469,7 @@ bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef M, EVT VT) const { // Currently no fixed length shuffles that require SVE are legal. - if (useSVEForFixedLengthVectorVT(VT)) + if (useSVEForFixedLengthVectorVT(VT, Subtarget->forceStreamingCompatibleSVE())) return false; if (VT.getVectorNumElements() == 4 && @@ -13994,6 +13994,11 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const { + // Skip if streaming compatible SVE is enabled, + // because it generates invalid code in streaming mode when SVE length is not specified. + if(Subtarget->forceStreamingCompatibleSVE()) + return false; + assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && "Invalid interleave factor"); diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-fixed-length-int-shifts.ll b/llvm/test/CodeGen/AArch64/sve-streaming-fixed-length-int-shifts.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-streaming-fixed-length-int-shifts.ll @@ -0,0 +1,640 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +; +; ASHR +; + +define <4 x i8> @ashr_v4i8(<4 x i8> %op1, <4 x i8> %op2) #0 { +; CHECK-LABEL: ashr_v4i8: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI0_0 +; CHECK-NEXT: adrp x9, .LCPI0_1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI0_0] +; CHECK-NEXT: ldr d3, [x9, :lo12:.LCPI0_1] +; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z2.h +; CHECK-NEXT: asr z0.h, p0/m, z0.h, z2.h +; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <4 x i8> %op1, %op2 + ret <4 x i8> %res +} + +define <8 x i8> @ashr_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { +; CHECK-LABEL: ashr_v8i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.b, vl8 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <8 x i8> %op1, %op2 + ret <8 x i8> %res +} + +define <16 x i8> @ashr_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { +; CHECK-LABEL: ashr_v16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <16 x i8> %op1, %op2 + ret <16 x i8> %res +} + +define void @ashr_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { +; CHECK-LABEL: ashr_v32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: asr z0.b, p0/m, z0.b, z2.b +; CHECK-NEXT: asr z1.b, p0/m, z1.b, z3.b +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <32 x i8>, <32 x i8>* %a + %op2 = load <32 x i8>, <32 x i8>* %b + %res = ashr <32 x i8> %op1, %op2 + store <32 x i8> %res, <32 x i8>* %a + ret void +} + +define <2 x i16> @ashr_v2i16(<2 x i16> %op1, <2 x i16> %op2) #0 { +; CHECK-LABEL: ashr_v2i16: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI4_0 +; CHECK-NEXT: adrp x9, .LCPI4_1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI4_0] +; CHECK-NEXT: ldr d3, [x9, :lo12:.LCPI4_1] +; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: asr z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <2 x i16> %op1, %op2 + ret <2 x i16> %res +} + +define <4 x i16> @ashr_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { +; CHECK-LABEL: ashr_v4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <4 x i16> %op1, %op2 + ret <4 x i16> %res +} + +define <8 x i16> @ashr_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { +; CHECK-LABEL: ashr_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <8 x i16> %op1, %op2 + ret <8 x i16> %res +} + +define void @ashr_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { +; CHECK-LABEL: ashr_v16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: asr z0.h, p0/m, z0.h, z2.h +; CHECK-NEXT: asr z1.h, p0/m, z1.h, z3.h +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <16 x i16>, <16 x i16>* %a + %op2 = load <16 x i16>, <16 x i16>* %b + %res = ashr <16 x i16> %op1, %op2 + store <16 x i16> %res, <16 x i16>* %a + ret void +} + +define <2 x i32> @ashr_v2i32(<2 x i32> %op1, <2 x i32> %op2) #0 { +; CHECK-LABEL: ashr_v2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <2 x i32> %op1, %op2 + ret <2 x i32> %res +} + +define <4 x i32> @ashr_v4i32(<4 x i32> %op1, <4 x i32> %op2) #0 { +; CHECK-LABEL: ashr_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <4 x i32> %op1, %op2 + ret <4 x i32> %res +} + +define void @ashr_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { +; CHECK-LABEL: ashr_v8i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: asr z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: asr z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <8 x i32>, <8 x i32>* %a + %op2 = load <8 x i32>, <8 x i32>* %b + %res = ashr <8 x i32> %op1, %op2 + store <8 x i32> %res, <8 x i32>* %a + ret void +} + +define <1 x i64> @ashr_v1i64(<1 x i64> %op1, <1 x i64> %op2) #0 { +; CHECK-LABEL: ashr_v1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <1 x i64> %op1, %op2 + ret <1 x i64> %res +} + +define <2 x i64> @ashr_v2i64(<2 x i64> %op1, <2 x i64> %op2) #0 { +; CHECK-LABEL: ashr_v2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = ashr <2 x i64> %op1, %op2 + ret <2 x i64> %res +} + +define void @ashr_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { +; CHECK-LABEL: ashr_v4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: asr z0.d, p0/m, z0.d, z2.d +; CHECK-NEXT: asr z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <4 x i64>, <4 x i64>* %a + %op2 = load <4 x i64>, <4 x i64>* %b + %res = ashr <4 x i64> %op1, %op2 + store <4 x i64> %res, <4 x i64>* %a + ret void +} + +; +; LSHR +; + +define <4 x i8> @lshr_v4i8(<4 x i8> %op1, <4 x i8> %op2) #0 { +; CHECK-LABEL: lshr_v4i8: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI14_0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI14_0] +; CHECK-NEXT: and z1.d, z1.d, z2.d +; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <4 x i8> %op1, %op2 + ret <4 x i8> %res +} + +define <8 x i8> @lshr_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { +; CHECK-LABEL: lshr_v8i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.b, vl8 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <8 x i8> %op1, %op2 + ret <8 x i8> %res +} + +define <16 x i8> @lshr_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { +; CHECK-LABEL: lshr_v16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <16 x i8> %op1, %op2 + ret <16 x i8> %res +} + +define void @lshr_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { +; CHECK-LABEL: lshr_v32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z2.b +; CHECK-NEXT: lsr z1.b, p0/m, z1.b, z3.b +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <32 x i8>, <32 x i8>* %a + %op2 = load <32 x i8>, <32 x i8>* %b + %res = lshr <32 x i8> %op1, %op2 + store <32 x i8> %res, <32 x i8>* %a + ret void +} + +define <2 x i16> @lshr_v2i16(<2 x i16> %op1, <2 x i16> %op2) #0 { +; CHECK-LABEL: lshr_v2i16: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI18_0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI18_0] +; CHECK-NEXT: and z1.d, z1.d, z2.d +; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <2 x i16> %op1, %op2 + ret <2 x i16> %res +} + +define <4 x i16> @lshr_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { +; CHECK-LABEL: lshr_v4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <4 x i16> %op1, %op2 + ret <4 x i16> %res +} + +define <8 x i16> @lshr_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { +; CHECK-LABEL: lshr_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <8 x i16> %op1, %op2 + ret <8 x i16> %res +} + +define void @lshr_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { +; CHECK-LABEL: lshr_v16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z2.h +; CHECK-NEXT: lsr z1.h, p0/m, z1.h, z3.h +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <16 x i16>, <16 x i16>* %a + %op2 = load <16 x i16>, <16 x i16>* %b + %res = lshr <16 x i16> %op1, %op2 + store <16 x i16> %res, <16 x i16>* %a + ret void +} + +define <2 x i32> @lshr_v2i32(<2 x i32> %op1, <2 x i32> %op2) #0 { +; CHECK-LABEL: lshr_v2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <2 x i32> %op1, %op2 + ret <2 x i32> %res +} + +define <4 x i32> @lshr_v4i32(<4 x i32> %op1, <4 x i32> %op2) #0 { +; CHECK-LABEL: lshr_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <4 x i32> %op1, %op2 + ret <4 x i32> %res +} + +define void @lshr_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { +; CHECK-LABEL: lshr_v8i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: lsr z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <8 x i32>, <8 x i32>* %a + %op2 = load <8 x i32>, <8 x i32>* %b + %res = lshr <8 x i32> %op1, %op2 + store <8 x i32> %res, <8 x i32>* %a + ret void +} + +define <1 x i64> @lshr_v1i64(<1 x i64> %op1, <1 x i64> %op2) #0 { +; CHECK-LABEL: lshr_v1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <1 x i64> %op1, %op2 + ret <1 x i64> %res +} + +define <2 x i64> @lshr_v2i64(<2 x i64> %op1, <2 x i64> %op2) #0 { +; CHECK-LABEL: lshr_v2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = lshr <2 x i64> %op1, %op2 + ret <2 x i64> %res +} + +define void @lshr_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { +; CHECK-LABEL: lshr_v4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z2.d +; CHECK-NEXT: lsr z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <4 x i64>, <4 x i64>* %a + %op2 = load <4 x i64>, <4 x i64>* %b + %res = lshr <4 x i64> %op1, %op2 + store <4 x i64> %res, <4 x i64>* %a + ret void +} + +; +; SHL +; + +define <2 x i8> @shl_v2i8(<2 x i8> %op1, <2 x i8> %op2) #0 { +; CHECK-LABEL: shl_v2i8: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI28_0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI28_0] +; CHECK-NEXT: and z1.d, z1.d, z2.d +; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shl <2 x i8> %op1, %op2 + ret <2 x i8> %res +} + +define <4 x i8> @shl_v4i8(<4 x i8> %op1, <4 x i8> %op2) #0 { +; CHECK-LABEL: shl_v4i8: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI29_0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI29_0] +; CHECK-NEXT: and z1.d, z1.d, z2.d +; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shl <4 x i8> %op1, %op2 + ret <4 x i8> %res +} + +define <8 x i8> @shl_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { +; CHECK-LABEL: shl_v8i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.b, vl8 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shl <8 x i8> %op1, %op2 + ret <8 x i8> %res +} + +define <16 x i8> @shl_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { +; CHECK-LABEL: shl_v16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shl <16 x i8> %op1, %op2 + ret <16 x i8> %res +} + +define void @shl_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { +; CHECK-LABEL: shl_v32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z2.b +; CHECK-NEXT: lsl z1.b, p0/m, z1.b, z3.b +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <32 x i8>, <32 x i8>* %a + %op2 = load <32 x i8>, <32 x i8>* %b + %res = shl <32 x i8> %op1, %op2 + store <32 x i8> %res, <32 x i8>* %a + ret void +} + +define <4 x i16> @shl_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { +; CHECK-LABEL: shl_v4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shl <4 x i16> %op1, %op2 + ret <4 x i16> %res +} + +define <8 x i16> @shl_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { +; CHECK-LABEL: shl_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shl <8 x i16> %op1, %op2 + ret <8 x i16> %res +} + +define void @shl_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { +; CHECK-LABEL: shl_v16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z2.h +; CHECK-NEXT: lsl z1.h, p0/m, z1.h, z3.h +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <16 x i16>, <16 x i16>* %a + %op2 = load <16 x i16>, <16 x i16>* %b + %res = shl <16 x i16> %op1, %op2 + store <16 x i16> %res, <16 x i16>* %a + ret void +} + +define <2 x i32> @shl_v2i32(<2 x i32> %op1, <2 x i32> %op2) #0 { +; CHECK-LABEL: shl_v2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shl <2 x i32> %op1, %op2 + ret <2 x i32> %res +} + +define <4 x i32> @shl_v4i32(<4 x i32> %op1, <4 x i32> %op2) #0 { +; CHECK-LABEL: shl_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shl <4 x i32> %op1, %op2 + ret <4 x i32> %res +} + +define void @shl_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { +; CHECK-LABEL: shl_v8i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: lsl z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <8 x i32>, <8 x i32>* %a + %op2 = load <8 x i32>, <8 x i32>* %b + %res = shl <8 x i32> %op1, %op2 + store <8 x i32> %res, <8 x i32>* %a + ret void +} + +define <1 x i64> @shl_v1i64(<1 x i64> %op1, <1 x i64> %op2) #0 { +; CHECK-LABEL: shl_v1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shl <1 x i64> %op1, %op2 + ret <1 x i64> %res +} + +define <2 x i64> @shl_v2i64(<2 x i64> %op1, <2 x i64> %op2) #0 { +; CHECK-LABEL: shl_v2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shl <2 x i64> %op1, %op2 + ret <2 x i64> %res +} + +define void @shl_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { +; CHECK-LABEL: shl_v4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z2.d +; CHECK-NEXT: lsl z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + %op1 = load <4 x i64>, <4 x i64>* %a + %op2 = load <4 x i64>, <4 x i64>* %b + %res = shl <4 x i64> %op1, %op2 + store <4 x i64> %res, <4 x i64>* %a + ret void +} + +attributes #0 = { "target-features"="+sve" } diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-build-vector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-build-vector.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-build-vector.ll @@ -0,0 +1,138 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +define void @build_vector_7_inc1_v4i1(ptr %a) #0 { +; CHECK-LABEL: build_vector_7_inc1_v4i1: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, #5 +; CHECK-NEXT: strb w8, [x0] +; CHECK-NEXT: ret + store <4 x i1> , ptr %a, align 1 + ret void +} + +define void @build_vector_7_inc1_v32i8(ptr %a) #0 { +; CHECK-LABEL: build_vector_7_inc1_v32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: index z0.b, #0, #1 +; CHECK-NEXT: mov z1.d, z0.d +; CHECK-NEXT: add z0.b, z0.b, #7 // =0x7 +; CHECK-NEXT: add z1.b, z1.b, #23 // =0x17 +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + store <32 x i8> , ptr %a, align 1 + ret void +} + +define void @build_vector_0_inc2_v16i16(ptr %a) #0 { +; CHECK-LABEL: build_vector_0_inc2_v16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: index z0.h, #0, #2 +; CHECK-NEXT: str q0, [x0] +; CHECK-NEXT: add z0.h, z0.h, #16 // =0x10 +; CHECK-NEXT: str q0, [x0, #16] +; CHECK-NEXT: ret + store <16 x i16> , ptr %a, align 2 + ret void +} + +; Negative const stride. +define void @build_vector_0_dec3_v8i32(ptr %a) #0 { +; CHECK-LABEL: build_vector_0_dec3_v8i32: +; CHECK: // %bb.0: +; CHECK-NEXT: index z0.s, #0, #-3 +; CHECK-NEXT: mov z1.s, #-12 // =0xfffffffffffffff4 +; CHECK-NEXT: add z1.s, z0.s, z1.s +; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: ret + store <8 x i32> , ptr %a, align 4 + ret void +} + +; Constant stride that's too big to be directly encoded into the index. +define void @build_vector_minus2_dec32_v4i64(ptr %a) #0 { +; CHECK-LABEL: build_vector_minus2_dec32_v4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: mov x8, #-32 +; CHECK-NEXT: mov z0.d, #-66 // =0xffffffffffffffbe +; CHECK-NEXT: mov z2.d, #-2 // =0xfffffffffffffffe +; CHECK-NEXT: index z1.d, #0, x8 +; CHECK-NEXT: add z0.d, z1.d, z0.d +; CHECK-NEXT: add z1.d, z1.d, z2.d +; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ret + store <4 x i64> , ptr %a, align 8 + ret void +} + +; Constant but not a sequence. +define void @build_vector_no_stride_v4i64(ptr %a) #0 { +; CHECK-LABEL: build_vector_no_stride_v4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: index z0.d, #1, #7 +; CHECK-NEXT: index z1.d, #0, #4 +; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ret + store <4 x i64> , ptr %a, align 8 + ret void +} + +define void @build_vector_0_inc2_v16f16(ptr %a) #0 { +; CHECK-LABEL: build_vector_0_inc2_v16f16: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI6_0 +; CHECK-NEXT: adrp x9, .LCPI6_1 +; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI6_0] +; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI6_1] +; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ret + store <16 x half> , ptr %a, align 2 + ret void +} + +; Negative const stride. +define void @build_vector_0_dec3_v8f32(ptr %a) #0 { +; CHECK-LABEL: build_vector_0_dec3_v8f32: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI7_0 +; CHECK-NEXT: adrp x9, .LCPI7_1 +; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI7_0] +; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI7_1] +; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ret + store <8 x float> , ptr %a, align 4 + ret void +} + +; Constant stride that's too big to be directly encoded into the index. +define void @build_vector_minus2_dec32_v4f64(ptr %a) #0 { +; CHECK-LABEL: build_vector_minus2_dec32_v4f64: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI8_0 +; CHECK-NEXT: adrp x9, .LCPI8_1 +; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI8_0] +; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI8_1] +; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ret + store <4 x double> , ptr %a, align 8 + ret void +} + +; Constant but not a sequence. +define void @build_vector_no_stride_v4f64(ptr %a) #0 { +; CHECK-LABEL: build_vector_no_stride_v4f64: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI9_0 +; CHECK-NEXT: adrp x9, .LCPI9_1 +; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI9_0] +; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI9_1] +; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ret + store <4 x double> , ptr %a, align 8 + ret void +} + + +attributes #0 = { "target-features"="+sve" } diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll @@ -0,0 +1,606 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +; +; i8 +; + +define <8 x i8> @concat_v8i8(<4 x i8> %op1, <4 x i8> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v8i8: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: mov z2.h, z1.h[3] +; CHECK-NEXT: mov z3.h, z1.h[2] +; CHECK-NEXT: mov z4.h, z1.h[1] +; CHECK-NEXT: fmov w10, s2 +; CHECK-NEXT: strb w8, [sp, #12] +; CHECK-NEXT: fmov w8, s3 +; CHECK-NEXT: strb w9, [sp, #8] +; CHECK-NEXT: fmov w9, s4 +; CHECK-NEXT: mov z1.h, z0.h[3] +; CHECK-NEXT: mov z5.h, z0.h[2] +; CHECK-NEXT: mov z0.h, z0.h[1] +; CHECK-NEXT: strb w10, [sp, #15] +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: strb w8, [sp, #14] +; CHECK-NEXT: fmov w8, s5 +; CHECK-NEXT: strb w9, [sp, #13] +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: strb w10, [sp, #11] +; CHECK-NEXT: strb w8, [sp, #10] +; CHECK-NEXT: strb w9, [sp, #9] +; CHECK-NEXT: ldr d0, [sp, #8] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %res = shufflevector <4 x i8> %op1, <4 x i8> %op2, <8 x i32> + ret <8 x i8> %res +} + +define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.b, vl8 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <8 x i8> %op1, <8 x i8> %op2, <16 x i32> + ret <16 x i8> %res +} + +define void @concat_v32i8(<16 x i8>* %a, <16 x i8>* %b, <32 x i8>* %c) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: ldr q1, [x0] +; CHECK-NEXT: splice z1.b, p0, z1.b, z0.b +; CHECK-NEXT: ptrue p0.b, vl32 +; CHECK-NEXT: st1b { z1.b }, p0, [x2] +; CHECK-NEXT: ret + %op1 = load <16 x i8>, <16 x i8>* %a + %op2 = load <16 x i8>, <16 x i8>* %b + %res = shufflevector <16 x i8> %op1, <16 x i8> %op2, <32 x i32> + store <32 x i8> %res, <32 x i8>* %c + ret void +} + +define void @concat_v64i8(<32 x i8>* %a, <32 x i8>* %b, <64 x i8>* %c) #0 { +; CHECK-LABEL: concat_v64i8: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: stp q0, q1, [x2, #32] +; CHECK-NEXT: stp q2, q3, [x2] +; CHECK-NEXT: ret + %op1 = load <32 x i8>, <32 x i8>* %a + %op2 = load <32 x i8>, <32 x i8>* %b + %res = shufflevector <32 x i8> %op1, <32 x i8> %op2, <64 x i32> + store <64 x i8> %res, <64 x i8>* %c + ret void +} + +; +; i16 +; + +define <4 x i16> @concat_v4i16(<2 x i16> %op1, <2 x i16> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: mov z1.s, z1.s[1] +; CHECK-NEXT: mov z0.s, z0.s[1] +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: fmov w11, s0 +; CHECK-NEXT: strh w8, [sp, #12] +; CHECK-NEXT: strh w9, [sp, #8] +; CHECK-NEXT: strh w10, [sp, #14] +; CHECK-NEXT: strh w11, [sp, #10] +; CHECK-NEXT: ldr d0, [sp, #8] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %res = shufflevector <2 x i16> %op1, <2 x i16> %op2, <4 x i32> + ret <4 x i16> %res +} + +; Don't use SVE for 128-bit vectors. +define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <4 x i16> %op1, <4 x i16> %op2, <8 x i32> + ret <8 x i16> %res +} + +define void @concat_v16i16(<8 x i16>* %a, <8 x i16>* %b, <16 x i16>* %c) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: ldr q1, [x0] +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z1.h }, p0, [x2] +; CHECK-NEXT: ret + %op1 = load <8 x i16>, <8 x i16>* %a + %op2 = load <8 x i16>, <8 x i16>* %b + %res = shufflevector <8 x i16> %op1, <8 x i16> %op2, <16 x i32> + store <16 x i16> %res, <16 x i16>* %c + ret void +} + +define void @concat_v32i16(<16 x i16>* %a, <16 x i16>* %b, <32 x i16>* %c) #0 { +; CHECK-LABEL: concat_v32i16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: stp q0, q1, [x2, #32] +; CHECK-NEXT: stp q2, q3, [x2] +; CHECK-NEXT: ret + %op1 = load <16 x i16>, <16 x i16>* %a + %op2 = load <16 x i16>, <16 x i16>* %b + %res = shufflevector <16 x i16> %op1, <16 x i16> %op2, <32 x i32> + store <32 x i16> %res, <32 x i16>* %c + ret void +} + +; +; i32 +; + +; Don't use SVE for 64-bit vectors. +define <2 x i32> @concat_v2i32(<1 x i32> %op1, <1 x i32> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: zip1 z0.s, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <1 x i32> %op1, <1 x i32> %op2, <2 x i32> + ret <2 x i32> %res +} + +; Don't use SVE for 128-bit vectors. +define <4 x i32> @concat_v4i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <2 x i32> %op1, <2 x i32> %op2, <4 x i32> + ret <4 x i32> %res +} + +define void @concat_v8i32(<4 x i32>* %a, <4 x i32>* %b, <8 x i32>* %c) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v8i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldr q1, [x0] +; CHECK-NEXT: splice z1.s, p0, z1.s, z0.s +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z1.s }, p0, [x2] +; CHECK-NEXT: ret + %op1 = load <4 x i32>, <4 x i32>* %a + %op2 = load <4 x i32>, <4 x i32>* %b + %res = shufflevector <4 x i32> %op1, <4 x i32> %op2, <8 x i32> + store <8 x i32> %res, <8 x i32>* %c + ret void +} + +define void @concat_v16i32(<8 x i32>* %a, <8 x i32>* %b, <16 x i32>* %c) #0 { +; CHECK-LABEL: concat_v16i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: stp q0, q1, [x2, #32] +; CHECK-NEXT: stp q2, q3, [x2] +; CHECK-NEXT: ret + %op1 = load <8 x i32>, <8 x i32>* %a + %op2 = load <8 x i32>, <8 x i32>* %b + %res = shufflevector <8 x i32> %op1, <8 x i32> %op2, <16 x i32> + store <16 x i32> %res, <16 x i32>* %c + ret void +} + +; +; i64 +; + +; Don't use SVE for 128-bit vectors. +define <2 x i64> @concat_v2i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <1 x i64> %op1, <1 x i64> %op2, <2 x i32> + ret <2 x i64> %res +} + +define void @concat_v4i64(<2 x i64>* %a, <2 x i64>* %b, <4 x i64>* %c) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: ldr q1, [x0] +; CHECK-NEXT: splice z1.d, p0, z1.d, z0.d +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z1.d }, p0, [x2] +; CHECK-NEXT: ret + %op1 = load <2 x i64>, <2 x i64>* %a + %op2 = load <2 x i64>, <2 x i64>* %b + %res = shufflevector <2 x i64> %op1, <2 x i64> %op2, <4 x i32> + store <4 x i64> %res, <4 x i64>* %c + ret void +} + +define void @concat_v8i64(<4 x i64>* %a, <4 x i64>* %b, <8 x i64>* %c) #0 { +; CHECK-LABEL: concat_v8i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: stp q0, q1, [x2, #32] +; CHECK-NEXT: stp q2, q3, [x2] +; CHECK-NEXT: ret + %op1 = load <4 x i64>, <4 x i64>* %a + %op2 = load <4 x i64>, <4 x i64>* %b + %res = shufflevector <4 x i64> %op1, <4 x i64> %op2, <8 x i32> + store <8 x i64> %res, <8 x i64>* %c + ret void +} + +; +; f16 +; + +define <4 x half> @concat_v4f16(<2 x half> %op1, <2 x half> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4f16: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: str h1, [sp, #12] +; CHECK-NEXT: str h0, [sp, #8] +; CHECK-NEXT: mov z1.h, z1.h[1] +; CHECK-NEXT: mov z0.h, z0.h[1] +; CHECK-NEXT: str h1, [sp, #14] +; CHECK-NEXT: str h0, [sp, #10] +; CHECK-NEXT: ldr d0, [sp, #8] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %res = shufflevector <2 x half> %op1, <2 x half> %op2, <4 x i32> + ret <4 x half> %res +} + +define <8 x half> @concat_v8f16(<4 x half> %op1, <4 x half> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v8f16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <4 x half> %op1, <4 x half> %op2, <8 x i32> + ret <8 x half> %res +} + +define void @concat_v16f16(<8 x half>* %a, <8 x half>* %b, <16 x half>* %c) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v16f16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: ldr q1, [x0] +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z1.h }, p0, [x2] +; CHECK-NEXT: ret + %op1 = load <8 x half>, <8 x half>* %a + %op2 = load <8 x half>, <8 x half>* %b + %res = shufflevector <8 x half> %op1, <8 x half> %op2, <16 x i32> + store <16 x half> %res, <16 x half>* %c + ret void +} + +define void @concat_v32f16(<16 x half>* %a, <16 x half>* %b, <32 x half>* %c) #0 { +; CHECK-LABEL: concat_v32f16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: stp q0, q1, [x2, #32] +; CHECK-NEXT: stp q2, q3, [x2] +; CHECK-NEXT: ret + %op1 = load <16 x half>, <16 x half>* %a + %op2 = load <16 x half>, <16 x half>* %b + %res = shufflevector <16 x half> %op1, <16 x half> %op2, <32 x i32> + store <32 x half> %res, <32 x half>* %c + ret void +} + +; +; i32 +; + +; Don't use SVE for 64-bit vectors. +define <2 x float> @concat_v2f32(<1 x float> %op1, <1 x float> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v2f32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: zip1 z0.s, z0.s, z1.s +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <1 x float> %op1, <1 x float> %op2, <2 x i32> + ret <2 x float> %res +} + +; Don't use SVE for 128-bit vectors. +define <4 x float> @concat_v4f32(<2 x float> %op1, <2 x float> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4f32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <2 x float> %op1, <2 x float> %op2, <4 x i32> + ret <4 x float> %res +} + +define void @concat_v8f32(<4 x float>* %a, <4 x float>* %b, <8 x float>* %c) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v8f32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldr q1, [x0] +; CHECK-NEXT: splice z1.s, p0, z1.s, z0.s +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z1.s }, p0, [x2] +; CHECK-NEXT: ret + %op1 = load <4 x float>, <4 x float>* %a + %op2 = load <4 x float>, <4 x float>* %b + %res = shufflevector <4 x float> %op1, <4 x float> %op2, <8 x i32> + store <8 x float> %res, <8 x float>* %c + ret void +} + +define void @concat_v16f32(<8 x float>* %a, <8 x float>* %b, <16 x float>* %c) #0 { +; CHECK-LABEL: concat_v16f32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: stp q0, q1, [x2, #32] +; CHECK-NEXT: stp q2, q3, [x2] +; CHECK-NEXT: ret + %op1 = load <8 x float>, <8 x float>* %a + %op2 = load <8 x float>, <8 x float>* %b + %res = shufflevector <8 x float> %op1, <8 x float> %op2, <16 x i32> + store <16 x float> %res, <16 x float>* %c + ret void +} + +; +; f64 +; + +; Don't use SVE for 128-bit vectors. +define <2 x double> @concat_v2f64(<1 x double> %op1, <1 x double> %op2) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v2f64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %res = shufflevector <1 x double> %op1, <1 x double> %op2, <2 x i32> + ret <2 x double> %res +} + +define void @concat_v4f64(<2 x double>* %a, <2 x double>* %b, <4 x double>* %c) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4f64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: ldr q1, [x0] +; CHECK-NEXT: splice z1.d, p0, z1.d, z0.d +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z1.d }, p0, [x2] +; CHECK-NEXT: ret + %op1 = load <2 x double>, <2 x double>* %a + %op2 = load <2 x double>, <2 x double>* %b + %res = shufflevector <2 x double> %op1, <2 x double> %op2, <4 x i32> + store <4 x double> %res, <4 x double>* %c + ret void +} + +define void @concat_v8f64(<4 x double>* %a, <4 x double>* %b, <8 x double>* %c) #0 { +; CHECK-LABEL: concat_v8f64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: stp q0, q1, [x2, #32] +; CHECK-NEXT: stp q2, q3, [x2] +; CHECK-NEXT: ret + %op1 = load <4 x double>, <4 x double>* %a + %op2 = load <4 x double>, <4 x double>* %b + %res = shufflevector <4 x double> %op1, <4 x double> %op2, <8 x i32> + store <8 x double> %res, <8 x double>* %c + ret void +} + +; +; undef +; + +define void @concat_v32i8_undef(<16 x i8>* %a, <32 x i8>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v32i8_undef: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl32 +; CHECK-NEXT: st1b { z0.b }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <16 x i8>, <16 x i8>* %a + %res = shufflevector <16 x i8> %op1, <16 x i8> undef, <32 x i32> + store <32 x i8> %res, <32 x i8>* %b + ret void +} + +define void @concat_v16i16_undef(<8 x i16>* %a, <16 x i16>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v16i16_undef: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z0.h }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <8 x i16>, <8 x i16>* %a + %res = shufflevector <8 x i16> %op1, <8 x i16> undef, <16 x i32> + store <16 x i16> %res, <16 x i16>* %b + ret void +} + +define void @concat_v8i32_undef(<4 x i32>* %a, <8 x i32>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v8i32_undef: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z0.s }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <4 x i32>, <4 x i32>* %a + %res = shufflevector <4 x i32> %op1, <4 x i32> undef, <8 x i32> + store <8 x i32> %res, <8 x i32>* %b + ret void +} + +define void @concat_v4i64_undef(<2 x i64>* %a, <4 x i64>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4i64_undef: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z0.d }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <2 x i64>, <2 x i64>* %a + %res = shufflevector <2 x i64> %op1, <2 x i64> undef, <4 x i32> + store <4 x i64> %res, <4 x i64>* %b + ret void +} + +; +; > 2 operands +; + +define void @concat_v32i8_4op(<8 x i8>* %a, <32 x i8>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v32i8_4op: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ptrue p0.b, vl32 +; CHECK-NEXT: st1b { z0.b }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <8 x i8>, <8 x i8>* %a + %shuffle = shufflevector <8 x i8> %op1, <8 x i8> undef, <16 x i32> + %res = shufflevector <16 x i8> %shuffle, <16 x i8> undef, <32 x i32> + store <32 x i8> %res, <32 x i8>* %b + ret void +} + +define void @concat_v16i16_4op(<4 x i16>* %a, <16 x i16>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v16i16_4op: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z0.h }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <4 x i16>, <4 x i16>* %a + %shuffle = shufflevector <4 x i16> %op1, <4 x i16> undef, <8 x i32> + %res = shufflevector <8 x i16> %shuffle, <8 x i16> undef, <16 x i32> + store <16 x i16> %res, <16 x i16>* %b + ret void +} + +define void @concat_v8i32_4op(<2 x i32>* %a, <8 x i32>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v8i32_4op: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z0.s }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <2 x i32>, <2 x i32>* %a + %shuffle = shufflevector <2 x i32> %op1, <2 x i32> undef, <4 x i32> + %res = shufflevector <4 x i32> %shuffle, <4 x i32> undef, <8 x i32> + store <8 x i32> %res, <8 x i32>* %b + ret void +} + +define void @concat_v4i64_4op(<1 x i64>* %a, <4 x i64>* %b) vscale_range(2,0) #0 { +; CHECK-LABEL: concat_v4i64_4op: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z0.d }, p0, [x1] +; CHECK-NEXT: ret + %op1 = load <1 x i64>, <1 x i64>* %a + %shuffle = shufflevector <1 x i64> %op1, <1 x i64> undef, <2 x i32> + %res = shufflevector <2 x i64> %shuffle, <2 x i64> undef, <4 x i32> + store <4 x i64> %res, <4 x i64>* %b + ret void +} + +attributes #0 = { "target-features"="+sve" } diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll @@ -0,0 +1,322 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +; i1 + +define <4 x i1> @extract_subvector_v8i1(<8 x i1> %op) #0 { +; CHECK-LABEL: extract_subvector_v8i1: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z1.b, z0.b[7] +; CHECK-NEXT: mov z2.b, z0.b[6] +; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: mov z1.b, z0.b[5] +; CHECK-NEXT: mov z0.b, z0.b[4] +; CHECK-NEXT: fmov w9, s2 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: fmov w11, s0 +; CHECK-NEXT: strh w8, [sp, #14] +; CHECK-NEXT: strh w9, [sp, #12] +; CHECK-NEXT: strh w10, [sp, #10] +; CHECK-NEXT: strh w11, [sp, #8] +; CHECK-NEXT: ldr d0, [sp, #8] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %ret = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> %op, i64 4) + ret <4 x i1> %ret +} + +; i8 + +define <4 x i8> @extract_subvector_v8i8(<8 x i8> %op) #0 { +; CHECK-LABEL: extract_subvector_v8i8: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z1.b, z0.b[7] +; CHECK-NEXT: mov z2.b, z0.b[6] +; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: mov z1.b, z0.b[5] +; CHECK-NEXT: mov z0.b, z0.b[4] +; CHECK-NEXT: fmov w9, s2 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: fmov w11, s0 +; CHECK-NEXT: strh w8, [sp, #14] +; CHECK-NEXT: strh w9, [sp, #12] +; CHECK-NEXT: strh w10, [sp, #10] +; CHECK-NEXT: strh w11, [sp, #8] +; CHECK-NEXT: ldr d0, [sp, #8] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %ret = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> %op, i64 4) + ret <4 x i8> %ret +} + +define <8 x i8> @extract_subvector_v16i8(<16 x i8> %op) #0 { +; CHECK-LABEL: extract_subvector_v16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> %op, i64 8) + ret <8 x i8> %ret +} + +define void @extract_subvector_v32i8(<32 x i8>* %a, <16 x i8>* %b) #0 { +; CHECK-LABEL: extract_subvector_v32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: str q0, [x1] +; CHECK-NEXT: ret + %op = load <32 x i8>, <32 x i8>* %a + %ret = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> %op, i64 16) + store <16 x i8> %ret, <16 x i8>* %b + ret void +} + +; i16 + +define <2 x i16> @extract_subvector_v4i16(<4 x i16> %op) #0 { +; CHECK-LABEL: extract_subvector_v4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <2 x i16> @llvm.vector.extract.v2i16.v4i16(<4 x i16> %op, i64 2) + ret <2 x i16> %ret +} + +define <4 x i16> @extract_subvector_v8i16(<8 x i16> %op) #0 { +; CHECK-LABEL: extract_subvector_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <4 x i16> @llvm.vector.extract.v4i16.v8i16(<8 x i16> %op, i64 4) + ret <4 x i16> %ret +} + +define void @extract_subvector_v16i16(<16 x i16>* %a, <8 x i16>* %b) #0 { +; CHECK-LABEL: extract_subvector_v16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: str q0, [x1] +; CHECK-NEXT: ret + %op = load <16 x i16>, <16 x i16>* %a + %ret = call <8 x i16> @llvm.vector.extract.v8i16.v16i16(<16 x i16> %op, i64 8) + store <8 x i16> %ret, <8 x i16>* %b + ret void +} + +; i32 + +define <1 x i32> @extract_subvector_v2i32(<2 x i32> %op) #0 { +; CHECK-LABEL: extract_subvector_v2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z0.s, z0.s[1] +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: insr z0.s, w8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <1 x i32> @llvm.vector.extract.v1i32.v2i32(<2 x i32> %op, i64 1) + ret <1 x i32> %ret +} + +define <2 x i32> @extract_subvector_v4i32(<4 x i32> %op) #0 { +; CHECK-LABEL: extract_subvector_v4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <2 x i32> @llvm.vector.extract.v2i32.v4i32(<4 x i32> %op, i64 2) + ret <2 x i32> %ret +} + +define void @extract_subvector_v8i32(<8 x i32>* %a, <4 x i32>* %b) #0 { +; CHECK-LABEL: extract_subvector_v8i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: str q0, [x1] +; CHECK-NEXT: ret + %op = load <8 x i32>, <8 x i32>* %a + %ret = call <4 x i32> @llvm.vector.extract.v4i32.v8i32(<8 x i32> %op, i64 4) + store <4 x i32> %ret, <4 x i32>* %b + ret void +} + +; i64 + +define <1 x i64> @extract_subvector_v2i64(<2 x i64> %op) #0 { +; CHECK-LABEL: extract_subvector_v2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <1 x i64> @llvm.vector.extract.v1i64.v2i64(<2 x i64> %op, i64 1) + ret <1 x i64> %ret +} + +define void @extract_subvector_v4i64(<4 x i64>* %a, <2 x i64>* %b) #0 { +; CHECK-LABEL: extract_subvector_v4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: str q0, [x1] +; CHECK-NEXT: ret + %op = load <4 x i64>, <4 x i64>* %a + %ret = call <2 x i64> @llvm.vector.extract.v2i64.v4i64(<4 x i64> %op, i64 2) + store <2 x i64> %ret, <2 x i64>* %b + ret void +} + +; f16 + +define <2 x half> @extract_subvector_v4f16(<4 x half> %op) #0 { +; CHECK-LABEL: extract_subvector_v4f16: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z1.h, z0.h[3] +; CHECK-NEXT: mov z0.h, z0.h[2] +; CHECK-NEXT: str h1, [sp, #10] +; CHECK-NEXT: str h0, [sp, #8] +; CHECK-NEXT: ldr d0, [sp, #8] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %ret = call <2 x half> @llvm.vector.extract.v2f16.v4f16(<4 x half> %op, i64 2) + ret <2 x half> %ret +} + +define <4 x half> @extract_subvector_v8f16(<8 x half> %op) #0 { +; CHECK-LABEL: extract_subvector_v8f16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <4 x half> @llvm.vector.extract.v4f16.v8f16(<8 x half> %op, i64 4) + ret <4 x half> %ret +} + +define void @extract_subvector_v16f16(<16 x half>* %a, <8 x half>* %b) #0 { +; CHECK-LABEL: extract_subvector_v16f16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: str q0, [x1] +; CHECK-NEXT: ret + %op = load <16 x half>, <16 x half>* %a + %ret = call <8 x half> @llvm.vector.extract.v8f16.v16f16(<16 x half> %op, i64 8) + store <8 x half> %ret, <8 x half>* %b + ret void +} + +; f32 + +define <1 x float> @extract_subvector_v2f32(<2 x float> %op) #0 { +; CHECK-LABEL: extract_subvector_v2f32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z0.s, z0.s[1] +; CHECK-NEXT: insr z0.s, s0 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <1 x float> @llvm.vector.extract.v1f32.v2f32(<2 x float> %op, i64 1) + ret <1 x float> %ret +} + +define <2 x float> @extract_subvector_v4f32(<4 x float> %op) #0 { +; CHECK-LABEL: extract_subvector_v4f32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <2 x float> @llvm.vector.extract.v2f32.v4f32(<4 x float> %op, i64 2) + ret <2 x float> %ret +} + +define void @extract_subvector_v8f32(<8 x float>* %a, <4 x float>* %b) #0 { +; CHECK-LABEL: extract_subvector_v8f32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: str q0, [x1] +; CHECK-NEXT: ret + %op = load <8 x float>, <8 x float>* %a + %ret = call <4 x float> @llvm.vector.extract.v4f32.v8f32(<8 x float> %op, i64 4) + store <4 x float> %ret, <4 x float>* %b + ret void +} + +; f64 + +define <1 x double> @extract_subvector_v2f64(<2 x double> %op) #0 { +; CHECK-LABEL: extract_subvector_v2f64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %ret = call <1 x double> @llvm.vector.extract.v1f64.v2f64(<2 x double> %op, i64 1) + ret <1 x double> %ret +} + +define void @extract_subvector_v4f64(<4 x double>* %a, <2 x double>* %b) #0 { +; CHECK-LABEL: extract_subvector_v4f64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: str q0, [x1] +; CHECK-NEXT: ret + %op = load <4 x double>, <4 x double>* %a + %ret = call <2 x double> @llvm.vector.extract.v2f64.v4f64(<4 x double> %op, i64 2) + store <2 x double> %ret, <2 x double>* %b + ret void +} + +declare <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1>, i64) + +declare <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8>, i64) +declare <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8>, i64) +declare <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8>, i64) +declare <32 x i8> @llvm.vector.extract.v32i8.v64i8(<64 x i8>, i64) + +declare <2 x i16> @llvm.vector.extract.v2i16.v4i16(<4 x i16>, i64) +declare <4 x i16> @llvm.vector.extract.v4i16.v8i16(<8 x i16>, i64) +declare <8 x i16> @llvm.vector.extract.v8i16.v16i16(<16 x i16>, i64) +declare <16 x i16> @llvm.vector.extract.v16i16.v32i16(<32 x i16>, i64) + +declare <1 x i32> @llvm.vector.extract.v1i32.v2i32(<2 x i32>, i64) +declare <2 x i32> @llvm.vector.extract.v2i32.v4i32(<4 x i32>, i64) +declare <4 x i32> @llvm.vector.extract.v4i32.v8i32(<8 x i32>, i64) +declare <8 x i32> @llvm.vector.extract.v8i32.v16i32(<16 x i32>, i64) + +declare <1 x i64> @llvm.vector.extract.v1i64.v2i64(<2 x i64>, i64) +declare <2 x i64> @llvm.vector.extract.v2i64.v4i64(<4 x i64>, i64) +declare <4 x i64> @llvm.vector.extract.v4i64.v8i64(<8 x i64>, i64) + +declare <2 x half> @llvm.vector.extract.v2f16.v4f16(<4 x half>, i64) +declare <4 x half> @llvm.vector.extract.v4f16.v8f16(<8 x half>, i64) +declare <8 x half> @llvm.vector.extract.v8f16.v16f16(<16 x half>, i64) +declare <16 x half> @llvm.vector.extract.v16f16.v32f16(<32 x half>, i64) + +declare <1 x float> @llvm.vector.extract.v1f32.v2f32(<2 x float>, i64) +declare <2 x float> @llvm.vector.extract.v2f32.v4f32(<4 x float>, i64) +declare <4 x float> @llvm.vector.extract.v4f32.v8f32(<8 x float>, i64) +declare <8 x float> @llvm.vector.extract.v8f32.v16f32(<16 x float>, i64) + +declare <1 x double> @llvm.vector.extract.v1f64.v2f64(<2 x double>, i64) +declare <2 x double> @llvm.vector.extract.v2f64.v4f64(<4 x double>, i64) +declare <4 x double> @llvm.vector.extract.v4f64.v8f64(<8 x double>, i64) + +attributes #0 = { "target-features"="+sve" } diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-vector-elt.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-vector-elt.ll @@ -0,0 +1,122 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +; +; extractelement +; + +define half @extractelement_v2f16(<2 x half> %op1) #0 { +; CHECK-LABEL: extractelement_v2f16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z0.h, z0.h[1] +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 +; CHECK-NEXT: ret + %r = extractelement <2 x half> %op1, i64 1 + ret half %r +} + +define half @extractelement_v4f16(<4 x half> %op1) #0 { +; CHECK-LABEL: extractelement_v4f16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z0.h, z0.h[3] +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 +; CHECK-NEXT: ret + %r = extractelement <4 x half> %op1, i64 3 + ret half %r +} + +define half @extractelement_v8f16(<8 x half> %op1) #0 { +; CHECK-LABEL: extractelement_v8f16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: mov z0.h, z0.h[7] +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 +; CHECK-NEXT: ret + %r = extractelement <8 x half> %op1, i64 7 + ret half %r +} + +define half @extractelement_v16f16(<16 x half>* %a) #0 { +; CHECK-LABEL: extractelement_v16f16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: mov z0.h, z0.h[7] +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 +; CHECK-NEXT: ret + %op1 = load <16 x half>, <16 x half>* %a + %r = extractelement <16 x half> %op1, i64 15 + ret half %r +} + +define float @extractelement_v2f32(<2 x float> %op1) #0 { +; CHECK-LABEL: extractelement_v2f32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: mov z0.s, z0.s[1] +; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 +; CHECK-NEXT: ret + %r = extractelement <2 x float> %op1, i64 1 + ret float %r +} + +define float @extractelement_v4f32(<4 x float> %op1) #0 { +; CHECK-LABEL: extractelement_v4f32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: mov z0.s, z0.s[3] +; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 +; CHECK-NEXT: ret + %r = extractelement <4 x float> %op1, i64 3 + ret float %r +} + +define float @extractelement_v8f32(<8 x float>* %a) #0 { +; CHECK-LABEL: extractelement_v8f32: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: mov z0.s, z0.s[3] +; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 +; CHECK-NEXT: ret + %op1 = load <8 x float>, <8 x float>* %a + %r = extractelement <8 x float> %op1, i64 7 + ret float %r +} + +define double @extractelement_v1f64(<1 x double> %op1) #0 { +; CHECK-LABEL: extractelement_v1f64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %r = extractelement <1 x double> %op1, i64 0 + ret double %r +} + +define double @extractelement_v2f64(<2 x double> %op1) #0 { +; CHECK-LABEL: extractelement_v2f64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: mov z0.d, z0.d[1] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %r = extractelement <2 x double> %op1, i64 1 + ret double %r +} + +define double @extractelement_v4f64(<4 x double>* %a) #0 { +; CHECK-LABEL: extractelement_v4f64: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr q0, [x0, #16] +; CHECK-NEXT: mov z0.d, z0.d[1] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: ret + %op1 = load <4 x double>, <4 x double>* %a + %r = extractelement <4 x double> %op1, i64 3 + ret double %r +} + +attributes #0 = { "target-features"="+sve" } diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll @@ -45,15 +45,11 @@ define void @add_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: add_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] -; CHECK-NEXT: add z1.b, z1.b, z3.b +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: add z0.b, z0.b, z2.b -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: add z1.b, z1.b, z3.b +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -65,23 +61,15 @@ define void @add_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 { ; CHECK-LABEL: add_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #32 -; CHECK-NEXT: mov w9, #48 -; CHECK-NEXT: mov w10, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, x9] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x10] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z4.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z5.b }, p0/z, [x1, x9] -; CHECK-NEXT: ld1b { z6.b }, p0/z, [x1, x10] -; CHECK-NEXT: ld1b { z7.b }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: add z0.b, z0.b, z4.b +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: add z1.b, z1.b, z5.b +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: add z0.b, z3.b, z7.b -; CHECK-NEXT: add z1.b, z2.b, z6.b +; CHECK-NEXT: add z0.b, z2.b, z6.b +; CHECK-NEXT: add z1.b, z3.b, z4.b ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <64 x i8>, <64 x i8>* %a @@ -130,15 +118,11 @@ define void @add_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: add_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] -; CHECK-NEXT: add z1.h, z1.h, z3.h +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: add z0.h, z0.h, z2.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: add z1.h, z1.h, z3.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -150,23 +134,15 @@ define void @add_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 { ; CHECK-LABEL: add_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #16 -; CHECK-NEXT: mov x9, #24 -; CHECK-NEXT: mov x10, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z4.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z5.h }, p0/z, [x1, x9, lsl #1] -; CHECK-NEXT: ld1h { z6.h }, p0/z, [x1, x10, lsl #1] -; CHECK-NEXT: ld1h { z7.h }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: add z0.h, z0.h, z4.h +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: add z1.h, z1.h, z5.h +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: add z0.h, z3.h, z7.h -; CHECK-NEXT: add z1.h, z2.h, z6.h +; CHECK-NEXT: add z0.h, z2.h, z6.h +; CHECK-NEXT: add z1.h, z3.h, z4.h ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i16>, <32 x i16>* %a @@ -203,15 +179,11 @@ define void @add_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: add_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: add z1.s, z1.s, z3.s +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: add z0.s, z0.s, z2.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: add z1.s, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -223,23 +195,15 @@ define void @add_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 { ; CHECK-LABEL: add_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: mov x9, #12 -; CHECK-NEXT: mov x10, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x9, lsl #2] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1, x9, lsl #2] -; CHECK-NEXT: ld1w { z6.s }, p0/z, [x1, x10, lsl #2] -; CHECK-NEXT: ld1w { z7.s }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: add z0.s, z0.s, z4.s +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: add z1.s, z1.s, z5.s +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: add z0.s, z3.s, z7.s -; CHECK-NEXT: add z1.s, z2.s, z6.s +; CHECK-NEXT: add z0.s, z2.s, z6.s +; CHECK-NEXT: add z1.s, z3.s, z4.s ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i32>, <16 x i32>* %a @@ -276,15 +240,11 @@ define void @add_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: add_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: add z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: add z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: add z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -296,23 +256,15 @@ define void @add_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 { ; CHECK-LABEL: add_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: mov x9, #6 -; CHECK-NEXT: mov x10, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, x9, lsl #3] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, x10, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x1, x9, lsl #3] -; CHECK-NEXT: ld1d { z6.d }, p0/z, [x1, x10, lsl #3] -; CHECK-NEXT: ld1d { z7.d }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: add z0.d, z0.d, z4.d +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: add z1.d, z1.d, z5.d +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: add z0.d, z3.d, z7.d -; CHECK-NEXT: add z1.d, z2.d, z6.d +; CHECK-NEXT: add z0.d, z2.d, z6.d +; CHECK-NEXT: add z1.d, z3.d, z4.d ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i64>, <8 x i64>* %a @@ -368,15 +320,12 @@ define void @mul_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: mul_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] -; CHECK-NEXT: mul z1.b, p0/m, z1.b, z3.b +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: mul z0.b, p0/m, z0.b, z2.b -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mul z1.b, p0/m, z1.b, z3.b +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -388,25 +337,18 @@ define void @mul_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 { ; CHECK-LABEL: mul_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #32 -; CHECK-NEXT: mov w9, #48 -; CHECK-NEXT: mov w10, #16 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, x9] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x10] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z4.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z5.b }, p0/z, [x1, x9] -; CHECK-NEXT: ld1b { z6.b }, p0/z, [x1, x10] -; CHECK-NEXT: ld1b { z7.b }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: mul z0.b, p0/m, z0.b, z4.b ; CHECK-NEXT: mul z1.b, p0/m, z1.b, z5.b +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: mul z0.b, p0/m, z0.b, z7.b -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: mul z1.b, p0/m, z1.b, z6.b +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: mul z0.b, p0/m, z0.b, z6.b +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: mul z1.b, p0/m, z1.b, z4.b ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <64 x i8>, <64 x i8>* %a @@ -458,15 +400,12 @@ define void @mul_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: mul_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] -; CHECK-NEXT: mul z1.h, p0/m, z1.h, z3.h +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z2.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mul z1.h, p0/m, z1.h, z3.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -478,25 +417,18 @@ define void @mul_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 { ; CHECK-LABEL: mul_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #16 -; CHECK-NEXT: mov x9, #24 -; CHECK-NEXT: mov x10, #8 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z4.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z5.h }, p0/z, [x1, x9, lsl #1] -; CHECK-NEXT: ld1h { z6.h }, p0/z, [x1, x10, lsl #1] -; CHECK-NEXT: ld1h { z7.h }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z4.h ; CHECK-NEXT: mul z1.h, p0/m, z1.h, z5.h +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: mul z0.h, p0/m, z0.h, z7.h -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: mul z1.h, p0/m, z1.h, z6.h +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: mul z0.h, p0/m, z0.h, z6.h +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: mul z1.h, p0/m, z1.h, z4.h ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i16>, <32 x i16>* %a @@ -535,15 +467,12 @@ define void @mul_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: mul_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: mul z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mul z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -555,25 +484,18 @@ define void @mul_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 { ; CHECK-LABEL: mul_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: mov x9, #12 -; CHECK-NEXT: mov x10, #4 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x9, lsl #2] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1, x9, lsl #2] -; CHECK-NEXT: ld1w { z6.s }, p0/z, [x1, x10, lsl #2] -; CHECK-NEXT: ld1w { z7.s }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z4.s ; CHECK-NEXT: mul z1.s, p0/m, z1.s, z5.s +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: mul z0.s, p0/m, z0.s, z7.s -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: mul z1.s, p0/m, z1.s, z6.s +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: mul z0.s, p0/m, z0.s, z6.s +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: mul z1.s, p0/m, z1.s, z4.s ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i32>, <16 x i32>* %a @@ -612,15 +534,12 @@ define void @mul_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: mul_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: mul z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mul z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -632,25 +551,18 @@ define void @mul_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 { ; CHECK-LABEL: mul_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: mov x9, #6 -; CHECK-NEXT: mov x10, #2 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, x9, lsl #3] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, x10, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x1, x9, lsl #3] -; CHECK-NEXT: ld1d { z6.d }, p0/z, [x1, x10, lsl #3] -; CHECK-NEXT: ld1d { z7.d }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z4.d ; CHECK-NEXT: mul z1.d, p0/m, z1.d, z5.d +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: mul z0.d, p0/m, z0.d, z7.d -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: mul z1.d, p0/m, z1.d, z6.d +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: mul z0.d, p0/m, z0.d, z6.d +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: mul z1.d, p0/m, z1.d, z4.d ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i64>, <8 x i64>* %a @@ -703,15 +615,11 @@ define void @sub_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: sub_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] -; CHECK-NEXT: sub z1.b, z1.b, z3.b +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: sub z0.b, z0.b, z2.b -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: sub z1.b, z1.b, z3.b +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -723,23 +631,15 @@ define void @sub_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 { ; CHECK-LABEL: sub_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #32 -; CHECK-NEXT: mov w9, #48 -; CHECK-NEXT: mov w10, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, x9] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x10] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z4.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z5.b }, p0/z, [x1, x9] -; CHECK-NEXT: ld1b { z6.b }, p0/z, [x1, x10] -; CHECK-NEXT: ld1b { z7.b }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: sub z0.b, z0.b, z4.b +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: sub z1.b, z1.b, z5.b +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: sub z0.b, z3.b, z7.b -; CHECK-NEXT: sub z1.b, z2.b, z6.b +; CHECK-NEXT: sub z0.b, z2.b, z6.b +; CHECK-NEXT: sub z1.b, z3.b, z4.b ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <64 x i8>, <64 x i8>* %a @@ -788,15 +688,11 @@ define void @sub_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: sub_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] -; CHECK-NEXT: sub z1.h, z1.h, z3.h +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: sub z0.h, z0.h, z2.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: sub z1.h, z1.h, z3.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -808,23 +704,15 @@ define void @sub_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 { ; CHECK-LABEL: sub_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #16 -; CHECK-NEXT: mov x9, #24 -; CHECK-NEXT: mov x10, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z4.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z5.h }, p0/z, [x1, x9, lsl #1] -; CHECK-NEXT: ld1h { z6.h }, p0/z, [x1, x10, lsl #1] -; CHECK-NEXT: ld1h { z7.h }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: sub z0.h, z0.h, z4.h +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: sub z1.h, z1.h, z5.h +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: sub z0.h, z3.h, z7.h -; CHECK-NEXT: sub z1.h, z2.h, z6.h +; CHECK-NEXT: sub z0.h, z2.h, z6.h +; CHECK-NEXT: sub z1.h, z3.h, z4.h ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i16>, <32 x i16>* %a @@ -861,15 +749,11 @@ define void @sub_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: sub_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: sub z1.s, z1.s, z3.s +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: sub z0.s, z0.s, z2.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: sub z1.s, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -881,23 +765,15 @@ define void @sub_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 { ; CHECK-LABEL: sub_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: mov x9, #12 -; CHECK-NEXT: mov x10, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x9, lsl #2] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1, x9, lsl #2] -; CHECK-NEXT: ld1w { z6.s }, p0/z, [x1, x10, lsl #2] -; CHECK-NEXT: ld1w { z7.s }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: sub z0.s, z0.s, z4.s +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: sub z1.s, z1.s, z5.s +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: sub z0.s, z3.s, z7.s -; CHECK-NEXT: sub z1.s, z2.s, z6.s +; CHECK-NEXT: sub z0.s, z2.s, z6.s +; CHECK-NEXT: sub z1.s, z3.s, z4.s ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i32>, <16 x i32>* %a @@ -934,15 +810,11 @@ define void @sub_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: sub_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: sub z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: sub z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: sub z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -954,23 +826,15 @@ define void @sub_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 { ; CHECK-LABEL: sub_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: mov x9, #6 -; CHECK-NEXT: mov x10, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, x9, lsl #3] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, x10, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x1, x9, lsl #3] -; CHECK-NEXT: ld1d { z6.d }, p0/z, [x1, x10, lsl #3] -; CHECK-NEXT: ld1d { z7.d }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: sub z0.d, z0.d, z4.d +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: sub z1.d, z1.d, z5.d +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: sub z0.d, z3.d, z7.d -; CHECK-NEXT: sub z1.d, z2.d, z6.d +; CHECK-NEXT: sub z0.d, z2.d, z6.d +; CHECK-NEXT: sub z1.d, z3.d, z4.d ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i64>, <8 x i64>* %a @@ -1028,13 +892,11 @@ define void @abs_v32i8(<32 x i8>* %a) #0 { ; CHECK-LABEL: abs_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: abs z1.b, p0/m, z1.b ; CHECK-NEXT: abs z0.b, p0/m, z0.b -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: abs z1.b, p0/m, z1.b +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %res = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %op1, i1 false) @@ -1045,21 +907,16 @@ define void @abs_v64i8(<64 x i8>* %a) #0 { ; CHECK-LABEL: abs_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #32 -; CHECK-NEXT: mov w9, #48 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: mov w10, #16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, x9] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x10] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x0] ; CHECK-NEXT: abs z0.b, p0/m, z0.b ; CHECK-NEXT: abs z1.b, p0/m, z1.b +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: abs z0.b, p0/m, z3.b -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: abs z1.b, p0/m, z2.b +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: abs z0.b, p0/m, z2.b +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: abs z1.b, p0/m, z3.b ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <64 x i8>, <64 x i8>* %a @@ -1111,13 +968,11 @@ define void @abs_v16i16(<16 x i16>* %a) #0 { ; CHECK-LABEL: abs_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: abs z1.h, p0/m, z1.h ; CHECK-NEXT: abs z0.h, p0/m, z0.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: abs z1.h, p0/m, z1.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %res = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %op1, i1 false) @@ -1128,21 +983,16 @@ define void @abs_v32i16(<32 x i16>* %a) #0 { ; CHECK-LABEL: abs_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #16 -; CHECK-NEXT: mov x9, #24 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: mov x10, #8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0] ; CHECK-NEXT: abs z0.h, p0/m, z0.h ; CHECK-NEXT: abs z1.h, p0/m, z1.h +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: abs z0.h, p0/m, z3.h -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: abs z1.h, p0/m, z2.h +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: abs z0.h, p0/m, z2.h +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: abs z1.h, p0/m, z3.h ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i16>, <32 x i16>* %a @@ -1178,13 +1028,11 @@ define void @abs_v8i32(<8 x i32>* %a) #0 { ; CHECK-LABEL: abs_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: abs z1.s, p0/m, z1.s ; CHECK-NEXT: abs z0.s, p0/m, z0.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: abs z1.s, p0/m, z1.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %res = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %op1, i1 false) @@ -1195,21 +1043,16 @@ define void @abs_v16i32(<16 x i32>* %a) #0 { ; CHECK-LABEL: abs_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: mov x9, #12 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: mov x10, #4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x9, lsl #2] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0] ; CHECK-NEXT: abs z0.s, p0/m, z0.s ; CHECK-NEXT: abs z1.s, p0/m, z1.s +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: abs z0.s, p0/m, z3.s -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: abs z1.s, p0/m, z2.s +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: abs z0.s, p0/m, z2.s +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: abs z1.s, p0/m, z3.s ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i32>, <16 x i32>* %a @@ -1245,13 +1088,11 @@ define void @abs_v4i64(<4 x i64>* %a) #0 { ; CHECK-LABEL: abs_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: abs z1.d, p0/m, z1.d ; CHECK-NEXT: abs z0.d, p0/m, z0.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: abs z1.d, p0/m, z1.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %res = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %op1, i1 false) @@ -1262,21 +1103,16 @@ define void @abs_v8i64(<8 x i64>* %a) #0 { ; CHECK-LABEL: abs_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: mov x9, #6 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: mov x10, #2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, x9, lsl #3] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, x10, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] ; CHECK-NEXT: abs z0.d, p0/m, z0.d ; CHECK-NEXT: abs z1.d, p0/m, z1.d +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: abs z0.d, p0/m, z3.d -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: abs z1.d, p0/m, z2.d +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: abs z0.d, p0/m, z2.d +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: abs z1.d, p0/m, z3.d ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i64>, <8 x i64>* %a diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll @@ -124,50 +124,46 @@ define void @sdiv_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: sdiv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] +; CHECK-NEXT: ldp q3, q0, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: sunpkhi z5.h, z0.b +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: sunpkhi z4.h, z0.b ; CHECK-NEXT: sunpklo z0.h, z0.b -; CHECK-NEXT: sunpkhi z4.h, z2.b -; CHECK-NEXT: sunpklo z2.h, z2.b ; CHECK-NEXT: sunpkhi z6.s, z4.h -; CHECK-NEXT: sunpkhi z7.s, z5.h ; CHECK-NEXT: sunpklo z4.s, z4.h +; CHECK-NEXT: sunpkhi z16.s, z0.h +; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: sunpkhi z5.h, z1.b +; CHECK-NEXT: sunpklo z1.h, z1.b +; CHECK-NEXT: sunpkhi z7.s, z5.h ; CHECK-NEXT: sunpklo z5.s, z5.h -; CHECK-NEXT: sunpkhi z16.s, z2.h ; CHECK-NEXT: sdivr z6.s, p0/m, z6.s, z7.s ; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: sunpkhi z5.s, z0.h -; CHECK-NEXT: sunpklo z2.s, z2.h -; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: sunpkhi z5.s, z1.h +; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z6.h -; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: sunpkhi z2.h, z3.b -; CHECK-NEXT: sunpkhi z6.h, z1.b +; CHECK-NEXT: sdivr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: sunpkhi z1.h, z3.b +; CHECK-NEXT: sunpkhi z6.h, z2.b ; CHECK-NEXT: sdiv z5.s, p0/m, z5.s, z16.s -; CHECK-NEXT: sunpkhi z7.s, z2.h +; CHECK-NEXT: sunpkhi z7.s, z1.h ; CHECK-NEXT: sunpkhi z16.s, z6.h -; CHECK-NEXT: sunpklo z2.s, z2.h +; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sunpklo z6.s, z6.h ; CHECK-NEXT: sunpklo z3.h, z3.b -; CHECK-NEXT: sunpklo z1.h, z1.b +; CHECK-NEXT: sunpklo z2.h, z2.b ; CHECK-NEXT: sdivr z7.s, p0/m, z7.s, z16.s -; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z6.s +; CHECK-NEXT: sdivr z1.s, p0/m, z1.s, z6.s ; CHECK-NEXT: sunpkhi z6.s, z3.h -; CHECK-NEXT: sunpkhi z16.s, z1.h +; CHECK-NEXT: sunpkhi z16.s, z2.h ; CHECK-NEXT: sunpklo z3.s, z3.h -; CHECK-NEXT: sunpklo z1.s, z1.h +; CHECK-NEXT: sunpklo z2.s, z2.h ; CHECK-NEXT: sdivr z6.s, p0/m, z6.s, z16.s -; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z3.s -; CHECK-NEXT: uzp1 z2.h, z2.h, z7.h -; CHECK-NEXT: uzp1 z1.h, z1.h, z6.h +; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z3.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z7.h +; CHECK-NEXT: uzp1 z2.h, z2.h, z6.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z5.h -; CHECK-NEXT: uzp1 z1.b, z1.b, z2.b +; CHECK-NEXT: uzp1 z1.b, z2.b, z1.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z4.b ; CHECK-NEXT: stp q1, q0, [x0] ; CHECK-NEXT: ret @@ -181,97 +177,89 @@ define void @sdiv_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 { ; CHECK-LABEL: sdiv_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #32 -; CHECK-NEXT: mov w9, #48 -; CHECK-NEXT: mov w10, #16 -; CHECK-NEXT: ptrue p1.b, vl16 +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0, x8] -; CHECK-NEXT: ld1b { z3.b }, p1/z, [x0, x9] -; CHECK-NEXT: ld1b { z4.b }, p1/z, [x0, x10] -; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] -; CHECK-NEXT: ld1b { z5.b }, p1/z, [x1, x10] -; CHECK-NEXT: ld1b { z7.b }, p1/z, [x1, x9] -; CHECK-NEXT: ld1b { z6.b }, p1/z, [x1, x8] -; CHECK-NEXT: sunpkhi z16.h, z4.b -; CHECK-NEXT: sunpklo z4.h, z4.b -; CHECK-NEXT: sunpkhi z1.h, z5.b -; CHECK-NEXT: sunpkhi z18.s, z16.h -; CHECK-NEXT: sunpkhi z17.s, z1.h -; CHECK-NEXT: sunpklo z1.s, z1.h -; CHECK-NEXT: sunpklo z16.s, z16.h -; CHECK-NEXT: sdivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: sdivr z1.s, p0/m, z1.s, z16.s +; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: sunpkhi z0.h, z3.b +; CHECK-NEXT: sunpklo z3.h, z3.b +; CHECK-NEXT: sunpkhi z16.s, z0.h +; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: sunpkhi z7.h, z5.b ; CHECK-NEXT: sunpklo z5.h, z5.b -; CHECK-NEXT: uzp1 z1.h, z1.h, z17.h -; CHECK-NEXT: sunpkhi z17.s, z5.h -; CHECK-NEXT: sunpkhi z18.s, z4.h +; CHECK-NEXT: sunpkhi z17.s, z7.h +; CHECK-NEXT: sunpklo z7.s, z7.h +; CHECK-NEXT: ldp q1, q6, [x0, #32] +; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: sdivr z0.s, p0/m, z0.s, z7.s +; CHECK-NEXT: uzp1 z0.h, z0.h, z16.h +; CHECK-NEXT: sunpkhi z16.s, z3.h +; CHECK-NEXT: sunpkhi z18.s, z5.h +; CHECK-NEXT: sunpklo z3.s, z3.h ; CHECK-NEXT: sunpklo z5.s, z5.h -; CHECK-NEXT: sunpklo z4.s, z4.h -; CHECK-NEXT: sdivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: sdiv z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: sunpkhi z5.h, z7.b -; CHECK-NEXT: sunpkhi z18.h, z3.b +; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z18.s +; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z5.s +; CHECK-NEXT: sunpkhi z5.h, z2.b +; CHECK-NEXT: ldp q7, q17, [x1, #32] +; CHECK-NEXT: sunpkhi z18.h, z4.b ; CHECK-NEXT: sunpkhi z19.s, z5.h ; CHECK-NEXT: sunpkhi z20.s, z18.h ; CHECK-NEXT: sunpklo z5.s, z5.h ; CHECK-NEXT: sunpklo z18.s, z18.h -; CHECK-NEXT: sunpklo z7.h, z7.b -; CHECK-NEXT: sunpklo z3.h, z3.b -; CHECK-NEXT: sdivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: sunpklo z2.h, z2.b +; CHECK-NEXT: sunpklo z4.h, z4.b ; CHECK-NEXT: sdivr z5.s, p0/m, z5.s, z18.s -; CHECK-NEXT: sunpkhi z18.s, z7.h -; CHECK-NEXT: sunpkhi z20.s, z3.h -; CHECK-NEXT: sunpklo z7.s, z7.h -; CHECK-NEXT: sunpklo z3.s, z3.h -; CHECK-NEXT: sdivr z18.s, p0/m, z18.s, z20.s -; CHECK-NEXT: sdiv z3.s, p0/m, z3.s, z7.s +; CHECK-NEXT: uzp1 z3.h, z3.h, z16.h +; CHECK-NEXT: sunpkhi z16.s, z2.h +; CHECK-NEXT: sunpkhi z18.s, z4.h +; CHECK-NEXT: sunpklo z2.s, z2.h +; CHECK-NEXT: sunpklo z4.s, z4.h +; CHECK-NEXT: sdivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z18.s +; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z4.s +; CHECK-NEXT: sunpkhi z4.h, z17.b +; CHECK-NEXT: sunpkhi z18.h, z6.b ; CHECK-NEXT: uzp1 z5.h, z5.h, z19.h -; CHECK-NEXT: uzp1 z3.h, z3.h, z18.h -; CHECK-NEXT: ld1b { z16.b }, p1/z, [x1] -; CHECK-NEXT: uzp1 z3.b, z3.b, z5.b -; CHECK-NEXT: sunpkhi z5.h, z6.b -; CHECK-NEXT: sunpkhi z7.h, z2.b -; CHECK-NEXT: uzp1 z4.h, z4.h, z17.h -; CHECK-NEXT: sunpkhi z17.s, z5.h -; CHECK-NEXT: sunpkhi z18.s, z7.h -; CHECK-NEXT: sunpklo z5.s, z5.h -; CHECK-NEXT: sunpklo z7.s, z7.h +; CHECK-NEXT: sunpkhi z19.s, z4.h +; CHECK-NEXT: sunpkhi z20.s, z18.h +; CHECK-NEXT: sunpklo z4.s, z4.h +; CHECK-NEXT: sunpklo z18.s, z18.h +; CHECK-NEXT: uzp1 z2.h, z2.h, z16.h +; CHECK-NEXT: sunpklo z16.h, z17.b ; CHECK-NEXT: sunpklo z6.h, z6.b -; CHECK-NEXT: sunpklo z2.h, z2.b -; CHECK-NEXT: sdivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: sdivr z5.s, p0/m, z5.s, z7.s -; CHECK-NEXT: sunpkhi z7.s, z6.h -; CHECK-NEXT: sunpkhi z18.s, z2.h +; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z18.s +; CHECK-NEXT: sunpkhi z17.s, z16.h +; CHECK-NEXT: sunpkhi z18.s, z6.h +; CHECK-NEXT: sunpklo z16.s, z16.h ; CHECK-NEXT: sunpklo z6.s, z6.h -; CHECK-NEXT: sunpklo z2.s, z2.h -; CHECK-NEXT: sdivr z7.s, p0/m, z7.s, z18.s -; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z6.s -; CHECK-NEXT: uzp1 z2.h, z2.h, z7.h -; CHECK-NEXT: sunpkhi z6.h, z16.b -; CHECK-NEXT: sunpkhi z7.h, z0.b -; CHECK-NEXT: uzp1 z5.h, z5.h, z17.h -; CHECK-NEXT: sunpkhi z17.s, z6.h +; CHECK-NEXT: sdivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: sdivr z17.s, p0/m, z17.s, z18.s +; CHECK-NEXT: sdiv z6.s, p0/m, z6.s, z16.s +; CHECK-NEXT: sunpkhi z16.h, z7.b +; CHECK-NEXT: sunpkhi z18.h, z1.b +; CHECK-NEXT: uzp1 z4.h, z4.h, z19.h +; CHECK-NEXT: sunpkhi z19.s, z16.h +; CHECK-NEXT: sunpkhi z20.s, z18.h +; CHECK-NEXT: sunpklo z16.s, z16.h +; CHECK-NEXT: sunpklo z18.s, z18.h +; CHECK-NEXT: sunpklo z7.h, z7.b +; CHECK-NEXT: sunpklo z1.h, z1.b +; CHECK-NEXT: sdivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z18.s ; CHECK-NEXT: sunpkhi z18.s, z7.h -; CHECK-NEXT: sunpklo z6.s, z6.h +; CHECK-NEXT: sunpkhi z20.s, z1.h ; CHECK-NEXT: sunpklo z7.s, z7.h -; CHECK-NEXT: sdivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: sdivr z6.s, p0/m, z6.s, z7.s -; CHECK-NEXT: uzp1 z2.b, z2.b, z5.b -; CHECK-NEXT: uzp1 z5.h, z6.h, z17.h -; CHECK-NEXT: sunpklo z6.h, z16.b -; CHECK-NEXT: sunpklo z0.h, z0.b -; CHECK-NEXT: sunpkhi z7.s, z6.h -; CHECK-NEXT: sunpkhi z16.s, z0.h -; CHECK-NEXT: sunpklo z6.s, z6.h -; CHECK-NEXT: sunpklo z0.s, z0.h -; CHECK-NEXT: sdivr z7.s, p0/m, z7.s, z16.s -; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z6.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z7.h -; CHECK-NEXT: uzp1 z1.b, z4.b, z1.b -; CHECK-NEXT: uzp1 z0.b, z0.b, z5.b -; CHECK-NEXT: stp q2, q3, [x0, #32] -; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: sunpklo z1.s, z1.h +; CHECK-NEXT: sdivr z18.s, p0/m, z18.s, z20.s +; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z7.s +; CHECK-NEXT: uzp1 z7.h, z16.h, z19.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z18.h +; CHECK-NEXT: uzp1 z6.h, z6.h, z17.h +; CHECK-NEXT: uzp1 z1.b, z1.b, z7.b +; CHECK-NEXT: uzp1 z4.b, z6.b, z4.b +; CHECK-NEXT: stp q1, q4, [x0, #32] +; CHECK-NEXT: uzp1 z1.b, z2.b, z5.b +; CHECK-NEXT: uzp1 z0.b, z3.b, z0.b +; CHECK-NEXT: stp q1, q0, [x0] ; CHECK-NEXT: ret %op1 = load <64 x i8>, <64 x i8>* %a %op2 = load <64 x i8>, <64 x i8>* %b @@ -350,28 +338,24 @@ define void @sdiv_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: sdiv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: sunpkhi z5.s, z0.h +; CHECK-NEXT: sunpkhi z6.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h -; CHECK-NEXT: sunpkhi z4.s, z2.h -; CHECK-NEXT: sunpkhi z6.s, z3.h -; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: sunpkhi z5.s, z1.h +; CHECK-NEXT: ldp q3, q2, [x0] +; CHECK-NEXT: sunpkhi z4.s, z1.h +; CHECK-NEXT: sunpklo z1.s, z1.h +; CHECK-NEXT: sunpkhi z5.s, z2.h ; CHECK-NEXT: sunpklo z2.s, z2.h +; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s +; CHECK-NEXT: sunpkhi z5.s, z3.h ; CHECK-NEXT: sunpklo z3.s, z3.h -; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sdiv z5.s, p0/m, z5.s, z6.s -; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z3.s -; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: uzp1 z1.h, z1.h, z5.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z4.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: sdivr z0.s, p0/m, z0.s, z3.s +; CHECK-NEXT: sdivr z1.s, p0/m, z1.s, z2.s +; CHECK-NEXT: uzp1 z0.h, z0.h, z5.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z4.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -383,50 +367,43 @@ define void @sdiv_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 { ; CHECK-LABEL: sdiv_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #16 -; CHECK-NEXT: mov x9, #24 -; CHECK-NEXT: mov x10, #8 -; CHECK-NEXT: ptrue p1.h, vl8 +; CHECK-NEXT: ldp q0, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p1/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z2.h }, p1/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p1/z, [x0] -; CHECK-NEXT: ld1h { z4.h }, p1/z, [x1, x10, lsl #1] -; CHECK-NEXT: ld1h { z5.h }, p1/z, [x1, x9, lsl #1] -; CHECK-NEXT: ld1h { z6.h }, p1/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z17.h }, p1/z, [x1] -; CHECK-NEXT: sunpkhi z18.s, z1.h +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: sunpkhi z6.s, z1.h ; CHECK-NEXT: sunpklo z1.s, z1.h -; CHECK-NEXT: sunpkhi z16.s, z2.h +; CHECK-NEXT: sunpkhi z18.s, z2.h ; CHECK-NEXT: sunpklo z2.s, z2.h -; CHECK-NEXT: sunpkhi z7.s, z4.h -; CHECK-NEXT: sunpklo z4.s, z4.h -; CHECK-NEXT: sdivr z7.s, p0/m, z7.s, z16.s -; CHECK-NEXT: sunpkhi z16.s, z5.h -; CHECK-NEXT: sunpklo z5.s, z5.h -; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z4.s -; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z5.s -; CHECK-NEXT: sunpkhi z4.s, z6.h -; CHECK-NEXT: sunpkhi z5.s, z0.h -; CHECK-NEXT: sunpklo z6.s, z6.h -; CHECK-NEXT: sunpklo z0.s, z0.h -; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z6.s -; CHECK-NEXT: sunpkhi z5.s, z17.h -; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z18.s -; CHECK-NEXT: sunpkhi z6.s, z3.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z4.h -; CHECK-NEXT: movprfx z4, z6 -; CHECK-NEXT: sdiv z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: sunpklo z5.s, z17.h +; CHECK-NEXT: ldp q4, q5, [x0, #32] +; CHECK-NEXT: sunpkhi z7.s, z3.h ; CHECK-NEXT: sunpklo z3.s, z3.h -; CHECK-NEXT: uzp1 z1.h, z1.h, z16.h -; CHECK-NEXT: sdiv z3.s, p0/m, z3.s, z5.s -; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: uzp1 z0.h, z3.h, z4.h -; CHECK-NEXT: uzp1 z1.h, z2.h, z7.h +; CHECK-NEXT: sdivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: sunpkhi z7.s, z0.h +; CHECK-NEXT: sdivr z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: sdivr z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z6.h +; CHECK-NEXT: ldp q16, q17, [x1, #32] +; CHECK-NEXT: movprfx z3, z18 +; CHECK-NEXT: sdiv z3.s, p0/m, z3.s, z7.s +; CHECK-NEXT: sunpkhi z18.s, z5.h +; CHECK-NEXT: sunpklo z5.s, z5.h +; CHECK-NEXT: uzp1 z0.h, z0.h, z3.h +; CHECK-NEXT: sunpkhi z7.s, z17.h +; CHECK-NEXT: movprfx z2, z18 +; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z7.s +; CHECK-NEXT: sunpkhi z7.s, z16.h +; CHECK-NEXT: sunpkhi z18.s, z4.h +; CHECK-NEXT: sunpklo z17.s, z17.h +; CHECK-NEXT: sunpklo z16.s, z16.h +; CHECK-NEXT: sunpklo z4.s, z4.h +; CHECK-NEXT: sdivr z7.s, p0/m, z7.s, z18.s +; CHECK-NEXT: sdiv z4.s, p0/m, z4.s, z16.s +; CHECK-NEXT: sdiv z5.s, p0/m, z5.s, z17.s +; CHECK-NEXT: uzp1 z4.h, z4.h, z7.h +; CHECK-NEXT: uzp1 z2.h, z5.h, z2.h ; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: stp q4, q2, [x0, #32] ; CHECK-NEXT: ret %op1 = load <32 x i16>, <32 x i16>* %a %op2 = load <32 x i16>, <32 x i16>* %b @@ -464,15 +441,12 @@ define void @sdiv_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: sdiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -484,25 +458,18 @@ define void @sdiv_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 { ; CHECK-LABEL: sdiv_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: mov x9, #12 -; CHECK-NEXT: mov x10, #4 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x9, lsl #2] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1, x9, lsl #2] -; CHECK-NEXT: ld1w { z6.s }, p0/z, [x1, x10, lsl #2] -; CHECK-NEXT: ld1w { z7.s }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z4.s ; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z5.s +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z7.s -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z6.s +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z6.s +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z4.s ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i32>, <16 x i32>* %a @@ -541,15 +508,12 @@ define void @sdiv_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: sdiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: sdiv z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: sdiv z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -561,25 +525,18 @@ define void @sdiv_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 { ; CHECK-LABEL: sdiv_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: mov x9, #6 -; CHECK-NEXT: mov x10, #2 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, x9, lsl #3] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, x10, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x1, x9, lsl #3] -; CHECK-NEXT: ld1d { z6.d }, p0/z, [x1, x10, lsl #3] -; CHECK-NEXT: ld1d { z7.d }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z4.d ; CHECK-NEXT: sdiv z1.d, p0/m, z1.d, z5.d +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z7.d -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: sdiv z1.d, p0/m, z1.d, z6.d +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z6.d +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: sdiv z1.d, p0/m, z1.d, z4.d ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i64>, <8 x i64>* %a @@ -707,50 +664,46 @@ define void @udiv_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: udiv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] +; CHECK-NEXT: ldp q3, q0, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: uunpkhi z5.h, z0.b +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: uunpkhi z4.h, z0.b ; CHECK-NEXT: uunpklo z0.h, z0.b -; CHECK-NEXT: uunpkhi z4.h, z2.b -; CHECK-NEXT: uunpklo z2.h, z2.b ; CHECK-NEXT: uunpkhi z6.s, z4.h -; CHECK-NEXT: uunpkhi z7.s, z5.h ; CHECK-NEXT: uunpklo z4.s, z4.h +; CHECK-NEXT: uunpkhi z16.s, z0.h +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: uunpkhi z5.h, z1.b +; CHECK-NEXT: uunpklo z1.h, z1.b +; CHECK-NEXT: uunpkhi z7.s, z5.h ; CHECK-NEXT: uunpklo z5.s, z5.h -; CHECK-NEXT: uunpkhi z16.s, z2.h ; CHECK-NEXT: udivr z6.s, p0/m, z6.s, z7.s ; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: uunpkhi z5.s, z0.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: uunpkhi z5.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z6.h -; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: uunpkhi z2.h, z3.b -; CHECK-NEXT: uunpkhi z6.h, z1.b +; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: uunpkhi z1.h, z3.b +; CHECK-NEXT: uunpkhi z6.h, z2.b ; CHECK-NEXT: udiv z5.s, p0/m, z5.s, z16.s -; CHECK-NEXT: uunpkhi z7.s, z2.h +; CHECK-NEXT: uunpkhi z7.s, z1.h ; CHECK-NEXT: uunpkhi z16.s, z6.h -; CHECK-NEXT: uunpklo z2.s, z2.h +; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z6.s, z6.h ; CHECK-NEXT: uunpklo z3.h, z3.b -; CHECK-NEXT: uunpklo z1.h, z1.b +; CHECK-NEXT: uunpklo z2.h, z2.b ; CHECK-NEXT: udivr z7.s, p0/m, z7.s, z16.s -; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z6.s +; CHECK-NEXT: udivr z1.s, p0/m, z1.s, z6.s ; CHECK-NEXT: uunpkhi z6.s, z3.h -; CHECK-NEXT: uunpkhi z16.s, z1.h +; CHECK-NEXT: uunpkhi z16.s, z2.h ; CHECK-NEXT: uunpklo z3.s, z3.h -; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpklo z2.s, z2.h ; CHECK-NEXT: udivr z6.s, p0/m, z6.s, z16.s -; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z3.s -; CHECK-NEXT: uzp1 z2.h, z2.h, z7.h -; CHECK-NEXT: uzp1 z1.h, z1.h, z6.h +; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z3.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z7.h +; CHECK-NEXT: uzp1 z2.h, z2.h, z6.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z5.h -; CHECK-NEXT: uzp1 z1.b, z1.b, z2.b +; CHECK-NEXT: uzp1 z1.b, z2.b, z1.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z4.b ; CHECK-NEXT: stp q1, q0, [x0] ; CHECK-NEXT: ret @@ -764,97 +717,89 @@ define void @udiv_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 { ; CHECK-LABEL: udiv_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #32 -; CHECK-NEXT: mov w9, #48 -; CHECK-NEXT: mov w10, #16 -; CHECK-NEXT: ptrue p1.b, vl16 +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0, x8] -; CHECK-NEXT: ld1b { z3.b }, p1/z, [x0, x9] -; CHECK-NEXT: ld1b { z4.b }, p1/z, [x0, x10] -; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] -; CHECK-NEXT: ld1b { z5.b }, p1/z, [x1, x10] -; CHECK-NEXT: ld1b { z7.b }, p1/z, [x1, x9] -; CHECK-NEXT: ld1b { z6.b }, p1/z, [x1, x8] -; CHECK-NEXT: uunpkhi z16.h, z4.b -; CHECK-NEXT: uunpklo z4.h, z4.b -; CHECK-NEXT: uunpkhi z1.h, z5.b -; CHECK-NEXT: uunpkhi z18.s, z16.h -; CHECK-NEXT: uunpkhi z17.s, z1.h -; CHECK-NEXT: uunpklo z1.s, z1.h -; CHECK-NEXT: uunpklo z16.s, z16.h -; CHECK-NEXT: udivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: udivr z1.s, p0/m, z1.s, z16.s +; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: uunpkhi z0.h, z3.b +; CHECK-NEXT: uunpklo z3.h, z3.b +; CHECK-NEXT: uunpkhi z16.s, z0.h +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: uunpkhi z7.h, z5.b ; CHECK-NEXT: uunpklo z5.h, z5.b -; CHECK-NEXT: uzp1 z1.h, z1.h, z17.h -; CHECK-NEXT: uunpkhi z17.s, z5.h -; CHECK-NEXT: uunpkhi z18.s, z4.h +; CHECK-NEXT: uunpkhi z17.s, z7.h +; CHECK-NEXT: uunpklo z7.s, z7.h +; CHECK-NEXT: ldp q1, q6, [x0, #32] +; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z7.s +; CHECK-NEXT: uzp1 z0.h, z0.h, z16.h +; CHECK-NEXT: uunpkhi z16.s, z3.h +; CHECK-NEXT: uunpkhi z18.s, z5.h +; CHECK-NEXT: uunpklo z3.s, z3.h ; CHECK-NEXT: uunpklo z5.s, z5.h -; CHECK-NEXT: uunpklo z4.s, z4.h -; CHECK-NEXT: udivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: udiv z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: uunpkhi z5.h, z7.b -; CHECK-NEXT: uunpkhi z18.h, z3.b +; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z18.s +; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z5.s +; CHECK-NEXT: uunpkhi z5.h, z2.b +; CHECK-NEXT: ldp q7, q17, [x1, #32] +; CHECK-NEXT: uunpkhi z18.h, z4.b ; CHECK-NEXT: uunpkhi z19.s, z5.h ; CHECK-NEXT: uunpkhi z20.s, z18.h ; CHECK-NEXT: uunpklo z5.s, z5.h ; CHECK-NEXT: uunpklo z18.s, z18.h -; CHECK-NEXT: uunpklo z7.h, z7.b -; CHECK-NEXT: uunpklo z3.h, z3.b -; CHECK-NEXT: udivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: uunpklo z2.h, z2.b +; CHECK-NEXT: uunpklo z4.h, z4.b ; CHECK-NEXT: udivr z5.s, p0/m, z5.s, z18.s -; CHECK-NEXT: uunpkhi z18.s, z7.h -; CHECK-NEXT: uunpkhi z20.s, z3.h -; CHECK-NEXT: uunpklo z7.s, z7.h -; CHECK-NEXT: uunpklo z3.s, z3.h -; CHECK-NEXT: udivr z18.s, p0/m, z18.s, z20.s -; CHECK-NEXT: udiv z3.s, p0/m, z3.s, z7.s +; CHECK-NEXT: uzp1 z3.h, z3.h, z16.h +; CHECK-NEXT: uunpkhi z16.s, z2.h +; CHECK-NEXT: uunpkhi z18.s, z4.h +; CHECK-NEXT: uunpklo z2.s, z2.h +; CHECK-NEXT: uunpklo z4.s, z4.h +; CHECK-NEXT: udivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z18.s +; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z4.s +; CHECK-NEXT: uunpkhi z4.h, z17.b +; CHECK-NEXT: uunpkhi z18.h, z6.b ; CHECK-NEXT: uzp1 z5.h, z5.h, z19.h -; CHECK-NEXT: uzp1 z3.h, z3.h, z18.h -; CHECK-NEXT: ld1b { z16.b }, p1/z, [x1] -; CHECK-NEXT: uzp1 z3.b, z3.b, z5.b -; CHECK-NEXT: uunpkhi z5.h, z6.b -; CHECK-NEXT: uunpkhi z7.h, z2.b -; CHECK-NEXT: uzp1 z4.h, z4.h, z17.h -; CHECK-NEXT: uunpkhi z17.s, z5.h -; CHECK-NEXT: uunpkhi z18.s, z7.h -; CHECK-NEXT: uunpklo z5.s, z5.h -; CHECK-NEXT: uunpklo z7.s, z7.h +; CHECK-NEXT: uunpkhi z19.s, z4.h +; CHECK-NEXT: uunpkhi z20.s, z18.h +; CHECK-NEXT: uunpklo z4.s, z4.h +; CHECK-NEXT: uunpklo z18.s, z18.h +; CHECK-NEXT: uzp1 z2.h, z2.h, z16.h +; CHECK-NEXT: uunpklo z16.h, z17.b ; CHECK-NEXT: uunpklo z6.h, z6.b -; CHECK-NEXT: uunpklo z2.h, z2.b -; CHECK-NEXT: udivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: udivr z5.s, p0/m, z5.s, z7.s -; CHECK-NEXT: uunpkhi z7.s, z6.h -; CHECK-NEXT: uunpkhi z18.s, z2.h +; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z18.s +; CHECK-NEXT: uunpkhi z17.s, z16.h +; CHECK-NEXT: uunpkhi z18.s, z6.h +; CHECK-NEXT: uunpklo z16.s, z16.h ; CHECK-NEXT: uunpklo z6.s, z6.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: udivr z7.s, p0/m, z7.s, z18.s -; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z6.s -; CHECK-NEXT: uzp1 z2.h, z2.h, z7.h -; CHECK-NEXT: uunpkhi z6.h, z16.b -; CHECK-NEXT: uunpkhi z7.h, z0.b -; CHECK-NEXT: uzp1 z5.h, z5.h, z17.h -; CHECK-NEXT: uunpkhi z17.s, z6.h +; CHECK-NEXT: udivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: udivr z17.s, p0/m, z17.s, z18.s +; CHECK-NEXT: udiv z6.s, p0/m, z6.s, z16.s +; CHECK-NEXT: uunpkhi z16.h, z7.b +; CHECK-NEXT: uunpkhi z18.h, z1.b +; CHECK-NEXT: uzp1 z4.h, z4.h, z19.h +; CHECK-NEXT: uunpkhi z19.s, z16.h +; CHECK-NEXT: uunpkhi z20.s, z18.h +; CHECK-NEXT: uunpklo z16.s, z16.h +; CHECK-NEXT: uunpklo z18.s, z18.h +; CHECK-NEXT: uunpklo z7.h, z7.b +; CHECK-NEXT: uunpklo z1.h, z1.b +; CHECK-NEXT: udivr z19.s, p0/m, z19.s, z20.s +; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z18.s ; CHECK-NEXT: uunpkhi z18.s, z7.h -; CHECK-NEXT: uunpklo z6.s, z6.h +; CHECK-NEXT: uunpkhi z20.s, z1.h ; CHECK-NEXT: uunpklo z7.s, z7.h -; CHECK-NEXT: udivr z17.s, p0/m, z17.s, z18.s -; CHECK-NEXT: udivr z6.s, p0/m, z6.s, z7.s -; CHECK-NEXT: uzp1 z2.b, z2.b, z5.b -; CHECK-NEXT: uzp1 z5.h, z6.h, z17.h -; CHECK-NEXT: uunpklo z6.h, z16.b -; CHECK-NEXT: uunpklo z0.h, z0.b -; CHECK-NEXT: uunpkhi z7.s, z6.h -; CHECK-NEXT: uunpkhi z16.s, z0.h -; CHECK-NEXT: uunpklo z6.s, z6.h -; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: udivr z7.s, p0/m, z7.s, z16.s -; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z6.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z7.h -; CHECK-NEXT: uzp1 z1.b, z4.b, z1.b -; CHECK-NEXT: uzp1 z0.b, z0.b, z5.b -; CHECK-NEXT: stp q2, q3, [x0, #32] -; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: udivr z18.s, p0/m, z18.s, z20.s +; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z7.s +; CHECK-NEXT: uzp1 z7.h, z16.h, z19.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z18.h +; CHECK-NEXT: uzp1 z6.h, z6.h, z17.h +; CHECK-NEXT: uzp1 z1.b, z1.b, z7.b +; CHECK-NEXT: uzp1 z4.b, z6.b, z4.b +; CHECK-NEXT: stp q1, q4, [x0, #32] +; CHECK-NEXT: uzp1 z1.b, z2.b, z5.b +; CHECK-NEXT: uzp1 z0.b, z3.b, z0.b +; CHECK-NEXT: stp q1, q0, [x0] ; CHECK-NEXT: ret %op1 = load <64 x i8>, <64 x i8>* %a %op2 = load <64 x i8>, <64 x i8>* %b @@ -931,28 +876,24 @@ define void @udiv_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: udiv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: uunpkhi z5.s, z0.h +; CHECK-NEXT: uunpkhi z6.s, z0.h ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: uunpkhi z4.s, z2.h -; CHECK-NEXT: uunpkhi z6.s, z3.h -; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: uunpkhi z5.s, z1.h +; CHECK-NEXT: ldp q3, q2, [x0] +; CHECK-NEXT: uunpkhi z4.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpkhi z5.s, z2.h ; CHECK-NEXT: uunpklo z2.s, z2.h +; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s +; CHECK-NEXT: uunpkhi z5.s, z3.h ; CHECK-NEXT: uunpklo z3.s, z3.h -; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: udiv z5.s, p0/m, z5.s, z6.s -; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z3.s -; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: uzp1 z1.h, z1.h, z5.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z4.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z3.s +; CHECK-NEXT: udivr z1.s, p0/m, z1.s, z2.s +; CHECK-NEXT: uzp1 z0.h, z0.h, z5.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z4.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -964,50 +905,43 @@ define void @udiv_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 { ; CHECK-LABEL: udiv_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #16 -; CHECK-NEXT: mov x9, #24 -; CHECK-NEXT: mov x10, #8 -; CHECK-NEXT: ptrue p1.h, vl8 +; CHECK-NEXT: ldp q0, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p1/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z2.h }, p1/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p1/z, [x0] -; CHECK-NEXT: ld1h { z4.h }, p1/z, [x1, x10, lsl #1] -; CHECK-NEXT: ld1h { z5.h }, p1/z, [x1, x9, lsl #1] -; CHECK-NEXT: ld1h { z6.h }, p1/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z17.h }, p1/z, [x1] -; CHECK-NEXT: uunpkhi z18.s, z1.h +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: uunpkhi z6.s, z1.h ; CHECK-NEXT: uunpklo z1.s, z1.h -; CHECK-NEXT: uunpkhi z16.s, z2.h +; CHECK-NEXT: uunpkhi z18.s, z2.h ; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpkhi z7.s, z4.h -; CHECK-NEXT: uunpklo z4.s, z4.h -; CHECK-NEXT: udivr z7.s, p0/m, z7.s, z16.s -; CHECK-NEXT: uunpkhi z16.s, z5.h -; CHECK-NEXT: uunpklo z5.s, z5.h -; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z4.s -; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z5.s -; CHECK-NEXT: uunpkhi z4.s, z6.h -; CHECK-NEXT: uunpkhi z5.s, z0.h -; CHECK-NEXT: uunpklo z6.s, z6.h -; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z6.s -; CHECK-NEXT: uunpkhi z5.s, z17.h -; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z18.s -; CHECK-NEXT: uunpkhi z6.s, z3.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z4.h -; CHECK-NEXT: movprfx z4, z6 -; CHECK-NEXT: udiv z4.s, p0/m, z4.s, z5.s -; CHECK-NEXT: uunpklo z5.s, z17.h +; CHECK-NEXT: ldp q4, q5, [x0, #32] +; CHECK-NEXT: uunpkhi z7.s, z3.h ; CHECK-NEXT: uunpklo z3.s, z3.h -; CHECK-NEXT: uzp1 z1.h, z1.h, z16.h -; CHECK-NEXT: udiv z3.s, p0/m, z3.s, z5.s -; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: uzp1 z0.h, z3.h, z4.h -; CHECK-NEXT: uzp1 z1.h, z2.h, z7.h +; CHECK-NEXT: udivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: uunpkhi z7.s, z0.h +; CHECK-NEXT: udivr z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z6.h +; CHECK-NEXT: ldp q16, q17, [x1, #32] +; CHECK-NEXT: movprfx z3, z18 +; CHECK-NEXT: udiv z3.s, p0/m, z3.s, z7.s +; CHECK-NEXT: uunpkhi z18.s, z5.h +; CHECK-NEXT: uunpklo z5.s, z5.h +; CHECK-NEXT: uzp1 z0.h, z0.h, z3.h +; CHECK-NEXT: uunpkhi z7.s, z17.h +; CHECK-NEXT: movprfx z2, z18 +; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z7.s +; CHECK-NEXT: uunpkhi z7.s, z16.h +; CHECK-NEXT: uunpkhi z18.s, z4.h +; CHECK-NEXT: uunpklo z17.s, z17.h +; CHECK-NEXT: uunpklo z16.s, z16.h +; CHECK-NEXT: uunpklo z4.s, z4.h +; CHECK-NEXT: udivr z7.s, p0/m, z7.s, z18.s +; CHECK-NEXT: udiv z4.s, p0/m, z4.s, z16.s +; CHECK-NEXT: udiv z5.s, p0/m, z5.s, z17.s +; CHECK-NEXT: uzp1 z4.h, z4.h, z7.h +; CHECK-NEXT: uzp1 z2.h, z5.h, z2.h ; CHECK-NEXT: stp q0, q1, [x0] +; CHECK-NEXT: stp q4, q2, [x0, #32] ; CHECK-NEXT: ret %op1 = load <32 x i16>, <32 x i16>* %a %op2 = load <32 x i16>, <32 x i16>* %b @@ -1045,15 +979,12 @@ define void @udiv_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: udiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -1065,25 +996,18 @@ define void @udiv_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 { ; CHECK-LABEL: udiv_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: mov x9, #12 -; CHECK-NEXT: mov x10, #4 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x9, lsl #2] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1, x9, lsl #2] -; CHECK-NEXT: ld1w { z6.s }, p0/z, [x1, x10, lsl #2] -; CHECK-NEXT: ld1w { z7.s }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z4.s ; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z5.s +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z7.s -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z6.s +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z6.s +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z4.s ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i32>, <16 x i32>* %a @@ -1122,15 +1046,12 @@ define void @udiv_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: udiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: udiv z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: udiv z1.d, p0/m, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -1142,25 +1063,18 @@ define void @udiv_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 { ; CHECK-LABEL: udiv_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: mov x9, #6 -; CHECK-NEXT: mov x10, #2 +; CHECK-NEXT: ldp q0, q1, [x0, #32] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, x9, lsl #3] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, x10, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x1, x9, lsl #3] -; CHECK-NEXT: ld1d { z6.d }, p0/z, [x1, x10, lsl #3] -; CHECK-NEXT: ld1d { z7.d }, p0/z, [x1] +; CHECK-NEXT: ldp q4, q5, [x1, #32] ; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z4.d ; CHECK-NEXT: udiv z1.d, p0/m, z1.d, z5.d +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q6, q4, [x1] ; CHECK-NEXT: stp q0, q1, [x0, #32] -; CHECK-NEXT: movprfx z0, z3 -; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z7.d -; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: udiv z1.d, p0/m, z1.d, z6.d +; CHECK-NEXT: movprfx z0, z2 +; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z6.d +; CHECK-NEXT: movprfx z1, z3 +; CHECK-NEXT: udiv z1.d, p0/m, z1.d, z4.d ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i64>, <8 x i64>* %a @@ -1173,28 +1087,23 @@ define void @udiv_constantsplat_v8i32(<8 x i32>* %a) #0 { ; CHECK-LABEL: udiv_constantsplat_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; CHECK-NEXT: adrp x8, .LCPI36_0 -; CHECK-NEXT: add x8, x8, :lo12:.LCPI36_0 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x8] -; CHECK-NEXT: adrp x8, .LCPI36_1 -; CHECK-NEXT: add x8, x8, :lo12:.LCPI36_1 -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x8] +; CHECK-NEXT: adrp x9, .LCPI36_1 +; CHECK-NEXT: ldp q1, q2, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI36_0] ; CHECK-NEXT: adrp x8, .LCPI36_2 -; CHECK-NEXT: add x8, x8, :lo12:.LCPI36_2 -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x8] +; CHECK-NEXT: ldr q3, [x9, :lo12:.LCPI36_1] ; CHECK-NEXT: movprfx z5, z1 -; CHECK-NEXT: umulh z5.s, p0/m, z5.s, z2.s -; CHECK-NEXT: umulh z2.s, p0/m, z2.s, z0.s +; CHECK-NEXT: umulh z5.s, p0/m, z5.s, z0.s ; CHECK-NEXT: sub z1.s, z1.s, z5.s -; CHECK-NEXT: sub z0.s, z0.s, z2.s +; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI36_2] +; CHECK-NEXT: sub z2.s, z2.s, z0.s ; CHECK-NEXT: lsr z1.s, p0/m, z1.s, z3.s -; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z3.s +; CHECK-NEXT: lsr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: add z1.s, z1.s, z5.s -; CHECK-NEXT: add z0.s, z0.s, z2.s +; CHECK-NEXT: add z0.s, z2.s, z0.s ; CHECK-NEXT: lsr z1.s, p0/m, z1.s, z4.s ; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z4.s ; CHECK-NEXT: stp q1, q0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll @@ -34,15 +34,11 @@ define void @and_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: and_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] -; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: and z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -78,15 +74,11 @@ define void @and_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: and_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] -; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: and z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -122,15 +114,11 @@ define void @and_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: and_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: and z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -166,15 +154,11 @@ define void @and_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: and_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: and z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: and z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -214,15 +198,11 @@ define void @or_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: or_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] -; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: orr z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -258,15 +238,11 @@ define void @or_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: or_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] -; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: orr z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -302,15 +278,11 @@ define void @or_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: or_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: orr z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -346,15 +318,11 @@ define void @or_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: or_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: orr z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: orr z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -394,15 +362,11 @@ define void @xor_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: xor_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] -; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: eor z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -438,15 +402,11 @@ define void @xor_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: xor_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] -; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: eor z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -482,15 +442,11 @@ define void @xor_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: xor_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: eor z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -526,15 +482,11 @@ define void @xor_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: xor_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] ; CHECK-NEXT: eor z0.d, z0.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: eor z1.d, z1.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll @@ -77,135 +77,129 @@ define void @smulh_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: smulh_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ptrue p1.h, vl8 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] +; CHECK-NEXT: sub sp, sp, #32 +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: adrp x8, .LCPI3_0 -; CHECK-NEXT: add x8, x8, :lo12:.LCPI3_0 -; CHECK-NEXT: sunpklo z5.h, z2.b +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: sunpklo z0.h, z2.b ; CHECK-NEXT: ext z2.b, z2.b, z2.b, #8 -; CHECK-NEXT: sunpklo z7.h, z3.b -; CHECK-NEXT: ld1h { z16.h }, p1/z, [x8] -; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 ; CHECK-NEXT: sunpklo z2.h, z2.b +; CHECK-NEXT: ldp q4, q5, [x1] +; CHECK-NEXT: sunpklo z6.h, z3.b +; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 ; CHECK-NEXT: sunpklo z3.h, z3.b -; CHECK-NEXT: mul z5.h, p1/m, z5.h, z7.h -; CHECK-NEXT: mul z2.h, p1/m, z2.h, z3.h -; CHECK-NEXT: movprfx z3, z5 -; CHECK-NEXT: lsr z3.h, p1/m, z3.h, z16.h +; CHECK-NEXT: sunpklo z1.h, z4.b +; CHECK-NEXT: ext z4.b, z4.b, z4.b, #8 +; CHECK-NEXT: sunpklo z4.h, z4.b +; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: sunpklo z7.h, z5.b +; CHECK-NEXT: ext z5.b, z5.b, z5.b, #8 +; CHECK-NEXT: ldr q16, [x8, :lo12:.LCPI3_0] +; CHECK-NEXT: sunpklo z5.h, z5.b +; CHECK-NEXT: mul z3.h, p0/m, z3.h, z5.h +; CHECK-NEXT: movprfx z5, z6 +; CHECK-NEXT: mul z5.h, p0/m, z5.h, z7.h +; CHECK-NEXT: mul z2.h, p0/m, z2.h, z4.h +; CHECK-NEXT: movprfx z4, z5 +; CHECK-NEXT: lsr z4.h, p0/m, z4.h, z16.h +; CHECK-NEXT: lsr z3.h, p0/m, z3.h, z16.h +; CHECK-NEXT: fmov w9, s4 ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: sunpklo z4.h, z0.b -; CHECK-NEXT: sunpklo z6.h, z1.b -; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 -; CHECK-NEXT: ext z1.b, z1.b, z1.b, #8 -; CHECK-NEXT: lsr z2.h, p1/m, z2.h, z16.h ; CHECK-NEXT: mov z5.h, z3.h[7] -; CHECK-NEXT: sunpklo z0.h, z0.b -; CHECK-NEXT: sunpklo z1.h, z1.b -; CHECK-NEXT: mul z4.h, p1/m, z4.h, z6.h ; CHECK-NEXT: mov z6.h, z3.h[6] ; CHECK-NEXT: mov z7.h, z3.h[5] -; CHECK-NEXT: mov z17.h, z3.h[4] -; CHECK-NEXT: mov z18.h, z3.h[3] -; CHECK-NEXT: mov z19.h, z3.h[2] -; CHECK-NEXT: mov z20.h, z3.h[1] -; CHECK-NEXT: mov z3.h, z2.h[7] -; CHECK-NEXT: mov z21.h, z2.h[6] -; CHECK-NEXT: mov z22.h, z2.h[5] -; CHECK-NEXT: mov z23.h, z2.h[4] -; CHECK-NEXT: mov z24.h, z2.h[3] -; CHECK-NEXT: mov z25.h, z2.h[2] -; CHECK-NEXT: mov z26.h, z2.h[1] -; CHECK-NEXT: fmov w9, s2 -; CHECK-NEXT: mul z0.h, p1/m, z0.h, z1.h ; CHECK-NEXT: fmov w10, s5 -; CHECK-NEXT: strb w8, [sp, #-32]! -; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: strb w9, [sp, #16] +; CHECK-NEXT: strb w8, [sp, #24] ; CHECK-NEXT: fmov w8, s6 -; CHECK-NEXT: strb w9, [sp, #8] ; CHECK-NEXT: fmov w9, s7 -; CHECK-NEXT: strb w10, [sp, #7] +; CHECK-NEXT: mov z17.h, z3.h[4] +; CHECK-NEXT: mov z18.h, z3.h[3] +; CHECK-NEXT: mov z19.h, z3.h[2] +; CHECK-NEXT: strb w10, [sp, #31] ; CHECK-NEXT: fmov w10, s17 -; CHECK-NEXT: lsr z0.h, p1/m, z0.h, z16.h -; CHECK-NEXT: strb w8, [sp, #6] +; CHECK-NEXT: strb w8, [sp, #30] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: strb w9, [sp, #5] +; CHECK-NEXT: strb w9, [sp, #29] ; CHECK-NEXT: fmov w9, s19 -; CHECK-NEXT: strb w10, [sp, #4] +; CHECK-NEXT: mov z20.h, z3.h[1] +; CHECK-NEXT: mov z3.h, z4.h[7] +; CHECK-NEXT: mov z21.h, z4.h[6] +; CHECK-NEXT: strb w10, [sp, #28] ; CHECK-NEXT: fmov w10, s20 -; CHECK-NEXT: strb w8, [sp, #3] +; CHECK-NEXT: strb w8, [sp, #27] ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: strb w9, [sp, #2] +; CHECK-NEXT: strb w9, [sp, #26] ; CHECK-NEXT: fmov w9, s21 -; CHECK-NEXT: strb w10, [sp, #1] +; CHECK-NEXT: mov z22.h, z4.h[5] +; CHECK-NEXT: mov z23.h, z4.h[4] +; CHECK-NEXT: mov z24.h, z4.h[3] +; CHECK-NEXT: strb w10, [sp, #25] ; CHECK-NEXT: fmov w10, s22 -; CHECK-NEXT: strb w8, [sp, #15] +; CHECK-NEXT: strb w8, [sp, #23] ; CHECK-NEXT: fmov w8, s23 -; CHECK-NEXT: strb w9, [sp, #14] +; CHECK-NEXT: strb w9, [sp, #22] ; CHECK-NEXT: fmov w9, s24 -; CHECK-NEXT: strb w10, [sp, #13] +; CHECK-NEXT: mov z25.h, z4.h[2] +; CHECK-NEXT: mov z26.h, z4.h[1] +; CHECK-NEXT: strb w10, [sp, #21] ; CHECK-NEXT: fmov w10, s25 -; CHECK-NEXT: strb w8, [sp, #12] +; CHECK-NEXT: strb w8, [sp, #20] +; CHECK-NEXT: movprfx z1, z2 +; CHECK-NEXT: lsr z1.h, p0/m, z1.h, z16.h +; CHECK-NEXT: strb w9, [sp, #19] ; CHECK-NEXT: fmov w8, s26 -; CHECK-NEXT: movprfx z1, z4 -; CHECK-NEXT: lsr z1.h, p1/m, z1.h, z16.h -; CHECK-NEXT: strb w9, [sp, #11] -; CHECK-NEXT: mov z2.h, z1.h[7] ; CHECK-NEXT: fmov w9, s1 -; CHECK-NEXT: strb w10, [sp, #10] +; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z16.h +; CHECK-NEXT: mov z2.h, z1.h[7] +; CHECK-NEXT: mov z3.h, z1.h[6] +; CHECK-NEXT: strb w10, [sp, #18] ; CHECK-NEXT: fmov w10, s0 -; CHECK-NEXT: strb w8, [sp, #9] +; CHECK-NEXT: strb w8, [sp, #17] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: mov z3.h, z1.h[6] +; CHECK-NEXT: strb w9, [sp, #8] +; CHECK-NEXT: fmov w9, s3 ; CHECK-NEXT: mov z4.h, z1.h[5] ; CHECK-NEXT: mov z5.h, z1.h[4] -; CHECK-NEXT: strb w9, [sp, #16] -; CHECK-NEXT: fmov w9, s3 -; CHECK-NEXT: strb w10, [sp, #24] +; CHECK-NEXT: mov z6.h, z1.h[3] +; CHECK-NEXT: strb w10, [sp] ; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: strb w8, [sp, #23] +; CHECK-NEXT: strb w8, [sp, #15] ; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: mov z6.h, z1.h[3] +; CHECK-NEXT: strb w9, [sp, #14] +; CHECK-NEXT: fmov w9, s6 ; CHECK-NEXT: mov z7.h, z1.h[2] ; CHECK-NEXT: mov z16.h, z1.h[1] -; CHECK-NEXT: strb w9, [sp, #22] -; CHECK-NEXT: fmov w9, s6 -; CHECK-NEXT: strb w10, [sp, #21] +; CHECK-NEXT: mov z1.h, z0.h[7] +; CHECK-NEXT: strb w10, [sp, #13] ; CHECK-NEXT: fmov w10, s7 -; CHECK-NEXT: strb w8, [sp, #20] +; CHECK-NEXT: strb w8, [sp, #12] ; CHECK-NEXT: fmov w8, s16 -; CHECK-NEXT: mov z1.h, z0.h[7] +; CHECK-NEXT: strb w9, [sp, #11] +; CHECK-NEXT: fmov w9, s1 ; CHECK-NEXT: mov z17.h, z0.h[6] ; CHECK-NEXT: mov z18.h, z0.h[5] -; CHECK-NEXT: strb w9, [sp, #19] -; CHECK-NEXT: fmov w9, s1 -; CHECK-NEXT: strb w10, [sp, #18] +; CHECK-NEXT: mov z19.h, z0.h[4] +; CHECK-NEXT: strb w10, [sp, #10] ; CHECK-NEXT: fmov w10, s17 -; CHECK-NEXT: strb w8, [sp, #17] +; CHECK-NEXT: strb w8, [sp, #9] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: mov z19.h, z0.h[4] +; CHECK-NEXT: strb w9, [sp, #7] +; CHECK-NEXT: fmov w9, s19 ; CHECK-NEXT: mov z20.h, z0.h[3] ; CHECK-NEXT: mov z21.h, z0.h[2] -; CHECK-NEXT: strb w9, [sp, #31] -; CHECK-NEXT: fmov w9, s19 -; CHECK-NEXT: strb w10, [sp, #30] +; CHECK-NEXT: mov z22.h, z0.h[1] +; CHECK-NEXT: strb w10, [sp, #6] ; CHECK-NEXT: fmov w10, s20 -; CHECK-NEXT: strb w8, [sp, #29] +; CHECK-NEXT: strb w8, [sp, #5] ; CHECK-NEXT: fmov w8, s21 -; CHECK-NEXT: mov z22.h, z0.h[1] -; CHECK-NEXT: strb w9, [sp, #28] +; CHECK-NEXT: strb w9, [sp, #4] ; CHECK-NEXT: fmov w9, s22 -; CHECK-NEXT: strb w10, [sp, #27] -; CHECK-NEXT: mov x10, sp -; CHECK-NEXT: strb w8, [sp, #26] -; CHECK-NEXT: add x8, sp, #16 -; CHECK-NEXT: strb w9, [sp, #25] -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x10] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8] +; CHECK-NEXT: strb w10, [sp, #3] +; CHECK-NEXT: strb w8, [sp, #2] +; CHECK-NEXT: strb w9, [sp, #1] +; CHECK-NEXT: ldp q0, q1, [sp] ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: add sp, sp, #32 ; CHECK-NEXT: ret @@ -281,26 +275,25 @@ define void @smulh_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: smulh_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: smulh z4.h, p0/m, z4.h, z3.h -; CHECK-NEXT: ext z1.b, z1.b, z1.b, #8 -; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 -; CHECK-NEXT: smulh z1.h, p0/m, z1.h, z3.h -; CHECK-NEXT: movprfx z3, z0 -; CHECK-NEXT: smulh z3.h, p0/m, z3.h, z2.h -; CHECK-NEXT: ext z2.b, z2.b, z2.b, #8 -; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: mov z5.d, z0.d +; CHECK-NEXT: ext z5.b, z5.b, z5.b, #8 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: ext z4.b, z4.b, z4.b, #8 +; CHECK-NEXT: mov z6.d, z2.d ; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z2.h -; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h -; CHECK-NEXT: splice z3.h, p0, z3.h, z0.h -; CHECK-NEXT: stp q4, q3, [x0] +; CHECK-NEXT: ext z6.b, z6.b, z6.b, #8 +; CHECK-NEXT: mov z2.d, z3.d +; CHECK-NEXT: smulh z1.h, p0/m, z1.h, z3.h +; CHECK-NEXT: ext z2.b, z2.b, z3.b, #8 +; CHECK-NEXT: movprfx z3, z5 +; CHECK-NEXT: smulh z3.h, p0/m, z3.h, z6.h +; CHECK-NEXT: smulh z2.h, p0/m, z2.h, z4.h +; CHECK-NEXT: splice z0.h, p0, z0.h, z3.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z2.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -350,26 +343,25 @@ define void @smulh_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: smulh_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl2 -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: smulh z4.s, p0/m, z4.s, z3.s -; CHECK-NEXT: ext z1.b, z1.b, z1.b, #8 -; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 -; CHECK-NEXT: smulh z1.s, p0/m, z1.s, z3.s -; CHECK-NEXT: movprfx z3, z0 -; CHECK-NEXT: smulh z3.s, p0/m, z3.s, z2.s -; CHECK-NEXT: ext z2.b, z2.b, z2.b, #8 -; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: mov z5.d, z0.d +; CHECK-NEXT: ext z5.b, z5.b, z5.b, #8 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: ext z4.b, z4.b, z4.b, #8 +; CHECK-NEXT: mov z6.d, z2.d ; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: splice z4.s, p0, z4.s, z1.s -; CHECK-NEXT: splice z3.s, p0, z3.s, z0.s -; CHECK-NEXT: stp q4, q3, [x0] +; CHECK-NEXT: ext z6.b, z6.b, z6.b, #8 +; CHECK-NEXT: mov z2.d, z3.d +; CHECK-NEXT: smulh z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: ext z2.b, z2.b, z3.b, #8 +; CHECK-NEXT: movprfx z3, z5 +; CHECK-NEXT: smulh z3.s, p0/m, z3.s, z6.s +; CHECK-NEXT: smulh z2.s, p0/m, z2.s, z4.s +; CHECK-NEXT: splice z0.s, p0, z0.s, z3.s +; CHECK-NEXT: splice z1.s, p0, z1.s, z2.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -421,33 +413,29 @@ define void @smulh_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: smulh_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl1 -; CHECK-NEXT: fmov x8, d0 -; CHECK-NEXT: mov z4.d, z0.d[1] -; CHECK-NEXT: fmov x10, d2 -; CHECK-NEXT: mov z0.d, z1.d[1] -; CHECK-NEXT: fmov x9, d1 -; CHECK-NEXT: mov z1.d, z2.d[1] -; CHECK-NEXT: mov z2.d, z3.d[1] -; CHECK-NEXT: fmov x11, d3 -; CHECK-NEXT: fmov x12, d0 -; CHECK-NEXT: fmov x13, d2 -; CHECK-NEXT: fmov x14, d4 -; CHECK-NEXT: smulh x8, x8, x10 +; CHECK-NEXT: fmov x9, d0 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: mov z4.d, z1.d[1] +; CHECK-NEXT: fmov x8, d1 +; CHECK-NEXT: mov z1.d, z0.d[1] +; CHECK-NEXT: fmov x13, d4 ; CHECK-NEXT: fmov x10, d1 -; CHECK-NEXT: smulh x9, x9, x11 -; CHECK-NEXT: smulh x12, x12, x13 -; CHECK-NEXT: smulh x10, x14, x10 -; CHECK-NEXT: fmov d2, x8 +; CHECK-NEXT: mov z0.d, z2.d[1] +; CHECK-NEXT: fmov x12, d2 +; CHECK-NEXT: fmov x11, d0 +; CHECK-NEXT: mov z0.d, z3.d[1] +; CHECK-NEXT: fmov x14, d0 +; CHECK-NEXT: smulh x9, x9, x12 +; CHECK-NEXT: smulh x10, x10, x11 +; CHECK-NEXT: fmov x11, d3 +; CHECK-NEXT: smulh x12, x13, x14 +; CHECK-NEXT: smulh x8, x8, x11 ; CHECK-NEXT: fmov d0, x9 -; CHECK-NEXT: fmov d1, x12 -; CHECK-NEXT: fmov d3, x10 +; CHECK-NEXT: fmov d1, x10 +; CHECK-NEXT: fmov d3, x12 +; CHECK-NEXT: fmov d2, x8 ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d ; CHECK-NEXT: splice z2.d, p0, z2.d, z3.d ; CHECK-NEXT: stp q0, q2, [x0] @@ -528,135 +516,129 @@ define void @umulh_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: umulh_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ptrue p1.h, vl8 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] +; CHECK-NEXT: sub sp, sp, #32 +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: adrp x8, .LCPI17_0 -; CHECK-NEXT: add x8, x8, :lo12:.LCPI17_0 -; CHECK-NEXT: uunpklo z5.h, z2.b +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: uunpklo z0.h, z2.b ; CHECK-NEXT: ext z2.b, z2.b, z2.b, #8 -; CHECK-NEXT: uunpklo z7.h, z3.b -; CHECK-NEXT: ld1h { z16.h }, p1/z, [x8] -; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 ; CHECK-NEXT: uunpklo z2.h, z2.b +; CHECK-NEXT: ldp q4, q5, [x1] +; CHECK-NEXT: uunpklo z6.h, z3.b +; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 ; CHECK-NEXT: uunpklo z3.h, z3.b -; CHECK-NEXT: mul z5.h, p1/m, z5.h, z7.h -; CHECK-NEXT: mul z2.h, p1/m, z2.h, z3.h -; CHECK-NEXT: movprfx z3, z5 -; CHECK-NEXT: lsr z3.h, p1/m, z3.h, z16.h +; CHECK-NEXT: uunpklo z1.h, z4.b +; CHECK-NEXT: ext z4.b, z4.b, z4.b, #8 +; CHECK-NEXT: uunpklo z4.h, z4.b +; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: uunpklo z7.h, z5.b +; CHECK-NEXT: ext z5.b, z5.b, z5.b, #8 +; CHECK-NEXT: ldr q16, [x8, :lo12:.LCPI17_0] +; CHECK-NEXT: uunpklo z5.h, z5.b +; CHECK-NEXT: mul z3.h, p0/m, z3.h, z5.h +; CHECK-NEXT: movprfx z5, z6 +; CHECK-NEXT: mul z5.h, p0/m, z5.h, z7.h +; CHECK-NEXT: mul z2.h, p0/m, z2.h, z4.h +; CHECK-NEXT: movprfx z4, z5 +; CHECK-NEXT: lsr z4.h, p0/m, z4.h, z16.h +; CHECK-NEXT: lsr z3.h, p0/m, z3.h, z16.h +; CHECK-NEXT: fmov w9, s4 ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: uunpklo z4.h, z0.b -; CHECK-NEXT: uunpklo z6.h, z1.b -; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 -; CHECK-NEXT: ext z1.b, z1.b, z1.b, #8 -; CHECK-NEXT: lsr z2.h, p1/m, z2.h, z16.h ; CHECK-NEXT: mov z5.h, z3.h[7] -; CHECK-NEXT: uunpklo z0.h, z0.b -; CHECK-NEXT: uunpklo z1.h, z1.b -; CHECK-NEXT: mul z4.h, p1/m, z4.h, z6.h ; CHECK-NEXT: mov z6.h, z3.h[6] ; CHECK-NEXT: mov z7.h, z3.h[5] -; CHECK-NEXT: mov z17.h, z3.h[4] -; CHECK-NEXT: mov z18.h, z3.h[3] -; CHECK-NEXT: mov z19.h, z3.h[2] -; CHECK-NEXT: mov z20.h, z3.h[1] -; CHECK-NEXT: mov z3.h, z2.h[7] -; CHECK-NEXT: mov z21.h, z2.h[6] -; CHECK-NEXT: mov z22.h, z2.h[5] -; CHECK-NEXT: mov z23.h, z2.h[4] -; CHECK-NEXT: mov z24.h, z2.h[3] -; CHECK-NEXT: mov z25.h, z2.h[2] -; CHECK-NEXT: mov z26.h, z2.h[1] -; CHECK-NEXT: fmov w9, s2 -; CHECK-NEXT: mul z0.h, p1/m, z0.h, z1.h ; CHECK-NEXT: fmov w10, s5 -; CHECK-NEXT: strb w8, [sp, #-32]! -; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: strb w9, [sp, #16] +; CHECK-NEXT: strb w8, [sp, #24] ; CHECK-NEXT: fmov w8, s6 -; CHECK-NEXT: strb w9, [sp, #8] ; CHECK-NEXT: fmov w9, s7 -; CHECK-NEXT: strb w10, [sp, #7] +; CHECK-NEXT: mov z17.h, z3.h[4] +; CHECK-NEXT: mov z18.h, z3.h[3] +; CHECK-NEXT: mov z19.h, z3.h[2] +; CHECK-NEXT: strb w10, [sp, #31] ; CHECK-NEXT: fmov w10, s17 -; CHECK-NEXT: lsr z0.h, p1/m, z0.h, z16.h -; CHECK-NEXT: strb w8, [sp, #6] +; CHECK-NEXT: strb w8, [sp, #30] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: strb w9, [sp, #5] +; CHECK-NEXT: strb w9, [sp, #29] ; CHECK-NEXT: fmov w9, s19 -; CHECK-NEXT: strb w10, [sp, #4] +; CHECK-NEXT: mov z20.h, z3.h[1] +; CHECK-NEXT: mov z3.h, z4.h[7] +; CHECK-NEXT: mov z21.h, z4.h[6] +; CHECK-NEXT: strb w10, [sp, #28] ; CHECK-NEXT: fmov w10, s20 -; CHECK-NEXT: strb w8, [sp, #3] +; CHECK-NEXT: strb w8, [sp, #27] ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: strb w9, [sp, #2] +; CHECK-NEXT: strb w9, [sp, #26] ; CHECK-NEXT: fmov w9, s21 -; CHECK-NEXT: strb w10, [sp, #1] +; CHECK-NEXT: mov z22.h, z4.h[5] +; CHECK-NEXT: mov z23.h, z4.h[4] +; CHECK-NEXT: mov z24.h, z4.h[3] +; CHECK-NEXT: strb w10, [sp, #25] ; CHECK-NEXT: fmov w10, s22 -; CHECK-NEXT: strb w8, [sp, #15] +; CHECK-NEXT: strb w8, [sp, #23] ; CHECK-NEXT: fmov w8, s23 -; CHECK-NEXT: strb w9, [sp, #14] +; CHECK-NEXT: strb w9, [sp, #22] ; CHECK-NEXT: fmov w9, s24 -; CHECK-NEXT: strb w10, [sp, #13] +; CHECK-NEXT: mov z25.h, z4.h[2] +; CHECK-NEXT: mov z26.h, z4.h[1] +; CHECK-NEXT: strb w10, [sp, #21] ; CHECK-NEXT: fmov w10, s25 -; CHECK-NEXT: strb w8, [sp, #12] +; CHECK-NEXT: strb w8, [sp, #20] +; CHECK-NEXT: movprfx z1, z2 +; CHECK-NEXT: lsr z1.h, p0/m, z1.h, z16.h +; CHECK-NEXT: strb w9, [sp, #19] ; CHECK-NEXT: fmov w8, s26 -; CHECK-NEXT: movprfx z1, z4 -; CHECK-NEXT: lsr z1.h, p1/m, z1.h, z16.h -; CHECK-NEXT: strb w9, [sp, #11] -; CHECK-NEXT: mov z2.h, z1.h[7] ; CHECK-NEXT: fmov w9, s1 -; CHECK-NEXT: strb w10, [sp, #10] +; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z16.h +; CHECK-NEXT: mov z2.h, z1.h[7] +; CHECK-NEXT: mov z3.h, z1.h[6] +; CHECK-NEXT: strb w10, [sp, #18] ; CHECK-NEXT: fmov w10, s0 -; CHECK-NEXT: strb w8, [sp, #9] +; CHECK-NEXT: strb w8, [sp, #17] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: mov z3.h, z1.h[6] +; CHECK-NEXT: strb w9, [sp, #8] +; CHECK-NEXT: fmov w9, s3 ; CHECK-NEXT: mov z4.h, z1.h[5] ; CHECK-NEXT: mov z5.h, z1.h[4] -; CHECK-NEXT: strb w9, [sp, #16] -; CHECK-NEXT: fmov w9, s3 -; CHECK-NEXT: strb w10, [sp, #24] +; CHECK-NEXT: mov z6.h, z1.h[3] +; CHECK-NEXT: strb w10, [sp] ; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: strb w8, [sp, #23] +; CHECK-NEXT: strb w8, [sp, #15] ; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: mov z6.h, z1.h[3] +; CHECK-NEXT: strb w9, [sp, #14] +; CHECK-NEXT: fmov w9, s6 ; CHECK-NEXT: mov z7.h, z1.h[2] ; CHECK-NEXT: mov z16.h, z1.h[1] -; CHECK-NEXT: strb w9, [sp, #22] -; CHECK-NEXT: fmov w9, s6 -; CHECK-NEXT: strb w10, [sp, #21] +; CHECK-NEXT: mov z1.h, z0.h[7] +; CHECK-NEXT: strb w10, [sp, #13] ; CHECK-NEXT: fmov w10, s7 -; CHECK-NEXT: strb w8, [sp, #20] +; CHECK-NEXT: strb w8, [sp, #12] ; CHECK-NEXT: fmov w8, s16 -; CHECK-NEXT: mov z1.h, z0.h[7] +; CHECK-NEXT: strb w9, [sp, #11] +; CHECK-NEXT: fmov w9, s1 ; CHECK-NEXT: mov z17.h, z0.h[6] ; CHECK-NEXT: mov z18.h, z0.h[5] -; CHECK-NEXT: strb w9, [sp, #19] -; CHECK-NEXT: fmov w9, s1 -; CHECK-NEXT: strb w10, [sp, #18] +; CHECK-NEXT: mov z19.h, z0.h[4] +; CHECK-NEXT: strb w10, [sp, #10] ; CHECK-NEXT: fmov w10, s17 -; CHECK-NEXT: strb w8, [sp, #17] +; CHECK-NEXT: strb w8, [sp, #9] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: mov z19.h, z0.h[4] +; CHECK-NEXT: strb w9, [sp, #7] +; CHECK-NEXT: fmov w9, s19 ; CHECK-NEXT: mov z20.h, z0.h[3] ; CHECK-NEXT: mov z21.h, z0.h[2] -; CHECK-NEXT: strb w9, [sp, #31] -; CHECK-NEXT: fmov w9, s19 -; CHECK-NEXT: strb w10, [sp, #30] +; CHECK-NEXT: mov z22.h, z0.h[1] +; CHECK-NEXT: strb w10, [sp, #6] ; CHECK-NEXT: fmov w10, s20 -; CHECK-NEXT: strb w8, [sp, #29] +; CHECK-NEXT: strb w8, [sp, #5] ; CHECK-NEXT: fmov w8, s21 -; CHECK-NEXT: mov z22.h, z0.h[1] -; CHECK-NEXT: strb w9, [sp, #28] +; CHECK-NEXT: strb w9, [sp, #4] ; CHECK-NEXT: fmov w9, s22 -; CHECK-NEXT: strb w10, [sp, #27] -; CHECK-NEXT: mov x10, sp -; CHECK-NEXT: strb w8, [sp, #26] -; CHECK-NEXT: add x8, sp, #16 -; CHECK-NEXT: strb w9, [sp, #25] -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x10] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8] +; CHECK-NEXT: strb w10, [sp, #3] +; CHECK-NEXT: strb w8, [sp, #2] +; CHECK-NEXT: strb w9, [sp, #1] +; CHECK-NEXT: ldp q0, q1, [sp] ; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: add sp, sp, #32 ; CHECK-NEXT: ret @@ -732,26 +714,25 @@ define void @umulh_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: umulh_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: umulh z4.h, p0/m, z4.h, z3.h -; CHECK-NEXT: ext z1.b, z1.b, z1.b, #8 -; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 -; CHECK-NEXT: umulh z1.h, p0/m, z1.h, z3.h -; CHECK-NEXT: movprfx z3, z0 -; CHECK-NEXT: umulh z3.h, p0/m, z3.h, z2.h -; CHECK-NEXT: ext z2.b, z2.b, z2.b, #8 -; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: mov z5.d, z0.d +; CHECK-NEXT: ext z5.b, z5.b, z5.b, #8 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: ext z4.b, z4.b, z4.b, #8 +; CHECK-NEXT: mov z6.d, z2.d ; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z2.h -; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h -; CHECK-NEXT: splice z3.h, p0, z3.h, z0.h -; CHECK-NEXT: stp q4, q3, [x0] +; CHECK-NEXT: ext z6.b, z6.b, z6.b, #8 +; CHECK-NEXT: mov z2.d, z3.d +; CHECK-NEXT: umulh z1.h, p0/m, z1.h, z3.h +; CHECK-NEXT: ext z2.b, z2.b, z3.b, #8 +; CHECK-NEXT: movprfx z3, z5 +; CHECK-NEXT: umulh z3.h, p0/m, z3.h, z6.h +; CHECK-NEXT: umulh z2.h, p0/m, z2.h, z4.h +; CHECK-NEXT: splice z0.h, p0, z0.h, z3.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z2.h +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -801,26 +782,25 @@ define void @umulh_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: umulh_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl2 -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: umulh z4.s, p0/m, z4.s, z3.s -; CHECK-NEXT: ext z1.b, z1.b, z1.b, #8 -; CHECK-NEXT: ext z3.b, z3.b, z3.b, #8 -; CHECK-NEXT: umulh z1.s, p0/m, z1.s, z3.s -; CHECK-NEXT: movprfx z3, z0 -; CHECK-NEXT: umulh z3.s, p0/m, z3.s, z2.s -; CHECK-NEXT: ext z2.b, z2.b, z2.b, #8 -; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8 +; CHECK-NEXT: mov z5.d, z0.d +; CHECK-NEXT: ext z5.b, z5.b, z5.b, #8 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: mov z4.d, z1.d +; CHECK-NEXT: ext z4.b, z4.b, z4.b, #8 +; CHECK-NEXT: mov z6.d, z2.d ; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: splice z4.s, p0, z4.s, z1.s -; CHECK-NEXT: splice z3.s, p0, z3.s, z0.s -; CHECK-NEXT: stp q4, q3, [x0] +; CHECK-NEXT: ext z6.b, z6.b, z6.b, #8 +; CHECK-NEXT: mov z2.d, z3.d +; CHECK-NEXT: umulh z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: ext z2.b, z2.b, z3.b, #8 +; CHECK-NEXT: movprfx z3, z5 +; CHECK-NEXT: umulh z3.s, p0/m, z3.s, z6.s +; CHECK-NEXT: umulh z2.s, p0/m, z2.s, z4.s +; CHECK-NEXT: splice z0.s, p0, z0.s, z3.s +; CHECK-NEXT: splice z1.s, p0, z1.s, z2.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -872,33 +852,29 @@ define void @umulh_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: umulh_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 -; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl1 -; CHECK-NEXT: fmov x8, d0 -; CHECK-NEXT: mov z4.d, z0.d[1] -; CHECK-NEXT: fmov x10, d2 -; CHECK-NEXT: mov z0.d, z1.d[1] -; CHECK-NEXT: fmov x9, d1 -; CHECK-NEXT: mov z1.d, z2.d[1] -; CHECK-NEXT: mov z2.d, z3.d[1] -; CHECK-NEXT: fmov x11, d3 -; CHECK-NEXT: fmov x12, d0 -; CHECK-NEXT: fmov x13, d2 -; CHECK-NEXT: fmov x14, d4 -; CHECK-NEXT: umulh x8, x8, x10 +; CHECK-NEXT: fmov x9, d0 +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: mov z4.d, z1.d[1] +; CHECK-NEXT: fmov x8, d1 +; CHECK-NEXT: mov z1.d, z0.d[1] +; CHECK-NEXT: fmov x13, d4 ; CHECK-NEXT: fmov x10, d1 -; CHECK-NEXT: umulh x9, x9, x11 -; CHECK-NEXT: umulh x12, x12, x13 -; CHECK-NEXT: umulh x10, x14, x10 -; CHECK-NEXT: fmov d2, x8 +; CHECK-NEXT: mov z0.d, z2.d[1] +; CHECK-NEXT: fmov x12, d2 +; CHECK-NEXT: fmov x11, d0 +; CHECK-NEXT: mov z0.d, z3.d[1] +; CHECK-NEXT: fmov x14, d0 +; CHECK-NEXT: umulh x9, x9, x12 +; CHECK-NEXT: umulh x10, x10, x11 +; CHECK-NEXT: fmov x11, d3 +; CHECK-NEXT: umulh x12, x13, x14 +; CHECK-NEXT: umulh x8, x8, x11 ; CHECK-NEXT: fmov d0, x9 -; CHECK-NEXT: fmov d1, x12 -; CHECK-NEXT: fmov d3, x10 +; CHECK-NEXT: fmov d1, x10 +; CHECK-NEXT: fmov d3, x12 +; CHECK-NEXT: fmov d2, x8 ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d ; CHECK-NEXT: splice z2.d, p0, z2.d, z3.d ; CHECK-NEXT: stp q0, q2, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll @@ -131,54 +131,51 @@ define void @srem_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: srem_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ptrue p1.s, vl4 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] +; CHECK-NEXT: ldp q2, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldp q3, q1, [x1] ; CHECK-NEXT: sunpkhi z5.h, z0.b ; CHECK-NEXT: sunpklo z7.h, z0.b -; CHECK-NEXT: sunpkhi z4.h, z2.b -; CHECK-NEXT: sunpklo z6.h, z2.b -; CHECK-NEXT: sunpkhi z16.s, z4.h ; CHECK-NEXT: sunpkhi z17.s, z5.h -; CHECK-NEXT: sunpklo z4.s, z4.h ; CHECK-NEXT: sunpklo z5.s, z5.h +; CHECK-NEXT: sunpkhi z4.h, z1.b +; CHECK-NEXT: sunpklo z6.h, z1.b +; CHECK-NEXT: sunpkhi z16.s, z4.h +; CHECK-NEXT: sunpklo z4.s, z4.h ; CHECK-NEXT: sunpkhi z18.s, z6.h -; CHECK-NEXT: sdivr z16.s, p1/m, z16.s, z17.s -; CHECK-NEXT: sdivr z4.s, p1/m, z4.s, z5.s +; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: sunpkhi z5.s, z7.h ; CHECK-NEXT: sunpklo z6.s, z6.h ; CHECK-NEXT: sunpklo z7.s, z7.h +; CHECK-NEXT: sdiv z5.s, p0/m, z5.s, z18.s +; CHECK-NEXT: sdivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: uzp1 z5.h, z6.h, z5.h +; CHECK-NEXT: sunpkhi z6.h, z3.b +; CHECK-NEXT: sunpkhi z7.h, z2.b ; CHECK-NEXT: uzp1 z4.h, z4.h, z16.h -; CHECK-NEXT: sdivr z6.s, p1/m, z6.s, z7.s -; CHECK-NEXT: sunpkhi z7.h, z3.b -; CHECK-NEXT: sunpkhi z16.h, z1.b -; CHECK-NEXT: sdiv z5.s, p1/m, z5.s, z18.s +; CHECK-NEXT: sunpkhi z16.s, z6.h ; CHECK-NEXT: sunpkhi z17.s, z7.h -; CHECK-NEXT: sunpkhi z18.s, z16.h +; CHECK-NEXT: sunpklo z6.s, z6.h ; CHECK-NEXT: sunpklo z7.s, z7.h -; CHECK-NEXT: sunpklo z16.s, z16.h -; CHECK-NEXT: sdivr z17.s, p1/m, z17.s, z18.s -; CHECK-NEXT: sdivr z7.s, p1/m, z7.s, z16.s -; CHECK-NEXT: sunpklo z16.h, z3.b -; CHECK-NEXT: sunpklo z18.h, z1.b -; CHECK-NEXT: sunpkhi z19.s, z16.h -; CHECK-NEXT: sunpkhi z20.s, z18.h -; CHECK-NEXT: sunpklo z16.s, z16.h -; CHECK-NEXT: sunpklo z18.s, z18.h -; CHECK-NEXT: sdivr z19.s, p1/m, z19.s, z20.s -; CHECK-NEXT: sdivr z16.s, p1/m, z16.s, z18.s -; CHECK-NEXT: uzp1 z7.h, z7.h, z17.h -; CHECK-NEXT: uzp1 z16.h, z16.h, z19.h -; CHECK-NEXT: uzp1 z5.h, z6.h, z5.h -; CHECK-NEXT: uzp1 z6.b, z16.b, z7.b +; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: sdivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: sunpklo z7.h, z3.b +; CHECK-NEXT: sunpklo z17.h, z2.b +; CHECK-NEXT: sunpkhi z18.s, z7.h +; CHECK-NEXT: sunpkhi z19.s, z17.h +; CHECK-NEXT: sunpklo z7.s, z7.h +; CHECK-NEXT: sunpklo z17.s, z17.h +; CHECK-NEXT: sdivr z18.s, p0/m, z18.s, z19.s +; CHECK-NEXT: sdivr z7.s, p0/m, z7.s, z17.s +; CHECK-NEXT: uzp1 z6.h, z6.h, z16.h +; CHECK-NEXT: uzp1 z7.h, z7.h, z18.h +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: uzp1 z6.b, z7.b, z6.b ; CHECK-NEXT: uzp1 z4.b, z5.b, z4.b -; CHECK-NEXT: mls z1.b, p0/m, z6.b, z3.b -; CHECK-NEXT: mls z0.b, p0/m, z4.b, z2.b -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mls z2.b, p0/m, z6.b, z3.b +; CHECK-NEXT: mls z0.b, p0/m, z4.b, z1.b +; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -244,30 +241,27 @@ define void @srem_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: srem_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ptrue p1.s, vl4 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] +; CHECK-NEXT: ldp q2, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: sunpkhi z17.s, z2.h +; CHECK-NEXT: ldp q3, q1, [x1] ; CHECK-NEXT: sunpkhi z5.s, z0.h -; CHECK-NEXT: sunpkhi z16.s, z1.h -; CHECK-NEXT: sunpkhi z4.s, z2.h -; CHECK-NEXT: sunpkhi z7.s, z3.h -; CHECK-NEXT: sdivr z4.s, p1/m, z4.s, z5.s +; CHECK-NEXT: sunpklo z7.s, z0.h +; CHECK-NEXT: sunpkhi z16.s, z3.h +; CHECK-NEXT: sdivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: sunpkhi z4.s, z1.h +; CHECK-NEXT: sunpklo z6.s, z1.h +; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: sunpklo z5.s, z3.h -; CHECK-NEXT: sdivr z7.s, p1/m, z7.s, z16.s -; CHECK-NEXT: sunpklo z16.s, z1.h -; CHECK-NEXT: sunpklo z6.s, z2.h -; CHECK-NEXT: sdivr z5.s, p1/m, z5.s, z16.s -; CHECK-NEXT: sunpklo z16.s, z0.h -; CHECK-NEXT: uzp1 z5.h, z5.h, z7.h -; CHECK-NEXT: sdivr z6.s, p1/m, z6.s, z16.s -; CHECK-NEXT: mls z1.h, p0/m, z5.h, z3.h +; CHECK-NEXT: sdivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: sunpklo z7.s, z2.h +; CHECK-NEXT: sdivr z5.s, p0/m, z5.s, z7.s +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: uzp1 z5.h, z5.h, z16.h ; CHECK-NEXT: uzp1 z4.h, z6.h, z4.h -; CHECK-NEXT: mls z0.h, p0/m, z4.h, z2.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mls z2.h, p0/m, z5.h, z3.h +; CHECK-NEXT: mls z0.h, p0/m, z4.h, z1.h +; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -309,19 +303,16 @@ define void @srem_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: srem_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: sdiv z4.s, p0/m, z4.s, z3.s -; CHECK-NEXT: movprfx z5, z0 -; CHECK-NEXT: sdiv z5.s, p0/m, z5.s, z2.s -; CHECK-NEXT: mls z1.s, p0/m, z4.s, z3.s -; CHECK-NEXT: mls z0.s, p0/m, z5.s, z2.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: sdiv z4.s, p0/m, z4.s, z2.s +; CHECK-NEXT: movprfx z5, z1 +; CHECK-NEXT: sdiv z5.s, p0/m, z5.s, z3.s +; CHECK-NEXT: mls z0.s, p0/m, z4.s, z2.s +; CHECK-NEXT: mls z1.s, p0/m, z5.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -363,19 +354,16 @@ define void @srem_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: srem_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: sdiv z4.d, p0/m, z4.d, z3.d -; CHECK-NEXT: movprfx z5, z0 -; CHECK-NEXT: sdiv z5.d, p0/m, z5.d, z2.d -; CHECK-NEXT: mls z1.d, p0/m, z4.d, z3.d -; CHECK-NEXT: mls z0.d, p0/m, z5.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: sdiv z4.d, p0/m, z4.d, z2.d +; CHECK-NEXT: movprfx z5, z1 +; CHECK-NEXT: sdiv z5.d, p0/m, z5.d, z3.d +; CHECK-NEXT: mls z0.d, p0/m, z4.d, z2.d +; CHECK-NEXT: mls z1.d, p0/m, z5.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b @@ -510,54 +498,51 @@ define void @urem_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 { ; CHECK-LABEL: urem_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #16 -; CHECK-NEXT: ptrue p0.b, vl16 -; CHECK-NEXT: ptrue p1.s, vl4 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8] -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1, x8] -; CHECK-NEXT: ld1b { z3.b }, p0/z, [x1] +; CHECK-NEXT: ldp q2, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: ldp q3, q1, [x1] ; CHECK-NEXT: uunpkhi z5.h, z0.b ; CHECK-NEXT: uunpklo z7.h, z0.b -; CHECK-NEXT: uunpkhi z4.h, z2.b -; CHECK-NEXT: uunpklo z6.h, z2.b -; CHECK-NEXT: uunpkhi z16.s, z4.h ; CHECK-NEXT: uunpkhi z17.s, z5.h -; CHECK-NEXT: uunpklo z4.s, z4.h ; CHECK-NEXT: uunpklo z5.s, z5.h +; CHECK-NEXT: uunpkhi z4.h, z1.b +; CHECK-NEXT: uunpklo z6.h, z1.b +; CHECK-NEXT: uunpkhi z16.s, z4.h +; CHECK-NEXT: uunpklo z4.s, z4.h ; CHECK-NEXT: uunpkhi z18.s, z6.h -; CHECK-NEXT: udivr z16.s, p1/m, z16.s, z17.s -; CHECK-NEXT: udivr z4.s, p1/m, z4.s, z5.s +; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: uunpkhi z5.s, z7.h ; CHECK-NEXT: uunpklo z6.s, z6.h ; CHECK-NEXT: uunpklo z7.s, z7.h +; CHECK-NEXT: udiv z5.s, p0/m, z5.s, z18.s +; CHECK-NEXT: udivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: uzp1 z5.h, z6.h, z5.h +; CHECK-NEXT: uunpkhi z6.h, z3.b +; CHECK-NEXT: uunpkhi z7.h, z2.b ; CHECK-NEXT: uzp1 z4.h, z4.h, z16.h -; CHECK-NEXT: udivr z6.s, p1/m, z6.s, z7.s -; CHECK-NEXT: uunpkhi z7.h, z3.b -; CHECK-NEXT: uunpkhi z16.h, z1.b -; CHECK-NEXT: udiv z5.s, p1/m, z5.s, z18.s +; CHECK-NEXT: uunpkhi z16.s, z6.h ; CHECK-NEXT: uunpkhi z17.s, z7.h -; CHECK-NEXT: uunpkhi z18.s, z16.h +; CHECK-NEXT: uunpklo z6.s, z6.h ; CHECK-NEXT: uunpklo z7.s, z7.h -; CHECK-NEXT: uunpklo z16.s, z16.h -; CHECK-NEXT: udivr z17.s, p1/m, z17.s, z18.s -; CHECK-NEXT: udivr z7.s, p1/m, z7.s, z16.s -; CHECK-NEXT: uunpklo z16.h, z3.b -; CHECK-NEXT: uunpklo z18.h, z1.b -; CHECK-NEXT: uunpkhi z19.s, z16.h -; CHECK-NEXT: uunpkhi z20.s, z18.h -; CHECK-NEXT: uunpklo z16.s, z16.h -; CHECK-NEXT: uunpklo z18.s, z18.h -; CHECK-NEXT: udivr z19.s, p1/m, z19.s, z20.s -; CHECK-NEXT: udivr z16.s, p1/m, z16.s, z18.s -; CHECK-NEXT: uzp1 z7.h, z7.h, z17.h -; CHECK-NEXT: uzp1 z16.h, z16.h, z19.h -; CHECK-NEXT: uzp1 z5.h, z6.h, z5.h -; CHECK-NEXT: uzp1 z6.b, z16.b, z7.b +; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: udivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: uunpklo z7.h, z3.b +; CHECK-NEXT: uunpklo z17.h, z2.b +; CHECK-NEXT: uunpkhi z18.s, z7.h +; CHECK-NEXT: uunpkhi z19.s, z17.h +; CHECK-NEXT: uunpklo z7.s, z7.h +; CHECK-NEXT: uunpklo z17.s, z17.h +; CHECK-NEXT: udivr z18.s, p0/m, z18.s, z19.s +; CHECK-NEXT: udivr z7.s, p0/m, z7.s, z17.s +; CHECK-NEXT: uzp1 z6.h, z6.h, z16.h +; CHECK-NEXT: uzp1 z7.h, z7.h, z18.h +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: uzp1 z6.b, z7.b, z6.b ; CHECK-NEXT: uzp1 z4.b, z5.b, z4.b -; CHECK-NEXT: mls z1.b, p0/m, z6.b, z3.b -; CHECK-NEXT: mls z0.b, p0/m, z4.b, z2.b -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mls z2.b, p0/m, z6.b, z3.b +; CHECK-NEXT: mls z0.b, p0/m, z4.b, z1.b +; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, <32 x i8>* %a %op2 = load <32 x i8>, <32 x i8>* %b @@ -623,30 +608,27 @@ define void @urem_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 { ; CHECK-LABEL: urem_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #8 -; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: ptrue p1.s, vl4 -; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1] -; CHECK-NEXT: ld1h { z3.h }, p0/z, [x1] +; CHECK-NEXT: ldp q2, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: uunpkhi z17.s, z2.h +; CHECK-NEXT: ldp q3, q1, [x1] ; CHECK-NEXT: uunpkhi z5.s, z0.h -; CHECK-NEXT: uunpkhi z16.s, z1.h -; CHECK-NEXT: uunpkhi z4.s, z2.h -; CHECK-NEXT: uunpkhi z7.s, z3.h -; CHECK-NEXT: udivr z4.s, p1/m, z4.s, z5.s +; CHECK-NEXT: uunpklo z7.s, z0.h +; CHECK-NEXT: uunpkhi z16.s, z3.h +; CHECK-NEXT: udivr z16.s, p0/m, z16.s, z17.s +; CHECK-NEXT: uunpkhi z4.s, z1.h +; CHECK-NEXT: uunpklo z6.s, z1.h +; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: uunpklo z5.s, z3.h -; CHECK-NEXT: udivr z7.s, p1/m, z7.s, z16.s -; CHECK-NEXT: uunpklo z16.s, z1.h -; CHECK-NEXT: uunpklo z6.s, z2.h -; CHECK-NEXT: udivr z5.s, p1/m, z5.s, z16.s -; CHECK-NEXT: uunpklo z16.s, z0.h -; CHECK-NEXT: uzp1 z5.h, z5.h, z7.h -; CHECK-NEXT: udivr z6.s, p1/m, z6.s, z16.s -; CHECK-NEXT: mls z1.h, p0/m, z5.h, z3.h +; CHECK-NEXT: udivr z6.s, p0/m, z6.s, z7.s +; CHECK-NEXT: uunpklo z7.s, z2.h +; CHECK-NEXT: udivr z5.s, p0/m, z5.s, z7.s +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: uzp1 z5.h, z5.h, z16.h ; CHECK-NEXT: uzp1 z4.h, z6.h, z4.h -; CHECK-NEXT: mls z0.h, p0/m, z4.h, z2.h -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: mls z2.h, p0/m, z5.h, z3.h +; CHECK-NEXT: mls z0.h, p0/m, z4.h, z1.h +; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, <16 x i16>* %a %op2 = load <16 x i16>, <16 x i16>* %b @@ -688,19 +670,16 @@ define void @urem_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 { ; CHECK-LABEL: urem_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #4 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: udiv z4.s, p0/m, z4.s, z3.s -; CHECK-NEXT: movprfx z5, z0 -; CHECK-NEXT: udiv z5.s, p0/m, z5.s, z2.s -; CHECK-NEXT: mls z1.s, p0/m, z4.s, z3.s -; CHECK-NEXT: mls z0.s, p0/m, z5.s, z2.s -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: udiv z4.s, p0/m, z4.s, z2.s +; CHECK-NEXT: movprfx z5, z1 +; CHECK-NEXT: udiv z5.s, p0/m, z5.s, z3.s +; CHECK-NEXT: mls z0.s, p0/m, z4.s, z2.s +; CHECK-NEXT: mls z1.s, p0/m, z5.s, z3.s +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, <8 x i32>* %a %op2 = load <8 x i32>, <8 x i32>* %b @@ -742,19 +721,16 @@ define void @urem_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 { ; CHECK-LABEL: urem_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: mov x8, #2 +; CHECK-NEXT: ldp q0, q1, [x0] ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] -; CHECK-NEXT: movprfx z4, z1 -; CHECK-NEXT: udiv z4.d, p0/m, z4.d, z3.d -; CHECK-NEXT: movprfx z5, z0 -; CHECK-NEXT: udiv z5.d, p0/m, z5.d, z2.d -; CHECK-NEXT: mls z1.d, p0/m, z4.d, z3.d -; CHECK-NEXT: mls z0.d, p0/m, z5.d, z2.d -; CHECK-NEXT: stp q1, q0, [x0] +; CHECK-NEXT: ldp q2, q3, [x1] +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: udiv z4.d, p0/m, z4.d, z2.d +; CHECK-NEXT: movprfx z5, z1 +; CHECK-NEXT: udiv z5.d, p0/m, z5.d, z3.d +; CHECK-NEXT: mls z0.d, p0/m, z4.d, z2.d +; CHECK-NEXT: mls z1.d, p0/m, z5.d, z3.d +; CHECK-NEXT: stp q0, q1, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, <4 x i64>* %a %op2 = load <4 x i64>, <4 x i64>* %b diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll @@ -44,12 +44,14 @@ define <2 x i16> @load_v2i16(<2 x i16>* %a) #0 { ; CHECK-LABEL: load_v2i16: ; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldrh w8, [x0, #2] -; CHECK-NEXT: ldrh w9, [x0] -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: fmov s1, w9 -; CHECK-NEXT: zip1 z0.s, z1.s, z0.s -; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: str w8, [sp, #12] +; CHECK-NEXT: ldrh w8, [x0] +; CHECK-NEXT: str w8, [sp, #8] +; CHECK-NEXT: ldr d0, [sp, #8] +; CHECK-NEXT: add sp, sp, #16 ; CHECK-NEXT: ret %load = load <2 x i16>, <2 x i16>* %a ret <2 x i16> %load diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll @@ -0,0 +1,47 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +; Currently there is no custom lowering for vector shuffles operating on types +; bigger than NEON. However, having no support opens us up to a code generator +; hang when expanding BUILD_VECTOR. Here we just validate the promblematic case +; successfully exits code generation. +define void @hang_when_merging_stores_after_legalisation(<8 x i32>* %a, <2 x i32> %b) #0 { +; CHECK-LABEL: hang_when_merging_stores_after_legalisation: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: stp w8, w8, [sp, #8] +; CHECK-NEXT: stp w8, w8, [sp] +; CHECK-NEXT: ldr q0, [sp] +; CHECK-NEXT: stp q0, q0, [x0] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <8 x i32> zeroinitializer + %interleaved.vec = shufflevector <8 x i32> %splat, <8 x i32> undef, <8 x i32> + store <8 x i32> %interleaved.vec, <8 x i32>* %a, align 4 + ret void +} + +; Ensure we don't crash when trying to lower a shuffle via an extract +define void @crash_when_lowering_extract_shuffle(<32 x i32>* %dst, i1 %cond) #0 { +; CHECK-LABEL: crash_when_lowering_extract_shuffle: +; CHECK: // %bb.0: +; CHECK-NEXT: ret + %broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer + br i1 %cond, label %exit, label %vector.body + +vector.body: + %1 = load <32 x i32>, <32 x i32>* %dst, align 16 + %predphi = select <32 x i1> %broadcast.splat, <32 x i32> zeroinitializer, <32 x i32> %1 + store <32 x i32> %predphi, <32 x i32>* %dst, align 16 + br label %exit + +exit: + ret void +} + +attributes #0 = { "target-features"="+sve" }