[AArch64][DAGCombiner]: combine <2xi64> mul add/sub.

[hassnaa-arm]

Summary

64-bit vector mul is not supported in NEON,
so we use the SVE's mul.
To improve the performance, we can go one step further,
and use SVE's add/sub, so that we can use SVE's mla/mls.
That works on these patterns:
This works on the patterns of:
add v1, (mul v2, v3)
// sub v1, (mul v2, v3)

[david-arm]

LGTM! Can you fix the formatting issue before landing the patch please? Thanks!

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17827

nit: I think there is a formatting issue here and should be if (N->getOpcode() == ISD::SUB)

[hassnaa-arm]

Hi @paulwalker-arm Are you okay with landing this patch ?

[paulwalker-arm]

In D148118#4277526, @hassnaa-arm wrote:

Hi @paulwalker-arm Are you okay with landing this patch ?

Hi @hassnaa-arm , this dropped off my radar since Dave already accepted the updated patch. I'll try to take a look later today but please don't feel you have to wait for me before landing the patch.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17817

Given the MUL_PRED node you care about is the result of operation legalisation it's perhaps worth having a !isBeforeLegalizeOps() or isAfterLegalizeDAG() check so the combine costs less compile time during the earlier code generation phases. It'll also mean you can be sure the types are legal just in case we get here via some weird route.

17826–17836

I think you'll cover more cases if you check N->getOperand(1) first. For example, consider the case when you have sub(extract(), extract(mul_pred())), with the current order (i.e. checking Op0 first) the combine will not fire?

I guess there's an argument it would be even better for the function to take the operands as parameters (or implement a lambda function) so you'd have something like:

if (res = performOpt(Opcode, Op0, Op1)
  return res;
if (Opcode == ISD::ADD)
  if (res = performOpt(Opcode, Op1, Op0)
    return res;

This way you'll cover the maximum number of combinations.

I'll leave the decision whether it's worth covering all cases to you but as a minimum I'd switch to check Op1 first as mention above.

[hassnaa-arm]

Enhance code readability.

[david-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17855

Sorry to be a pain @hassnaa-arm, but this generates warnings about missing brackets () around the res = performOpt... code. I think you can silence the warnings by rewriting this as:

if (SDValue res = performOpt(N->getOperand(0), N->getOperand(1)))
  return res;
else if (N->getOpcode() == ISD::ADD)
  return performOpt(N->getOperand(1), N->getOperand(0));

return SDValue();

[hassnaa-arm]

Fix warning.

[paulwalker-arm]

Looks good to me but please add a couple of tests to show the new operand flexibility before landing the patch. For example:

define <2 x i64> @test(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c, <2 x i64> %d) {
  %mul1 = mul <2 x i64> %a, %b
  %mul2= mul <2 x i64> %c, %d
  %sub = sub <2 x i64> %mul, %mul2
  ret <2 x i64> %sub
}

[hassnaa-arm]

Add extra test cases that have 'extract_subvector' in both operands.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (M2.getOpcode() != ISD::MUL && M2.getOpcode() != AArch64ISD::SMULL &&
       M2.getOpcode() != AArch64ISD::UMULL)
     return SDValue();
 
   EVT VT = N->getValueType(0);
   SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, X, M1);
   return DAG.getNode(ISD::SUB, SDLoc(N), VT, Sub, M2);
 }
 
+// This works on the patterns of:
+//   add v1, (mul v2, v3)
+//   sub v1, (mul v2, v3)
+// for vectors of type <1 x i64> and <2 x i64> when SVE is available.
+// It will transform the add/sub to a scalable version, so that we can
+// make use of SVE's MLA/MLS that will be generated for that pattern
+static SDValue performMulAddSubCombine(SDNode *N, SelectionDAG &DAG) {
+  // Before using SVE's features, check first if it's available.
+  if (!DAG.getSubtarget<AArch64Subtarget>().hasSVE())
+    return SDValue();
+
+  if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::SUB)
+    return SDValue();
+
+  if (!N->getValueType(0).isFixedLengthVector())
+    return SDValue();
+
+  SDValue MulValue, Op, ExtractIndexValue, ExtractOp;
+
+  if (N->getOperand(0)->getOpcode() == ISD::EXTRACT_SUBVECTOR) {
+    if(N->getOpcode() == ISD::SUB)
+      // That invalidate the pattern for mls
+      return SDValue();
+    ExtractOp = N->getOperand(0);
+    Op = N->getOperand(1);
+  } else if (N->getOperand(1)->getOpcode() == ISD::EXTRACT_SUBVECTOR) {
+    ExtractOp = N->getOperand(1);
+    Op = N->getOperand(0);
+  } else
+    return SDValue();
+
+  MulValue = ExtractOp.getOperand(0);
+  ExtractIndexValue = ExtractOp.getOperand(1);
+
+  if (!ExtractOp.hasOneUse() || !MulValue.hasOneUse())
+    return SDValue();
+
+  // If the Opcode is NOT MUL, then that is NOT the expected pattern:
+  if (MulValue.getOpcode() != AArch64ISD::MUL_PRED)
+    return SDValue();
+
+  // If the Mul value type is NOT scalable vector, then that is NOT the expected
+  // pattern:
+  EVT VT = MulValue.getValueType();
+  if (!VT.isScalableVector())
+    return SDValue();
+
+  // If the ConstValue is NOT 0, then that is NOT the expected pattern:
+  if (!cast<ConstantSDNode>(ExtractIndexValue)->isZero())
+    return SDValue();
+
+  SDValue ScaledOp = convertToScalableVector(DAG, VT, Op);
+  SDValue NewValue = DAG.getNode(N->getOpcode(), SDLoc(N), VT, {ScaledOp, MulValue});
+  return convertFromScalableVector(DAG, N->getValueType(0), NewValue);
+}
+
 static SDValue performAddSubCombine(SDNode *N,
                                     TargetLowering::DAGCombinerInfo &DCI,
                                     SelectionDAG &DAG) {
+  if (SDValue Val = performMulAddSubCombine(N, DAG))
+    return Val;
   // Try to change sum of two reductions.
   if (SDValue Val = performAddUADDVCombine(N, DAG))
     return Val;
   if (SDValue Val = performAddDotCombine(N, DAG))
     return Val;
   if (SDValue Val = performAddCSelIntoCSinc(N, DAG))
     return Val;
   if (SDValue Val = performNegCSelCombine(N, DAG))

llvm/test/CodeGen/AArch64/aarch64-combine-add-sub-mul.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+sve | FileCheck %s
+
+define <2 x i64> @test_mul_add_2x64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_mul_add_2x64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    mla z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    ret
+  %mul = mul <2 x i64> %b, %c
+  %add = add <2 x i64> %a, %mul
+  ret <2 x i64> %add
+}
+
+define <1 x i64> @test_mul_add_1x64(<1 x i64> %a, <1 x i64> %b, <1 x i64> %c) {
+; CHECK-LABEL: test_mul_add_1x64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $z2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    mla z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    ret
+  %mul = mul <1 x i64> %b, %c
+  %add = add <1 x i64> %mul, %a
+  ret <1 x i64> %add
+}
+
+define <2 x i64> @test_mul_sub_2x64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: test_mul_sub_2x64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    mls z0.d, p0/m, z1.d, z2.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    ret
+  %mul = mul <2 x i64> %b, %c
+  %sub = sub <2 x i64> %a, %mul
+  ret <2 x i64> %sub
+}
+
+define <1 x i64> @test_mul_sub_1x64(<1 x i64> %a, <1 x i64> %b, <1 x i64> %c) {
+; CHECK-LABEL: test_mul_sub_1x64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $z2
+; CHECK-NEXT:    mul z1.d, p0/m, z1.d, z2.d
+; CHECK-NEXT:    sub d0, d1, d0
+; CHECK-NEXT:    ret
+  %mul = mul <1 x i64> %b, %c
+  %sub = sub <1 x i64> %mul, %a
+  ret <1 x i64> %sub
+}

llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll

 ; VBITS_GE_128-NEXT:    mls v2.4s, v5.4s, v7.4s
 ; VBITS_GE_128-NEXT:    mls v3.4s, v4.4s, v6.4s
 ; VBITS_GE_128-NEXT:    stp q0, q1, [x0, #32]
 ; VBITS_GE_128-NEXT:    stp q2, q3, [x0]
 ; VBITS_GE_128-NEXT:    ret
 ;
 ; VBITS_GE_256-LABEL: srem_v16i32:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z3.s }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    movprfx z4, z0
 ; VBITS_GE_256-NEXT:    sdiv z4.s, p0/m, z4.s, z2.s
 ; VBITS_GE_256-NEXT:    movprfx z5, z1
; CHECK-NEXT:    ret
   ret void
 }
 
 ; Vector i64 sdiv are not legal for NEON so use SVE when available.
 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
 define <1 x i64> @srem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: srem_v1i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    ptrue p0.d, vl1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
-; CHECK-NEXT:    mul z1.d, p0/m, z1.d, z2.d
-; CHECK-NEXT:    sub d0, d0, d1
+; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = srem <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
 
 ; Vector i64 sdiv are not legal for NEON so use SVE when available.
 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
 define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: srem_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
-; CHECK-NEXT:    mul z1.d, p0/m, z1.d, z2.d
-; CHECK-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = srem <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
 
 define void @srem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-LABEL: srem_v4i64:
 ; CHECK:       // %bb.0:
; CHECK-NEXT:    ret
   %res = srem <4 x i64> %op1, %op2
   store <4 x i64> %res, ptr %a
   ret void
 }
 
 define void @srem_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_128-LABEL: srem_v8i64:
 ; VBITS_GE_128:       // %bb.0:
-; VBITS_GE_128-NEXT:    ldp q4, q5, [x1]
-; VBITS_GE_128-NEXT:    ptrue p0.d, vl2
-; VBITS_GE_128-NEXT:    ldp q7, q6, [x1, #32]
 ; VBITS_GE_128-NEXT:    ldp q0, q1, [x0, #32]
-; VBITS_GE_128-NEXT:    ldp q2, q3, [x0]
-; VBITS_GE_128-NEXT:    movprfx z16, z3
-; VBITS_GE_128-NEXT:    sdiv z16.d, p0/m, z16.d, z5.d
-; VBITS_GE_128-NEXT:    movprfx z17, z2
-; VBITS_GE_128-NEXT:    sdiv z17.d, p0/m, z17.d, z4.d
-; VBITS_GE_128-NEXT:    mul z5.d, p0/m, z5.d, z16.d
+; VBITS_GE_128-NEXT:    ptrue p0.d, vl2
+; VBITS_GE_128-NEXT:    ldp q2, q3, [x1, #32]
 ; VBITS_GE_128-NEXT:    movprfx z16, z1
+; VBITS_GE_128-NEXT:    sdiv z16.d, p0/m, z16.d, z3.d
+; VBITS_GE_128-NEXT:    mls z1.d, p0/m, z16.d, z3.d
+; VBITS_GE_128-NEXT:    movprfx z3, z0
+; VBITS_GE_128-NEXT:    sdiv z3.d, p0/m, z3.d, z2.d
+; VBITS_GE_128-NEXT:    mls z0.d, p0/m, z3.d, z2.d
+; VBITS_GE_128-NEXT:    ldp q4, q5, [x0]
+; VBITS_GE_128-NEXT:    ldp q7, q6, [x1]
+; VBITS_GE_128-NEXT:    movprfx z16, z5
 ; VBITS_GE_128-NEXT:    sdiv z16.d, p0/m, z16.d, z6.d
-; VBITS_GE_128-NEXT:    mul z4.d, p0/m, z4.d, z17.d
-; VBITS_GE_128-NEXT:    movprfx z17, z0
-; VBITS_GE_128-NEXT:    sdiv z17.d, p0/m, z17.d, z7.d
-; VBITS_GE_128-NEXT:    mul z6.d, p0/m, z6.d, z16.d
-; VBITS_GE_128-NEXT:    mul z7.d, p0/m, z7.d, z17.d
-; VBITS_GE_128-NEXT:    sub v0.2d, v0.2d, v7.2d
-; VBITS_GE_128-NEXT:    sub v1.2d, v1.2d, v6.2d
-; VBITS_GE_128-NEXT:    sub v2.2d, v2.2d, v4.2d
+; VBITS_GE_128-NEXT:    movprfx z2, z4
+; VBITS_GE_128-NEXT:    sdiv z2.d, p0/m, z2.d, z7.d
 ; VBITS_GE_128-NEXT:    stp q0, q1, [x0, #32]
-; VBITS_GE_128-NEXT:    sub v0.2d, v3.2d, v5.2d
-; VBITS_GE_128-NEXT:    stp q2, q0, [x0]
+; VBITS_GE_128-NEXT:    movprfx z0, z4
+; VBITS_GE_128-NEXT:    mls z0.d, p0/m, z2.d, z7.d
+; VBITS_GE_128-NEXT:    movprfx z1, z5
+; VBITS_GE_128-NEXT:    mls z1.d, p0/m, z16.d, z6.d
+; VBITS_GE_128-NEXT:    stp q0, q1, [x0]
 ; VBITS_GE_128-NEXT:    ret
 ;
 ; VBITS_GE_256-LABEL: srem_v8i64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z3.d }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    movprfx z4, z0
 ; VBITS_GE_256-NEXT:    sdiv z4.d, p0/m, z4.d, z2.d
 ; VBITS_GE_256-NEXT:    movprfx z5, z1

 ; VBITS_GE_128-NEXT:    mls v2.4s, v5.4s, v7.4s
 ; VBITS_GE_128-NEXT:    mls v3.4s, v4.4s, v6.4s
 ; VBITS_GE_128-NEXT:    stp q0, q1, [x0, #32]
 ; VBITS_GE_128-NEXT:    stp q2, q3, [x0]
 ; VBITS_GE_128-NEXT:    ret
 ;
 ; VBITS_GE_256-LABEL: urem_v16i32:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z3.s }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    movprfx z4, z0
 ; VBITS_GE_256-NEXT:    udiv z4.s, p0/m, z4.s, z2.s
 ; VBITS_GE_256-NEXT:    movprfx z5, z1
; CHECK-NEXT:    ret
   ret void
 }
 
 ; Vector i64 udiv are not legal for NEON so use SVE when available.
 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
 define <1 x i64> @urem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: urem_v1i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
-; CHECK-NEXT:    ptrue p0.d, vl1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
-; CHECK-NEXT:    mul z1.d, p0/m, z1.d, z2.d
-; CHECK-NEXT:    sub d0, d0, d1
+; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = urem <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
 
 ; Vector i64 udiv are not legal for NEON so use SVE when available.
 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
 define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: urem_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
-; CHECK-NEXT:    mul z1.d, p0/m, z1.d, z2.d
-; CHECK-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = urem <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
 
 define void @urem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-LABEL: urem_v4i64:
 ; CHECK:       // %bb.0:
; CHECK-NEXT:    ret
   %res = urem <4 x i64> %op1, %op2
   store <4 x i64> %res, ptr %a
   ret void
 }
 
 define void @urem_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_128-LABEL: urem_v8i64:
 ; VBITS_GE_128:       // %bb.0:
-; VBITS_GE_128-NEXT:    ldp q4, q5, [x1]
-; VBITS_GE_128-NEXT:    ptrue p0.d, vl2
-; VBITS_GE_128-NEXT:    ldp q7, q6, [x1, #32]
 ; VBITS_GE_128-NEXT:    ldp q0, q1, [x0, #32]
-; VBITS_GE_128-NEXT:    ldp q2, q3, [x0]
-; VBITS_GE_128-NEXT:    movprfx z16, z3
-; VBITS_GE_128-NEXT:    udiv z16.d, p0/m, z16.d, z5.d
-; VBITS_GE_128-NEXT:    movprfx z17, z2
-; VBITS_GE_128-NEXT:    udiv z17.d, p0/m, z17.d, z4.d
-; VBITS_GE_128-NEXT:    mul z5.d, p0/m, z5.d, z16.d
+; VBITS_GE_128-NEXT:    ptrue p0.d, vl2
+; VBITS_GE_128-NEXT:    ldp q2, q3, [x1, #32]
 ; VBITS_GE_128-NEXT:    movprfx z16, z1
+; VBITS_GE_128-NEXT:    udiv z16.d, p0/m, z16.d, z3.d
+; VBITS_GE_128-NEXT:    mls z1.d, p0/m, z16.d, z3.d
+; VBITS_GE_128-NEXT:    movprfx z3, z0
+; VBITS_GE_128-NEXT:    udiv z3.d, p0/m, z3.d, z2.d
+; VBITS_GE_128-NEXT:    mls z0.d, p0/m, z3.d, z2.d
+; VBITS_GE_128-NEXT:    ldp q4, q5, [x0]
+; VBITS_GE_128-NEXT:    ldp q7, q6, [x1]
+; VBITS_GE_128-NEXT:    movprfx z16, z5
 ; VBITS_GE_128-NEXT:    udiv z16.d, p0/m, z16.d, z6.d
-; VBITS_GE_128-NEXT:    mul z4.d, p0/m, z4.d, z17.d
-; VBITS_GE_128-NEXT:    movprfx z17, z0
-; VBITS_GE_128-NEXT:    udiv z17.d, p0/m, z17.d, z7.d
-; VBITS_GE_128-NEXT:    mul z6.d, p0/m, z6.d, z16.d
-; VBITS_GE_128-NEXT:    mul z7.d, p0/m, z7.d, z17.d
-; VBITS_GE_128-NEXT:    sub v0.2d, v0.2d, v7.2d
-; VBITS_GE_128-NEXT:    sub v1.2d, v1.2d, v6.2d
-; VBITS_GE_128-NEXT:    sub v2.2d, v2.2d, v4.2d
+; VBITS_GE_128-NEXT:    movprfx z2, z4
+; VBITS_GE_128-NEXT:    udiv z2.d, p0/m, z2.d, z7.d
 ; VBITS_GE_128-NEXT:    stp q0, q1, [x0, #32]
-; VBITS_GE_128-NEXT:    sub v0.2d, v3.2d, v5.2d
-; VBITS_GE_128-NEXT:    stp q2, q0, [x0]
+; VBITS_GE_128-NEXT:    movprfx z0, z4
+; VBITS_GE_128-NEXT:    mls z0.d, p0/m, z2.d, z7.d
+; VBITS_GE_128-NEXT:    movprfx z1, z5
+; VBITS_GE_128-NEXT:    mls z1.d, p0/m, z16.d, z6.d
+; VBITS_GE_128-NEXT:    stp q0, q1, [x0]
 ; VBITS_GE_128-NEXT:    ret
 ;
 ; VBITS_GE_256-LABEL: urem_v8i64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z3.d }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    movprfx z4, z0
 ; VBITS_GE_256-NEXT:    udiv z4.d, p0/m, z4.d, z2.d
 ; VBITS_GE_256-NEXT:    movprfx z5, z1