[CodeGen] Add pre-commit tests for D152022

[igor.kirillov]

Update tests comments

[igor.kirillov]

Add reduction tests with predication

[NickGuy]

Test pre-commits probably don't need to go through review, just remember to squish them back into the main commit before they land

llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64-arm-none-eabi"
+
+%"class.std::complex" = type { { double, double } }
+
+; Zero initialized reduction. The IR is generated with predicated tail folding (-prefer-predicate-over-epilogue=predicate-dont-vectorize)
+;
+;   complex<double> x = 0.0 + 0.0i;
+;   for (int i = 0; i < 100; ++i)
+;       x += a[i] * b[i];
+;
+define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
+; CHECK-LABEL: complex_mul_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w9, #100 // =0x64
+; CHECK-NEXT:    cntd x10
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    mov x11, x10
+; CHECK-NEXT:    mov z1.d, #0 // =0x0
+; CHECK-NEXT:    rdvl x12, #2
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    whilelo p1.d, xzr, x9
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:  .LBB0_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x13, x0, x8
+; CHECK-NEXT:    add x14, x1, x8
+; CHECK-NEXT:    zip1 p2.d, p1.d, p1.d
+; CHECK-NEXT:    zip2 p3.d, p1.d, p1.d
+; CHECK-NEXT:    add x8, x8, x12
+; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
+; CHECK-NEXT:    uzp2 z6.d, z3.d, z2.d
+; CHECK-NEXT:    uzp1 z2.d, z3.d, z2.d
+; CHECK-NEXT:    uzp1 z7.d, z5.d, z4.d
+; CHECK-NEXT:    uzp2 z4.d, z5.d, z4.d
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    fmla z3.d, p0/m, z7.d, z6.d
+; CHECK-NEXT:    fmad z7.d, p0/m, z2.d, z0.d
+; CHECK-NEXT:    fmad z2.d, p0/m, z4.d, z3.d
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    fmls z3.d, p0/m, z4.d, z6.d
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    mov z0.d, p1/m, z3.d
+; CHECK-NEXT:    whilelo p1.d, x11, x9
+; CHECK-NEXT:    add x11, x11, x10
+; CHECK-NEXT:    b.mi .LBB0_1
+; CHECK-NEXT:  // %bb.2: // %exit.block
+; CHECK-NEXT:    faddv d0, p0, z0.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT:    ret
+entry:
+  %active.lane.mask.entry = tail call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 100)
+  %0 = tail call i64 @llvm.vscale.i64()
+  %1 = shl i64 %0, 1
+  %2 = shl nuw nsw i64 %0, 5
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv35 = phi i64 [ %lsr.iv.next36, %vector.body ], [ %1, %entry ]
+  %lsr.iv = phi i64 [ %lsr.iv.next, %vector.body ], [ 0, %entry ]
+  %active.lane.mask = phi <vscale x 2 x i1> [ %active.lane.mask.entry, %entry ], [ %active.lane.mask.next, %vector.body ]
+  %vec.phi = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %15, %vector.body ]
+  %vec.phi27 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %16, %vector.body ]
+  %scevgep = getelementptr i8, ptr %a, i64 %lsr.iv
+  %scevgep34 = getelementptr i8, ptr %b, i64 %lsr.iv
+  %interleaved.mask = tail call <vscale x 4 x i1> @llvm.experimental.vector.interleave2.nxv4i1(<vscale x 2 x i1> %active.lane.mask, <vscale x 2 x i1> %active.lane.mask)
+  %wide.masked.vec = tail call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr %scevgep, i32 8, <vscale x 4 x i1> %interleaved.mask, <vscale x 4 x double> poison)
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.masked.vec)
+  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %interleaved.mask28 = tail call <vscale x 4 x i1> @llvm.experimental.vector.interleave2.nxv4i1(<vscale x 2 x i1> %active.lane.mask, <vscale x 2 x i1> %active.lane.mask)
+  %wide.masked.vec29 = tail call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr %scevgep34, i32 8, <vscale x 4 x i1> %interleaved.mask28, <vscale x 4 x double> poison)
+  %strided.vec30 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.masked.vec29)
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec30, 0
+  %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec30, 1
+  %7 = fmul fast <vscale x 2 x double> %6, %3
+  %8 = fmul fast <vscale x 2 x double> %5, %4
+  %9 = fmul fast <vscale x 2 x double> %5, %3
+  %10 = fadd fast <vscale x 2 x double> %9, %vec.phi27
+  %11 = fmul fast <vscale x 2 x double> %6, %4
+  %12 = fsub fast <vscale x 2 x double> %10, %11
+  %13 = fadd fast <vscale x 2 x double> %8, %vec.phi
+  %14 = fadd fast <vscale x 2 x double> %13, %7
+  %15 = select fast <vscale x 2 x i1> %active.lane.mask, <vscale x 2 x double> %14, <vscale x 2 x double> %vec.phi
+  %16 = select fast <vscale x 2 x i1> %active.lane.mask, <vscale x 2 x double> %12, <vscale x 2 x double> %vec.phi27
+  %active.lane.mask.next = tail call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 %lsr.iv35, i64 100)
+  %17 = extractelement <vscale x 2 x i1> %active.lane.mask.next, i64 0
+  %lsr.iv.next = add i64 %lsr.iv, %2
+  %lsr.iv.next36 = add i64 %lsr.iv35, %1
+  br i1 %17, label %vector.body, label %exit.block
+
+exit.block:                                     ; preds = %vector.body
+  %18 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %16)
+  %19 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %15)
+  %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %18, 0, 0
+  %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %19, 0, 1
+  ret %"class.std::complex" %.fca.0.1.insert
+}
+
+; Zero initialized reduction with conditional block. The IR is generated with scalar tail folding (-prefer-predicate-over-epilogue=scalar-epilogue)
+;
+;   complex<double> x = 0.0 + 0.0i;
+;   for (int i = 0; i < 100; ++i)
+;       if (cond[i])
+;           x += a[i] * b[i];
+;
+define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr %cond) {
+; CHECK-LABEL: complex_mul_predicated_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntd x10
+; CHECK-NEXT:    mov w12, #100 // =0x64
+; CHECK-NEXT:    neg x11, x10
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    mov x9, xzr
+; CHECK-NEXT:    and x11, x11, x12
+; CHECK-NEXT:    mov z1.d, #0 // =0x0
+; CHECK-NEXT:    rdvl x12, #2
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:  .LBB1_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ld1w { z2.d }, p0/z, [x2, x9, lsl #2]
+; CHECK-NEXT:    add x13, x0, x8
+; CHECK-NEXT:    add x14, x1, x8
+; CHECK-NEXT:    add x9, x9, x10
+; CHECK-NEXT:    add x8, x8, x12
+; CHECK-NEXT:    cmpne p1.d, p0/z, z2.d, #0
+; CHECK-NEXT:    zip1 p2.d, p1.d, p1.d
+; CHECK-NEXT:    zip2 p3.d, p1.d, p1.d
+; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
+; CHECK-NEXT:    cmp x11, x9
+; CHECK-NEXT:    uzp2 z6.d, z3.d, z2.d
+; CHECK-NEXT:    uzp1 z2.d, z3.d, z2.d
+; CHECK-NEXT:    uzp1 z3.d, z5.d, z4.d
+; CHECK-NEXT:    movprfx z7, z0
+; CHECK-NEXT:    fmla z7.d, p0/m, z3.d, z2.d
+; CHECK-NEXT:    fmad z3.d, p0/m, z6.d, z1.d
+; CHECK-NEXT:    uzp2 z4.d, z5.d, z4.d
+; CHECK-NEXT:    fmad z2.d, p0/m, z4.d, z3.d
+; CHECK-NEXT:    movprfx z5, z7
+; CHECK-NEXT:    fmls z5.d, p0/m, z4.d, z6.d
+; CHECK-NEXT:    mov z0.d, p1/m, z5.d
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    b.ne .LBB1_1
+; CHECK-NEXT:  // %bb.2: // %exit.block
+; CHECK-NEXT:    faddv d0, p0, z0.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.vscale.i64()
+  %1 = shl nuw nsw i64 %0, 1
+  %n.mod.vf = urem i64 100, %1
+  %n.vec = sub i64 100, %n.mod.vf
+  %2 = shl nuw nsw i64 %0, 5
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv48 = phi i64 [ %lsr.iv.next, %vector.body ], [ 0, %entry ]
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %predphi34, %vector.body ]
+  %vec.phi30 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %predphi, %vector.body ]
+  %3 = shl i64 %index, 2
+  %scevgep47 = getelementptr i8, ptr %cond, i64 %3
+  %wide.load = load <vscale x 2 x i32>, ptr %scevgep47, align 4
+  %4 = icmp ne <vscale x 2 x i32> %wide.load, zeroinitializer
+  %scevgep49 = getelementptr i8, ptr %a, i64 %lsr.iv48
+  %scevgep50 = getelementptr i8, ptr %b, i64 %lsr.iv48
+  %interleaved.mask = tail call <vscale x 4 x i1> @llvm.experimental.vector.interleave2.nxv4i1(<vscale x 2 x i1> %4, <vscale x 2 x i1> %4)
+  %wide.masked.vec = tail call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr %scevgep49, i32 8, <vscale x 4 x i1> %interleaved.mask, <vscale x 4 x double> poison)
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.masked.vec)
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %wide.masked.vec32 = tail call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr %scevgep50, i32 8, <vscale x 4 x i1> %interleaved.mask, <vscale x 4 x double> poison)
+  %strided.vec33 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.masked.vec32)
+  %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec33, 0
+  %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec33, 1
+  %9 = fmul fast <vscale x 2 x double> %8, %5
+  %10 = fmul fast <vscale x 2 x double> %7, %6
+  %11 = fmul fast <vscale x 2 x double> %7, %5
+  %12 = fadd fast <vscale x 2 x double> %11, %vec.phi30
+  %13 = fmul fast <vscale x 2 x double> %8, %6
+  %14 = fsub fast <vscale x 2 x double> %12, %13
+  %15 = fadd fast <vscale x 2 x double> %10, %vec.phi
+  %16 = fadd fast <vscale x 2 x double> %15, %9
+  %predphi = select <vscale x 2 x i1> %4, <vscale x 2 x double> %14, <vscale x 2 x double> %vec.phi30
+  %predphi34 = select <vscale x 2 x i1> %4, <vscale x 2 x double> %16, <vscale x 2 x double> %vec.phi
+  %index.next = add nuw i64 %index, %1
+  %lsr.iv.next = add i64 %lsr.iv48, %2
+  %17 = icmp eq i64 %n.vec, %index.next
+  br i1 %17, label %exit.block, label %vector.body
+
+exit.block:                                     ; preds = %vector.body
+  %18 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %predphi)
+  %19 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %predphi34)
+  %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %18, 0, 0
+  %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %19, 0, 1
+  ret %"class.std::complex" %.fca.0.1.insert
+}
+
+; Zero initialized reduction with conditional block. The IR is generated with scalar tail folding (-predicate-over-epilogue=predicate-dont-vectorize)
+;
+;   complex<double> x = 0.0 + 0.0i;
+;   for (int i = 0; i < 100; ++i)
+;       if (cond[i])
+;           x += a[i] * b[i];
+;
+define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, ptr %cond) {
+; CHECK-LABEL: complex_mul_predicated_x2_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w10, #100 // =0x64
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    mov x9, xzr
+; CHECK-NEXT:    mov z1.d, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    cntd x11
+; CHECK-NEXT:    whilelo p1.d, xzr, x10
+; CHECK-NEXT:    rdvl x12, #2
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:  .LBB2_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ld1w { z2.d }, p1/z, [x2, x9, lsl #2]
+; CHECK-NEXT:    add x13, x0, x8
+; CHECK-NEXT:    add x14, x1, x8
+; CHECK-NEXT:    add x9, x9, x11
+; CHECK-NEXT:    add x8, x8, x12
+; CHECK-NEXT:    cmpne p2.d, p1/z, z2.d, #0
+; CHECK-NEXT:    zip1 p1.d, p2.d, p2.d
+; CHECK-NEXT:    zip2 p3.d, p2.d, p2.d
+; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p1/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p1/z, [x14]
+; CHECK-NEXT:    whilelo p1.d, x9, x10
+; CHECK-NEXT:    uzp1 z6.d, z3.d, z2.d
+; CHECK-NEXT:    uzp2 z2.d, z3.d, z2.d
+; CHECK-NEXT:    uzp1 z7.d, z5.d, z4.d
+; CHECK-NEXT:    uzp2 z4.d, z5.d, z4.d
+; CHECK-NEXT:    movprfx z3, z0
+; CHECK-NEXT:    fmla z3.d, p0/m, z7.d, z6.d
+; CHECK-NEXT:    fmad z7.d, p0/m, z2.d, z1.d
+; CHECK-NEXT:    fmsb z2.d, p0/m, z4.d, z3.d
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    fmla z3.d, p0/m, z4.d, z6.d
+; CHECK-NEXT:    mov z1.d, p2/m, z3.d
+; CHECK-NEXT:    mov z0.d, p2/m, z2.d
+; CHECK-NEXT:    b.mi .LBB2_1
+; CHECK-NEXT:  // %bb.2: // %exit.block
+; CHECK-NEXT:    faddv d0, p0, z0.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT:    ret
+entry:
+  %active.lane.mask.entry = tail call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 100)
+  %0 = tail call i64 @llvm.vscale.i64()
+  %1 = shl i64 %0, 1
+  %2 = shl nuw nsw i64 %0, 5
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv = phi i64 [ %lsr.iv.next, %vector.body ], [ 0, %entry ]
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %active.lane.mask = phi <vscale x 2 x i1> [ %active.lane.mask.entry, %entry ], [ %active.lane.mask.next, %vector.body ]
+  %vec.phi = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %19, %vector.body ]
+  %vec.phi30 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %21, %vector.body ]
+  %3 = shl i64 %index, 2
+  %scevgep = getelementptr i8, ptr %cond, i64 %3
+  %wide.masked.load = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr %scevgep, i32 4, <vscale x 2 x i1> %active.lane.mask, <vscale x 2 x i32> poison)
+  %4 = icmp ne <vscale x 2 x i32> %wide.masked.load, zeroinitializer
+  %scevgep38 = getelementptr i8, ptr %a, i64 %lsr.iv
+  %scevgep39 = getelementptr i8, ptr %b, i64 %lsr.iv
+  %5 = select <vscale x 2 x i1> %active.lane.mask, <vscale x 2 x i1> %4, <vscale x 2 x i1> zeroinitializer
+  %interleaved.mask = tail call <vscale x 4 x i1> @llvm.experimental.vector.interleave2.nxv4i1(<vscale x 2 x i1> %5, <vscale x 2 x i1> %5)
+  %wide.masked.vec = tail call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr %scevgep38, i32 8, <vscale x 4 x i1> %interleaved.mask, <vscale x 4 x double> poison)
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.masked.vec)
+  %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %interleaved.mask31 = tail call <vscale x 4 x i1> @llvm.experimental.vector.interleave2.nxv4i1(<vscale x 2 x i1> %5, <vscale x 2 x i1> %5)
+  %wide.masked.vec32 = tail call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr %scevgep39, i32 8, <vscale x 4 x i1> %interleaved.mask31, <vscale x 4 x double> poison)
+  %strided.vec33 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.masked.vec32)
+  %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec33, 0
+  %9 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec33, 1
+  %10 = fmul fast <vscale x 2 x double> %9, %6
+  %11 = fmul fast <vscale x 2 x double> %8, %7
+  %12 = fmul fast <vscale x 2 x double> %8, %6
+  %13 = fadd fast <vscale x 2 x double> %12, %vec.phi30
+  %14 = fmul fast <vscale x 2 x double> %9, %7
+  %15 = fsub fast <vscale x 2 x double> %13, %14
+  %16 = fadd fast <vscale x 2 x double> %11, %vec.phi
+  %17 = fadd fast <vscale x 2 x double> %16, %10
+  %18 = select <vscale x 2 x i1> %active.lane.mask, <vscale x 2 x i1> %4, <vscale x 2 x i1> zeroinitializer
+  %19 = select fast <vscale x 2 x i1> %18, <vscale x 2 x double> %17, <vscale x 2 x double> %vec.phi
+  %20 = select <vscale x 2 x i1> %active.lane.mask, <vscale x 2 x i1> %4, <vscale x 2 x i1> zeroinitializer
+  %21 = select fast <vscale x 2 x i1> %20, <vscale x 2 x double> %15, <vscale x 2 x double> %vec.phi30
+  %index.next = add i64 %index, %1
+  %22 = add i64 %1, %index
+  %active.lane.mask.next = tail call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 %22, i64 100)
+  %23 = extractelement <vscale x 2 x i1> %active.lane.mask.next, i64 0
+  %lsr.iv.next = add i64 %lsr.iv, %2
+  br i1 %23, label %vector.body, label %exit.block
+
+exit.block:                                     ; preds = %vector.body
+  %24 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %21)
+  %25 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %19)
+  %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %24, 0, 0
+  %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %25, 0, 1
+  ret %"class.std::complex" %.fca.0.1.insert
+}
+
+declare i64 @llvm.vscale.i64()
+declare <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64, i64)
+declare <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr nocapture, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr nocapture, i32 immarg, <vscale x 4 x i1>, <vscale x 4 x double>)
+declare <vscale x 4 x i1> @llvm.experimental.vector.interleave2.nxv4i1(<vscale x 2 x i1>, <vscale x 2 x i1>)
+declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double>)
+declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)

llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64-arm-none-eabi"
+
+%"class.std::complex" = type { { double, double } }
+
+; Zero initialized reduction
+;
+;   complex<double> x = 0.0 + 0.0i;
+;   for (int i = 0; i < 100; ++i)
+;       x += a[i] * b[i];
+;
+define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
+; CHECK-LABEL: complex_mul_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    mov w11, #100 // =0x64
+; CHECK-NEXT:    neg x10, x9
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    and x10, x10, x11
+; CHECK-NEXT:    mov z1.d, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    rdvl x11, #2
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:  .LBB0_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x12, x0, x8
+; CHECK-NEXT:    add x13, x1, x8
+; CHECK-NEXT:    ld1b { z2.b }, p1/z, [x0, x8]
+; CHECK-NEXT:    subs x10, x10, x9
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x12, #1, mul vl]
+; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x1, x8]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x13, #1, mul vl]
+; CHECK-NEXT:    add x8, x8, x11
+; CHECK-NEXT:    uzp2 z6.d, z2.d, z3.d
+; CHECK-NEXT:    uzp1 z2.d, z2.d, z3.d
+; CHECK-NEXT:    uzp1 z3.d, z4.d, z5.d
+; CHECK-NEXT:    fmla z0.d, p0/m, z3.d, z2.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z3.d, z6.d
+; CHECK-NEXT:    uzp2 z3.d, z4.d, z5.d
+; CHECK-NEXT:    fmls z0.d, p0/m, z3.d, z6.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z3.d, z2.d
+; CHECK-NEXT:    b.ne .LBB0_1
+; CHECK-NEXT:  // %bb.2: // %exit.block
+; CHECK-NEXT:    faddv d0, p0, z0.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.vscale.i64()
+  %1 = shl nuw nsw i64 %0, 1
+  %n.mod.vf = urem i64 100, %1
+  %n.vec = sub nuw nsw i64 100, %n.mod.vf
+  %2 = shl nuw nsw i64 %0, 5
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv31 = phi i64 [ %lsr.iv.next32, %vector.body ], [ %n.vec, %entry ]
+  %lsr.iv27 = phi i64 [ %lsr.iv.next28, %vector.body ], [ 0, %entry ]
+  %vec.phi = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %16, %vector.body ]
+  %vec.phi12 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %14, %vector.body ]
+  %scevgep46 = getelementptr i8, ptr %a, i64 %lsr.iv27
+  %scevgep47 = getelementptr i8, ptr %b, i64 %lsr.iv27
+  %wide.vec = load <vscale x 4 x double>, ptr %scevgep46, align 8
+  %3 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec)
+  %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 0
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 1
+  %wide.vec30 = load <vscale x 4 x double>, ptr %scevgep47, align 8
+  %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec30)
+  %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 0
+  %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 1
+  %9 = fmul fast <vscale x 2 x double> %8, %4
+  %10 = fmul fast <vscale x 2 x double> %7, %5
+  %11 = fmul fast <vscale x 2 x double> %7, %4
+  %12 = fadd fast <vscale x 2 x double> %11, %vec.phi12
+  %13 = fmul fast <vscale x 2 x double> %8, %5
+  %14 = fsub fast <vscale x 2 x double> %12, %13
+  %15 = fadd fast <vscale x 2 x double> %10, %vec.phi
+  %16 = fadd fast <vscale x 2 x double> %15, %9
+  %lsr.iv.next28 = add i64 %lsr.iv27, %2
+  %lsr.iv.next32 = sub i64 %lsr.iv31, %1
+  %17 = icmp eq i64 %lsr.iv.next32, 0
+  br i1 %17, label %exit.block, label %vector.body
+
+exit.block:                                     ; preds = %vector.body
+  %18 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %14)
+  %19 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %16)
+  %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %18, 0, 0
+  %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %19, 0, 1
+  ret %"class.std::complex" %.fca.0.1.insert
+}
+
+; Fixed value initialized reduction
+;
+;   complex<double> x = 2.0 + 1.0i;
+;   for (int i = 0; i < 100; ++i)
+;       x += a[i] * b[i];
+;
+define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
+; CHECK-LABEL: complex_mul_nonzero_init_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    fmov d0, #2.00000000
+; CHECK-NEXT:    neg x10, x9
+; CHECK-NEXT:    mov w11, #100 // =0x64
+; CHECK-NEXT:    fmov d1, #1.00000000
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    and x10, x10, x11
+; CHECK-NEXT:    mov z2.d, #0 // =0x0
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    rdvl x11, #2
+; CHECK-NEXT:    sel z0.d, p0, z0.d, z2.d
+; CHECK-NEXT:    sel z1.d, p0, z1.d, z2.d
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:  .LBB1_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x12, x0, x8
+; CHECK-NEXT:    add x13, x1, x8
+; CHECK-NEXT:    ld1b { z2.b }, p1/z, [x0, x8]
+; CHECK-NEXT:    subs x10, x10, x9
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x12, #1, mul vl]
+; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x1, x8]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x13, #1, mul vl]
+; CHECK-NEXT:    add x8, x8, x11
+; CHECK-NEXT:    uzp2 z6.d, z2.d, z3.d
+; CHECK-NEXT:    uzp1 z2.d, z2.d, z3.d
+; CHECK-NEXT:    uzp1 z3.d, z4.d, z5.d
+; CHECK-NEXT:    fmla z0.d, p0/m, z3.d, z2.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z3.d, z6.d
+; CHECK-NEXT:    uzp2 z3.d, z4.d, z5.d
+; CHECK-NEXT:    fmls z0.d, p0/m, z3.d, z6.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z3.d, z2.d
+; CHECK-NEXT:    b.ne .LBB1_1
+; CHECK-NEXT:  // %bb.2: // %exit.block
+; CHECK-NEXT:    faddv d0, p0, z0.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.vscale.i64()
+  %1 = shl nuw nsw i64 %0, 1
+  %n.mod.vf = urem i64 100, %1
+  %n.vec = sub nuw nsw i64 100, %n.mod.vf
+  %2 = shl nuw nsw i64 %0, 5
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv31 = phi i64 [ %lsr.iv.next32, %vector.body ], [ %n.vec, %entry ]
+  %lsr.iv27 = phi i64 [ %lsr.iv.next28, %vector.body ], [ 0, %entry ]
+  %vec.phi = phi <vscale x 2 x double> [ insertelement (<vscale x 2 x double> zeroinitializer, double 1.000000e+00, i32 0), %entry ], [ %16, %vector.body ]
+  %vec.phi12 = phi <vscale x 2 x double> [ insertelement (<vscale x 2 x double> zeroinitializer, double 2.000000e+0, i32 0), %entry ], [ %14, %vector.body ]
+  %scevgep46 = getelementptr i8, ptr %a, i64 %lsr.iv27
+  %scevgep47 = getelementptr i8, ptr %b, i64 %lsr.iv27
+  %wide.vec = load <vscale x 4 x double>, ptr %scevgep46, align 8
+  %3 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec)
+  %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 0
+  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 1
+  %wide.vec30 = load <vscale x 4 x double>, ptr %scevgep47, align 8
+  %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec30)
+  %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 0
+  %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 1
+  %9 = fmul fast <vscale x 2 x double> %8, %4
+  %10 = fmul fast <vscale x 2 x double> %7, %5
+  %11 = fmul fast <vscale x 2 x double> %7, %4
+  %12 = fadd fast <vscale x 2 x double> %11, %vec.phi12
+  %13 = fmul fast <vscale x 2 x double> %8, %5
+  %14 = fsub fast <vscale x 2 x double> %12, %13
+  %15 = fadd fast <vscale x 2 x double> %10, %vec.phi
+  %16 = fadd fast <vscale x 2 x double> %15, %9
+  %lsr.iv.next28 = add i64 %lsr.iv27, %2
+  %lsr.iv.next32 = sub i64 %lsr.iv31, %1
+  %17 = icmp eq i64 %lsr.iv.next32, 0
+  br i1 %17, label %exit.block, label %vector.body
+
+exit.block:                                     ; preds = %vector.body
+  %18 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %14)
+  %19 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %16)
+  %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %18, 0, 0
+  %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %19, 0, 1
+  ret %"class.std::complex" %.fca.0.1.insert
+}
+
+; Loop unrolled with factor 2
+;
+define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
+; CHECK-LABEL: complex_mul_v2f64_unrolled:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntw x9
+; CHECK-NEXT:    mov w11, #1000 // =0x3e8
+; CHECK-NEXT:    neg x10, x9
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    and x10, x10, x11
+; CHECK-NEXT:    mov z0.d, #0 // =0x0
+; CHECK-NEXT:    rdvl x11, #4
+; CHECK-NEXT:    mov z1.d, z0.d
+; CHECK-NEXT:    mov z2.d, z0.d
+; CHECK-NEXT:    mov z3.d, z0.d
+; CHECK-NEXT:    addvl x12, x1, #2
+; CHECK-NEXT:    addvl x13, x0, #2
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:  .LBB2_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x14, x0, x8
+; CHECK-NEXT:    add x15, x13, x8
+; CHECK-NEXT:    add x16, x1, x8
+; CHECK-NEXT:    add x17, x12, x8
+; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x0, x8]
+; CHECK-NEXT:    subs x10, x10, x9
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x13, x8]
+; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x15, #1, mul vl]
+; CHECK-NEXT:    ld1b { z16.b }, p1/z, [x1, x8]
+; CHECK-NEXT:    ld1d { z17.d }, p0/z, [x16, #1, mul vl]
+; CHECK-NEXT:    ld1b { z18.b }, p1/z, [x12, x8]
+; CHECK-NEXT:    ld1d { z19.d }, p0/z, [x17, #1, mul vl]
+; CHECK-NEXT:    add x8, x8, x11
+; CHECK-NEXT:    uzp2 z20.d, z4.d, z5.d
+; CHECK-NEXT:    uzp1 z4.d, z4.d, z5.d
+; CHECK-NEXT:    uzp2 z5.d, z6.d, z7.d
+; CHECK-NEXT:    uzp1 z6.d, z6.d, z7.d
+; CHECK-NEXT:    uzp1 z7.d, z16.d, z17.d
+; CHECK-NEXT:    uzp1 z21.d, z18.d, z19.d
+; CHECK-NEXT:    fmla z2.d, p0/m, z7.d, z4.d
+; CHECK-NEXT:    fmla z3.d, p0/m, z21.d, z6.d
+; CHECK-NEXT:    fmla z0.d, p0/m, z7.d, z20.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z21.d, z5.d
+; CHECK-NEXT:    uzp2 z7.d, z16.d, z17.d
+; CHECK-NEXT:    uzp2 z16.d, z18.d, z19.d
+; CHECK-NEXT:    fmls z2.d, p0/m, z7.d, z20.d
+; CHECK-NEXT:    fmls z3.d, p0/m, z16.d, z5.d
+; CHECK-NEXT:    fmla z0.d, p0/m, z7.d, z4.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z16.d, z6.d
+; CHECK-NEXT:    b.ne .LBB2_1
+; CHECK-NEXT:  // %bb.2: // %exit.block
+; CHECK-NEXT:    fadd z2.d, z3.d, z2.d
+; CHECK-NEXT:    fadd z1.d, z1.d, z0.d
+; CHECK-NEXT:    faddv d0, p0, z2.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.vscale.i64()
+  %1 = shl nuw nsw i64 %0, 2
+  %n.mod.vf = urem i64 1000, %1
+  %n.vec = sub i64 1000, %n.mod.vf
+  %2 = shl nuw nsw i64 %0, 6
+  %3 = shl nuw nsw i64 %0, 5
+  %scevgep61 = getelementptr i8, ptr %b, i64 %3
+  %scevgep63 = getelementptr i8, ptr %a, i64 %3
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv38 = phi i64 [ %lsr.iv.next39, %vector.body ], [ %n.vec, %entry ]
+  %lsr.iv34 = phi i64 [ %lsr.iv.next35, %vector.body ], [ 0, %entry ]
+  %vec.phi = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %30, %vector.body ]
+  %vec.phi12 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %31, %vector.body ]
+  %vec.phi13 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %26, %vector.body ]
+  %vec.phi14 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %27, %vector.body ]
+  %scevgep57 = getelementptr i8, ptr %a, i64 %lsr.iv34
+  %scevgep64 = getelementptr i8, ptr %scevgep63, i64 %lsr.iv34
+  %scevgep58 = getelementptr i8, ptr %b, i64 %lsr.iv34
+  %scevgep62 = getelementptr i8, ptr %scevgep61, i64 %lsr.iv34
+  %wide.vec = load <vscale x 4 x double>, ptr %scevgep57, align 8
+  %wide.vec32 = load <vscale x 4 x double>, ptr %scevgep64, align 8
+  %4 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec)
+  %5 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec32)
+  %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 0
+  %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %5, 0
+  %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 1
+  %9 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %5, 1
+  %wide.vec34 = load <vscale x 4 x double>, ptr %scevgep58, align 8
+  %wide.vec35 = load <vscale x 4 x double>, ptr %scevgep62, align 8
+  %10 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec34)
+  %11 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec35)
+  %12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %10, 0
+  %13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %11, 0
+  %14 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %10, 1
+  %15 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %11, 1
+  %16 = fmul fast <vscale x 2 x double> %14, %6
+  %17 = fmul fast <vscale x 2 x double> %15, %7
+  %18 = fmul fast <vscale x 2 x double> %12, %8
+  %19 = fmul fast <vscale x 2 x double> %13, %9
+  %20 = fmul fast <vscale x 2 x double> %12, %6
+  %21 = fmul fast <vscale x 2 x double> %13, %7
+  %22 = fadd fast <vscale x 2 x double> %20, %vec.phi13
+  %23 = fadd fast <vscale x 2 x double> %21, %vec.phi14
+  %24 = fmul fast <vscale x 2 x double> %14, %8
+  %25 = fmul fast <vscale x 2 x double> %15, %9
+  %26 = fsub fast <vscale x 2 x double> %22, %24
+  %27 = fsub fast <vscale x 2 x double> %23, %25
+  %28 = fadd fast <vscale x 2 x double> %18, %vec.phi
+  %29 = fadd fast <vscale x 2 x double> %19, %vec.phi12
+  %30 = fadd fast <vscale x 2 x double> %28, %16
+  %31 = fadd fast <vscale x 2 x double> %29, %17
+  %lsr.iv.next35 = add i64 %lsr.iv34, %2
+  %lsr.iv.next39 = sub i64 %lsr.iv38, %1
+  %32 = icmp eq i64 %lsr.iv.next39, 0
+  br i1 %32, label %exit.block, label %vector.body
+
+exit.block:                                     ; preds = %vector.body
+  %bin.rdx15 = fadd fast <vscale x 2 x double> %27, %26
+  %33 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %bin.rdx15)
+  %bin.rdx = fadd fast <vscale x 2 x double> %31, %30
+  %34 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %bin.rdx)
+  %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %33, 0, 0
+  %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %34, 0, 1
+  ret %"class.std::complex" %.fca.0.1.insert
+}
+
+
+declare i64 @llvm.vscale.i64()
+declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double>)
+declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)

llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+complxnum,+neon -o - | FileCheck %s
+
+target triple = "aarch64-arm-none-eabi"
+
+%"struct.std::complex" = type { { double, double } }
+
+; Zero initialized reduction
+;
+;   complex<double> x = 0.0 + 0.0i;
+;   for (int i = 0; i < 100; ++i)
+;       x += a[i] * b[i];
+;
+define dso_local %"struct.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
+; CHECK-LABEL: complex_mul_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:  .LBB0_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x9, x0, x8
+; CHECK-NEXT:    ld2 { v2.2d, v3.2d }, [x9]
+; CHECK-NEXT:    add x9, x1, x8
+; CHECK-NEXT:    add x8, x8, #32
+; CHECK-NEXT:    cmp x8, #1600
+; CHECK-NEXT:    ld2 { v4.2d, v5.2d }, [x9]
+; CHECK-NEXT:    fmla v0.2d, v2.2d, v4.2d
+; CHECK-NEXT:    fmla v1.2d, v3.2d, v4.2d
+; CHECK-NEXT:    fmls v0.2d, v3.2d, v5.2d
+; CHECK-NEXT:    fmla v1.2d, v2.2d, v5.2d
+; CHECK-NEXT:    b.ne .LBB0_1
+; CHECK-NEXT:  // %bb.2: // %middle.block
+; CHECK-NEXT:    faddp d0, v0.2d
+; CHECK-NEXT:    faddp d1, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv = phi i64 [ %lsr.iv.next, %vector.body ], [ 0, %entry ]
+  %vec.phi = phi <2 x double> [ zeroinitializer, %entry ], [ %7, %vector.body ]
+  %vec.phi27 = phi <2 x double> [ zeroinitializer, %entry ], [ %5, %vector.body ]
+  %scevgep = getelementptr i8, ptr %a, i64 %lsr.iv
+  %scevgep35 = getelementptr i8, ptr %b, i64 %lsr.iv
+  %wide.vec = load <4 x double>, ptr %scevgep, align 8
+  %strided.vec = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec28 = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %wide.vec29 = load <4 x double>, ptr %scevgep35, align 8
+  %strided.vec30 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec31 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %0 = fmul fast <2 x double> %strided.vec31, %strided.vec
+  %1 = fmul fast <2 x double> %strided.vec30, %strided.vec28
+  %2 = fmul fast <2 x double> %strided.vec30, %strided.vec
+  %3 = fadd fast <2 x double> %2, %vec.phi27
+  %4 = fmul fast <2 x double> %strided.vec31, %strided.vec28
+  %5 = fsub fast <2 x double> %3, %4
+  %6 = fadd fast <2 x double> %1, %vec.phi
+  %7 = fadd fast <2 x double> %6, %0
+  %lsr.iv.next = add nuw nsw i64 %lsr.iv, 32
+  %8 = icmp eq i64 %lsr.iv.next, 1600
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %9 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %5)
+  %10 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %7)
+  %.fca.0.0.insert = insertvalue %"struct.std::complex" poison, double %9, 0, 0
+  %.fca.0.1.insert = insertvalue %"struct.std::complex" %.fca.0.0.insert, double %10, 0, 1
+  ret %"struct.std::complex" %.fca.0.1.insert
+}
+
+; Fixed value initialized reduction
+;
+;   complex<double> x = 2.0 + 1.0i;
+;   for (int i = 0; i < 100; ++i)
+;       x += a[i] * b[i];
+;
+define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
+; CHECK-LABEL: complex_mul_nonzero_init_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x9, .LCPI1_0
+; CHECK-NEXT:    adrp x10, .LCPI1_1
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    ldr q0, [x9, :lo12:.LCPI1_0]
+; CHECK-NEXT:    ldr q1, [x10, :lo12:.LCPI1_1]
+; CHECK-NEXT:  .LBB1_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x9, x0, x8
+; CHECK-NEXT:    ld2 { v2.2d, v3.2d }, [x9]
+; CHECK-NEXT:    add x9, x1, x8
+; CHECK-NEXT:    add x8, x8, #32
+; CHECK-NEXT:    cmp x8, #1600
+; CHECK-NEXT:    ld2 { v4.2d, v5.2d }, [x9]
+; CHECK-NEXT:    fmla v0.2d, v2.2d, v4.2d
+; CHECK-NEXT:    fmla v1.2d, v3.2d, v4.2d
+; CHECK-NEXT:    fmls v0.2d, v3.2d, v5.2d
+; CHECK-NEXT:    fmla v1.2d, v2.2d, v5.2d
+; CHECK-NEXT:    b.ne .LBB1_1
+; CHECK-NEXT:  // %bb.2: // %middle.block
+; CHECK-NEXT:    faddp d0, v0.2d
+; CHECK-NEXT:    faddp d1, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv = phi i64 [ %lsr.iv.next, %vector.body ], [ 0, %entry ]
+  %vec.phi = phi <2 x double> [ <double 1.000000e+00, double 0.000000e+00>, %entry ], [ %7, %vector.body ]
+  %vec.phi27 = phi <2 x double> [ <double 2.000000e+00, double 0.000000e+00>, %entry ], [ %5, %vector.body ]
+  %scevgep = getelementptr i8, ptr %a, i64 %lsr.iv
+  %scevgep35 = getelementptr i8, ptr %b, i64 %lsr.iv
+  %wide.vec = load <4 x double>, ptr %scevgep, align 8
+  %strided.vec = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec28 = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %wide.vec29 = load <4 x double>, ptr %scevgep35, align 8
+  %strided.vec30 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec31 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %0 = fmul fast <2 x double> %strided.vec31, %strided.vec
+  %1 = fmul fast <2 x double> %strided.vec30, %strided.vec28
+  %2 = fmul fast <2 x double> %strided.vec30, %strided.vec
+  %3 = fadd fast <2 x double> %2, %vec.phi27
+  %4 = fmul fast <2 x double> %strided.vec31, %strided.vec28
+  %5 = fsub fast <2 x double> %3, %4
+  %6 = fadd fast <2 x double> %1, %vec.phi
+  %7 = fadd fast <2 x double> %6, %0
+  %lsr.iv.next = add nuw nsw i64 %lsr.iv, 32
+  %8 = icmp eq i64 %lsr.iv.next, 1600
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %9 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %5)
+  %10 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %7)
+  %.fca.0.0.insert = insertvalue %"struct.std::complex" poison, double %9, 0, 0
+  %.fca.0.1.insert = insertvalue %"struct.std::complex" %.fca.0.0.insert, double %10, 0, 1
+  ret %"struct.std::complex" %.fca.0.1.insert
+}
+
+; Loop unrolled with factor 2
+;
+define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
+; CHECK-LABEL: complex_mul_v2f64_unrolled:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x9, .LCPI2_0
+; CHECK-NEXT:    adrp x10, .LCPI2_1
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    ldr q3, [x9, :lo12:.LCPI2_0]
+; CHECK-NEXT:    ldr q1, [x10, :lo12:.LCPI2_1]
+; CHECK-NEXT:  .LBB2_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x9, x0, x8
+; CHECK-NEXT:    ld2 { v4.2d, v5.2d }, [x9], #32
+; CHECK-NEXT:    ld2 { v6.2d, v7.2d }, [x9]
+; CHECK-NEXT:    add x9, x1, x8
+; CHECK-NEXT:    add x8, x8, #64
+; CHECK-NEXT:    cmp x8, #1600
+; CHECK-NEXT:    ld2 { v16.2d, v17.2d }, [x9], #32
+; CHECK-NEXT:    fmla v3.2d, v4.2d, v16.2d
+; CHECK-NEXT:    fmla v1.2d, v5.2d, v16.2d
+; CHECK-NEXT:    fmls v3.2d, v5.2d, v17.2d
+; CHECK-NEXT:    fmla v1.2d, v4.2d, v17.2d
+; CHECK-NEXT:    ld2 { v18.2d, v19.2d }, [x9]
+; CHECK-NEXT:    fmla v2.2d, v6.2d, v18.2d
+; CHECK-NEXT:    fmla v0.2d, v7.2d, v18.2d
+; CHECK-NEXT:    fmls v2.2d, v7.2d, v19.2d
+; CHECK-NEXT:    fmla v0.2d, v6.2d, v19.2d
+; CHECK-NEXT:    b.ne .LBB2_1
+; CHECK-NEXT:  // %bb.2: // %middle.block
+; CHECK-NEXT:    fadd v2.2d, v2.2d, v3.2d
+; CHECK-NEXT:    fadd v1.2d, v0.2d, v1.2d
+; CHECK-NEXT:    faddp d0, v2.2d
+; CHECK-NEXT:    faddp d1, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %scevgep = getelementptr i8, ptr %a, i64 32
+  %scevgep49 = getelementptr i8, ptr %b, i64 32
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %lsr.iv54 = phi i64 [ %lsr.iv.next, %vector.body ], [ 100, %entry ]
+  %lsr.iv50 = phi ptr [ %scevgep51, %vector.body ], [ %scevgep49, %entry ]
+  %lsr.iv = phi ptr [ %scevgep48, %vector.body ], [ %scevgep, %entry ]
+  %vec.phi = phi <2 x double> [ <double 1.000000e+00, double 0.000000e+00>, %entry ], [ %14, %vector.body ]
+  %vec.phi27 = phi <2 x double> [ zeroinitializer, %entry ], [ %15, %vector.body ]
+  %vec.phi28 = phi <2 x double> [ <double 2.000000e+00, double 0.000000e+00>, %entry ], [ %10, %vector.body ]
+  %vec.phi29 = phi <2 x double> [ zeroinitializer, %entry ], [ %11, %vector.body ]
+  %scevgep52 = getelementptr i8, ptr %lsr.iv, i64 -32
+  %scevgep53 = getelementptr i8, ptr %lsr.iv50, i64 -32
+  %wide.vec = load <4 x double>, ptr %scevgep52, align 8
+  %wide.vec30 = load <4 x double>, ptr %lsr.iv, align 8
+  %strided.vec = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec31 = shufflevector <4 x double> %wide.vec30, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec32 = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %strided.vec33 = shufflevector <4 x double> %wide.vec30, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %wide.vec34 = load <4 x double>, ptr %scevgep53, align 8
+  %wide.vec35 = load <4 x double>, ptr %lsr.iv50, align 8
+  %strided.vec36 = shufflevector <4 x double> %wide.vec34, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec37 = shufflevector <4 x double> %wide.vec35, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec38 = shufflevector <4 x double> %wide.vec34, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %strided.vec39 = shufflevector <4 x double> %wide.vec35, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %0 = fmul fast <2 x double> %strided.vec38, %strided.vec
+  %1 = fmul fast <2 x double> %strided.vec39, %strided.vec31
+  %2 = fmul fast <2 x double> %strided.vec36, %strided.vec32
+  %3 = fmul fast <2 x double> %strided.vec37, %strided.vec33
+  %4 = fmul fast <2 x double> %strided.vec36, %strided.vec
+  %5 = fmul fast <2 x double> %strided.vec37, %strided.vec31
+  %6 = fadd fast <2 x double> %4, %vec.phi28
+  %7 = fadd fast <2 x double> %5, %vec.phi29
+  %8 = fmul fast <2 x double> %strided.vec38, %strided.vec32
+  %9 = fmul fast <2 x double> %strided.vec39, %strided.vec33
+  %10 = fsub fast <2 x double> %6, %8
+  %11 = fsub fast <2 x double> %7, %9
+  %12 = fadd fast <2 x double> %2, %vec.phi
+  %13 = fadd fast <2 x double> %3, %vec.phi27
+  %14 = fadd fast <2 x double> %12, %0
+  %15 = fadd fast <2 x double> %13, %1
+  %scevgep48 = getelementptr i8, ptr %lsr.iv, i64 64
+  %scevgep51 = getelementptr i8, ptr %lsr.iv50, i64 64
+  %lsr.iv.next = add nsw i64 %lsr.iv54, -4
+  %16 = icmp eq i64 %lsr.iv.next, 0
+  br i1 %16, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %bin.rdx40 = fadd fast <2 x double> %11, %10
+  %17 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %bin.rdx40)
+  %bin.rdx = fadd fast <2 x double> %15, %14
+  %18 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %bin.rdx)
+  %.fca.0.0.insert = insertvalue %"struct.std::complex" poison, double %17, 0, 0
+  %.fca.0.1.insert = insertvalue %"struct.std::complex" %.fca.0.0.insert, double %18, 0, 1
+  ret %"struct.std::complex" %.fca.0.1.insert
+}
+declare double @llvm.vector.reduce.fadd.v2f64(double, <2 x double>)