[AArch64][CodeGen] Add AArch64 support for complex deinterleaving

[NickGuy]

Added context missing from initial patch

[dmgreen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

23249

I think ComplxNum requires NEON, so checking for the first should be enough.

23296

There is quite a lot of debug message left here. I assume it will be removed in the final version?

llvm/test/CodeGen/AArch64/ComplexArithmetic/complex-arithmetic-f16-add.ll

1 ↗

(On Diff #442050)

These don't need to be in a "ComplexArithmetic" directory - they are already easy to identify from the file prefix.

6 ↗

(On Diff #442050)

Can you try and clean up these newlines too. There are gaps in places, but some of the functions are a little jammed together.

[NickGuy]

Addressed comments, cleaned up debug outputs, and implemented vector splitting

[NickGuy]

Addressed comments, and rebased to align with latest revision of D114174 (Diff 16)

[dmgreen]

Can you add tests cases for some of the awkward cases we found in D114174, like the <12 x > vectors (if they are not present already).

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

23248

Can you add some extra newlines between methods

23249

This I don't think needs the NEON check.

23264

Formatting, here and below.

23279

Add a message to the assert

[NickGuy]

Addressed comments, and added more tests

[SjoerdMeijer]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

23286

It isn't clear to me why TyWidth can have any value greater than 128. Don't we expect it to be a multiple of something?

23300

Accumulator always has a value at this point, don't need to check it?

[NickGuy]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

23286

We do, but it's also backed up by the if ((VTyWidth < 128 && VTyWidth != 64) || !llvm::isPowerOf2_32(VTyWidth)) above. Though changing the assert to reflect this restriction might be best, rather than assuming that it's met.

23300

This check is needed, however the previous assignment is not. I've moved it down to where it is needed

[SjoerdMeijer]

Cool, thanks.
Looks like a nice change to me. Will leave further reviews/approval up to Dave.

[dmgreen]

I was hoping we could get some correctness results for the MVE side overnight, but out internal testing seems pretty broken for other reasons. I don't see anything that is complex number related, so hopefully this is OK.

So long as you have done some testing, LGTM.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

23280–23281

Formatting, and it probably wants brackets around the ||

llvm/lib/Target/AArch64/AArch64ISelLowering.h

bool shouldTransformSignedTruncationCheck(EVT XVT,
     MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);
     return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
   }
 
   bool preferIncOfAddToSubOfNot(EVT VT) const override;
 
   bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;
 
+  bool isComplexDeinterleavingSupported() const override;
+  bool isComplexDeinterleavingOperationSupported(
+      ComplexDeinterleavingOperation Operation, Type *Ty) const override;
+
+  Value *createComplexDeinterleavingIR(
+      Instruction *I, ComplexDeinterleavingOperation OperationType,
+      ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
+      Value *Accumulator = nullptr) const override;
+
   bool hasBitPreservingFPLogic(EVT VT) const override {
     // FIXME: Is this always true? It should be true for vectors at least.
     return VT == MVT::f32 || VT == MVT::f64;
   }
 
   bool supportSplitCSR(MachineFunction *MF) const override {
     return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
            MF->getFunction().hasFnAttribute(Attribute::NoUnwind);

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

return Op.getOpcode() == AArch64ISD::DUP ||
           Op.getOperand(0).getOpcode() == AArch64ISD::DUP) ||
          TargetLowering::isTargetCanonicalConstantNode(Op);
 }
 
 bool AArch64TargetLowering::isConstantUnsignedBitfieldExtractLegal(
     unsigned Opc, LLT Ty1, LLT Ty2) const {
   return Ty1 == Ty2 && (Ty1 == LLT::scalar(32) || Ty1 == LLT::scalar(64));
 }
+
+bool AArch64TargetLowering::isComplexDeinterleavingSupported() const {
+  return Subtarget->hasComplxNum();
+}
+
+bool AArch64TargetLowering::isComplexDeinterleavingOperationSupported(
+    ComplexDeinterleavingOperation Operation, Type *Ty) const {
+  auto *VTy = dyn_cast<FixedVectorType>(Ty);
+  if (!VTy)
+    return false;
+
+  auto *ScalarTy = VTy->getScalarType();
+  unsigned NumElements = VTy->getNumElements();
+
+  unsigned VTyWidth = VTy->getScalarSizeInBits() * NumElements;
+  if ((VTyWidth < 128 && VTyWidth != 64) || !llvm::isPowerOf2_32(VTyWidth))
+    return false;
+
+  return (ScalarTy->isHalfTy() && Subtarget->hasFullFP16()) ||
+         ScalarTy->isFloatTy() || ScalarTy->isDoubleTy();
+}
+
+Value *AArch64TargetLowering::createComplexDeinterleavingIR(
+    Instruction *I, ComplexDeinterleavingOperation OperationType,
+    ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
+    Value *Accumulator) const {
+  FixedVectorType *Ty = cast<FixedVectorType>(InputA->getType());
+
+  IRBuilder<> B(I);
+
+  unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
+
+  assert((TyWidth >= 128 && llvm::isPowerOf2_32(TyWidth)) ||
+         TyWidth == 64 && "Vector type must be either 64 or a power of 2 that is at least 128");
+
+  if (TyWidth > 128) {
+    int Stride = Ty->getNumElements() / 2;
+    auto SplitSeq = llvm::seq<int>(0, Ty->getNumElements());
+    auto SplitSeqVec = llvm::to_vector(SplitSeq);
+    ArrayRef<int> LowerSplitMask(&SplitSeqVec[0], Stride);
+    ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
+
+    auto *LowerSplitA = B.CreateShuffleVector(InputA, LowerSplitMask);
+    auto *LowerSplitB = B.CreateShuffleVector(InputB, LowerSplitMask);
+    auto *UpperSplitA = B.CreateShuffleVector(InputA, UpperSplitMask);
+    auto *UpperSplitB = B.CreateShuffleVector(InputB, UpperSplitMask);
+    Value *LowerSplitAcc = nullptr;
+    Value *UpperSplitAcc = nullptr;
+
+    if (Accumulator) {
+      LowerSplitAcc = B.CreateShuffleVector(Accumulator, LowerSplitMask);
+      UpperSplitAcc = B.CreateShuffleVector(Accumulator, UpperSplitMask);
+    }
+
+    auto *LowerSplitInt = createComplexDeinterleavingIR(
+        I, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
+    auto *UpperSplitInt = createComplexDeinterleavingIR(
+        I, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
+
+    ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
+    return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
+  }
+
+  if (OperationType == ComplexDeinterleavingOperation::CMulPartial) {
+    Intrinsic::ID IdMap[4] = {Intrinsic::aarch64_neon_vcmla_rot0,
+                              Intrinsic::aarch64_neon_vcmla_rot90,
+                              Intrinsic::aarch64_neon_vcmla_rot180,
+                              Intrinsic::aarch64_neon_vcmla_rot270};
+
+    if (Accumulator == nullptr)
+      Accumulator = ConstantFP::get(Ty, 0);
+
+    return B.CreateIntrinsic(IdMap[(int)Rotation], Ty,
+                             {Accumulator, InputB, InputA});
+  }
+
+  if (OperationType == ComplexDeinterleavingOperation::CAdd) {
+    Intrinsic::ID IntId = Intrinsic::not_intrinsic;
+    if (Rotation == ComplexDeinterleavingRotation::Rotation_90)
+      IntId = Intrinsic::aarch64_neon_vcadd_rot90;
+    else if (Rotation == ComplexDeinterleavingRotation::Rotation_270)
+      IntId = Intrinsic::aarch64_neon_vcadd_rot270;
+
+    if (IntId == Intrinsic::not_intrinsic)
+      return nullptr;
+
+    return B.CreateIntrinsic(IntId, Ty, {InputA, InputB});
+  }
+
+  return nullptr;
+}

llvm/lib/Target/AArch64/AArch64TargetMachine.cpp

if (TM->getOptLevel() == CodeGenOpt::Aggressive && EnableGEPOpt) {
     addPass(createLICMPass());
   }
 
   TargetPassConfig::addIRPasses();
 
   addPass(createAArch64StackTaggingPass(
       /*IsOptNone=*/TM->getOptLevel() == CodeGenOpt::None));
 
+  // Match complex arithmetic patterns
+  if (TM->getOptLevel() >= CodeGenOpt::Default)
+    addPass(createComplexDeinterleavingPass(TM));
+
   // Match interleaved memory accesses to ldN/stN intrinsics.
   if (TM->getOptLevel() != CodeGenOpt::None) {
     addPass(createInterleavedLoadCombinePass());
     addPass(createInterleavedAccessPass());
   }
 
   // Expand any functions marked with SME attributes which require special
   // changes for the calling convention or that require the lazy-saving

llvm/test/CodeGen/AArch64/O3-pipeline.ll

 ; CHECK-NEXT:         Natural Loop Information
 ; CHECK-NEXT:         Scalar Evolution Analysis
 ; CHECK-NEXT:         Stack Safety Local Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       AArch64 Stack Tagging
+; CHECK-NEXT:       Complex Deinterleaving Pass
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Memory SSA
 ; CHECK-NEXT:       Interleaved Load Combine Pass
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Interleaved Access Pass
 ; CHECK-NEXT:       SME ABI Pass
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information

llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-add.ll

entry:
   %interleaved.vec = shufflevector <1 x half> %0, <1 x half> %1, <2 x i32> <i32 0, i32 1>
   ret <2 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x half> @complex_add_v4f16(<4 x half> %a, <4 x half> %b) {
 ; CHECK-LABEL: complex_add_v4f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp1 v2.4h, v0.4h, v0.4h
-; CHECK-NEXT:    uzp2 v0.4h, v0.4h, v0.4h
-; CHECK-NEXT:    uzp1 v3.4h, v1.4h, v0.4h
-; CHECK-NEXT:    uzp2 v1.4h, v1.4h, v0.4h
-; CHECK-NEXT:    fsub v0.4h, v3.4h, v0.4h
-; CHECK-NEXT:    fadd v1.4h, v1.4h, v2.4h
-; CHECK-NEXT:    zip1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    fcadd v0.4h, v1.4h, v0.4h, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <4 x half> %a, <4 x half> zeroinitializer, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x half> %a, <4 x half> zeroinitializer, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x half> %b, <4 x half> zeroinitializer, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x half> %b, <4 x half> zeroinitializer, <2 x i32> <i32 1, i32 3>
   %0 = fsub fast <2 x half> %b.real, %a.imag
   %1 = fadd fast <2 x half> %b.imag, %a.real
   %interleaved.vec = shufflevector <2 x half> %0, <2 x half> %1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <8 x half> @complex_add_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-LABEL: complex_add_v8f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    uzp1 v4.4h, v0.4h, v2.4h
-; CHECK-NEXT:    uzp2 v0.4h, v0.4h, v2.4h
-; CHECK-NEXT:    uzp1 v2.4h, v1.4h, v3.4h
-; CHECK-NEXT:    uzp2 v1.4h, v1.4h, v3.4h
-; CHECK-NEXT:    fsub v0.4h, v2.4h, v0.4h
-; CHECK-NEXT:    fadd v1.4h, v1.4h, v4.4h
-; CHECK-NEXT:    zip2 v2.4h, v0.4h, v1.4h
-; CHECK-NEXT:    zip1 v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    fcadd v0.8h, v1.8h, v0.8h, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <8 x half> %a, <8 x half> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %a.imag = shufflevector <8 x half> %a, <8 x half> zeroinitializer, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %b.real = shufflevector <8 x half> %b, <8 x half> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %b.imag = shufflevector <8 x half> %b, <8 x half> zeroinitializer, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %0 = fsub fast <4 x half> %b.real, %a.imag
   %1 = fadd fast <4 x half> %b.imag, %a.real
   %interleaved.vec = shufflevector <4 x half> %0, <4 x half> %1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <16 x half> @complex_add_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-LABEL: complex_add_v16f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp1 v4.8h, v2.8h, v3.8h
-; CHECK-NEXT:    uzp1 v5.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp2 v1.8h, v2.8h, v3.8h
-; CHECK-NEXT:    fsub v2.8h, v4.8h, v0.8h
-; CHECK-NEXT:    fadd v1.8h, v1.8h, v5.8h
-; CHECK-NEXT:    zip1 v0.8h, v2.8h, v1.8h
-; CHECK-NEXT:    zip2 v1.8h, v2.8h, v1.8h
+; CHECK-NEXT:    fcadd v0.8h, v2.8h, v0.8h, #90
+; CHECK-NEXT:    fcadd v1.8h, v3.8h, v1.8h, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <16 x half> %a, <16 x half> zeroinitializer, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %a.imag = shufflevector <16 x half> %a, <16 x half> zeroinitializer, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %b.real = shufflevector <16 x half> %b, <16 x half> zeroinitializer, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %b.imag = shufflevector <16 x half> %b, <16 x half> zeroinitializer, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %0 = fsub fast <8 x half> %b.real, %a.imag
   %1 = fadd fast <8 x half> %b.imag, %a.real
   %interleaved.vec = shufflevector <8 x half> %0, <8 x half> %1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <16 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <32 x half> @complex_add_v32f16(<32 x half> %a, <32 x half> %b) {
 ; CHECK-LABEL: complex_add_v32f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp1 v16.8h, v4.8h, v5.8h
-; CHECK-NEXT:    uzp1 v17.8h, v2.8h, v3.8h
-; CHECK-NEXT:    uzp1 v18.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp2 v1.8h, v2.8h, v3.8h
-; CHECK-NEXT:    uzp2 v2.8h, v4.8h, v5.8h
-; CHECK-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-NEXT:    uzp2 v4.8h, v6.8h, v7.8h
-; CHECK-NEXT:    fsub v5.8h, v16.8h, v0.8h
-; CHECK-NEXT:    fadd v2.8h, v2.8h, v18.8h
-; CHECK-NEXT:    fsub v3.8h, v3.8h, v1.8h
-; CHECK-NEXT:    fadd v4.8h, v4.8h, v17.8h
-; CHECK-NEXT:    zip1 v0.8h, v5.8h, v2.8h
-; CHECK-NEXT:    zip2 v1.8h, v5.8h, v2.8h
-; CHECK-NEXT:    zip1 v2.8h, v3.8h, v4.8h
-; CHECK-NEXT:    zip2 v3.8h, v3.8h, v4.8h
+; CHECK-NEXT:    fcadd v2.8h, v6.8h, v2.8h, #90
+; CHECK-NEXT:    fcadd v0.8h, v4.8h, v0.8h, #90
+; CHECK-NEXT:    fcadd v1.8h, v5.8h, v1.8h, #90
+; CHECK-NEXT:    fcadd v3.8h, v7.8h, v3.8h, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <32 x half> %a, <32 x half> zeroinitializer, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %a.imag = shufflevector <32 x half> %a, <32 x half> zeroinitializer, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %b.real = shufflevector <32 x half> %b, <32 x half> zeroinitializer, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %b.imag = shufflevector <32 x half> %b, <32 x half> zeroinitializer, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %0 = fsub fast <16 x half> %b.real, %a.imag
   %1 = fadd fast <16 x half> %b.imag, %a.real
   %interleaved.vec = shufflevector <16 x half> %0, <16 x half> %1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   ret <32 x half> %interleaved.vec
 }

llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul.ll

entry:
   %interleaved.vec = shufflevector <1 x half> %5, <1 x half> %2, <2 x i32> <i32 0, i32 1>
   ret <2 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x half> @complex_mul_v4f16(<4 x half> %a, <4 x half> %b) {
 ; CHECK-LABEL: complex_mul_v4f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp2 v2.4h, v0.4h, v0.4h
-; CHECK-NEXT:    uzp2 v3.4h, v1.4h, v0.4h
-; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-NEXT:    fmul v4.4h, v2.4h, v3.4h
-; CHECK-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
-; CHECK-NEXT:    fmul v3.4h, v3.4h, v0.4h
-; CHECK-NEXT:    fneg v4.4h, v4.4h
-; CHECK-NEXT:    fmla v3.4h, v2.4h, v1.4h
-; CHECK-NEXT:    fmla v4.4h, v0.4h, v1.4h
-; CHECK-NEXT:    zip1 v0.4h, v4.4h, v3.4h
+; CHECK-NEXT:    movi d2, #0000000000000000
+; CHECK-NEXT:    fcmla v2.4h, v0.4h, v1.4h, #0
+; CHECK-NEXT:    fcmla v2.4h, v0.4h, v1.4h, #90
+; CHECK-NEXT:    fmov d0, d2
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <4 x half> %a, <4 x half> poison, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x half> %a, <4 x half> poison, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x half> %b, <4 x half> poison, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x half> %b, <4 x half> poison, <2 x i32> <i32 1, i32 3>
   %0 = fmul fast <2 x half> %b.imag, %a.real
   %1 = fmul fast <2 x half> %b.real, %a.imag
   %2 = fadd fast <2 x half> %1, %0
   %3 = fmul fast <2 x half> %b.real, %a.real
   %4 = fmul fast <2 x half> %a.imag, %b.imag
   %5 = fsub fast <2 x half> %3, %4
   %interleaved.vec = shufflevector <2 x half> %5, <2 x half> %2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <8 x half> @complex_mul_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-LABEL: complex_mul_v8f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    uzp2 v4.4h, v0.4h, v2.4h
-; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v2.4h
-; CHECK-NEXT:    uzp2 v5.4h, v1.4h, v3.4h
-; CHECK-NEXT:    uzp1 v1.4h, v1.4h, v3.4h
-; CHECK-NEXT:    fmul v2.4h, v4.4h, v5.4h
-; CHECK-NEXT:    fmul v3.4h, v5.4h, v0.4h
-; CHECK-NEXT:    fneg v2.4h, v2.4h
-; CHECK-NEXT:    fmla v3.4h, v4.4h, v1.4h
-; CHECK-NEXT:    fmla v2.4h, v0.4h, v1.4h
-; CHECK-NEXT:    zip2 v1.4h, v2.4h, v3.4h
-; CHECK-NEXT:    zip1 v0.4h, v2.4h, v3.4h
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v2.8h, v0.8h, v1.8h, #0
+; CHECK-NEXT:    fcmla v2.8h, v0.8h, v1.8h, #90
+; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %a.imag = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %b.real = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %b.imag = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %0 = fmul fast <4 x half> %b.imag, %a.real
   %1 = fmul fast <4 x half> %b.real, %a.imag
   %2 = fadd fast <4 x half> %1, %0
   %3 = fmul fast <4 x half> %b.real, %a.real
   %4 = fmul fast <4 x half> %a.imag, %b.imag
   %5 = fsub fast <4 x half> %3, %4
   %interleaved.vec = shufflevector <4 x half> %5, <4 x half> %2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <16 x half> @complex_mul_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-LABEL: complex_mul_v16f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp2 v4.8h, v2.8h, v3.8h
-; CHECK-NEXT:    uzp2 v5.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-NEXT:    fmul v1.8h, v5.8h, v4.8h
-; CHECK-NEXT:    fmul v3.8h, v4.8h, v0.8h
-; CHECK-NEXT:    fneg v1.8h, v1.8h
-; CHECK-NEXT:    fmla v3.8h, v5.8h, v2.8h
-; CHECK-NEXT:    fmla v1.8h, v0.8h, v2.8h
-; CHECK-NEXT:    zip1 v0.8h, v1.8h, v3.8h
-; CHECK-NEXT:    zip2 v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.8h, v0.8h, v2.8h, #0
+; CHECK-NEXT:    fcmla v5.8h, v1.8h, v3.8h, #0
+; CHECK-NEXT:    fcmla v4.8h, v0.8h, v2.8h, #90
+; CHECK-NEXT:    fcmla v5.8h, v1.8h, v3.8h, #90
+; CHECK-NEXT:    mov v0.16b, v4.16b
+; CHECK-NEXT:    mov v1.16b, v5.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <16 x half> %a, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %a.imag = shufflevector <16 x half> %a, <16 x half> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %b.real = shufflevector <16 x half> %b, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %b.imag = shufflevector <16 x half> %b, <16 x half> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %0 = fmul fast <8 x half> %b.imag, %a.real
   %1 = fmul fast <8 x half> %b.real, %a.imag
   %2 = fadd fast <8 x half> %1, %0
   %3 = fmul fast <8 x half> %b.real, %a.real
   %4 = fmul fast <8 x half> %a.imag, %b.imag
   %5 = fsub fast <8 x half> %3, %4
   %interleaved.vec = shufflevector <8 x half> %5, <8 x half> %2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <16 x half> %interleaved.vec
 }
 
 ; Expected to transform
 define <32 x half> @complex_mul_v32f16(<32 x half> %a, <32 x half> %b) {
 ; CHECK-LABEL: complex_mul_v32f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp2 v16.8h, v4.8h, v5.8h
-; CHECK-NEXT:    uzp1 v17.8h, v2.8h, v3.8h
-; CHECK-NEXT:    uzp1 v18.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp2 v1.8h, v2.8h, v3.8h
-; CHECK-NEXT:    uzp2 v2.8h, v6.8h, v7.8h
-; CHECK-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
-; CHECK-NEXT:    fmul v4.8h, v0.8h, v16.8h
-; CHECK-NEXT:    uzp1 v5.8h, v6.8h, v7.8h
-; CHECK-NEXT:    fmul v6.8h, v1.8h, v2.8h
-; CHECK-NEXT:    fmul v7.8h, v16.8h, v18.8h
-; CHECK-NEXT:    fneg v4.8h, v4.8h
-; CHECK-NEXT:    fmul v16.8h, v2.8h, v17.8h
-; CHECK-NEXT:    fneg v6.8h, v6.8h
-; CHECK-NEXT:    fmla v7.8h, v0.8h, v3.8h
-; CHECK-NEXT:    fmla v4.8h, v18.8h, v3.8h
-; CHECK-NEXT:    fmla v16.8h, v1.8h, v5.8h
-; CHECK-NEXT:    fmla v6.8h, v17.8h, v5.8h
-; CHECK-NEXT:    zip1 v0.8h, v4.8h, v7.8h
-; CHECK-NEXT:    zip2 v1.8h, v4.8h, v7.8h
-; CHECK-NEXT:    zip1 v2.8h, v6.8h, v16.8h
-; CHECK-NEXT:    zip2 v3.8h, v6.8h, v16.8h
+; CHECK-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v16.8h, v0.8h, v4.8h, #0
+; CHECK-NEXT:    fcmla v17.8h, v1.8h, v5.8h, #0
+; CHECK-NEXT:    fcmla v18.8h, v2.8h, v6.8h, #0
+; CHECK-NEXT:    fcmla v19.8h, v3.8h, v7.8h, #0
+; CHECK-NEXT:    fcmla v16.8h, v0.8h, v4.8h, #90
+; CHECK-NEXT:    fcmla v17.8h, v1.8h, v5.8h, #90
+; CHECK-NEXT:    fcmla v18.8h, v2.8h, v6.8h, #90
+; CHECK-NEXT:    fcmla v19.8h, v3.8h, v7.8h, #90
+; CHECK-NEXT:    mov v0.16b, v16.16b
+; CHECK-NEXT:    mov v1.16b, v17.16b
+; CHECK-NEXT:    mov v2.16b, v18.16b
+; CHECK-NEXT:    mov v3.16b, v19.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <32 x half> %a, <32 x half> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %a.imag = shufflevector <32 x half> %a, <32 x half> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %b.real = shufflevector <32 x half> %b, <32 x half> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %b.imag = shufflevector <32 x half> %b, <32 x half> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %0 = fmul fast <16 x half> %b.imag, %a.real
   %1 = fmul fast <16 x half> %b.real, %a.imag
   %2 = fadd fast <16 x half> %1, %0
   %3 = fmul fast <16 x half> %b.real, %a.real
   %4 = fmul fast <16 x half> %a.imag, %b.imag
   %5 = fsub fast <16 x half> %3, %4
   %interleaved.vec = shufflevector <16 x half> %5, <16 x half> %2, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   ret <32 x half> %interleaved.vec
 }

llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-add.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+complxnum,+neon,+fullfp16 -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 
 ; Expected to transform
 define <2 x float> @complex_add_v2f32(<2 x float> %a, <2 x float> %b) {
 ; CHECK-LABEL: complex_add_v2f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    dup v2.2s, v0.s[1]
-; CHECK-NEXT:    dup v3.2s, v1.s[1]
-; CHECK-NEXT:    fsub v1.2s, v1.2s, v2.2s
-; CHECK-NEXT:    fadd v0.2s, v3.2s, v0.2s
-; CHECK-NEXT:    zip1 v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    fcadd v0.2s, v1.2s, v0.2s, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <2 x float> %a, <2 x float> zeroinitializer, <1 x i32> <i32 0>
   %a.imag = shufflevector <2 x float> %a, <2 x float> zeroinitializer, <1 x i32> <i32 1>
   %b.real = shufflevector <2 x float> %b, <2 x float> zeroinitializer, <1 x i32> <i32 0>
   %b.imag = shufflevector <2 x float> %b, <2 x float> zeroinitializer, <1 x i32> <i32 1>
   %0 = fsub fast <1 x float> %b.real, %a.imag
   %1 = fadd fast <1 x float> %b.imag, %a.real
   %interleaved.vec = shufflevector <1 x float> %0, <1 x float> %1, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x float> @complex_add_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: complex_add_v4f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v2.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    fsub v0.2s, v2.2s, v0.2s
-; CHECK-NEXT:    fadd v1.2s, v1.2s, v4.2s
-; CHECK-NEXT:    zip2 v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    fcadd v0.4s, v1.4s, v0.4s, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <4 x float> %a, <4 x float> zeroinitializer, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x float> %a, <4 x float> zeroinitializer, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x float> %b, <4 x float> zeroinitializer, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x float> %b, <4 x float> zeroinitializer, <2 x i32> <i32 1, i32 3>
   %0 = fsub fast <2 x float> %b.real, %a.imag
   %1 = fadd fast <2 x float> %b.imag, %a.real
   %interleaved.vec = shufflevector <2 x float> %0, <2 x float> %1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <8 x float> @complex_add_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: complex_add_v8f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp1 v4.4s, v2.4s, v3.4s
-; CHECK-NEXT:    uzp1 v5.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp2 v1.4s, v2.4s, v3.4s
-; CHECK-NEXT:    fsub v2.4s, v4.4s, v0.4s
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    zip1 v0.4s, v2.4s, v1.4s
-; CHECK-NEXT:    zip2 v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    fcadd v0.4s, v2.4s, v0.4s, #90
+; CHECK-NEXT:    fcadd v1.4s, v3.4s, v1.4s, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <8 x float> %a, <8 x float> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %a.imag = shufflevector <8 x float> %a, <8 x float> zeroinitializer, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %b.real = shufflevector <8 x float> %b, <8 x float> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %b.imag = shufflevector <8 x float> %b, <8 x float> zeroinitializer, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %0 = fsub fast <4 x float> %b.real, %a.imag
   %1 = fadd fast <4 x float> %b.imag, %a.real
   %interleaved.vec = shufflevector <4 x float> %0, <4 x float> %1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <16 x float> @complex_add_v16f32(<16 x float> %a, <16 x float> %b) {
 ; CHECK-LABEL: complex_add_v16f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp1 v16.4s, v4.4s, v5.4s
-; CHECK-NEXT:    uzp1 v17.4s, v2.4s, v3.4s
-; CHECK-NEXT:    uzp1 v18.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp2 v1.4s, v2.4s, v3.4s
-; CHECK-NEXT:    uzp2 v2.4s, v4.4s, v5.4s
-; CHECK-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-NEXT:    uzp2 v4.4s, v6.4s, v7.4s
-; CHECK-NEXT:    fsub v5.4s, v16.4s, v0.4s
-; CHECK-NEXT:    fadd v2.4s, v2.4s, v18.4s
-; CHECK-NEXT:    fsub v3.4s, v3.4s, v1.4s
-; CHECK-NEXT:    fadd v4.4s, v4.4s, v17.4s
-; CHECK-NEXT:    zip1 v0.4s, v5.4s, v2.4s
-; CHECK-NEXT:    zip2 v1.4s, v5.4s, v2.4s
-; CHECK-NEXT:    zip1 v2.4s, v3.4s, v4.4s
-; CHECK-NEXT:    zip2 v3.4s, v3.4s, v4.4s
+; CHECK-NEXT:    fcadd v2.4s, v6.4s, v2.4s, #90
+; CHECK-NEXT:    fcadd v0.4s, v4.4s, v0.4s, #90
+; CHECK-NEXT:    fcadd v1.4s, v5.4s, v1.4s, #90
+; CHECK-NEXT:    fcadd v3.4s, v7.4s, v3.4s, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <16 x float> %a, <16 x float> zeroinitializer, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %a.imag = shufflevector <16 x float> %a, <16 x float> zeroinitializer, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %b.real = shufflevector <16 x float> %b, <16 x float> zeroinitializer, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %b.imag = shufflevector <16 x float> %b, <16 x float> zeroinitializer, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %0 = fsub fast <8 x float> %b.real, %a.imag
   %1 = fadd fast <8 x float> %b.imag, %a.real
   %interleaved.vec = shufflevector <8 x float> %0, <8 x float> %1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <16 x float> %interleaved.vec
 }

llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+complxnum,+neon,+fullfp16 -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 ; Expected to transform
 define <2 x float> @complex_mul_v2f32(<2 x float> %a, <2 x float> %b) {
 ; CHECK-LABEL: complex_mul_v2f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    dup v2.2s, v1.s[1]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    dup v3.2s, v0.s[1]
-; CHECK-NEXT:    fmul v4.2s, v0.2s, v1.s[1]
-; CHECK-NEXT:    fmul v2.2s, v2.2s, v0.s[1]
-; CHECK-NEXT:    fmla v4.2s, v3.2s, v1.2s
-; CHECK-NEXT:    fneg v2.2s, v2.2s
-; CHECK-NEXT:    fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip1 v0.2s, v2.2s, v4.2s
+; CHECK-NEXT:    movi d2, #0000000000000000
+; CHECK-NEXT:    fcmla v2.2s, v0.2s, v1.2s, #0
+; CHECK-NEXT:    fcmla v2.2s, v0.2s, v1.2s, #90
+; CHECK-NEXT:    fmov d0, d2
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <2 x float> %a, <2 x float> poison, <1 x i32> <i32 0>
   %a.imag = shufflevector <2 x float> %a, <2 x float> poison, <1 x i32> <i32 1>
   %b.real = shufflevector <2 x float> %b, <2 x float> poison, <1 x i32> <i32 0>
   %b.imag = shufflevector <2 x float> %b, <2 x float> poison, <1 x i32> <i32 1>
   %0 = fmul fast <1 x float> %b.imag, %a.real
   %1 = fmul fast <1 x float> %b.real, %a.imag
   %2 = fadd fast <1 x float> %1, %0
   %3 = fmul fast <1 x float> %b.real, %a.real
   %4 = fmul fast <1 x float> %a.imag, %b.imag
   %5 = fsub fast <1 x float> %3, %4
   %interleaved.vec = shufflevector <1 x float> %5, <1 x float> %2, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x float> @complex_mul_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: complex_mul_v4f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip1 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT:    fmul v3.2s, v5.2s, v0.2s
-; CHECK-NEXT:    fneg v2.2s, v2.2s
-; CHECK-NEXT:    fmla v3.2s, v4.2s, v1.2s
-; CHECK-NEXT:    fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip2 v1.2s, v2.2s, v3.2s
-; CHECK-NEXT:    zip1 v0.2s, v2.2s, v3.2s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v2.4s, v0.4s, v1.4s, #0
+; CHECK-NEXT:    fcmla v2.4s, v0.4s, v1.4s, #90
+; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %0 = fmul fast <2 x float> %b.imag, %a.real
   %1 = fmul fast <2 x float> %b.real, %a.imag
   %2 = fadd fast <2 x float> %1, %0
   %3 = fmul fast <2 x float> %b.real, %a.real
   %4 = fmul fast <2 x float> %a.imag, %b.imag
   %5 = fsub fast <2 x float> %3, %4
   %interleaved.vec = shufflevector <2 x float> %5, <2 x float> %2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <8 x float> @complex_mul_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: complex_mul_v8f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp2 v4.4s, v2.4s, v3.4s
-; CHECK-NEXT:    uzp2 v5.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-NEXT:    fmul v1.4s, v5.4s, v4.4s
-; CHECK-NEXT:    fmul v3.4s, v4.4s, v0.4s
-; CHECK-NEXT:    fneg v1.4s, v1.4s
-; CHECK-NEXT:    fmla v3.4s, v5.4s, v2.4s
-; CHECK-NEXT:    fmla v1.4s, v0.4s, v2.4s
-; CHECK-NEXT:    zip1 v0.4s, v1.4s, v3.4s
-; CHECK-NEXT:    zip2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.4s, v0.4s, v2.4s, #0
+; CHECK-NEXT:    fcmla v5.4s, v1.4s, v3.4s, #0
+; CHECK-NEXT:    fcmla v4.4s, v0.4s, v2.4s, #90
+; CHECK-NEXT:    fcmla v5.4s, v1.4s, v3.4s, #90
+; CHECK-NEXT:    mov v0.16b, v4.16b
+; CHECK-NEXT:    mov v1.16b, v5.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %a.imag = shufflevector <8 x float> %a, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %b.real = shufflevector <8 x float> %b, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %b.imag = shufflevector <8 x float> %b, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %0 = fmul fast <4 x float> %b.imag, %a.real
   %1 = fmul fast <4 x float> %b.real, %a.imag
   %2 = fadd fast <4 x float> %1, %0
   %3 = fmul fast <4 x float> %b.real, %a.real
   %4 = fmul fast <4 x float> %a.imag, %b.imag
   %5 = fsub fast <4 x float> %3, %4
   %interleaved.vec = shufflevector <4 x float> %5, <4 x float> %2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <16 x float> @complex_mul_v16f32(<16 x float> %a, <16 x float> %b) {
 ; CHECK-LABEL: complex_mul_v16f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp2 v16.4s, v4.4s, v5.4s
-; CHECK-NEXT:    uzp1 v17.4s, v2.4s, v3.4s
-; CHECK-NEXT:    uzp1 v18.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp2 v1.4s, v2.4s, v3.4s
-; CHECK-NEXT:    uzp2 v2.4s, v6.4s, v7.4s
-; CHECK-NEXT:    uzp1 v3.4s, v4.4s, v5.4s
-; CHECK-NEXT:    fmul v4.4s, v0.4s, v16.4s
-; CHECK-NEXT:    uzp1 v5.4s, v6.4s, v7.4s
-; CHECK-NEXT:    fmul v6.4s, v1.4s, v2.4s
-; CHECK-NEXT:    fmul v7.4s, v16.4s, v18.4s
-; CHECK-NEXT:    fneg v4.4s, v4.4s
-; CHECK-NEXT:    fmul v16.4s, v2.4s, v17.4s
-; CHECK-NEXT:    fneg v6.4s, v6.4s
-; CHECK-NEXT:    fmla v7.4s, v0.4s, v3.4s
-; CHECK-NEXT:    fmla v4.4s, v18.4s, v3.4s
-; CHECK-NEXT:    fmla v16.4s, v1.4s, v5.4s
-; CHECK-NEXT:    fmla v6.4s, v17.4s, v5.4s
-; CHECK-NEXT:    zip1 v0.4s, v4.4s, v7.4s
-; CHECK-NEXT:    zip2 v1.4s, v4.4s, v7.4s
-; CHECK-NEXT:    zip1 v2.4s, v6.4s, v16.4s
-; CHECK-NEXT:    zip2 v3.4s, v6.4s, v16.4s
+; CHECK-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v16.4s, v0.4s, v4.4s, #0
+; CHECK-NEXT:    fcmla v17.4s, v1.4s, v5.4s, #0
+; CHECK-NEXT:    fcmla v18.4s, v2.4s, v6.4s, #0
+; CHECK-NEXT:    fcmla v19.4s, v3.4s, v7.4s, #0
+; CHECK-NEXT:    fcmla v16.4s, v0.4s, v4.4s, #90
+; CHECK-NEXT:    fcmla v17.4s, v1.4s, v5.4s, #90
+; CHECK-NEXT:    fcmla v18.4s, v2.4s, v6.4s, #90
+; CHECK-NEXT:    fcmla v19.4s, v3.4s, v7.4s, #90
+; CHECK-NEXT:    mov v0.16b, v16.16b
+; CHECK-NEXT:    mov v1.16b, v17.16b
+; CHECK-NEXT:    mov v2.16b, v18.16b
+; CHECK-NEXT:    mov v3.16b, v19.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %a.imag = shufflevector <16 x float> %a, <16 x float> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %b.real = shufflevector <16 x float> %b, <16 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %b.imag = shufflevector <16 x float> %b, <16 x float> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %0 = fmul fast <8 x float> %b.imag, %a.real
   %1 = fmul fast <8 x float> %b.real, %a.imag
   %2 = fadd fast <8 x float> %1, %0
   %3 = fmul fast <8 x float> %b.real, %a.real
   %4 = fmul fast <8 x float> %a.imag, %b.imag
   %5 = fsub fast <8 x float> %3, %4
   %interleaved.vec = shufflevector <8 x float> %5, <8 x float> %2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <16 x float> %interleaved.vec
 }

llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-add.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+complxnum,+neon,+fullfp16 -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 
 ; Expected to transform
 define <2 x double> @complex_add_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: complex_add_v2f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    fsub d1, d1, d2
-; CHECK-NEXT:    fadd d0, d3, d0
-; CHECK-NEXT:    mov v1.d[1], v0.d[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    fcadd v0.2d, v1.2d, v0.2d, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <1 x i32> <i32 0>
   %a.imag = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <1 x i32> <i32 1>
   %b.real = shufflevector <2 x double> %b, <2 x double> zeroinitializer, <1 x i32> <i32 0>
   %b.imag = shufflevector <2 x double> %b, <2 x double> zeroinitializer, <1 x i32> <i32 1>
   %0 = fsub fast <1 x double> %b.real, %a.imag
   %1 = fadd fast <1 x double> %b.imag, %a.real
   %interleaved.vec = shufflevector <1 x double> %0, <1 x double> %1, <2 x i32> <i32 0, i32 1>
   ret <2 x double> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x double> @complex_add_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: complex_add_v4f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    zip1 v4.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip1 v5.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v1.2d, v2.2d, v3.2d
-; CHECK-NEXT:    fsub v2.2d, v4.2d, v0.2d
-; CHECK-NEXT:    fadd v1.2d, v1.2d, v5.2d
-; CHECK-NEXT:    zip1 v0.2d, v2.2d, v1.2d
-; CHECK-NEXT:    zip2 v1.2d, v2.2d, v1.2d
+; CHECK-NEXT:    fcadd v0.2d, v2.2d, v0.2d, #90
+; CHECK-NEXT:    fcadd v1.2d, v3.2d, v1.2d, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <4 x double> %a, <4 x double> zeroinitializer, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x double> %a, <4 x double> zeroinitializer, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x double> %b, <4 x double> zeroinitializer, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x double> %b, <4 x double> zeroinitializer, <2 x i32> <i32 1, i32 3>
   %0 = fsub fast <2 x double> %b.real, %a.imag
   %1 = fadd fast <2 x double> %b.imag, %a.real
   %interleaved.vec = shufflevector <2 x double> %0, <2 x double> %1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x double> %interleaved.vec
 }
 
 ; Expected to transform
 define <8 x double> @complex_add_v8f64(<8 x double> %a, <8 x double> %b) {
 ; CHECK-LABEL: complex_add_v8f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    zip1 v16.2d, v4.2d, v5.2d
-; CHECK-NEXT:    zip1 v17.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip1 v18.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v1.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip2 v2.2d, v4.2d, v5.2d
-; CHECK-NEXT:    zip1 v3.2d, v6.2d, v7.2d
-; CHECK-NEXT:    zip2 v4.2d, v6.2d, v7.2d
-; CHECK-NEXT:    fsub v5.2d, v16.2d, v0.2d
-; CHECK-NEXT:    fadd v2.2d, v2.2d, v18.2d
-; CHECK-NEXT:    fsub v3.2d, v3.2d, v1.2d
-; CHECK-NEXT:    fadd v4.2d, v4.2d, v17.2d
-; CHECK-NEXT:    zip1 v0.2d, v5.2d, v2.2d
-; CHECK-NEXT:    zip2 v1.2d, v5.2d, v2.2d
-; CHECK-NEXT:    zip1 v2.2d, v3.2d, v4.2d
-; CHECK-NEXT:    zip2 v3.2d, v3.2d, v4.2d
+; CHECK-NEXT:    fcadd v2.2d, v6.2d, v2.2d, #90
+; CHECK-NEXT:    fcadd v0.2d, v4.2d, v0.2d, #90
+; CHECK-NEXT:    fcadd v1.2d, v5.2d, v1.2d, #90
+; CHECK-NEXT:    fcadd v3.2d, v7.2d, v3.2d, #90
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <8 x double> %a, <8 x double> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %a.imag = shufflevector <8 x double> %a, <8 x double> zeroinitializer, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %b.real = shufflevector <8 x double> %b, <8 x double> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %b.imag = shufflevector <8 x double> %b, <8 x double> zeroinitializer, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %0 = fsub fast <4 x double> %b.real, %a.imag
   %1 = fadd fast <4 x double> %b.imag, %a.real
   %interleaved.vec = shufflevector <4 x double> %0, <4 x double> %1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x double> %interleaved.vec
 }

llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+complxnum,+neon,+fullfp16 -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 ; Expected to transform
 define <2 x double> @complex_mul_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: complex_mul_v2f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    fmul d4, d3, d0
-; CHECK-NEXT:    fmul d3, d2, d3
-; CHECK-NEXT:    fmadd d2, d1, d2, d4
-; CHECK-NEXT:    fnmsub d0, d1, d0, d3
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v2.2d, v0.2d, v1.2d, #0
+; CHECK-NEXT:    fcmla v2.2d, v0.2d, v1.2d, #90
+; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <2 x double> %a, <2 x double> poison, <1 x i32> <i32 0>
   %a.imag = shufflevector <2 x double> %a, <2 x double> poison, <1 x i32> <i32 1>
   %b.real = shufflevector <2 x double> %b, <2 x double> poison, <1 x i32> <i32 0>
   %b.imag = shufflevector <2 x double> %b, <2 x double> poison, <1 x i32> <i32 1>
   %0 = fmul fast <1 x double> %b.imag, %a.real
   %1 = fmul fast <1 x double> %b.real, %a.imag
   %2 = fadd fast <1 x double> %1, %0
   %3 = fmul fast <1 x double> %b.real, %a.real
   %4 = fmul fast <1 x double> %a.imag, %b.imag
   %5 = fsub fast <1 x double> %3, %4
   %interleaved.vec = shufflevector <1 x double> %5, <1 x double> %2, <2 x i32> <i32 0, i32 1>
   ret <2 x double> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x double> @complex_mul_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: complex_mul_v4f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    zip2 v4.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip2 v5.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip1 v2.2d, v2.2d, v3.2d
-; CHECK-NEXT:    fmul v1.2d, v5.2d, v4.2d
-; CHECK-NEXT:    fmul v3.2d, v4.2d, v0.2d
-; CHECK-NEXT:    fneg v1.2d, v1.2d
-; CHECK-NEXT:    fmla v3.2d, v5.2d, v2.2d
-; CHECK-NEXT:    fmla v1.2d, v0.2d, v2.2d
-; CHECK-NEXT:    zip1 v0.2d, v1.2d, v3.2d
-; CHECK-NEXT:    zip2 v1.2d, v1.2d, v3.2d
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.2d, v0.2d, v2.2d, #0
+; CHECK-NEXT:    fcmla v5.2d, v1.2d, v3.2d, #0
+; CHECK-NEXT:    fcmla v4.2d, v0.2d, v2.2d, #90
+; CHECK-NEXT:    fcmla v5.2d, v1.2d, v3.2d, #90
+; CHECK-NEXT:    mov v0.16b, v4.16b
+; CHECK-NEXT:    mov v1.16b, v5.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 1, i32 3>
   %0 = fmul fast <2 x double> %b.imag, %a.real
   %1 = fmul fast <2 x double> %b.real, %a.imag
   %2 = fadd fast <2 x double> %1, %0
   %3 = fmul fast <2 x double> %b.real, %a.real
   %4 = fmul fast <2 x double> %a.imag, %b.imag
   %5 = fsub fast <2 x double> %3, %4
   %interleaved.vec = shufflevector <2 x double> %5, <2 x double> %2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x double> %interleaved.vec
 }
 
 ; Expected to transform
 define <8 x double> @complex_mul_v8f64(<8 x double> %a, <8 x double> %b) {
 ; CHECK-LABEL: complex_mul_v8f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    zip2 v16.2d, v4.2d, v5.2d
-; CHECK-NEXT:    zip1 v17.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip1 v18.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v1.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip2 v2.2d, v6.2d, v7.2d
-; CHECK-NEXT:    zip1 v3.2d, v4.2d, v5.2d
-; CHECK-NEXT:    fmul v4.2d, v0.2d, v16.2d
-; CHECK-NEXT:    zip1 v5.2d, v6.2d, v7.2d
-; CHECK-NEXT:    fmul v6.2d, v1.2d, v2.2d
-; CHECK-NEXT:    fmul v7.2d, v16.2d, v18.2d
-; CHECK-NEXT:    fneg v4.2d, v4.2d
-; CHECK-NEXT:    fmul v16.2d, v2.2d, v17.2d
-; CHECK-NEXT:    fneg v6.2d, v6.2d
-; CHECK-NEXT:    fmla v7.2d, v0.2d, v3.2d
-; CHECK-NEXT:    fmla v4.2d, v18.2d, v3.2d
-; CHECK-NEXT:    fmla v16.2d, v1.2d, v5.2d
-; CHECK-NEXT:    fmla v6.2d, v17.2d, v5.2d
-; CHECK-NEXT:    zip1 v0.2d, v4.2d, v7.2d
-; CHECK-NEXT:    zip2 v1.2d, v4.2d, v7.2d
-; CHECK-NEXT:    zip1 v2.2d, v6.2d, v16.2d
-; CHECK-NEXT:    zip2 v3.2d, v6.2d, v16.2d
+; CHECK-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v16.2d, v0.2d, v4.2d, #0
+; CHECK-NEXT:    fcmla v17.2d, v1.2d, v5.2d, #0
+; CHECK-NEXT:    fcmla v18.2d, v2.2d, v6.2d, #0
+; CHECK-NEXT:    fcmla v19.2d, v3.2d, v7.2d, #0
+; CHECK-NEXT:    fcmla v16.2d, v0.2d, v4.2d, #90
+; CHECK-NEXT:    fcmla v17.2d, v1.2d, v5.2d, #90
+; CHECK-NEXT:    fcmla v18.2d, v2.2d, v6.2d, #90
+; CHECK-NEXT:    fcmla v19.2d, v3.2d, v7.2d, #90
+; CHECK-NEXT:    mov v0.16b, v16.16b
+; CHECK-NEXT:    mov v1.16b, v17.16b
+; CHECK-NEXT:    mov v2.16b, v18.16b
+; CHECK-NEXT:    mov v3.16b, v19.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real   = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %a.imag = shufflevector <8 x double> %a, <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %b.real = shufflevector <8 x double> %b, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %b.imag = shufflevector <8 x double> %b, <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %0 = fmul fast <4 x double> %b.imag, %a.real
   %1 = fmul fast <4 x double> %b.real, %a.imag
   %2 = fadd fast <4 x double> %1, %0
   %3 = fmul fast <4 x double> %b.real, %a.real
   %4 = fmul fast <4 x double> %a.imag, %b.imag
   %5 = fsub fast <4 x double> %3, %4
   %interleaved.vec = shufflevector <4 x double> %5, <4 x double> %2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x double> %interleaved.vec
 }

llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+complxnum,+neon,+fullfp16 -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 ; Expected to transform
 define <4 x float> @mul_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; CHECK-LABEL: mul_mul:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    zip2 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip1 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip1 v6.2s, v0.2s, v4.2s
-; CHECK-NEXT:    zip2 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT:    ext v4.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    fmul v3.2s, v5.2s, v6.2s
-; CHECK-NEXT:    fmul v5.2s, v5.2s, v0.2s
-; CHECK-NEXT:    fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT:    fneg v0.2s, v5.2s
-; CHECK-NEXT:    zip2 v5.2s, v2.2s, v4.2s
-; CHECK-NEXT:    zip1 v2.2s, v2.2s, v4.2s
-; CHECK-NEXT:    fmla v0.2s, v6.2s, v1.2s
-; CHECK-NEXT:    fmul v1.2s, v3.2s, v5.2s
-; CHECK-NEXT:    fmul v4.2s, v0.2s, v5.2s
-; CHECK-NEXT:    fneg v1.2s, v1.2s
-; CHECK-NEXT:    fmla v4.2s, v2.2s, v3.2s
-; CHECK-NEXT:    fmla v1.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v2.2s, v1.2s, v4.2s
-; CHECK-NEXT:    zip1 v0.2s, v1.2s, v4.2s
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.4s, v0.4s, v1.4s, #0
+; CHECK-NEXT:    fcmla v4.4s, v0.4s, v1.4s, #90
+; CHECK-NEXT:    fcmla v3.4s, v4.4s, v2.4s, #0
+; CHECK-NEXT:    fcmla v3.4s, v4.4s, v2.4s, #90
+; CHECK-NEXT:    mov v0.16b, v3.16b
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec151 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec153 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec154 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %0 = fmul fast <2 x float> %strided.vec154, %strided.vec151
   %1 = fmul fast <2 x float> %strided.vec153, %strided.vec
entry:
   ret <4 x float> %interleaved.vec
 }
 
 ; (a * b) * a
 ; Expected to transform
 define <4 x float> @mul_triangle(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: mul_triangle:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT:    fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT:    fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    fneg v1.2s, v3.2s
-; CHECK-NEXT:    fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT:    fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT:    fmul v2.2s, v2.2s, v0.2s
-; CHECK-NEXT:    fneg v3.2s, v3.2s
-; CHECK-NEXT:    fmla v2.2s, v4.2s, v1.2s
-; CHECK-NEXT:    fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip2 v1.2s, v3.2s, v2.2s
-; CHECK-NEXT:    zip1 v0.2s, v3.2s, v2.2s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v3.4s, v1.4s, v0.4s, #0
+; CHECK-NEXT:    fcmla v3.4s, v1.4s, v0.4s, #90
+; CHECK-NEXT:    fcmla v2.4s, v0.4s, v3.4s, #0
+; CHECK-NEXT:    fcmla v2.4s, v0.4s, v3.4s, #90
+; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec35 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec37 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec38 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %0 = fmul fast <2 x float> %strided.vec37, %strided.vec
   %1 = fmul fast <2 x float> %strided.vec38, %strided.vec35

 }
 
 
 ; d * (b * a) * (c * a)
 ; Expected to transform
 define <4 x float> @mul_diamond(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d) {
 ; CHECK-LABEL: mul_diamond:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v5.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v6.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    ext v4.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT:    zip2 v7.2s, v0.2s, v5.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v5.2s
-; CHECK-NEXT:    zip2 v16.2s, v1.2s, v6.2s
-; CHECK-NEXT:    zip1 v1.2s, v1.2s, v6.2s
-; CHECK-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    fmul v17.2s, v16.2s, v7.2s
-; CHECK-NEXT:    fmul v18.2s, v7.2s, v1.2s
-; CHECK-NEXT:    fneg v5.2s, v17.2s
-; CHECK-NEXT:    zip2 v17.2s, v3.2s, v4.2s
-; CHECK-NEXT:    zip1 v3.2s, v3.2s, v4.2s
-; CHECK-NEXT:    fmla v18.2s, v16.2s, v0.2s
-; CHECK-NEXT:    fmla v5.2s, v1.2s, v0.2s
-; CHECK-NEXT:    zip1 v1.2s, v2.2s, v6.2s
-; CHECK-NEXT:    zip2 v2.2s, v2.2s, v6.2s
-; CHECK-NEXT:    fmul v16.2s, v18.2s, v17.2s
-; CHECK-NEXT:    fmul v4.2s, v5.2s, v17.2s
-; CHECK-NEXT:    fmul v6.2s, v1.2s, v7.2s
-; CHECK-NEXT:    fmul v7.2s, v2.2s, v7.2s
-; CHECK-NEXT:    fmla v4.2s, v18.2s, v3.2s
-; CHECK-NEXT:    fmla v6.2s, v0.2s, v2.2s
-; CHECK-NEXT:    fneg v2.2s, v7.2s
-; CHECK-NEXT:    fneg v7.2s, v16.2s
-; CHECK-NEXT:    fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    fmul v0.2s, v6.2s, v4.2s
-; CHECK-NEXT:    fmla v7.2s, v5.2s, v3.2s
-; CHECK-NEXT:    fmul v1.2s, v2.2s, v4.2s
-; CHECK-NEXT:    fneg v0.2s, v0.2s
-; CHECK-NEXT:    fmla v1.2s, v7.2s, v6.2s
-; CHECK-NEXT:    fmla v0.2s, v7.2s, v2.2s
-; CHECK-NEXT:    zip2 v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.4s, v1.4s, v0.4s, #0
+; CHECK-NEXT:    fcmla v6.4s, v2.4s, v0.4s, #0
+; CHECK-NEXT:    fcmla v4.4s, v1.4s, v0.4s, #90
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v6.4s, v2.4s, v0.4s, #90
+; CHECK-NEXT:    fcmla v5.4s, v4.4s, v3.4s, #0
+; CHECK-NEXT:    fcmla v5.4s, v4.4s, v3.4s, #90
+; CHECK-NEXT:    fcmla v1.4s, v6.4s, v5.4s, #0
+; CHECK-NEXT:    fcmla v1.4s, v6.4s, v5.4s, #90
+; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %c.real = shufflevector <4 x float> %c, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %c.imag = shufflevector <4 x float> %c, <4 x float> poison, <2 x i32> <i32 1, i32 3>
entry:
   %interleaved.vec = shufflevector <2 x float> %23, <2 x float> %20, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x float> @mul_add90_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; CHECK-LABEL: mul_add90_mul:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v7.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    zip2 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip1 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v6.2s, v0.2s, v4.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT:    zip2 v16.2s, v2.2s, v7.2s
-; CHECK-NEXT:    zip1 v2.2s, v2.2s, v7.2s
-; CHECK-NEXT:    fmul v17.2s, v5.2s, v6.2s
-; CHECK-NEXT:    fmul v7.2s, v1.2s, v6.2s
-; CHECK-NEXT:    fmul v4.2s, v16.2s, v6.2s
-; CHECK-NEXT:    fneg v3.2s, v17.2s
-; CHECK-NEXT:    fmla v7.2s, v0.2s, v5.2s
-; CHECK-NEXT:    fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT:    fneg v1.2s, v4.2s
-; CHECK-NEXT:    fmla v1.2s, v0.2s, v2.2s
-; CHECK-NEXT:    fmla v3.2s, v6.2s, v2.2s
-; CHECK-NEXT:    fmla v3.2s, v0.2s, v16.2s
-; CHECK-NEXT:    fsub v0.2s, v1.2s, v7.2s
-; CHECK-NEXT:    zip2 v1.2s, v0.2s, v3.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v3.4s, v1.4s, v0.4s, #0
+; CHECK-NEXT:    fcmla v4.4s, v2.4s, v0.4s, #0
+; CHECK-NEXT:    fcmla v3.4s, v1.4s, v0.4s, #90
+; CHECK-NEXT:    fcmla v4.4s, v2.4s, v0.4s, #90
+; CHECK-NEXT:    fcadd v0.4s, v4.4s, v3.4s, #90
 ; CHECK-NEXT:    ret
 entry:
   %ar = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %ai = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %br = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %bi = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %cr = shufflevector <4 x float> %c, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %ci = shufflevector <4 x float> %c, <4 x float> poison, <2 x i32> <i32 1, i32 3>

llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+complxnum,+neon,+fullfp16 -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 ; Expected to transform
 define <4 x float> @simple_mul(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: simple_mul:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip1 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT:    fmul v3.2s, v5.2s, v0.2s
-; CHECK-NEXT:    fneg v2.2s, v2.2s
-; CHECK-NEXT:    fmla v3.2s, v4.2s, v1.2s
-; CHECK-NEXT:    fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip2 v1.2s, v2.2s, v3.2s
-; CHECK-NEXT:    zip1 v0.2s, v2.2s, v3.2s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v2.4s, v0.4s, v1.4s, #0
+; CHECK-NEXT:    fcmla v2.4s, v0.4s, v1.4s, #90
+; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec17 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec19 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec20 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %0 = fmul fast <2 x float> %strided.vec20, %strided.vec
   %1 = fmul fast <2 x float> %strided.vec19, %strided.vec17
entry:
   %interleaved.vec = shufflevector <2 x float> %5, <2 x float> %2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x float> @three_way_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; CHECK-LABEL: three_way_mul:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v6.2s, v0.2s, v4.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT:    ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    fmul v4.2s, v6.2s, v5.2s
-; CHECK-NEXT:    fmul v6.2s, v1.2s, v6.2s
-; CHECK-NEXT:    fmla v4.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip2 v1.2s, v2.2s, v3.2s
-; CHECK-NEXT:    fneg v6.2s, v6.2s
-; CHECK-NEXT:    zip1 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT:    fmul v3.2s, v4.2s, v1.2s
-; CHECK-NEXT:    fmla v6.2s, v0.2s, v5.2s
-; CHECK-NEXT:    fmul v0.2s, v4.2s, v2.2s
-; CHECK-NEXT:    fneg v3.2s, v3.2s
-; CHECK-NEXT:    fmla v0.2s, v1.2s, v6.2s
-; CHECK-NEXT:    fmla v3.2s, v2.2s, v6.2s
-; CHECK-NEXT:    zip2 v1.2s, v3.2s, v0.2s
-; CHECK-NEXT:    zip1 v0.2s, v3.2s, v0.2s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.4s, v1.4s, v0.4s, #0
+; CHECK-NEXT:    fcmla v4.4s, v1.4s, v0.4s, #90
+; CHECK-NEXT:    fcmla v3.4s, v2.4s, v4.4s, #0
+; CHECK-NEXT:    fcmla v3.4s, v2.4s, v4.4s, #90
+; CHECK-NEXT:    mov v0.16b, v3.16b
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec39 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec41 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec42 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec44 = shufflevector <4 x float> %c, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec45 = shufflevector <4 x float> %c, <4 x float> poison, <2 x i32> <i32 1, i32 3>
entry:
   %interleaved.vec = shufflevector <2 x float> %11, <2 x float> %8, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x float> @simple_add_90(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: simple_add_90:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v2.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    fsub v0.2s, v2.2s, v0.2s
-; CHECK-NEXT:    fadd v1.2s, v1.2s, v4.2s
-; CHECK-NEXT:    zip2 v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    fcadd v0.4s, v1.4s, v0.4s, #90
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec17 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec19 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec20 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %0 = fsub fast <2 x float> %strided.vec19, %strided.vec17
   %1 = fadd fast <2 x float> %strided.vec20, %strided.vec
entry:
   %interleaved.vec = shufflevector <2 x float> %0, <2 x float> %1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x float> %interleaved.vec
 }
 
 ; Expected to transform
 define <4 x float> @simple_add_270_true(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: simple_add_270_true:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v2.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    fadd v1.2s, v4.2s, v1.2s
-; CHECK-NEXT:    fsub v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v2.2s, v1.2s, v0.2s
-; CHECK-NEXT:    zip1 v0.2s, v1.2s, v0.2s
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    fcadd v0.4s, v0.4s, v1.4s, #270
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec17 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec19 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec20 = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %0 = fadd fast <2 x float> %strided.vec, %strided.vec20
   %1 = fsub fast <2 x float> %strided.vec17, %strided.vec19
entry:
   %interleaved.vec2 = shufflevector <4 x float> %interleaved.vec, <4 x float> %dup, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %interleaved.vec2
 }
 
 ; Expected to transform
 define <4 x float> @mul_mul_with_fneg(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: mul_mul_with_fneg:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT:    fmul v3.2s, v4.2s, v1.2s
-; CHECK-NEXT:    fmul v4.2s, v4.2s, v5.2s
-; CHECK-NEXT:    fneg v2.2s, v3.2s
-; CHECK-NEXT:    fneg v3.2s, v4.2s
-; CHECK-NEXT:    fmls v2.2s, v5.2s, v0.2s
-; CHECK-NEXT:    fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT:    zip2 v1.2s, v3.2s, v2.2s
-; CHECK-NEXT:    zip1 v0.2s, v3.2s, v2.2s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v2.4s, v1.4s, v0.4s, #270
+; CHECK-NEXT:    fcmla v2.4s, v1.4s, v0.4s, #180
+; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ret
 entry:
   %a.real = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %a.imag = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %b.real = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %b.imag = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   %0 = fneg fast <2 x float> %a.imag
   %1 = fmul fast <2 x float> %b.real, %0