[CodeGen] Add support for Splats in ComplexDeinterleaving pass

[igor.kirillov]

Summary

This commit allows generating of complex number intrinsics for expressions
with constants or loops invariants, which are represented as splats.
For instance, after vectorizing loops in the following code snippets,
the ComplexDeinterleaving pass will be able to generate complex number
intrinsics:

complex<> x = ...;
for (int i = 0; i < N; ++i)
    c[i] = a[i] * b[i] * x;

or

for (int i = 0; i < N; ++i)
    c[i] = a[i] * b[i] * (11.0 + 3.0i);

Depends on D153446 and D153856

[igor.kirillov]

Refactor part of the code into separate revision (D153446)

[igor.kirillov]

Rebase

[NickGuy]

Looks good to me, but can you precommit the tests so we can see the expected differences?

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll

53

What's the reason for the square brackets here?

[igor.kirillov]

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll

53

That's how the complex<double> type variable looks when translated into the IR. So I thought it is better to leave it as aggregate.

[igor.kirillov]

Precommit tests

[glandium]

This causes a segfault when linking Firefox for arm64 mac with LTO. I can provide a reproducer if necessary, but it's large (several GB). The stack trace looks like this:

Stack dump:
0.	Running pass 'Function Pass Manager' on module 'builds/worker/workspace/obj-build/aarch64-apple-darwin/release/libgkrust.a(qcms-9781f4d323403c02.qcms.81a5ab1e-cgu.0.rcgu.o)56213408'.
1.	Running pass 'Complex Deinterleaving Pass' on function '@qcms_transform_data_rgb_out_lut_neon'
 #0 0x00007fb7032c6068 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) /tmp/llvm/llvm/lib/Support/Unix/Signals.inc:602:13
 #1 0x00007fb7032c4440 llvm::sys::RunSignalHandlers() /tmp/llvm/llvm/lib/Support/Signals.cpp:105:18
 #2 0x00007fb7032c66e8 SignalHandler(int) /tmp/llvm/llvm/lib/Support/Unix/Signals.inc:413:1
 #3 0x00007fb70245af90 (/lib/x86_64-linux-gnu/libc.so.6+0x3bf90)
 #4 0x00007fb70334be8e llvm::User::getNumOperands() const /tmp/llvm/llvm/include/llvm/IR/User.h:191:44
 #5 0x00007fb70334be8e llvm::HungoffOperandTraits<2u>::operands(llvm::User const*) /tmp/llvm/llvm/include/llvm/IR/OperandTraits.h:103:15
 #6 0x00007fb70334be8e llvm::PHINode::getNumOperands() const /tmp/llvm/llvm/include/llvm/IR/Instructions.h:2973:1
 #7 0x00007fb70334be8e llvm::PHINode::addIncoming(llvm::Value*, llvm::BasicBlock*) /tmp/llvm/llvm/include/llvm/IR/Instructions.h:2886:9
 #8 0x00007fb703545e92 (anonymous namespace)::ComplexDeinterleavingGraph::processReductionOperation(llvm::Value*, (anonymous namespace)::ComplexDeinterleavingCompositeNode*) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:1943:45
 #9 0x00007fb703545e92 (anonymous namespace)::ComplexDeinterleavingGraph::replaceNode(llvm::IRBuilderBase&, (anonymous namespace)::ComplexDeinterleavingCompositeNode*) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:1900:5
#10 0x00007fb70353d1e2 (anonymous namespace)::ComplexDeinterleavingGraph::replaceNodes() /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:1976:29
#11 0x00007fb70353d1e2 (anonymous namespace)::ComplexDeinterleaving::evaluateBasicBlock(llvm::BasicBlock*) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:520:11
#12 0x00007fb70353d1e2 (anonymous namespace)::ComplexDeinterleaving::runOnFunction(llvm::Function&) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:479:16
#13 0x00007fb70353e35f (anonymous namespace)::ComplexDeinterleavingLegacyPass::runOnFunction(llvm::Function&) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:460:42
#14 0x00007fb7033f3216 llvm::FPPassManager::runOnFunction(llvm::Function&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:1435:27
#15 0x00007fb7033f8b13 llvm::FPPassManager::runOnModule(llvm::Module&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:1481:13
#16 0x00007fb7033f389e (anonymous namespace)::MPPassManager::runOnModule(llvm::Module&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:0:27
#17 0x00007fb7033f389e llvm::legacy::PassManagerImpl::run(llvm::Module&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:535:44
#18 0x00007fb7047898ee std::unique_ptr<llvm::ToolOutputFile, std::default_delete<llvm::ToolOutputFile>>::operator bool() const /usr/lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/unique_ptr.h:479:22
#19 0x00007fb7047898ee codegen(llvm::lto::Config const&, llvm::TargetMachine*, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, unsigned int, llvm::Module&, llvm::ModuleSummaryIndex const&) /tmp/llvm/llvm/lib/LTO/LTOBackend.cpp:413:31
#20 0x00007fb70478a7a2 std::_Function_base::~_Function_base() /usr/lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:243:11
#21 0x00007fb70478a7a2 llvm::lto::thinBackend(llvm::lto::Config const&, unsigned int, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, llvm::Module&, llvm::ModuleSummaryIndex const&, llvm::StringMap<std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>, llvm::MallocAllocator> const&, llvm::DenseMap<unsigned long, llvm::GlobalValueSummary*, llvm::DenseMapInfo<unsigned long, void>, llvm::detail::DenseMapPair<unsigned long, llvm::GlobalValueSummary*>> const&, llvm::MapVector<llvm::StringRef, llvm::BitcodeModule, llvm::DenseMap<llvm::StringRef, unsigned int, llvm::DenseMapInfo<llvm::StringRef, void>, llvm::detail::DenseMapPair<llvm::StringRef, unsigned int>>, std::vector<std::pair<llvm::StringRef, llvm::BitcodeModule>, std::allocator<std::pair<llvm::StringRef, llvm::BitcodeModule>>>>*, std::vector<unsigned char, std::allocator<unsigned char>> const&)::$_1::operator()(llvm::Module&, llvm::TargetMachine*, std::unique_ptr<llvm::ToolOutputFile, std::default_delete<llvm::ToolOutputFile>>) const /tmp/llvm/llvm/lib/LTO/LTOBackend.cpp:585:9
#22 0x00007fb70478a63d llvm::lto::thinBackend(llvm::lto::Config const&, unsigned int, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, llvm::Module&, llvm::ModuleSummaryIndex const&, llvm::StringMap<std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>, llvm::MallocAllocator> const&, llvm::DenseMap<unsigned long, llvm::GlobalValueSummary*, llvm::DenseMapInfo<unsigned long, void>, llvm::detail::DenseMapPair<unsigned long, llvm::GlobalValueSummary*>> const&, llvm::MapVector<llvm::StringRef, llvm::BitcodeModule, llvm::DenseMap<llvm::StringRef, unsigned int, llvm::DenseMapInfo<llvm::StringRef, void>, llvm::detail::DenseMapPair<llvm::StringRef, unsigned int>>, std::vector<std::pair<llvm::StringRef, llvm::BitcodeModule>, std::allocator<std::pair<llvm::StringRef, llvm::BitcodeModule>>>>*, std::vector<unsigned char, std::allocator<unsigned char>> const&) /tmp/llvm/llvm/lib/LTO/LTOBackend.cpp:0:10

[igor.kirillov]

@glandium, thanks. I've reproduced and prepared the solution - https://reviews.llvm.org/D154598

llvm/include/llvm/CodeGen/ComplexDeinterleavingPass.h

 };
 
 enum class ComplexDeinterleavingOperation {
   CAdd,
   CMulPartial,
   // The following 'operations' are used to represent internal states. Backends
   // are not expected to try and support these in any capacity.
   Deinterleave,
+  Splat,
   Symmetric,
   ReductionPHI,
   ReductionOperation,
   ReductionSelect,
 };
 
 enum class ComplexDeinterleavingRotation {
   Rotation_0 = 0,
   Rotation_90 = 1,
   Rotation_180 = 2,
   Rotation_270 = 3,
 };
 
 } // namespace llvm
 
 #endif // LLVM_CODEGEN_COMPLEXDEINTERLEAVING_H

llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp

 private:
   const TargetMachine *TM;
 };
 
 class ComplexDeinterleavingGraph;
 struct ComplexDeinterleavingCompositeNode {
 
   ComplexDeinterleavingCompositeNode(ComplexDeinterleavingOperation Op,
-                                     Instruction *R, Instruction *I)
+                                     Value *R, Value *I)
       : Operation(Op), Real(R), Imag(I) {}
 
 private:
   friend class ComplexDeinterleavingGraph;
   using NodePtr = std::shared_ptr<ComplexDeinterleavingCompositeNode>;
   using RawNodePtr = ComplexDeinterleavingCompositeNode *;
 
 public:
   ComplexDeinterleavingOperation Operation;
-  Instruction *Real;
-  Instruction *Imag;
+  Value *Real;
+  Value *Imag;
 
   // This two members are required exclusively for generating
   // ComplexDeinterleavingOperation::Symmetric operations.
   unsigned Opcode;
   FastMathFlags Flags;
 
   ComplexDeinterleavingRotation Rotation =
       ComplexDeinterleavingRotation::Rotation_0;
for (const auto &Op : Operands) {
       PrintNodeRef(Op);
     }
   }
 };
 
 class ComplexDeinterleavingGraph {
 public:
   struct Product {
-    Instruction *Multiplier;
-    Instruction *Multiplicand;
+    Value *Multiplier;
+    Value *Multiplicand;
     bool IsPositive;
   };
 
-  using Addend = std::pair<Instruction *, bool>;
+  using Addend = std::pair<Value *, bool>;
   using NodePtr = ComplexDeinterleavingCompositeNode::NodePtr;
   using RawNodePtr = ComplexDeinterleavingCompositeNode::RawNodePtr;
 
   // Helper struct for holding info about potential partial multiplication
   // candidates
   struct PartialMulCandidate {
-    Instruction *Common;
+    Value *Common;
     NodePtr Node;
     unsigned RealIdx;
     unsigned ImagIdx;
     bool IsNodeInverted;
   };
 
   explicit ComplexDeinterleavingGraph(const TargetLowering *TL,
                                       const TargetLibraryInfo *TLI)
private:
   /// The new PHINode corresponds to a vector of deinterleaved complex numbers.
   /// This mapping is populated during
   /// ComplexDeinterleavingOperation::ReductionPHI node replacement. It is then
   /// used in the ComplexDeinterleavingOperation::ReductionOperation node
   /// replacement process.
   std::map<PHINode *, PHINode *> OldToNewPHI;
 
   NodePtr prepareCompositeNode(ComplexDeinterleavingOperation Operation,
-                               Instruction *R, Instruction *I) {
+                               Value *R, Value *I) {
     assert(((Operation != ComplexDeinterleavingOperation::ReductionPHI &&
              Operation != ComplexDeinterleavingOperation::ReductionOperation) ||
             (R && I)) &&
            "Reduction related nodes must have Real and Imaginary parts");
     return std::make_shared<ComplexDeinterleavingCompositeNode>(Operation, R,
                                                                 I);
   }
 
private:
   ///      i: ci - ar * bi
   /// 270: r: cr + ai * bi
   ///      i: ci - ai * br
   NodePtr identifyPartialMul(Instruction *Real, Instruction *Imag);
 
   /// Identify the other branch of a Partial Mul, taking the CommonOperandI that
   /// is partially known from identifyPartialMul, filling in the other half of
   /// the complex pair.
-  NodePtr identifyNodeWithImplicitAdd(
-      Instruction *I, Instruction *J,
-      std::pair<Instruction *, Instruction *> &CommonOperandI);
+  NodePtr
+  identifyNodeWithImplicitAdd(Instruction *I, Instruction *J,
+                              std::pair<Value *, Value *> &CommonOperandI);
 
   /// Identifies a complex add pattern and its rotation, based on the following
   /// patterns.
   ///
   /// 90:  r: ar - bi
   ///      i: ai + br
   /// 270: r: ar + bi
   ///      i: ai - br
   NodePtr identifyAdd(Instruction *Real, Instruction *Imag);
   NodePtr identifySymmetricOperation(Instruction *Real, Instruction *Imag);
 
-  NodePtr identifyNode(Instruction *I, Instruction *J);
+  NodePtr identifyNode(Value *R, Value *I);
 
   /// Determine if a sum of complex numbers can be formed from \p RealAddends
   /// and \p ImagAddens. If \p Accumulator is not null, add the result to it.
   /// Return nullptr if it is not possible to construct a complex number.
   /// \p Flags are needed to generate symmetric Add and Sub operations.
   NodePtr identifyAdditions(std::list<Addend> &RealAddends,
                             std::list<Addend> &ImagAddends, FastMathFlags Flags,
                             NodePtr Accumulator);
private:
   /// complex numbers. There are two ways to represent the Deinterleave
   /// operation:
   /// * Using two shufflevectors with even indices for /pReal instruction and
   /// odd indices for /pImag instructions (only for fixed-width vectors)
   /// * Using two extractvalue instructions applied to `vector.deinterleave2`
   /// intrinsic (for both fixed and scalable vectors)
   NodePtr identifyDeinterleave(Instruction *Real, Instruction *Imag);
 
+  /// identifying the operation that represents a complex number repeated in a
+  /// Splat vector. There are two possible types of splats: ConstantExpr with
+  /// the opcode ShuffleVector and ShuffleVectorInstr. Both should have an
+  /// initialization mask with all values set to zero.
+  NodePtr identifySplat(Value *Real, Value *Imag);
+
   NodePtr identifyPHINode(Instruction *Real, Instruction *Imag);
 
   /// Identifies SelectInsts in a loop that has reduction with predication masks
   /// and/or predicated tail folding
   NodePtr identifySelectNode(Instruction *Real, Instruction *Imag);
 
   Value *replaceNode(IRBuilderBase &Builder, RawNodePtr Node);
 
bool ComplexDeinterleaving::evaluateBasicBlock(BasicBlock *B) {
   }
 
   return false;
 }
 
 ComplexDeinterleavingGraph::NodePtr
 ComplexDeinterleavingGraph::identifyNodeWithImplicitAdd(
     Instruction *Real, Instruction *Imag,
-    std::pair<Instruction *, Instruction *> &PartialMatch) {
+    std::pair<Value *, Value *> &PartialMatch) {
   LLVM_DEBUG(dbgs() << "identifyNodeWithImplicitAdd " << *Real << " / " << *Imag
                     << "\n");
 
   if (!Real->hasOneUse() || !Imag->hasOneUse()) {
     LLVM_DEBUG(dbgs() << "  - Mul operand has multiple uses.\n");
     return nullptr;
   }
 
   if (Real->getOpcode() != Instruction::FMul ||
       Imag->getOpcode() != Instruction::FMul) {
     LLVM_DEBUG(dbgs() << "  - Real or imaginary instruction is not fmul\n");
     return nullptr;
   }
 
-  Instruction *R0 = dyn_cast<Instruction>(Real->getOperand(0));
-  Instruction *R1 = dyn_cast<Instruction>(Real->getOperand(1));
-  Instruction *I0 = dyn_cast<Instruction>(Imag->getOperand(0));
-  Instruction *I1 = dyn_cast<Instruction>(Imag->getOperand(1));
-  if (!R0 || !R1 || !I0 || !I1) {
-    LLVM_DEBUG(dbgs() << "  - Mul operand not Instruction\n");
-    return nullptr;
-  }
+  Value *R0 = Real->getOperand(0);
+  Value *R1 = Real->getOperand(1);
+  Value *I0 = Imag->getOperand(0);
+  Value *I1 = Imag->getOperand(1);
 
   // A +/+ has a rotation of 0. If any of the operands are fneg, we flip the
   // rotations and use the operand.
   unsigned Negs = 0;
   SmallVector<Instruction *> FNegs;
-  if (R0->getOpcode() == Instruction::FNeg ||
-      R1->getOpcode() == Instruction::FNeg) {
+  Value *Op;
+  if (match(R0, m_Neg(m_Value(Op)))) {
     Negs |= 1;
-    if (R0->getOpcode() == Instruction::FNeg) {
-      FNegs.push_back(R0);
-      R0 = dyn_cast<Instruction>(R0->getOperand(0));
-    } else {
-      FNegs.push_back(R1);
-      R1 = dyn_cast<Instruction>(R1->getOperand(0));
-    }
-    if (!R0 || !R1)
-      return nullptr;
+    FNegs.push_back(cast<Instruction>(R0));
+    R0 = Op;
+  } else if (match(R1, m_Neg(m_Value(Op)))) {
+    Negs |= 1;
+    FNegs.push_back(cast<Instruction>(R1));
+    R1 = Op;
   }
-  if (I0->getOpcode() == Instruction::FNeg ||
-      I1->getOpcode() == Instruction::FNeg) {
+
+  if (match(I0, m_Neg(m_Value(Op)))) {
     Negs |= 2;
     Negs ^= 1;
-    if (I0->getOpcode() == Instruction::FNeg) {
-      FNegs.push_back(I0);
-      I0 = dyn_cast<Instruction>(I0->getOperand(0));
-    } else {
-      FNegs.push_back(I1);
-      I1 = dyn_cast<Instruction>(I1->getOperand(0));
-    }
-    if (!I0 || !I1)
-      return nullptr;
+    FNegs.push_back(cast<Instruction>(I0));
+    I0 = Op;
+  } else if (match(I1, m_Neg(m_Value(Op)))) {
+    Negs |= 2;
+    Negs ^= 1;
+    FNegs.push_back(cast<Instruction>(I0));
+    I1 = Op;
   }
 
   ComplexDeinterleavingRotation Rotation = (ComplexDeinterleavingRotation)Negs;
 
-  Instruction *CommonOperand;
-  Instruction *UncommonRealOp;
-  Instruction *UncommonImagOp;
+  Value *CommonOperand;
+  Value *UncommonRealOp;
+  Value *UncommonImagOp;
 
   if (R0 == I0 || R0 == I1) {
     CommonOperand = R0;
     UncommonRealOp = R1;
   } else if (R1 == I0 || R1 == I1) {
     CommonOperand = R1;
     UncommonRealOp = R0;
   } else {
ComplexDeinterleavingGraph::identifyPartialMul(Instruction *Real,
   if (!ImagMulI)
     return nullptr;
 
   if (!RealMulI->hasOneUse() || !ImagMulI->hasOneUse()) {
     LLVM_DEBUG(dbgs() << "  - Mul instruction has multiple uses\n");
     return nullptr;
   }
 
-  Instruction *R0 = dyn_cast<Instruction>(RealMulI->getOperand(0));
-  Instruction *R1 = dyn_cast<Instruction>(RealMulI->getOperand(1));
-  Instruction *I0 = dyn_cast<Instruction>(ImagMulI->getOperand(0));
-  Instruction *I1 = dyn_cast<Instruction>(ImagMulI->getOperand(1));
-  if (!R0 || !R1 || !I0 || !I1) {
-    LLVM_DEBUG(dbgs() << "  - Mul operand not Instruction\n");
-    return nullptr;
-  }
-
-  Instruction *CommonOperand;
-  Instruction *UncommonRealOp;
-  Instruction *UncommonImagOp;
+  Value *R0 = RealMulI->getOperand(0);
+  Value *R1 = RealMulI->getOperand(1);
+  Value *I0 = ImagMulI->getOperand(0);
+  Value *I1 = ImagMulI->getOperand(1);
+
+  Value *CommonOperand;
+  Value *UncommonRealOp;
+  Value *UncommonImagOp;
 
   if (R0 == I0 || R0 == I1) {
     CommonOperand = R0;
     UncommonRealOp = R1;
   } else if (R1 == I0 || R1 == I1) {
     CommonOperand = R1;
     UncommonRealOp = R0;
   } else {
     LLVM_DEBUG(dbgs() << "  - No equal operand\n");
     return nullptr;
   }
 
   UncommonImagOp = (CommonOperand == I0) ? I1 : I0;
   if (Rotation == ComplexDeinterleavingRotation::Rotation_90 ||
       Rotation == ComplexDeinterleavingRotation::Rotation_270)
     std::swap(UncommonRealOp, UncommonImagOp);
 
-  std::pair<Instruction *, Instruction *> PartialMatch(
+  std::pair<Value *, Value *> PartialMatch(
       (Rotation == ComplexDeinterleavingRotation::Rotation_0 ||
        Rotation == ComplexDeinterleavingRotation::Rotation_180)
           ? CommonOperand
           : nullptr,
       (Rotation == ComplexDeinterleavingRotation::Rotation_90 ||
        Rotation == ComplexDeinterleavingRotation::Rotation_270)
           ? CommonOperand
           : nullptr);
ComplexDeinterleavingGraph::identifySymmetricOperation(Instruction *Real,
                                                        Instruction *Imag) {
   if (Real->getOpcode() != Imag->getOpcode())
     return nullptr;
 
   if (!isInstructionPotentiallySymmetric(Real) ||
       !isInstructionPotentiallySymmetric(Imag))
     return nullptr;
 
-  auto *R0 = dyn_cast<Instruction>(Real->getOperand(0));
-  auto *I0 = dyn_cast<Instruction>(Imag->getOperand(0));
-
-  if (!R0 || !I0)
-    return nullptr;
+  auto *R0 = Real->getOperand(0);
+  auto *I0 = Imag->getOperand(0);
 
   NodePtr Op0 = identifyNode(R0, I0);
   NodePtr Op1 = nullptr;
   if (Op0 == nullptr)
     return nullptr;
 
   if (Real->isBinaryOp()) {
-    auto *R1 = dyn_cast<Instruction>(Real->getOperand(1));
-    auto *I1 = dyn_cast<Instruction>(Imag->getOperand(1));
-    if (!R1 || !I1)
-      return nullptr;
-
+    auto *R1 = Real->getOperand(1);
+    auto *I1 = Imag->getOperand(1);
     Op1 = identifyNode(R1, I1);
     if (Op1 == nullptr)
       return nullptr;
   }
 
   if (isa<FPMathOperator>(Real) &&
       Real->getFastMathFlags() != Imag->getFastMathFlags())
     return nullptr;
 
   auto Node = prepareCompositeNode(ComplexDeinterleavingOperation::Symmetric,
                                    Real, Imag);
   Node->Opcode = Real->getOpcode();
   if (isa<FPMathOperator>(Real))
     Node->Flags = Real->getFastMathFlags();
 
   Node->addOperand(Op0);
   if (Real->isBinaryOp())
     Node->addOperand(Op1);
 
   return submitCompositeNode(Node);
 }
 
 ComplexDeinterleavingGraph::NodePtr
-ComplexDeinterleavingGraph::identifyNode(Instruction *Real, Instruction *Imag) {
-  LLVM_DEBUG(dbgs() << "identifyNode on " << *Real << " / " << *Imag << "\n");
-  if (NodePtr CN = getContainingComposite(Real, Imag)) {
+ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I) {
+  LLVM_DEBUG(dbgs() << "identifyNode on " << *R << " / " << *I << "\n");
+  if (NodePtr CN = getContainingComposite(R, I)) {
     LLVM_DEBUG(dbgs() << " - Folding to existing node\n");
     return CN;
   }
 
+  if (NodePtr CN = identifySplat(R, I))
+    return CN;
+
+  auto *Real = dyn_cast<Instruction>(R);
+  auto *Imag = dyn_cast<Instruction>(I);
+  if (!Real || !Imag)
+    return nullptr;
+
   if (NodePtr CN = identifyDeinterleave(Real, Imag))
     return CN;
 
   if (NodePtr CN = identifyPHINode(Real, Imag))
     return CN;
 
   if (NodePtr CN = identifySelectNode(Real, Imag))
     return CN;
ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I) {
 
   LLVM_DEBUG(dbgs() << "  - Not recognised as a valid pattern.\n");
   return nullptr;
 }
 
 ComplexDeinterleavingGraph::NodePtr
 ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
                                                  Instruction *Imag) {
+
   if ((Real->getOpcode() != Instruction::FAdd &&
        Real->getOpcode() != Instruction::FSub &&
        Real->getOpcode() != Instruction::FNeg) ||
       (Imag->getOpcode() != Instruction::FAdd &&
        Imag->getOpcode() != Instruction::FSub &&
        Imag->getOpcode() != Instruction::FNeg))
     return nullptr;
 
auto Collect = [&Flags](Instruction *Insn, std::vector<Product> &Muls,
     SmallPtrSet<Value *, 8> Visited;
     while (!Worklist.empty()) {
       auto [V, IsPositive] = Worklist.back();
       Worklist.pop_back();
       if (!Visited.insert(V).second)
         continue;
 
       Instruction *I = dyn_cast<Instruction>(V);
-      if (!I)
-        return false;
+      if (!I) {
+        Addends.emplace_back(V, IsPositive);
+        continue;
+      }
 
       // If an instruction has more than one user, it indicates that it either
       // has an external user, which will be later checked by the checkNodes
       // function, or it is a subexpression utilized by multiple expressions. In
       // the latter case, we will attempt to separately identify the complex
       // operation from here in order to create a shared
       // ComplexDeinterleavingCompositeNode.
       if (I != Insn && I->getNumUses() > 1) {
         LLVM_DEBUG(dbgs() << "Found potential sub-expression: " << *I << "\n");
         Addends.emplace_back(I, IsPositive);
         continue;
       }
 
       if (I->getOpcode() == Instruction::FAdd) {
         Worklist.emplace_back(I->getOperand(1), IsPositive);
         Worklist.emplace_back(I->getOperand(0), IsPositive);
       } else if (I->getOpcode() == Instruction::FSub) {
         Worklist.emplace_back(I->getOperand(1), !IsPositive);
         Worklist.emplace_back(I->getOperand(0), IsPositive);
       } else if (I->getOpcode() == Instruction::FMul) {
-        auto *A = dyn_cast<Instruction>(I->getOperand(0));
-        if (A && A->getOpcode() == Instruction::FNeg) {
-          A = dyn_cast<Instruction>(A->getOperand(0));
+        Value *A, *B;
+        if (match(I->getOperand(0), m_FNeg(m_Value(A)))) {
           IsPositive = !IsPositive;
+        } else {
+          A = I->getOperand(0);
         }
-        if (!A)
-          return false;
-        auto *B = dyn_cast<Instruction>(I->getOperand(1));
-        if (B && B->getOpcode() == Instruction::FNeg) {
-          B = dyn_cast<Instruction>(B->getOperand(0));
+
+        if (match(I->getOperand(1), m_FNeg(m_Value(B)))) {
           IsPositive = !IsPositive;
+        } else {
+          B = I->getOperand(1);
         }
-        if (!B)
-          return false;
         Muls.push_back(Product{A, B, IsPositive});
       } else if (I->getOpcode() == Instruction::FNeg) {
         Worklist.emplace_back(I->getOperand(0), !IsPositive);
       } else {
         Addends.emplace_back(I, IsPositive);
         continue;
       }
 
ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
   return FinalNode;
 }
 
 bool ComplexDeinterleavingGraph::collectPartialMuls(
     const std::vector<Product> &RealMuls, const std::vector<Product> &ImagMuls,
     std::vector<PartialMulCandidate> &PartialMulCandidates) {
   // Helper function to extract a common operand from two products
   auto FindCommonInstruction = [](const Product &Real,
-                                  const Product &Imag) -> Instruction * {
+                                  const Product &Imag) -> Value * {
     if (Real.Multiplicand == Imag.Multiplicand ||
         Real.Multiplicand == Imag.Multiplier)
       return Real.Multiplicand;
 
     if (Real.Multiplier == Imag.Multiplicand ||
         Real.Multiplier == Imag.Multiplier)
       return Real.Multiplier;
 
for (unsigned j = 0; j < ImagMuls.size(); ++j) {
       if (!Common)
         continue;
 
       auto *A = RealMuls[i].Multiplicand == Common ? RealMuls[i].Multiplier
                                                    : RealMuls[i].Multiplicand;
       auto *B = ImagMuls[j].Multiplicand == Common ? ImagMuls[j].Multiplier
                                                    : ImagMuls[j].Multiplicand;
 
-      bool Inverted = false;
       auto Node = identifyNode(A, B);
-      if (!Node) {
-        std::swap(A, B);
-        Inverted = true;
-        Node = identifyNode(A, B);
+      if (Node) {
+        FoundCommon = true;
+        PartialMulCandidates.push_back({Common, Node, i, j, false});
       }
-      if (!Node)
-        continue;
 
+      Node = identifyNode(B, A);
+      if (Node) {
         FoundCommon = true;
-      PartialMulCandidates.push_back({Common, Node, i, j, Inverted});
+        PartialMulCandidates.push_back({Common, Node, i, j, true});
+      }
     }
     if (!FoundCommon)
       return false;
   }
   return true;
 }
 
 ComplexDeinterleavingGraph::NodePtr
 ComplexDeinterleavingGraph::identifyMultiplications(
     std::vector<Product> &RealMuls, std::vector<Product> &ImagMuls,
     NodePtr Accumulator = nullptr) {
   if (RealMuls.size() != ImagMuls.size())
     return nullptr;
 
   std::vector<PartialMulCandidate> Info;
   if (!collectPartialMuls(RealMuls, ImagMuls, Info))
     return nullptr;
 
   // Map to store common instruction to node pointers
-  std::map<Instruction *, NodePtr> CommonToNode;
+  std::map<Value *, NodePtr> CommonToNode;
   std::vector<bool> Processed(Info.size(), false);
   for (unsigned I = 0; I < Info.size(); ++I) {
     if (Processed[I])
       continue;
 
     PartialMulCandidate &InfoA = Info[I];
     for (unsigned J = I + 1; J < Info.size(); ++J) {
       if (Processed[J])
NodePtr PlaceholderNode =
                            RealShuffle, ImagShuffle);
   PlaceholderNode->ReplacementNode = RealShuffle->getOperand(0);
   FinalInstructions.insert(RealShuffle);
   FinalInstructions.insert(ImagShuffle);
   return submitCompositeNode(PlaceholderNode);
 }
 
 ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::identifySplat(Value *R, Value *I) {
+  auto IsSplat = [](Value *V) -> bool {
+    // Fixed-width vector with constants
+    if (isa<ConstantDataVector>(V))
+      return true;
+
+    VectorType *VTy;
+    ArrayRef<int> Mask;
+    // Splats are represented differently depending on whether the repeated
+    // value is a constant or an Instruction
+    if (auto *Const = dyn_cast<ConstantExpr>(V)) {
+      if (Const->getOpcode() != Instruction::ShuffleVector)
+        return false;
+      VTy = cast<VectorType>(Const->getType());
+      Mask = Const->getShuffleMask();
+    } else if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
+      VTy = Shuf->getType();
+      Mask = Shuf->getShuffleMask();
+    } else {
+      return false;
+    }
+
+    // When the data type is <1 x Type>, it's not possible to differentiate
+    // between the ComplexDeinterleaving::Deinterleave and
+    // ComplexDeinterleaving::Splat operations.
+    if (!VTy->isScalableTy() && VTy->getElementCount().getKnownMinValue() == 1)
+      return false;
+
+    return all_equal(Mask) && Mask[0] == 0;
+  };
+
+  if (!IsSplat(R) || !IsSplat(I))
+    return nullptr;
+
+  auto *Real = dyn_cast<Instruction>(R);
+  auto *Imag = dyn_cast<Instruction>(I);
+  if ((!Real && Imag) || (Real && !Imag))
+    return nullptr;
+
+  if (Real && Imag) {
+    // Non-constant splats should be in the same basic block
+    if (Real->getParent() != Imag->getParent())
+      return nullptr;
+
+    FinalInstructions.insert(Real);
+    FinalInstructions.insert(Imag);
+  }
+  NodePtr PlaceholderNode =
+      prepareCompositeNode(ComplexDeinterleavingOperation::Splat, R, I);
+  return submitCompositeNode(PlaceholderNode);
+}
+
+ComplexDeinterleavingGraph::NodePtr
 ComplexDeinterleavingGraph::identifyPHINode(Instruction *Real,
                                             Instruction *Imag) {
   if (Real != RealPHI || Imag != ImagPHI)
     return nullptr;
 
   NodePtr PlaceholderNode = prepareCompositeNode(
       ComplexDeinterleavingOperation::ReductionPHI, Real, Imag);
   return submitCompositeNode(PlaceholderNode);
else
       ReplacementNode = TL->createComplexDeinterleavingIR(
           Builder, Node->Operation, Node->Rotation, Input0, Input1,
           Accumulator);
     break;
   }
   case ComplexDeinterleavingOperation::Deinterleave:
     llvm_unreachable("Deinterleave node should already have ReplacementNode");
     break;
+  case ComplexDeinterleavingOperation::Splat: {
+    auto *NewTy = VectorType::getDoubleElementsVectorType(
+        cast<VectorType>(Node->Real->getType()));
+    auto *R = dyn_cast<Instruction>(Node->Real);
+    auto *I = dyn_cast<Instruction>(Node->Imag);
+    if (R && I) {
+      // Splats that are not constant are interleaved where they are located
+      Instruction *InsertPoint = (I->comesBefore(R) ? R : I)->getNextNode();
+      IRBuilder<> IRB(InsertPoint);
+      ReplacementNode =
+          IRB.CreateIntrinsic(Intrinsic::experimental_vector_interleave2, NewTy,
+                              {Node->Real, Node->Imag});
+    } else {
+      ReplacementNode =
+          Builder.CreateIntrinsic(Intrinsic::experimental_vector_interleave2,
+                                  NewTy, {Node->Real, Node->Imag});
+    }
+    break;
+  }
   case ComplexDeinterleavingOperation::ReductionPHI: {
     // If Operation is ReductionPHI, a new empty PHINode is created.
     // It is filled later when the ReductionOperation is processed.
     auto *VTy = cast<VectorType>(Node->Real->getType());
     auto *NewVTy = VectorType::getDoubleElementsVectorType(VTy);
     auto *NewPHI = PHINode::Create(NewVTy, 0, "", BackEdge->getFirstNonPHI());
     OldToNewPHI[dyn_cast<PHINode>(Node->Real)] = NewPHI;
     ReplacementNode = NewPHI;
     break;
   }
   case ComplexDeinterleavingOperation::ReductionOperation:
     ReplacementNode = replaceNode(Builder, Node->Operands[0]);
     processReductionOperation(ReplacementNode, Node);
     break;
   case ComplexDeinterleavingOperation::ReductionSelect: {
-    auto *MaskReal = Node->Real->getOperand(0);
-    auto *MaskImag = Node->Imag->getOperand(0);
+    auto *MaskReal = cast<Instruction>(Node->Real)->getOperand(0);
+    auto *MaskImag = cast<Instruction>(Node->Imag)->getOperand(0);
     auto *A = replaceNode(Builder, Node->Operands[0]);
     auto *B = replaceNode(Builder, Node->Operands[1]);
     auto *NewMaskTy = VectorType::getDoubleElementsVectorType(
         cast<VectorType>(MaskReal->getType()));
     auto *NewMask =
         Builder.CreateIntrinsic(Intrinsic::experimental_vector_interleave2,
                                 NewMaskTy, {MaskReal, MaskImag});
     ReplacementNode = Builder.CreateSelect(NewMask, A, B);
     break;
   }
   default:
     llvm_unreachable(
         "Unhandled case in ComplexDeinterleavingGraph::replaceNode");
     break;
   }
 
   assert(ReplacementNode && "Target failed to create Intrinsic call.");
   NumComplexTransformations += 1;
   Node->ReplacementNode = ReplacementNode;
   return ReplacementNode;
 }
 
 void ComplexDeinterleavingGraph::processReductionOperation(
     Value *OperationReplacement, RawNodePtr Node) {
-  auto *OldPHIReal = ReductionInfo[Node->Real].first;
-  auto *OldPHIImag = ReductionInfo[Node->Imag].first;
+  auto *Real = cast<Instruction>(Node->Real);
+  auto *Imag = cast<Instruction>(Node->Imag);
+  auto *OldPHIReal = ReductionInfo[Real].first;
+  auto *OldPHIImag = ReductionInfo[Imag].first;
   auto *NewPHI = OldToNewPHI[OldPHIReal];
 
-  auto *VTy = cast<VectorType>(Node->Real->getType());
+  auto *VTy = cast<VectorType>(Real->getType());
   auto *NewVTy = VectorType::getDoubleElementsVectorType(VTy);
 
   // We have to interleave initial origin values coming from IncomingBlock
   Value *InitReal = OldPHIReal->getIncomingValueForBlock(Incoming);
   Value *InitImag = OldPHIImag->getIncomingValueForBlock(Incoming);
 
   IRBuilder<> Builder(Incoming->getTerminator());
   auto *NewInit = Builder.CreateIntrinsic(
       Intrinsic::experimental_vector_interleave2, NewVTy, {InitReal, InitImag});
 
   NewPHI->addIncoming(NewInit, Incoming);
   NewPHI->addIncoming(OperationReplacement, BackEdge);
 
   // Deinterleave complex vector outside of loop so that it can be finally
   // reduced
-  auto *FinalReductionReal = ReductionInfo[Node->Real].second;
-  auto *FinalReductionImag = ReductionInfo[Node->Imag].second;
+  auto *FinalReductionReal = ReductionInfo[Real].second;
+  auto *FinalReductionImag = ReductionInfo[Imag].second;
 
   Builder.SetInsertPoint(
       &*FinalReductionReal->getParent()->getFirstInsertionPt());
   auto *Deinterleave = Builder.CreateIntrinsic(
       Intrinsic::experimental_vector_deinterleave2,
       OperationReplacement->getType(), OperationReplacement);
 
   auto *NewReal = Builder.CreateExtractValue(Deinterleave, (uint64_t)0);
-  FinalReductionReal->replaceUsesOfWith(Node->Real, NewReal);
+  FinalReductionReal->replaceUsesOfWith(Real, NewReal);
 
   Builder.SetInsertPoint(FinalReductionImag);
   auto *NewImag = Builder.CreateExtractValue(Deinterleave, 1);
-  FinalReductionImag->replaceUsesOfWith(Node->Imag, NewImag);
+  FinalReductionImag->replaceUsesOfWith(Imag, NewImag);
 }
 
 void ComplexDeinterleavingGraph::replaceNodes() {
   SmallVector<Instruction *, 16> DeadInstrRoots;
   for (auto *RootInstruction : OrderedRoots) {
     // Check if this potential root went through check process and we can
     // deinterleave it
     if (!RootToNode.count(RootInstruction))
       continue;
 
     IRBuilder<> Builder(RootInstruction);
     auto RootNode = RootToNode[RootInstruction];
     Value *R = replaceNode(Builder, RootNode.get());
 
     if (RootNode->Operation ==
         ComplexDeinterleavingOperation::ReductionOperation) {
-      ReductionInfo[RootNode->Real].first->removeIncomingValue(BackEdge);
-      ReductionInfo[RootNode->Imag].first->removeIncomingValue(BackEdge);
-      DeadInstrRoots.push_back(RootNode->Real);
-      DeadInstrRoots.push_back(RootNode->Imag);
+      auto *RootReal = cast<Instruction>(RootNode->Real);
+      auto *RootImag = cast<Instruction>(RootNode->Imag);
+      ReductionInfo[RootReal].first->removeIncomingValue(BackEdge);
+      ReductionInfo[RootImag].first->removeIncomingValue(BackEdge);
+      DeadInstrRoots.push_back(cast<Instruction>(RootReal));
+      DeadInstrRoots.push_back(cast<Instruction>(RootImag));
     } else {
       assert(R && "Unable to find replacement for RootInstruction");
       DeadInstrRoots.push_back(RootInstruction);
       RootInstruction->replaceAllUsesWith(R);
     }
   }
 
   for (auto *I : DeadInstrRoots)
     RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
 }

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64-arm-none-eabi"
+
+; a[i] * b[i] * (11.0 + 3.0.i);
+;
+define <vscale x 4 x double> @complex_mul_const(<vscale x 4 x double> %a, <vscale x 4 x double> %b) {
+; CHECK-LABEL: complex_mul_const:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z4.d, #0 // =0x0
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z5.d, z4.d
+; CHECK-NEXT:    mov z6.d, z4.d
+; CHECK-NEXT:    fcmla z5.d, p0/m, z0.d, z2.d, #0
+; CHECK-NEXT:    fcmla z6.d, p0/m, z1.d, z3.d, #0
+; CHECK-NEXT:    fcmla z5.d, p0/m, z0.d, z2.d, #90
+; CHECK-NEXT:    fcmla z6.d, p0/m, z1.d, z3.d, #90
+; CHECK-NEXT:    fmov z1.d, #3.00000000
+; CHECK-NEXT:    fmov z2.d, #11.00000000
+; CHECK-NEXT:    zip2 z3.d, z2.d, z1.d
+; CHECK-NEXT:    mov z0.d, z4.d
+; CHECK-NEXT:    zip1 z1.d, z2.d, z1.d
+; CHECK-NEXT:    fcmla z4.d, p0/m, z6.d, z3.d, #0
+; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z1.d, #0
+; CHECK-NEXT:    fcmla z4.d, p0/m, z6.d, z3.d, #90
+; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z1.d, #90
+; CHECK-NEXT:    mov z1.d, z4.d
+; CHECK-NEXT:    ret
+entry:
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %a)
+  %0 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %strided.vec48 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %b)
+  %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec48, 0
+  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec48, 1
+  %4 = fmul fast <vscale x 2 x double> %3, %0
+  %5 = fmul fast <vscale x 2 x double> %2, %1
+  %6 = fadd fast <vscale x 2 x double> %4, %5
+  %7 = fmul fast <vscale x 2 x double> %2, %0
+  %8 = fmul fast <vscale x 2 x double> %3, %1
+  %9 = fsub fast <vscale x 2 x double> %7, %8
+  %10 = fmul fast <vscale x 2 x double> %9, shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double 3.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+  %11 = fmul fast <vscale x 2 x double> %6, shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double 1.100000e+01, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+  %12 = fadd fast <vscale x 2 x double> %10, %11
+  %13 = fmul fast <vscale x 2 x double> %9, shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double 1.100000e+01, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+  %14 = fmul fast <vscale x 2 x double> %6, shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double 3.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+  %15 = fsub fast <vscale x 2 x double> %13, %14
+  %interleaved.vec = tail call <vscale x 4 x double> @llvm.experimental.vector.interleave2.nxv4f64(<vscale x 2 x double> %15, <vscale x 2 x double> %12)
+  ret <vscale x 4 x double> %interleaved.vec
+}
+
+; a[i] * b[i] * c;
+;
+define <vscale x 4 x double> @complex_mul_non_const(<vscale x 4 x double> %a, <vscale x 4 x double> %b, [2 x double] %c) {
+; CHECK-LABEL: complex_mul_non_const:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z6.d, #0 // =0x0
+; CHECK-NEXT:    // kill: def $d5 killed $d5 def $z5
+; CHECK-NEXT:    // kill: def $d4 killed $d4 def $z4
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z7.d, z6.d
+; CHECK-NEXT:    mov z24.d, z6.d
+; CHECK-NEXT:    mov z5.d, d5
+; CHECK-NEXT:    mov z4.d, d4
+; CHECK-NEXT:    fcmla z7.d, p0/m, z0.d, z2.d, #0
+; CHECK-NEXT:    fcmla z24.d, p0/m, z1.d, z3.d, #0
+; CHECK-NEXT:    fcmla z7.d, p0/m, z0.d, z2.d, #90
+; CHECK-NEXT:    zip2 z2.d, z4.d, z5.d
+; CHECK-NEXT:    fcmla z24.d, p0/m, z1.d, z3.d, #90
+; CHECK-NEXT:    mov z0.d, z6.d
+; CHECK-NEXT:    zip1 z4.d, z4.d, z5.d
+; CHECK-NEXT:    fcmla z6.d, p0/m, z24.d, z2.d, #0
+; CHECK-NEXT:    fcmla z0.d, p0/m, z7.d, z4.d, #0
+; CHECK-NEXT:    fcmla z6.d, p0/m, z24.d, z2.d, #90
+; CHECK-NEXT:    fcmla z0.d, p0/m, z7.d, z4.d, #90
+; CHECK-NEXT:    mov z1.d, z6.d
+; CHECK-NEXT:    ret
+entry:
+  %c.coerce.fca.0.extract = extractvalue [2 x double] %c, 0
+  %c.coerce.fca.1.extract = extractvalue [2 x double] %c, 1
+  %broadcast.splatinsert = insertelement <vscale x 2 x double> poison, double %c.coerce.fca.1.extract, i64 0
+  %broadcast.splat = shufflevector <vscale x 2 x double> %broadcast.splatinsert, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+  %broadcast.splatinsert49 = insertelement <vscale x 2 x double> poison, double %c.coerce.fca.0.extract, i64 0
+  %broadcast.splat50 = shufflevector <vscale x 2 x double> %broadcast.splatinsert49, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %a)
+  %0 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %strided.vec48 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %b)
+  %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec48, 0
+  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec48, 1
+  %4 = fmul fast <vscale x 2 x double> %3, %0
+  %5 = fmul fast <vscale x 2 x double> %2, %1
+  %6 = fadd fast <vscale x 2 x double> %4, %5
+  %7 = fmul fast <vscale x 2 x double> %2, %0
+  %8 = fmul fast <vscale x 2 x double> %3, %1
+  %9 = fsub fast <vscale x 2 x double> %7, %8
+  %10 = fmul fast <vscale x 2 x double> %9, %broadcast.splat
+  %11 = fmul fast <vscale x 2 x double> %6, %broadcast.splat50
+  %12 = fadd fast <vscale x 2 x double> %10, %11
+  %13 = fmul fast <vscale x 2 x double> %9, %broadcast.splat50
+  %14 = fmul fast <vscale x 2 x double> %6, %broadcast.splat
+  %15 = fsub fast <vscale x 2 x double> %13, %14
+  %interleaved.vec = tail call <vscale x 4 x double> @llvm.experimental.vector.interleave2.nxv4f64(<vscale x 2 x double> %15, <vscale x 2 x double> %12)
+  ret <vscale x 4 x double> %interleaved.vec
+}
+
+declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double>)
+declare <vscale x 4 x double> @llvm.experimental.vector.interleave2.nxv4f64(<vscale x 2 x double>, <vscale x 2 x double>)

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+complxnum -o - | FileCheck %s
+
+target triple = "aarch64-arm-none-eabi"
+
+
+; a[i] * b[i] * (11.0 + 3.0.i);
+;
+define <4 x double> @complex_mul_const(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: complex_mul_const:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v6.2d, v3.2d, v1.2d, #0
+; CHECK-NEXT:    fcmla v5.2d, v2.2d, v0.2d, #0
+; CHECK-NEXT:    fcmla v6.2d, v3.2d, v1.2d, #90
+; CHECK-NEXT:    fcmla v5.2d, v2.2d, v0.2d, #90
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.2d, v2.2d, v6.2d, #0
+; CHECK-NEXT:    fcmla v0.2d, v2.2d, v5.2d, #0
+; CHECK-NEXT:    fcmla v4.2d, v2.2d, v6.2d, #90
+; CHECK-NEXT:    fcmla v0.2d, v2.2d, v5.2d, #90
+; CHECK-NEXT:    mov v1.16b, v4.16b
+; CHECK-NEXT:    ret
+entry:
+  %strided.vec = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec47 = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %strided.vec49 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec50 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %0 = fmul fast <2 x double> %strided.vec50, %strided.vec
+  %1 = fmul fast <2 x double> %strided.vec49, %strided.vec47
+  %2 = fadd fast <2 x double> %0, %1
+  %3 = fmul fast <2 x double> %strided.vec49, %strided.vec
+  %4 = fmul fast <2 x double> %strided.vec50, %strided.vec47
+  %5 = fsub fast <2 x double> %3, %4
+  %6 = fmul fast <2 x double> %5, <double 3.000000e+00, double 3.000000e+00>
+  %7 = fmul fast <2 x double> %2, <double 1.100000e+01, double 1.100000e+01>
+  %8 = fadd fast <2 x double> %6, %7
+  %9 = fmul fast <2 x double> %5, <double 1.100000e+01, double 1.100000e+01>
+  %10 = fmul fast <2 x double> %2, <double 3.000000e+00, double 3.000000e+00>
+  %11 = fsub fast <2 x double> %9, %10
+  %interleaved.vec = shufflevector <2 x double> %11, <2 x double> %8, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  ret <4 x double> %interleaved.vec
+}
+
+
+; a[i] * b[i] * c;
+;
+define <4 x double> @complex_mul_non_const(<4 x double> %a, <4 x double> %b, [2 x double] %c) {
+; CHECK-LABEL: complex_mul_non_const:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-NEXT:    fcmla v6.2d, v2.2d, v0.2d, #0
+; CHECK-NEXT:    fcmla v7.2d, v3.2d, v1.2d, #0
+; CHECK-NEXT:    fcmla v6.2d, v2.2d, v0.2d, #90
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v7.2d, v3.2d, v1.2d, #90
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v2.2d, v4.2d, v7.2d, #0
+; CHECK-NEXT:    fcmla v0.2d, v4.2d, v6.2d, #0
+; CHECK-NEXT:    fcmla v2.2d, v4.2d, v7.2d, #90
+; CHECK-NEXT:    fcmla v0.2d, v4.2d, v6.2d, #90
+; CHECK-NEXT:    mov v1.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %c.coerce.fca.1.extract = extractvalue [2 x double] %c, 1
+  %c.coerce.fca.0.extract = extractvalue [2 x double] %c, 0
+  %broadcast.splatinsert = insertelement <2 x double> poison, double %c.coerce.fca.1.extract, i64 0
+  %broadcast.splat = shufflevector <2 x double> %broadcast.splatinsert, <2 x double> poison, <2 x i32> zeroinitializer
+  %broadcast.splatinsert51 = insertelement <2 x double> poison, double %c.coerce.fca.0.extract, i64 0
+  %broadcast.splat52 = shufflevector <2 x double> %broadcast.splatinsert51, <2 x double> poison, <2 x i32> zeroinitializer
+  %strided.vec = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec47 = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %strided.vec49 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %strided.vec50 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %0 = fmul fast <2 x double> %strided.vec50, %strided.vec
+  %1 = fmul fast <2 x double> %strided.vec49, %strided.vec47
+  %2 = fadd fast <2 x double> %0, %1
+  %3 = fmul fast <2 x double> %strided.vec49, %strided.vec
+  %4 = fmul fast <2 x double> %strided.vec50, %strided.vec47
+  %5 = fsub fast <2 x double> %3, %4
+  %6 = fmul fast <2 x double> %5, %broadcast.splat
+  %7 = fmul fast <2 x double> %2, %broadcast.splat52
+  %8 = fadd fast <2 x double> %6, %7
+  %9 = fmul fast <2 x double> %5, %broadcast.splat52
+  %10 = fmul fast <2 x double> %2, %broadcast.splat
+  %11 = fsub fast <2 x double> %9, %10
+  %interleaved.vec = shufflevector <2 x double> %11, <2 x double> %8, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  ret <4 x double> %interleaved.vec
+}