[CodeGen] Add support for Splats in ComplexDeinterleaving pass

[igor.kirillov]

Summary

This commit allows generating of complex number intrinsics for expressions
with constants or loops invariants, which are represented as splats.
For instance, after vectorizing loops in the following code snippets,
the ComplexDeinterleaving pass will be able to generate complex number
intrinsics:

complex<> x = ...;
for (int i = 0; i < N; ++i)
    c[i] = a[i] * b[i] * x;

or

for (int i = 0; i < N; ++i)
    c[i] = a[i] * b[i] * (11.0 + 3.0i);

Depends on D153446 and D153856

[igor.kirillov]

Refactor part of the code into separate revision (D153446)

[igor.kirillov]

Rebase

[NickGuy]

Looks good to me, but can you precommit the tests so we can see the expected differences?

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll

52

What's the reason for the square brackets here?

[igor.kirillov]

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll

52

That's how the complex<double> type variable looks when translated into the IR. So I thought it is better to leave it as aggregate.

[igor.kirillov]

Precommit tests

[glandium]

This causes a segfault when linking Firefox for arm64 mac with LTO. I can provide a reproducer if necessary, but it's large (several GB). The stack trace looks like this:

Stack dump:
0.	Running pass 'Function Pass Manager' on module 'builds/worker/workspace/obj-build/aarch64-apple-darwin/release/libgkrust.a(qcms-9781f4d323403c02.qcms.81a5ab1e-cgu.0.rcgu.o)56213408'.
1.	Running pass 'Complex Deinterleaving Pass' on function '@qcms_transform_data_rgb_out_lut_neon'
 #0 0x00007fb7032c6068 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) /tmp/llvm/llvm/lib/Support/Unix/Signals.inc:602:13
 #1 0x00007fb7032c4440 llvm::sys::RunSignalHandlers() /tmp/llvm/llvm/lib/Support/Signals.cpp:105:18
 #2 0x00007fb7032c66e8 SignalHandler(int) /tmp/llvm/llvm/lib/Support/Unix/Signals.inc:413:1
 #3 0x00007fb70245af90 (/lib/x86_64-linux-gnu/libc.so.6+0x3bf90)
 #4 0x00007fb70334be8e llvm::User::getNumOperands() const /tmp/llvm/llvm/include/llvm/IR/User.h:191:44
 #5 0x00007fb70334be8e llvm::HungoffOperandTraits<2u>::operands(llvm::User const*) /tmp/llvm/llvm/include/llvm/IR/OperandTraits.h:103:15
 #6 0x00007fb70334be8e llvm::PHINode::getNumOperands() const /tmp/llvm/llvm/include/llvm/IR/Instructions.h:2973:1
 #7 0x00007fb70334be8e llvm::PHINode::addIncoming(llvm::Value*, llvm::BasicBlock*) /tmp/llvm/llvm/include/llvm/IR/Instructions.h:2886:9
 #8 0x00007fb703545e92 (anonymous namespace)::ComplexDeinterleavingGraph::processReductionOperation(llvm::Value*, (anonymous namespace)::ComplexDeinterleavingCompositeNode*) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:1943:45
 #9 0x00007fb703545e92 (anonymous namespace)::ComplexDeinterleavingGraph::replaceNode(llvm::IRBuilderBase&, (anonymous namespace)::ComplexDeinterleavingCompositeNode*) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:1900:5
#10 0x00007fb70353d1e2 (anonymous namespace)::ComplexDeinterleavingGraph::replaceNodes() /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:1976:29
#11 0x00007fb70353d1e2 (anonymous namespace)::ComplexDeinterleaving::evaluateBasicBlock(llvm::BasicBlock*) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:520:11
#12 0x00007fb70353d1e2 (anonymous namespace)::ComplexDeinterleaving::runOnFunction(llvm::Function&) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:479:16
#13 0x00007fb70353e35f (anonymous namespace)::ComplexDeinterleavingLegacyPass::runOnFunction(llvm::Function&) /tmp/llvm/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp:460:42
#14 0x00007fb7033f3216 llvm::FPPassManager::runOnFunction(llvm::Function&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:1435:27
#15 0x00007fb7033f8b13 llvm::FPPassManager::runOnModule(llvm::Module&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:1481:13
#16 0x00007fb7033f389e (anonymous namespace)::MPPassManager::runOnModule(llvm::Module&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:0:27
#17 0x00007fb7033f389e llvm::legacy::PassManagerImpl::run(llvm::Module&) /tmp/llvm/llvm/lib/IR/LegacyPassManager.cpp:535:44
#18 0x00007fb7047898ee std::unique_ptr<llvm::ToolOutputFile, std::default_delete<llvm::ToolOutputFile>>::operator bool() const /usr/lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/unique_ptr.h:479:22
#19 0x00007fb7047898ee codegen(llvm::lto::Config const&, llvm::TargetMachine*, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, unsigned int, llvm::Module&, llvm::ModuleSummaryIndex const&) /tmp/llvm/llvm/lib/LTO/LTOBackend.cpp:413:31
#20 0x00007fb70478a7a2 std::_Function_base::~_Function_base() /usr/lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:243:11
#21 0x00007fb70478a7a2 llvm::lto::thinBackend(llvm::lto::Config const&, unsigned int, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, llvm::Module&, llvm::ModuleSummaryIndex const&, llvm::StringMap<std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>, llvm::MallocAllocator> const&, llvm::DenseMap<unsigned long, llvm::GlobalValueSummary*, llvm::DenseMapInfo<unsigned long, void>, llvm::detail::DenseMapPair<unsigned long, llvm::GlobalValueSummary*>> const&, llvm::MapVector<llvm::StringRef, llvm::BitcodeModule, llvm::DenseMap<llvm::StringRef, unsigned int, llvm::DenseMapInfo<llvm::StringRef, void>, llvm::detail::DenseMapPair<llvm::StringRef, unsigned int>>, std::vector<std::pair<llvm::StringRef, llvm::BitcodeModule>, std::allocator<std::pair<llvm::StringRef, llvm::BitcodeModule>>>>*, std::vector<unsigned char, std::allocator<unsigned char>> const&)::$_1::operator()(llvm::Module&, llvm::TargetMachine*, std::unique_ptr<llvm::ToolOutputFile, std::default_delete<llvm::ToolOutputFile>>) const /tmp/llvm/llvm/lib/LTO/LTOBackend.cpp:585:9
#22 0x00007fb70478a63d llvm::lto::thinBackend(llvm::lto::Config const&, unsigned int, std::function<llvm::Expected<std::unique_ptr<llvm::CachedFileStream, std::default_delete<llvm::CachedFileStream>>> (unsigned int, llvm::Twine const&)>, llvm::Module&, llvm::ModuleSummaryIndex const&, llvm::StringMap<std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long>>, llvm::MallocAllocator> const&, llvm::DenseMap<unsigned long, llvm::GlobalValueSummary*, llvm::DenseMapInfo<unsigned long, void>, llvm::detail::DenseMapPair<unsigned long, llvm::GlobalValueSummary*>> const&, llvm::MapVector<llvm::StringRef, llvm::BitcodeModule, llvm::DenseMap<llvm::StringRef, unsigned int, llvm::DenseMapInfo<llvm::StringRef, void>, llvm::detail::DenseMapPair<llvm::StringRef, unsigned int>>, std::vector<std::pair<llvm::StringRef, llvm::BitcodeModule>, std::allocator<std::pair<llvm::StringRef, llvm::BitcodeModule>>>>*, std::vector<unsigned char, std::allocator<unsigned char>> const&) /tmp/llvm/llvm/lib/LTO/LTOBackend.cpp:0:10

[igor.kirillov]

@glandium, thanks. I've reproduced and prepared the solution - https://reviews.llvm.org/D154598

llvm/include/llvm/CodeGen/ComplexDeinterleavingPass.h

 };
 
 enum class ComplexDeinterleavingOperation {
   CAdd,
   CMulPartial,
   // The following 'operations' are used to represent internal states. Backends
   // are not expected to try and support these in any capacity.
   Deinterleave,
+  Splat,
   Symmetric,
   ReductionPHI,
   ReductionOperation,
   ReductionSelect,
 };
 
 enum class ComplexDeinterleavingRotation {
   Rotation_0 = 0,
   Rotation_90 = 1,
   Rotation_180 = 2,
   Rotation_270 = 3,
 };
 
 } // namespace llvm
 
 #endif // LLVM_CODEGEN_COMPLEXDEINTERLEAVING_H

llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp

private:
   /// complex numbers. There are two ways to represent the Deinterleave
   /// operation:
   /// * Using two shufflevectors with even indices for /pReal instruction and
   /// odd indices for /pImag instructions (only for fixed-width vectors)
   /// * Using two extractvalue instructions applied to `vector.deinterleave2`
   /// intrinsic (for both fixed and scalable vectors)
   NodePtr identifyDeinterleave(Instruction *Real, Instruction *Imag);
 
+  /// identifying the operation that represents a complex number repeated in a
+  /// Splat vector. There are two possible types of splats: ConstantExpr with
+  /// the opcode ShuffleVector and ShuffleVectorInstr. Both should have an
+  /// initialization mask with all values set to zero.
+  NodePtr identifySplat(Value *Real, Value *Imag);
+
   NodePtr identifyPHINode(Instruction *Real, Instruction *Imag);
 
   /// Identifies SelectInsts in a loop that has reduction with predication masks
   /// and/or predicated tail folding
   NodePtr identifySelectNode(Instruction *Real, Instruction *Imag);
 
   Value *replaceNode(IRBuilderBase &Builder, RawNodePtr Node);
 

 ComplexDeinterleavingGraph::NodePtr
 ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I) {
   LLVM_DEBUG(dbgs() << "identifyNode on " << *R << " / " << *I << "\n");
   if (NodePtr CN = getContainingComposite(R, I)) {
     LLVM_DEBUG(dbgs() << " - Folding to existing node\n");
     return CN;
   }
 
+  if (NodePtr CN = identifySplat(R, I))
+    return CN;
+
   auto *Real = dyn_cast<Instruction>(R);
   auto *Imag = dyn_cast<Instruction>(I);
   if (!Real || !Imag)
     return nullptr;
 
   if (NodePtr CN = identifyDeinterleave(Real, Imag))
     return CN;
 
NodePtr PlaceholderNode =
                            RealShuffle, ImagShuffle);
   PlaceholderNode->ReplacementNode = RealShuffle->getOperand(0);
   FinalInstructions.insert(RealShuffle);
   FinalInstructions.insert(ImagShuffle);
   return submitCompositeNode(PlaceholderNode);
 }
 
 ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::identifySplat(Value *R, Value *I) {
+  auto IsSplat = [](Value *V) -> bool {
+    // Fixed-width vector with constants
+    if (isa<ConstantDataVector>(V))
+      return true;
+
+    VectorType *VTy;
+    ArrayRef<int> Mask;
+    // Splats are represented differently depending on whether the repeated
+    // value is a constant or an Instruction
+    if (auto *Const = dyn_cast<ConstantExpr>(V)) {
+      if (Const->getOpcode() != Instruction::ShuffleVector)
+        return false;
+      VTy = cast<VectorType>(Const->getType());
+      Mask = Const->getShuffleMask();
+    } else if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
+      VTy = Shuf->getType();
+      Mask = Shuf->getShuffleMask();
+    } else {
+      return false;
+    }
+
+    // When the data type is <1 x Type>, it's not possible to differentiate
+    // between the ComplexDeinterleaving::Deinterleave and
+    // ComplexDeinterleaving::Splat operations.
+    if (!VTy->isScalableTy() && VTy->getElementCount().getKnownMinValue() == 1)
+      return false;
+
+    return all_equal(Mask) && Mask[0] == 0;
+  };
+
+  if (!IsSplat(R) || !IsSplat(I))
+    return nullptr;
+
+  auto *Real = dyn_cast<Instruction>(R);
+  auto *Imag = dyn_cast<Instruction>(I);
+  if ((!Real && Imag) || (Real && !Imag))
+    return nullptr;
+
+  if (Real && Imag) {
+    // Non-constant splats should be in the same basic block
+    if (Real->getParent() != Imag->getParent())
+      return nullptr;
+
+    FinalInstructions.insert(Real);
+    FinalInstructions.insert(Imag);
+  }
+  NodePtr PlaceholderNode =
+      prepareCompositeNode(ComplexDeinterleavingOperation::Splat, R, I);
+  return submitCompositeNode(PlaceholderNode);
+}
+
+ComplexDeinterleavingGraph::NodePtr
 ComplexDeinterleavingGraph::identifyPHINode(Instruction *Real,
                                             Instruction *Imag) {
   if (Real != RealPHI || Imag != ImagPHI)
     return nullptr;
 
   NodePtr PlaceholderNode = prepareCompositeNode(
       ComplexDeinterleavingOperation::ReductionPHI, Real, Imag);
   return submitCompositeNode(PlaceholderNode);
else
       ReplacementNode = TL->createComplexDeinterleavingIR(
           Builder, Node->Operation, Node->Rotation, Input0, Input1,
           Accumulator);
     break;
   }
   case ComplexDeinterleavingOperation::Deinterleave:
     llvm_unreachable("Deinterleave node should already have ReplacementNode");
     break;
+  case ComplexDeinterleavingOperation::Splat: {
+    auto *NewTy = VectorType::getDoubleElementsVectorType(
+        cast<VectorType>(Node->Real->getType()));
+    auto *R = dyn_cast<Instruction>(Node->Real);
+    auto *I = dyn_cast<Instruction>(Node->Imag);
+    if (R && I) {
+      // Splats that are not constant are interleaved where they are located
+      Instruction *InsertPoint = (I->comesBefore(R) ? R : I)->getNextNode();
+      IRBuilder<> IRB(InsertPoint);
+      ReplacementNode =
+          IRB.CreateIntrinsic(Intrinsic::experimental_vector_interleave2, NewTy,
+                              {Node->Real, Node->Imag});
+    } else {
+      ReplacementNode =
+          Builder.CreateIntrinsic(Intrinsic::experimental_vector_interleave2,
+                                  NewTy, {Node->Real, Node->Imag});
+    }
+    break;
+  }
   case ComplexDeinterleavingOperation::ReductionPHI: {
     // If Operation is ReductionPHI, a new empty PHINode is created.
     // It is filled later when the ReductionOperation is processed.
     auto *VTy = cast<VectorType>(Node->Real->getType());
     auto *NewVTy = VectorType::getDoubleElementsVectorType(VTy);
     auto *NewPHI = PHINode::Create(NewVTy, 0, "", BackEdge->getFirstNonPHI());
     OldToNewPHI[dyn_cast<PHINode>(Node->Real)] = NewPHI;
     ReplacementNode = NewPHI;

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 ; a[i] * b[i] * (11.0 + 3.0.i);
 ;
 define <vscale x 4 x double> @complex_mul_const(<vscale x 4 x double> %a, <vscale x 4 x double> %b) {
 ; CHECK-LABEL: complex_mul_const:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp1 z4.d, z2.d, z3.d
-; CHECK-NEXT:    uzp1 z5.d, z0.d, z1.d
-; CHECK-NEXT:    uzp2 z0.d, z0.d, z1.d
+; CHECK-NEXT:    mov z4.d, #0 // =0x0
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uzp2 z1.d, z2.d, z3.d
-; CHECK-NEXT:    fmul z2.d, z4.d, z0.d
-; CHECK-NEXT:    fmla z2.d, p0/m, z1.d, z5.d
-; CHECK-NEXT:    fmul z0.d, z1.d, z0.d
-; CHECK-NEXT:    fmov z1.d, #11.00000000
-; CHECK-NEXT:    fnmls z0.d, p0/m, z4.d, z5.d
-; CHECK-NEXT:    fmov z3.d, #3.00000000
-; CHECK-NEXT:    fmul z4.d, z2.d, z1.d
-; CHECK-NEXT:    fmul z2.d, z2.d, z3.d
-; CHECK-NEXT:    fmla z4.d, p0/m, z0.d, z3.d
-; CHECK-NEXT:    fnmsb z1.d, p0/m, z0.d, z2.d
-; CHECK-NEXT:    zip1 z0.d, z1.d, z4.d
-; CHECK-NEXT:    zip2 z1.d, z1.d, z4.d
+; CHECK-NEXT:    mov z5.d, z4.d
+; CHECK-NEXT:    mov z6.d, z4.d
+; CHECK-NEXT:    fcmla z5.d, p0/m, z0.d, z2.d, #0
+; CHECK-NEXT:    fcmla z6.d, p0/m, z1.d, z3.d, #0
+; CHECK-NEXT:    fcmla z5.d, p0/m, z0.d, z2.d, #90
+; CHECK-NEXT:    fcmla z6.d, p0/m, z1.d, z3.d, #90
+; CHECK-NEXT:    fmov z1.d, #3.00000000
+; CHECK-NEXT:    fmov z2.d, #11.00000000
+; CHECK-NEXT:    zip2 z3.d, z2.d, z1.d
+; CHECK-NEXT:    mov z0.d, z4.d
+; CHECK-NEXT:    zip1 z1.d, z2.d, z1.d
+; CHECK-NEXT:    fcmla z4.d, p0/m, z6.d, z3.d, #0
+; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z1.d, #0
+; CHECK-NEXT:    fcmla z4.d, p0/m, z6.d, z3.d, #90
+; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z1.d, #90
+; CHECK-NEXT:    mov z1.d, z4.d
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %a)
   %0 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
   %1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
   %strided.vec48 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.experimental.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %b)
   %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec48, 0
   %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec48, 1
entry:
   ret <vscale x 4 x double> %interleaved.vec
 }
 
 ; a[i] * b[i] * c;
 ;
 define <vscale x 4 x double> @complex_mul_non_const(<vscale x 4 x double> %a, <vscale x 4 x double> %b, [2 x double] %c) {
 ; CHECK-LABEL: complex_mul_non_const:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uzp1 z6.d, z2.d, z3.d
-; CHECK-NEXT:    uzp1 z7.d, z0.d, z1.d
-; CHECK-NEXT:    uzp2 z0.d, z0.d, z1.d
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uzp2 z1.d, z2.d, z3.d
-; CHECK-NEXT:    fmul z2.d, z6.d, z0.d
-; CHECK-NEXT:    fmla z2.d, p0/m, z1.d, z7.d
+; CHECK-NEXT:    mov z6.d, #0 // =0x0
+; CHECK-NEXT:    // kill: def $d5 killed $d5 def $z5
 ; CHECK-NEXT:    // kill: def $d4 killed $d4 def $z4
-; CHECK-NEXT:    fmul z0.d, z1.d, z0.d
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z7.d, z6.d
+; CHECK-NEXT:    mov z24.d, z6.d
+; CHECK-NEXT:    mov z5.d, d5
 ; CHECK-NEXT:    mov z4.d, d4
-; CHECK-NEXT:    // kill: def $d5 killed $d5 def $z5
-; CHECK-NEXT:    mov z3.d, d5
-; CHECK-NEXT:    fnmls z0.d, p0/m, z6.d, z7.d
-; CHECK-NEXT:    fmul z1.d, z2.d, z4.d
-; CHECK-NEXT:    fmul z2.d, z2.d, z3.d
-; CHECK-NEXT:    fmla z1.d, p0/m, z0.d, z3.d
-; CHECK-NEXT:    fnmls z2.d, p0/m, z0.d, z4.d
-; CHECK-NEXT:    zip1 z0.d, z2.d, z1.d
-; CHECK-NEXT:    zip2 z1.d, z2.d, z1.d
+; CHECK-NEXT:    fcmla z7.d, p0/m, z0.d, z2.d, #0
+; CHECK-NEXT:    fcmla z24.d, p0/m, z1.d, z3.d, #0
+; CHECK-NEXT:    fcmla z7.d, p0/m, z0.d, z2.d, #90
+; CHECK-NEXT:    zip2 z2.d, z4.d, z5.d
+; CHECK-NEXT:    fcmla z24.d, p0/m, z1.d, z3.d, #90
+; CHECK-NEXT:    mov z0.d, z6.d
+; CHECK-NEXT:    zip1 z4.d, z4.d, z5.d
+; CHECK-NEXT:    fcmla z6.d, p0/m, z24.d, z2.d, #0
+; CHECK-NEXT:    fcmla z0.d, p0/m, z7.d, z4.d, #0
+; CHECK-NEXT:    fcmla z6.d, p0/m, z24.d, z2.d, #90
+; CHECK-NEXT:    fcmla z0.d, p0/m, z7.d, z4.d, #90
+; CHECK-NEXT:    mov z1.d, z6.d
 ; CHECK-NEXT:    ret
 entry:
   %c.coerce.fca.0.extract = extractvalue [2 x double] %c, 0
   %c.coerce.fca.1.extract = extractvalue [2 x double] %c, 1
   %broadcast.splatinsert = insertelement <vscale x 2 x double> poison, double %c.coerce.fca.1.extract, i64 0
   %broadcast.splat = shufflevector <vscale x 2 x double> %broadcast.splatinsert, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
   %broadcast.splatinsert49 = insertelement <vscale x 2 x double> poison, double %c.coerce.fca.0.extract, i64 0
   %broadcast.splat50 = shufflevector <vscale x 2 x double> %broadcast.splatinsert49, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer

llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+complxnum -o - | FileCheck %s
 
 target triple = "aarch64-arm-none-eabi"
 
 
 ; a[i] * b[i] * (11.0 + 3.0.i);
 ;
 define <4 x double> @complex_mul_const(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: complex_mul_const:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    zip1 v5.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip2 v6.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v2.2d, v2.2d, v3.2d
-; CHECK-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    fmov v4.2d, #3.00000000
-; CHECK-NEXT:    fmul v1.2d, v5.2d, v6.2d
-; CHECK-NEXT:    fmul v3.2d, v2.2d, v6.2d
-; CHECK-NEXT:    fmla v1.2d, v0.2d, v2.2d
-; CHECK-NEXT:    fneg v2.2d, v3.2d
-; CHECK-NEXT:    fmov v3.2d, #11.00000000
-; CHECK-NEXT:    fmul v6.2d, v1.2d, v4.2d
-; CHECK-NEXT:    fmla v2.2d, v0.2d, v5.2d
-; CHECK-NEXT:    fmul v1.2d, v1.2d, v3.2d
-; CHECK-NEXT:    fneg v5.2d, v6.2d
-; CHECK-NEXT:    fmla v1.2d, v4.2d, v2.2d
-; CHECK-NEXT:    fmla v5.2d, v3.2d, v2.2d
-; CHECK-NEXT:    zip1 v0.2d, v5.2d, v1.2d
-; CHECK-NEXT:    zip2 v1.2d, v5.2d, v1.2d
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v6.2d, v3.2d, v1.2d, #0
+; CHECK-NEXT:    fcmla v5.2d, v2.2d, v0.2d, #0
+; CHECK-NEXT:    fcmla v6.2d, v3.2d, v1.2d, #90
+; CHECK-NEXT:    fcmla v5.2d, v2.2d, v0.2d, #90
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v4.2d, v2.2d, v6.2d, #0
+; CHECK-NEXT:    fcmla v0.2d, v2.2d, v5.2d, #0
+; CHECK-NEXT:    fcmla v4.2d, v2.2d, v6.2d, #90
+; CHECK-NEXT:    fcmla v0.2d, v2.2d, v5.2d, #90
+; CHECK-NEXT:    mov v1.16b, v4.16b
 ; CHECK-NEXT:    ret
 entry:
   %strided.vec = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec47 = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 1, i32 3>
   %strided.vec49 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 0, i32 2>
   %strided.vec50 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 1, i32 3>
   %0 = fmul fast <2 x double> %strided.vec50, %strided.vec
   %1 = fmul fast <2 x double> %strided.vec49, %strided.vec47
entry:
   %11 = fsub fast <2 x double> %9, %10
   %interleaved.vec = shufflevector <2 x double> %11, <2 x double> %8, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x double> %interleaved.vec
 }
 
 
 ; a[i] * b[i] * c;
 ;
 define <4 x double> @complex_mul_non_const(<4 x double> %a, <4 x double> %b, [2 x double] %c) {
 ; CHECK-LABEL: complex_mul_non_const:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    zip1 v6.2d, v2.2d, v3.2d
-; CHECK-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
 ; CHECK-NEXT:    // kill: def $d4 killed $d4 def $q4
-; CHECK-NEXT:    zip2 v7.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    zip2 v1.2d, v2.2d, v3.2d
-; CHECK-NEXT:    fmul v2.2d, v6.2d, v7.2d
-; CHECK-NEXT:    fmul v3.2d, v1.2d, v7.2d
-; CHECK-NEXT:    fmla v2.2d, v0.2d, v1.2d
-; CHECK-NEXT:    fneg v1.2d, v3.2d
-; CHECK-NEXT:    fmul v3.2d, v2.2d, v5.d[0]
-; CHECK-NEXT:    fmul v2.2d, v2.2d, v4.d[0]
-; CHECK-NEXT:    fmla v1.2d, v0.2d, v6.2d
-; CHECK-NEXT:    fneg v3.2d, v3.2d
-; CHECK-NEXT:    fmla v2.2d, v1.2d, v5.d[0]
-; CHECK-NEXT:    fmla v3.2d, v1.2d, v4.d[0]
-; CHECK-NEXT:    zip1 v0.2d, v3.2d, v2.2d
-; CHECK-NEXT:    zip2 v1.2d, v3.2d, v2.2d
+; CHECK-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-NEXT:    fcmla v6.2d, v2.2d, v0.2d, #0
+; CHECK-NEXT:    fcmla v7.2d, v3.2d, v1.2d, #0
+; CHECK-NEXT:    fcmla v6.2d, v2.2d, v0.2d, #90
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v7.2d, v3.2d, v1.2d, #90
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    fcmla v2.2d, v4.2d, v7.2d, #0
+; CHECK-NEXT:    fcmla v0.2d, v4.2d, v6.2d, #0
+; CHECK-NEXT:    fcmla v2.2d, v4.2d, v7.2d, #90
+; CHECK-NEXT:    fcmla v0.2d, v4.2d, v6.2d, #90
+; CHECK-NEXT:    mov v1.16b, v2.16b
 ; CHECK-NEXT:    ret
 entry:
   %c.coerce.fca.1.extract = extractvalue [2 x double] %c, 1
   %c.coerce.fca.0.extract = extractvalue [2 x double] %c, 0
   %broadcast.splatinsert = insertelement <2 x double> poison, double %c.coerce.fca.1.extract, i64 0
   %broadcast.splat = shufflevector <2 x double> %broadcast.splatinsert, <2 x double> poison, <2 x i32> zeroinitializer
   %broadcast.splatinsert51 = insertelement <2 x double> poison, double %c.coerce.fca.0.extract, i64 0
   %broadcast.splat52 = shufflevector <2 x double> %broadcast.splatinsert51, <2 x double> poison, <2 x i32> zeroinitializer