[ARM] Teach DemandedVectorElts about VMOVN lanes

[dmgreen]

Summary

The class of instructions that write to narrow top/bottom lanes only demand the even or odd elements of the input lanes. Which means that VMOVNT; VMOVNB demands no lanes from the original input. This teaches that to instcombine from the target hooks available through ARMTTIImpl.

[SjoerdMeijer]

Before I continue reading the rest, just wanted to check this:

The class of instructions that write to narrow top/bottom lanes only demand the even or odd elements of the input lanes. Which means that VMOVNT; VMOVNB demands no lanes from the original input.

I don't follow the conclusion here that the VMOVNT; VMOVNB don't demand lanes from the original input, because we do read from the original input?

[dmgreen]

In D109325#2989713, @SjoerdMeijer wrote:

Before I continue reading the rest, just wanted to check this:

The class of instructions that write to narrow top/bottom lanes only demand the even or odd elements of the input lanes. Which means that VMOVNT; VMOVNB demands no lanes from the original input.

I don't follow the conclusion here that the VMOVNT; VMOVNB don't demand lanes from the original input, because we do read from the original input?

Oh yeah, good point - the predicated intrinsics will not work this way if they are not acting on all lanes. I had forgotten that the first input is also the passthrough value. I'll update the patch to remove those.

For the non-predicated cases I mean to say - if you have overwritten the top lanes and you have overwritten the bottom lanes - then no value from the original input are demanded. You have overridden all the lanes. So a VMOVNT will demand the bottom (even) lanes from the first input and insert a new value into into the top (odd) lanes. A VMOVNB will demand the top (odd) lanes and write new values into the bottom. A pair of X=VMOVNT A, B; Y= VMOVNB X, C will use none of the lanes of A.

[dmgreen]

Remove predicated instructions.

[SjoerdMeijer]

Ah, thanks, got it, that's clear! Looks like a good change to me.

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

276

The constants 2, 4, and 7 here were not immediately obvious to me. Perhaps a comment would help less familiar readers.

llvm/lib/Target/ARM/ARMTargetTransformInfo.h

public:
   /// SIMD instructions remains unchanged from ARMv7. Only AArch64 SIMD
   /// and Arm MVE are IEEE-754 compliant.
   bool isFPVectorizationPotentiallyUnsafe() {
     return !ST->isTargetDarwin() && !ST->hasMVEFloatOps();
   }
 
   Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
                                                IntrinsicInst &II) const;
+  Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) const;
 
   /// \name Scalar TTI Implementations
   /// @{
 
   InstructionCost getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
                                         const APInt &Imm, Type *Ty);
 
   using BaseT::getIntImmCost;

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

if (I->hasOneUse()) {
       }
     }
     return None;
   }
   }
   return None;
 }
 
+Optional<Value *> ARMTTIImpl::simplifyDemandedVectorEltsIntrinsic(
+    InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
+    APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
+    std::function<void(Instruction *, unsigned, APInt, APInt &)>
+        SimplifyAndSetOp) const {
+
+  auto SimplifyNarrowInstrTopBottom =[&](unsigned TopOpc) {
+    unsigned NumElts = cast<FixedVectorType>(II.getType())->getNumElements();
+    unsigned IsTop = cast<ConstantInt>(II.getOperand(TopOpc))->getZExtValue();
+
+    // The only odd/even lanes of operand 0 will only be demanded depending
+    // on whether this is a top/bottom instruction.
+    APInt DemandedElts =
+        APInt::getSplat(NumElts, IsTop ? APInt::getLowBitsSet(2, 1)
+                                       : APInt::getHighBitsSet(2, 1));
+    SimplifyAndSetOp(&II, 0, OrigDemandedElts & DemandedElts, UndefElts);
+    // The other lanes will be defined from the inserted elements.
+    UndefElts &= APInt::getSplat(NumElts, !IsTop ? APInt::getLowBitsSet(2, 1)
+                                                 : APInt::getHighBitsSet(2, 1));
+    return None;
+  };
+
+  switch (II.getIntrinsicID()) {
+  default:
+    break;
+  case Intrinsic::arm_mve_vmovn_predicated:
+  case Intrinsic::arm_mve_vcvt_narrow:
+  case Intrinsic::arm_mve_vcvt_narrow_predicated:
+    SimplifyNarrowInstrTopBottom(2);
+    break;
+  case Intrinsic::arm_mve_vqmovn:
+  case Intrinsic::arm_mve_vqmovn_predicated:
+    SimplifyNarrowInstrTopBottom(4);
+    break;
+  case Intrinsic::arm_mve_vshrn:
+  case Intrinsic::arm_mve_vshrn_predicated:
+    SimplifyNarrowInstrTopBottom(7);
+    break;
+  }
+
+  return None;
+}
+
 InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
                                           TTI::TargetCostKind CostKind) {
   assert(Ty->isIntegerTy());
 
  unsigned Bits = Ty->getPrimitiveSizeInBits();
  if (Bits == 0 || Imm.getActiveBits() >= 64)
    return 4;
 

llvm/test/Transforms/InstCombine/ARM/mve-narrow.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
 ; Various patterns testing v8i16
 
 define <8 x i16> @test_shrn_v8i16_t1(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d) {
 ; CHECK-LABEL: @test_shrn_v8i16_t1(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1>
+; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 1, i16 poison, i16 1, i16 poison, i16 1, i16 poison, i16 1, i16 poison>
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[X]], <4 x i32> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, <i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1>
   %z = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %x, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
   ret <8 x i16> %z
 }
 
 define <8 x i16> @test_shrn_v8i16_t2(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d) {
 ; CHECK-LABEL: @test_shrn_v8i16_t2(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1>
+; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 -1, i16 poison, i16 -1, i16 poison, i16 -1, i16 poison, i16 -1, i16 poison>
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[X]], <4 x i32> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, <i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1>
   %z = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %x, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
   ret <8 x i16> %z
 }
 
 define <8 x i16> @test_shrn_v8i16_b1(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d) {
 ; CHECK-LABEL: @test_shrn_v8i16_b1(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1>
+; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 poison, i16 -1, i16 poison, i16 -1, i16 poison, i16 -1, i16 poison, i16 -1>
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[X]], <4 x i32> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, <i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1>
   %z = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %x, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
   ret <8 x i16> %z
 }
 
 define <8 x i16> @test_shrn_v8i16_b2(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d) {
 ; CHECK-LABEL: @test_shrn_v8i16_b2(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1>
+; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], <i16 poison, i16 1, i16 poison, i16 1, i16 poison, i16 1, i16 poison, i16 1>
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[X]], <4 x i32> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, <i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1, i16 -1, i16 1>
   %z = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %x, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
   ret <8 x i16> %z
 }
 
 define <8 x i16> @test_shrn_v8i16_bt(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d) {
 ; CHECK-LABEL: @test_shrn_v8i16_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[X]], <4 x i32> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> poison, <4 x i32> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[Y]], <4 x i32> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, %b
   %y = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %x, <4 x i32> %c, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
   %z = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %y, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
   ret <8 x i16> %z
 }
 
 define <8 x i16> @test_shrn_v8i16_tb(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d) {
 ; CHECK-LABEL: @test_shrn_v8i16_tb(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[X]], <4 x i32> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> poison, <4 x i32> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> [[Y]], <4 x i32> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, %b
   %y = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %x, <4 x i32> %c, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
   %z = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %y, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
   ret <8 x i16> %z
 }
;
   %z = call <8 x i16> @llvm.arm.mve.vshrn.v8i16.v4i32(<8 x i16> %y, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
   ret <8 x i16> %z
 }
 
 ; Other types and sizes
 
 define <16 x i8> @test_shrn_v16i8_bt(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, <8 x i16> %d) {
 ; CHECK-LABEL: @test_shrn_v16i8_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <16 x i8> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> [[X]], <8 x i16> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
+; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> poison, <8 x i16> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
 ; CHECK-NEXT:    [[Z:%.*]] = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> [[Y]], <8 x i16> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
 ; CHECK-NEXT:    ret <16 x i8> [[Z]]
 ;
   %x = add <16 x i8> %a, %b
   %y = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %x, <8 x i16> %c, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0)
   %z = call <16 x i8> @llvm.arm.mve.vshrn.v16i8.v8i16(<16 x i8> %y, <8 x i16> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1)
   ret <16 x i8> %z
 }
 
 define <8 x i16> @test_shrnp_v8i16_bt(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d, <4 x i1> %p) {
 ; CHECK-LABEL: @test_shrnp_v8i16_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> [[X]], <4 x i32> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i1> [[P:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> poison, <4 x i32> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i1> [[P:%.*]])
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> [[Y]], <4 x i32> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1, <4 x i1> [[P]])
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, %b
   %y = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %x, <4 x i32> %c, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, <4 x i1> %p)
   %z = call <8 x i16> @llvm.arm.mve.vshrn.predicated.v8i16.v4i32.v4i1(<8 x i16> %y, <4 x i32> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1, <4 x i1> %p)
   ret <8 x i16> %z
 }
 
 define <16 x i8> @test_shrnp_v16i8_bt(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, <8 x i16> %d, <8 x i1> %p) {
 ; CHECK-LABEL: @test_shrnp_v16i8_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <16 x i8> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> [[X]], <8 x i16> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, <8 x i1> [[P:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> poison, <8 x i16> [[C:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, <8 x i1> [[P:%.*]])
 ; CHECK-NEXT:    [[Z:%.*]] = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> [[Y]], <8 x i16> [[D:%.*]], i32 16, i32 0, i32 0, i32 0, i32 0, i32 1, <8 x i1> [[P]])
 ; CHECK-NEXT:    ret <16 x i8> [[Z]]
 ;
   %x = add <16 x i8> %a, %b
   %y = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %x, <8 x i16> %c, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, <8 x i1> %p)
   %z = call <16 x i8> @llvm.arm.mve.vshrn.predicated.v16i8.v8i16.v8i1(<16 x i8> %y, <8 x i16> %d, i32 16, i32 0, i32 0, i32 0, i32 0, i32 1, <8 x i1> %p)
   ret <16 x i8> %z
 }
 
 
 define <8 x i16> @test_movnp_v8i16_bt(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d, <4 x i1> %p) {
 ; CHECK-LABEL: @test_movnp_v8i16_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> [[X]], <4 x i32> [[C:%.*]], i32 0, <4 x i1> [[P:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> poison, <4 x i32> [[C:%.*]], i32 0, <4 x i1> [[P:%.*]])
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> [[Y]], <4 x i32> [[D:%.*]], i32 1, <4 x i1> [[P]])
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, %b
   %y = call <8 x i16> @llvm.arm.mve.vmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> %x, <4 x i32> %c, i32 0, <4 x i1> %p)
   %z = call <8 x i16> @llvm.arm.mve.vmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> %y, <4 x i32> %d, i32 1, <4 x i1> %p)
   ret <8 x i16> %z
 }
 
 define <16 x i8> @test_movnp_v16i8_bt(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, <8 x i16> %d, <8 x i1> %p) {
 ; CHECK-LABEL: @test_movnp_v16i8_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <16 x i8> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> [[X]], <8 x i16> [[C:%.*]], i32 0, <8 x i1> [[P:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> poison, <8 x i16> [[C:%.*]], i32 0, <8 x i1> [[P:%.*]])
 ; CHECK-NEXT:    [[Z:%.*]] = call <16 x i8> @llvm.arm.mve.vmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> [[Y]], <8 x i16> [[D:%.*]], i32 1, <8 x i1> [[P]])
 ; CHECK-NEXT:    ret <16 x i8> [[Z]]
 ;
   %x = add <16 x i8> %a, %b
   %y = call <16 x i8> @llvm.arm.mve.vmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> %x, <8 x i16> %c, i32 0, <8 x i1> %p)
   %z = call <16 x i8> @llvm.arm.mve.vmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> %y, <8 x i16> %d, i32 1, <8 x i1> %p)
   ret <16 x i8> %z
 }
 
 define <8 x i16> @test_qmovn_v8i16_bt(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d) {
 ; CHECK-LABEL: @test_qmovn_v8i16_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vqmovn.v8i16.v4i32(<8 x i16> [[X]], <4 x i32> [[C:%.*]], i32 0, i32 0, i32 0)
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vqmovn.v8i16.v4i32(<8 x i16> poison, <4 x i32> [[C:%.*]], i32 0, i32 0, i32 0)
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vqmovn.v8i16.v4i32(<8 x i16> [[Y]], <4 x i32> [[D:%.*]], i32 0, i32 0, i32 1)
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, %b
   %y = call <8 x i16> @llvm.arm.mve.vqmovn.v8i16.v4i32(<8 x i16> %x, <4 x i32> %c, i32 0, i32 0, i32 0)
   %z = call <8 x i16> @llvm.arm.mve.vqmovn.v8i16.v4i32(<8 x i16> %y, <4 x i32> %d, i32 0, i32 0, i32 1)
   ret <8 x i16> %z
 }
 
 define <16 x i8> @test_qmovn_v16i8_bt(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, <8 x i16> %d) {
 ; CHECK-LABEL: @test_qmovn_v16i8_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <16 x i8> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vqmovn.v16i8.v8i16(<16 x i8> [[X]], <8 x i16> [[C:%.*]], i32 0, i32 0, i32 0)
+; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vqmovn.v16i8.v8i16(<16 x i8> poison, <8 x i16> [[C:%.*]], i32 0, i32 0, i32 0)
 ; CHECK-NEXT:    [[Z:%.*]] = call <16 x i8> @llvm.arm.mve.vqmovn.v16i8.v8i16(<16 x i8> [[Y]], <8 x i16> [[D:%.*]], i32 0, i32 0, i32 1)
 ; CHECK-NEXT:    ret <16 x i8> [[Z]]
 ;
   %x = add <16 x i8> %a, %b
   %y = call <16 x i8> @llvm.arm.mve.vqmovn.v16i8.v8i16(<16 x i8> %x, <8 x i16> %c, i32 0, i32 0, i32 0)
   %z = call <16 x i8> @llvm.arm.mve.vqmovn.v16i8.v8i16(<16 x i8> %y, <8 x i16> %d, i32 0, i32 0, i32 1)
   ret <16 x i8> %z
 }
 
 define <8 x i16> @test_qmovnp_v8i16_bt(<8 x i16> %a, <8 x i16> %b, <4 x i32> %c, <4 x i32> %d, <4 x i1> %p) {
 ; CHECK-LABEL: @test_qmovnp_v8i16_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <8 x i16> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vqmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> [[X]], <4 x i32> [[C:%.*]], i32 0, i32 0, i32 0, <4 x i1> [[P:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x i16> @llvm.arm.mve.vqmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> poison, <4 x i32> [[C:%.*]], i32 0, i32 0, i32 0, <4 x i1> [[P:%.*]])
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x i16> @llvm.arm.mve.vqmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> [[Y]], <4 x i32> [[D:%.*]], i32 0, i32 0, i32 1, <4 x i1> [[P]])
 ; CHECK-NEXT:    ret <8 x i16> [[Z]]
 ;
   %x = add <8 x i16> %a, %b
   %y = call <8 x i16> @llvm.arm.mve.vqmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> %x, <4 x i32> %c, i32 0, i32 0, i32 0, <4 x i1> %p)
   %z = call <8 x i16> @llvm.arm.mve.vqmovn.predicated.v8i16.v4i32.v4i1(<8 x i16> %y, <4 x i32> %d, i32 0, i32 0, i32 1, <4 x i1> %p)
   ret <8 x i16> %z
 }
 
 define <16 x i8> @test_qmovnp_v16i8_bt(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, <8 x i16> %d, <8 x i1> %p) {
 ; CHECK-LABEL: @test_qmovnp_v16i8_bt(
-; CHECK-NEXT:    [[X:%.*]] = add <16 x i8> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vqmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> [[X]], <8 x i16> [[C:%.*]], i32 0, i32 0, i32 0, <8 x i1> [[P:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = call <16 x i8> @llvm.arm.mve.vqmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> poison, <8 x i16> [[C:%.*]], i32 0, i32 0, i32 0, <8 x i1> [[P:%.*]])
 ; CHECK-NEXT:    [[Z:%.*]] = call <16 x i8> @llvm.arm.mve.vqmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> [[Y]], <8 x i16> [[D:%.*]], i32 0, i32 0, i32 1, <8 x i1> [[P]])
 ; CHECK-NEXT:    ret <16 x i8> [[Z]]
 ;
   %x = add <16 x i8> %a, %b
   %y = call <16 x i8> @llvm.arm.mve.vqmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> %x, <8 x i16> %c, i32 0, i32 0, i32 0, <8 x i1> %p)
   %z = call <16 x i8> @llvm.arm.mve.vqmovn.predicated.v16i8.v8i16.v8i1(<16 x i8> %y, <8 x i16> %d, i32 0, i32 0, i32 1, <8 x i1> %p)
   ret <16 x i8> %z
 }
 
 define <8 x half> @test_cvtn_v8i16_bt(<8 x half> %a, <8 x half> %b, <4 x float> %c, <4 x float> %d) {
 ; CHECK-LABEL: @test_cvtn_v8i16_bt(
-; CHECK-NEXT:    [[X:%.*]] = fadd <8 x half> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> [[X]], <4 x float> [[C:%.*]], i32 0)
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> poison, <4 x float> [[C:%.*]], i32 0)
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> [[Y]], <4 x float> [[D:%.*]], i32 1)
 ; CHECK-NEXT:    ret <8 x half> [[Z]]
 ;
   %x = fadd <8 x half> %a, %b
   %y = call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> %x, <4 x float> %c, i32 0)
   %z = call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> %y, <4 x float> %d, i32 1)
   ret <8 x half> %z
 }
 
 define <8 x half> @test_cvtnp_v8i16_bt(<8 x half> %a, <8 x half> %b, <4 x float> %c, <4 x float> %d, <4 x i1> %p) {
 ; CHECK-LABEL: @test_cvtnp_v8i16_bt(
-; CHECK-NEXT:    [[X:%.*]] = fadd <8 x half> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[Y:%.*]] = call <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half> [[X]], <4 x float> [[C:%.*]], i32 0, <4 x i1> [[P:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = call <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half> poison, <4 x float> [[C:%.*]], i32 0, <4 x i1> [[P:%.*]])
 ; CHECK-NEXT:    [[Z:%.*]] = call <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half> [[Y]], <4 x float> [[D:%.*]], i32 1, <4 x i1> [[P]])
 ; CHECK-NEXT:    ret <8 x half> [[Z]]
 ;
   %x = fadd <8 x half> %a, %b
   %y = call <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half> %x, <4 x float> %c, i32 0, <4 x i1> %p)
   %z = call <8 x half> @llvm.arm.mve.vcvt.narrow.predicated(<8 x half> %y, <4 x float> %d, i32 1, <4 x i1> %p)
   ret <8 x half> %z
 }