[AArch64] Don't use tbl lowering if ZExt can be folded into user.

[fhahn]

Summary

If the ZExt can be lowered to a single ZExt to the next power-of-2 and
the remaining ZExt folded into the user, don't use tbl lowering.

Fixes #62620.

[efriedma]

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

1818

Does this code do the right thing when the "Args" includes a cast with the wrong source type? (In this case, the source type is i16, but the type of the operand of the cast would actually be i8.)

[fhahn]

Generalize code to properly handling mull case with casts with different source types, thanks!

[fhahn]

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

1818

I don't think so, thanks! I updated the code to pass both source types in that case, not sure if there's a better way of handling that.

[fhahn]

ping :)

[efriedma]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

14422

Maybe just explicitly check "DstWidth != 16 && DstWidth != 32"?

[fhahn]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

14422

The current codegen also works for like i24 and there are existing tests for that; I left the check as is for now to avoid changing behavior unrelated to the patch

[v01dXYZ]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

14622

Sorry the question is dumb but why does it use TruncType instead of DoubleSrcType := SrcType with Element type width multiplied by 2 ?

14629

This assignment is not necessary.

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

1869

There is a typo (I think).

[v01dXYZ]

@fhahn ping

[v01dXYZ]

ping

[fhahn]

Thanks for taking a look. Comments should be addressed in baebe719a52acbe0a5aaa04dcf1abcfd4035bf1f and response inline.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

14622

TBL lowering is not profitable if only one zext step is saved. If the zext to the next smaller type is free (i.e. it can be folded into the user), then only 1 step will be needed and no tbl should be generated

14629

Thanks, removed!

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

1869

Yes, this should check I's source type, should be fixed.

llvm/include/llvm/CodeGen/TargetLowering.h

public:
   /// come first).
   virtual bool shouldSinkOperands(Instruction *I,
                                   SmallVectorImpl<Use *> &Ops) const {
     return false;
   }
 
   /// Try to optimize extending or truncating conversion instructions (like
   /// zext, trunc, fptoui, uitofp) for the target.
-  virtual bool optimizeExtendOrTruncateConversion(Instruction *I,
-                                                  Loop *L) const {
+  virtual bool
+  optimizeExtendOrTruncateConversion(Instruction *I, Loop *L,
+                                     const TargetTransformInfo &TTI) const {
     return false;
   }
 
   /// Return true if the target supplies and combines to a paired load
   /// two loaded values of type LoadedType next to each other in memory.
   /// RequiredAlignment gives the minimal alignment constraints that must be met
   /// to be able to select this paired load.
   ///

llvm/lib/CodeGen/CodeGenPrepare.cpp

if (CastInst *CI = dyn_cast<CastInst>(I)) {
     // want to forward-subst the cast.
     if (isa<Constant>(CI->getOperand(0)))
       return false;
 
     if (OptimizeNoopCopyExpression(CI, *TLI, *DL))
       return true;
 
     if ((isa<UIToFPInst>(I) || isa<FPToUIInst>(I) || isa<TruncInst>(I)) &&
-        TLI->optimizeExtendOrTruncateConversion(I,
-                                                LI->getLoopFor(I->getParent())))
+        TLI->optimizeExtendOrTruncateConversion(
+            I, LI->getLoopFor(I->getParent()), *TTI))
       return true;
 
     if (isa<ZExtInst>(I) || isa<SExtInst>(I)) {
       /// Sink a zext or sext into its user blocks if the target type doesn't
       /// fit in one register
       if (TLI->getTypeAction(CI->getContext(),
                              TLI->getValueType(*DL, CI->getType())) ==
           TargetLowering::TypeExpandInteger) {
         return SinkCast(CI);
       } else {
         if (TLI->optimizeExtendOrTruncateConversion(
-                I, LI->getLoopFor(I->getParent())))
+                I, LI->getLoopFor(I->getParent()), *TTI))
           return true;
 
         bool MadeChange = optimizeExt(I);
         return MadeChange | optimizeExtUses(I);
       }
     }
     return false;
   }

llvm/lib/Target/AArch64/AArch64ISelLowering.h

public:
 
   bool isZExtFree(Type *Ty1, Type *Ty2) const override;
   bool isZExtFree(EVT VT1, EVT VT2) const override;
   bool isZExtFree(SDValue Val, EVT VT2) const override;
 
   bool shouldSinkOperands(Instruction *I,
                           SmallVectorImpl<Use *> &Ops) const override;
 
-  bool optimizeExtendOrTruncateConversion(Instruction *I,
-                                          Loop *L) const override;
+  bool optimizeExtendOrTruncateConversion(
+      Instruction *I, Loop *L, const TargetTransformInfo &TTI) const override;
 
   bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override;
 
   unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
 
   bool lowerInterleavedLoad(LoadInst *LI,
                             ArrayRef<ShuffleVectorInst *> Shuffles,
                             ArrayRef<unsigned> Indices,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 }
 
 static void createTblShuffleForZExt(ZExtInst *ZExt, bool IsLittleEndian) {
   Value *Op = ZExt->getOperand(0);
   auto *SrcTy = cast<FixedVectorType>(Op->getType());
   auto *DstTy = cast<FixedVectorType>(ZExt->getType());
   auto SrcWidth = cast<IntegerType>(SrcTy->getElementType())->getBitWidth();
   auto DstWidth = cast<IntegerType>(DstTy->getElementType())->getBitWidth();
   assert(DstWidth % SrcWidth == 0 &&
          "TBL lowering is not supported for a ZExt instruction with this "
          "source & destination element type.");
   unsigned ZExtFactor = DstWidth / SrcWidth;
   unsigned NumElts = SrcTy->getNumElements();
   IRBuilder<> Builder(ZExt);
   SmallVector<int> Mask;
   // Create a mask that selects <0,...,Op[i]> for each lane of the destination
   // vector to replace the original ZExt. This can later be lowered to a set of
if (Results.size() == 1) {
     FinalResult =
         Builder.CreateShuffleVector(Results[0], Results[1], FinalMask);
   }
 
   TI->replaceAllUsesWith(FinalResult);
   TI->eraseFromParent();
 }
 
-bool AArch64TargetLowering::optimizeExtendOrTruncateConversion(Instruction *I,
-                                                               Loop *L) const {
+bool AArch64TargetLowering::optimizeExtendOrTruncateConversion(
+    Instruction *I, Loop *L, const TargetTransformInfo &TTI) const {
   // shuffle_vector instructions are serialized when targeting SVE,
   // see LowerSPLAT_VECTOR. This peephole is not beneficial.
   if (Subtarget->useSVEForFixedLengthVectors())
     return false;
 
   // Try to optimize conversions using tbl. This requires materializing constant
   // index vectors, which can increase code size and add loads. Skip the
   // transform unless the conversion is in a loop block guaranteed to execute
   // and we are not optimizing for size.
   Function *F = I->getParent()->getParent();
   if (!L || L->getHeader() != I->getParent() || F->hasMinSize() ||
       F->hasOptSize())
     return false;
 
   auto *SrcTy = dyn_cast<FixedVectorType>(I->getOperand(0)->getType());
   auto *DstTy = dyn_cast<FixedVectorType>(I->getType());
   if (!SrcTy || !DstTy)
     return false;
 
   // Convert 'zext <Y x i8> %x to <Y x i8X>' to a shuffle that can be
   // lowered to tbl instructions to insert the original i8 elements
   // into i8x lanes. This is enabled for cases where it is beneficial.
   auto *ZExt = dyn_cast<ZExtInst>(I);
   if (ZExt && SrcTy->getElementType()->isIntegerTy(8)) {
-    auto DstWidth = cast<IntegerType>(DstTy->getElementType())->getBitWidth();
+    // If the ZExt can be lowered to a single ZExt to the next power-of-2 and
+    // the remaining ZExt folded into the user, don't use tbl lowering.
+    auto SrcWidth = SrcTy->getElementType()->getScalarSizeInBits();
+    auto DstWidth = DstTy->getElementType()->getScalarSizeInBits();
+    if (SrcWidth * 4 == DstWidth) {
+      if (TTI.getCastInstrCost(I->getOpcode(), DstTy,
+                               VectorType::getExtendedElementVectorType(SrcTy),
+                               TargetTransformInfo::getCastContextHint(I),
+                               TTI::TCK_SizeAndLatency, I) == TTI::TCC_Free)
+        return false;
+    }
+
     if (DstWidth % 8 == 0 && DstWidth > 16 && DstWidth < 64) {
       createTblShuffleForZExt(ZExt, Subtarget->isLittleEndian());
       return true;
     }
   }
 
   auto *UIToFP = dyn_cast<UIToFPInst>(I);
   if (UIToFP && SrcTy->getElementType()->isIntegerTy(8) &&
       DstTy->getElementType()->isFloatTy()) {
     IRBuilder<> Builder(I);
     auto *ZExt = cast<ZExtInst>(
         Builder.CreateZExt(I->getOperand(0), VectorType::getInteger(DstTy)));
     auto *UI = Builder.CreateUIToFP(ZExt, DstTy);
     I->replaceAllUsesWith(UI);
     I->eraseFromParent();
     createTblShuffleForZExt(ZExt, Subtarget->isLittleEndian());
     return true;
   }
 
   // Convert 'fptoui <(8|16) x float> to <(8|16) x i8>' to a wide fptoui
   // followed by a truncate lowered to using tbl.4.
   auto *FPToUI = dyn_cast<FPToUIInst>(I);
   if (FPToUI &&
       (SrcTy->getNumElements() == 8 || SrcTy->getNumElements() == 16) &&
       SrcTy->getElementType()->isFloatTy() &&
       DstTy->getElementType()->isIntegerTy(8)) {
     IRBuilder<> Builder(I);

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h

class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   const AArch64TargetLowering *getTLI() const { return TLI; }
 
   enum MemIntrinsicType {
     VECTOR_LDST_TWO_ELEMENTS,
     VECTOR_LDST_THREE_ELEMENTS,
     VECTOR_LDST_FOUR_ELEMENTS
   };
 
-  bool isWideningInstruction(Type *Ty, unsigned Opcode,
+  bool isWideningInstruction(Type *DstTy, Type *SrcTy, unsigned Opcode,
                              ArrayRef<const Value *> Args);
 
   // A helper function called by 'getVectorInstrCost'.
   //
   // 'Val' and 'Index' are forwarded from 'getVectorInstrCost'; 'HasRealUse'
   // indicates whether the vector instruction is available in the input IR or
   // just imaginary in vectorizer passes.
   InstructionCost getVectorInstrCostHelper(const Instruction *I, Type *Val,

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

case TargetTransformInfo::RGK_ScalableVector:
     if (!ST->isStreamingSVEModeDisabled() && !EnableScalableAutovecInStreamingMode)
       return TypeSize::getScalable(0);
 
     return TypeSize::getScalable(ST->hasSVE() ? 128 : 0);
   }
   llvm_unreachable("Unsupported register kind");
 }
 
-bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
+bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, Type *SrcTy,
+                                           unsigned Opcode,
                                            ArrayRef<const Value *> Args) {
 
   // A helper that returns a vector type from the given type. The number of
   // elements in type Ty determines the vector width.
   auto toVectorTy = [&](Type *ArgTy) {
     return VectorType::get(ArgTy->getScalarType(),
                            cast<VectorType>(DstTy)->getElementCount());
   };
if (Args.size() != 2 ||
     return false;
   auto *Extend = cast<CastInst>(Args[1]);
   auto *Arg0 = dyn_cast<CastInst>(Args[0]);
 
   // A mul only has a mull version (not like addw). Both operands need to be
   // extending and the same type.
   if (Opcode == Instruction::Mul &&
       (!Arg0 || Arg0->getOpcode() != Extend->getOpcode() ||
        Arg0->getOperand(0)->getType() != Extend->getOperand(0)->getType()))
     return false;
 
   // Legalize the destination type and ensure it can be used in a widening
   // operation.
   auto DstTyL = getTypeLegalizationCost(DstTy);
   unsigned DstElTySize = DstTyL.second.getScalarSizeInBits();
   if (!DstTyL.second.isVector() || DstElTySize != DstTy->getScalarSizeInBits())
     return false;
 
   // Legalize the source type and ensure it can be used in a widening
   // operation.
-  auto *SrcTy = toVectorTy(Extend->getSrcTy());
+  if (!SrcTy)
+    SrcTy = toVectorTy(Extend->getSrcTy());
+
   auto SrcTyL = getTypeLegalizationCost(SrcTy);
   unsigned SrcElTySize = SrcTyL.second.getScalarSizeInBits();
   if (!SrcTyL.second.isVector() || SrcElTySize != SrcTy->getScalarSizeInBits())
     return false;
 
   // Get the total number of vector elements in the legalized types.
   InstructionCost NumDstEls =
       DstTyL.first * DstTyL.second.getVectorMinNumElements();
InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
   // If the cast is observable, and it is used by a widening instruction (e.g.,
   // uaddl, saddw, etc.), it may be free.
   if (I && I->hasOneUser()) {
     auto *SingleUser = cast<Instruction>(*I->user_begin());
     SmallVector<const Value *, 4> Operands(SingleUser->operand_values());
-    if (isWideningInstruction(Dst, SingleUser->getOpcode(), Operands)) {
+    if (isWideningInstruction(Dst, Src, SingleUser->getOpcode(), Operands)) {
       // If the cast is the second operand, it is free. We will generate either
       // a "wide" or "long" version of the widening instruction.
       if (I == SingleUser->getOperand(1))
         return 0;
       // If the cast is not the second operand, it will be free if it looks the
       // same as the second operand. In this case, we will generate a "long"
       // version of the widening instruction.
       if (auto *Cast = dyn_cast<CastInst>(SingleUser->getOperand(1)))
         if (I->getOpcode() == unsigned(Cast->getOpcode()) &&
             cast<CastInst>(I)->getSrcTy() == Cast->getSrcTy())
           return 0;
     }
   }
 
   // TODO: Allow non-throughput costs that aren't binary.
case ISD::MUL:
     // estimate the cost for a i64 vector directly here, which is:
     // - four 2-cost i64 extracts,
     // - two 2-cost i64 inserts, and
     // - two 1-cost muls.
     // So, for a v2i64 with LT.First = 1 the cost is 14, and for a v4i64 with
     // LT.first = 2 the cost is 28. If both operands are extensions it will not
     // need to scalarize so the cost can be cheaper (smull or umull).
     // so the cost can be cheaper (smull or umull).
-    if (LT.second != MVT::v2i64 || isWideningInstruction(Ty, Opcode, Args))
+    if (LT.second != MVT::v2i64 ||
+        isWideningInstruction(Ty, nullptr, Opcode, Args))
       return LT.first;
     return LT.first * 14;
   case ISD::ADD:
   case ISD::XOR:
   case ISD::OR:
   case ISD::AND:
   case ISD::SRL:
   case ISD::SRA:

llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll

 
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -opaque-pointers -aarch64-sve-vector-bits-min=256 -verify-machineinstrs | FileCheck %s --check-prefixes=SVE256
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -opaque-pointers -aarch64-sve-vector-bits-min=128 -verify-machineinstrs | FileCheck %s --check-prefixes=NEON
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-n1 -O3 -opaque-pointers -verify-machineinstrs | FileCheck %s --check-prefixes=NEON
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v2 -O3 -opaque-pointers -verify-machineinstrs | FileCheck %s --check-prefixes=NEON
 
 define internal i32 @test(ptr nocapture readonly %p1, i32 %i1, ptr nocapture readonly %p2, i32 %i2) {
 ; SVE256-LABEL: test:
-; SVE256:       ld1b  { z0.h }, p0/z,
-; SVE256:       ld1b  { z1.h }, p0/z,
-; SVE256:       sub z0.h, z0.h, z1.h
-; SVE256-NEXT:  sunpklo z1.s, z0.h
-; SVE256-NEXT:  ext z0.b, z0.b, z0.b, #16
-; SVE256-NEXT:  sunpklo z0.s, z0.h
-; SVE256-NEXT:  add z0.s, z1.s, z0.s
-; SVE256-NEXT:  uaddv d0, p1, z0.s
+; SVE256:       ld1b    { z0.h }, p0/z,
+; SVE256:       ld1b    { z1.h }, p0/z,
+; SVE256:       sub z0.h, z0.h, z1.h
+; SVE256-NEXT:  sunpklo z1.s, z0.h
+; SVE256-NEXT:  ext z0.b, z0.b, z0.b, #16
+; SVE256-NEXT:  sunpklo z0.s, z0.h
+; SVE256-NEXT:  add z0.s, z1.s, z0.s
+; SVE256-NEXT:  uaddv   d0, p1, z0.s
 
 ; NEON-LABEL: test:
-; NEON:         tbl
-; NEON-NEXT:    tbl
-; NEON-NEXT:    tbl
-; NEON-NEXT:    tbl
-; NEON-NEXT:    tbl
-; NEON-NEXT:    tbl
-; NEON-NEXT:    tbl
-; NEON-NEXT:    tbl
-; NEON:         addv
-
+; NEON:       ldr q0, [x0, w9, sxtw]
+; NEON:       ldr q1, [x2, w10, sxtw]
+; NEON:       usubl2  v2.8h, v0.16b, v1.16b
+; NEON-NEXT:  usubl   v0.8h, v0.8b, v1.8b
+; NEON:       saddl2  v1.4s, v0.8h, v2.8h
+; NEON-NEXT:  saddl   v0.4s, v0.4h, v2.4h
+; NEON-NEXT:  add v0.4s, v0.4s, v1.4s
+; NEON-NEXT:  addv    s0, v0.4s
 
 L.entry:
   br label %L1
 
 L1:                                          ; preds = %L1, %L.entry
   %a = phi i32 [ 16, %L.entry ], [ %14, %L1 ]
   %b = phi i32 [ 0, %L.entry ], [ %13, %L1 ]
   %i = phi i32 [ 0, %L.entry ], [ %12, %L1 ]

llvm/test/CodeGen/AArch64/zext-to-tbl.ll

 exit:
   ret void
 }
 
 ; FIXME: Widening instructions should be used instead of tbl.
 define i32 @test_pr62620_widening_instr(ptr %p1, ptr %p2, i64 %lx, i32 %h) {
 ; CHECK-LABEL: test_pr62620_widening_instr:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:  Lloh38:
-; CHECK-NEXT:    adrp x9, lCPI23_0@PAGE
-; CHECK-NEXT:  Lloh39:
-; CHECK-NEXT:    adrp x10, lCPI23_1@PAGE
-; CHECK-NEXT:  Lloh40:
-; CHECK-NEXT:    adrp x11, lCPI23_2@PAGE
-; CHECK-NEXT:  Lloh41:
-; CHECK-NEXT:    adrp x12, lCPI23_3@PAGE
 ; CHECK-NEXT:    mov x8, x0
 ; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:  Lloh42:
-; CHECK-NEXT:    ldr q0, [x9, lCPI23_0@PAGEOFF]
 ; CHECK-NEXT:    lsl x9, x2, #4
-; CHECK-NEXT:  Lloh43:
-; CHECK-NEXT:    ldr q1, [x10, lCPI23_1@PAGEOFF]
-; CHECK-NEXT:  Lloh44:
-; CHECK-NEXT:    ldr q2, [x11, lCPI23_2@PAGEOFF]
-; CHECK-NEXT:  Lloh45:
-; CHECK-NEXT:    ldr q3, [x12, lCPI23_3@PAGEOFF]
 ; CHECK-NEXT:  LBB23_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr q4, [x8, x9]
+; CHECK-NEXT:    ldr q0, [x8, x9]
 ; CHECK-NEXT:    subs w3, w3, #1
-; CHECK-NEXT:    ldr q5, [x1, x9]
-; CHECK-NEXT:    tbl.16b v6, { v4 }, v0
-; CHECK-NEXT:    tbl.16b v7, { v4 }, v1
-; CHECK-NEXT:    tbl.16b v16, { v4 }, v2
-; CHECK-NEXT:    tbl.16b v4, { v4 }, v3
-; CHECK-NEXT:    tbl.16b v17, { v5 }, v2
-; CHECK-NEXT:    tbl.16b v18, { v5 }, v3
-; CHECK-NEXT:    tbl.16b v19, { v5 }, v0
-; CHECK-NEXT:    tbl.16b v5, { v5 }, v1
-; CHECK-NEXT:    sabd.4s v16, v16, v17
-; CHECK-NEXT:    sabd.4s v4, v4, v18
-; CHECK-NEXT:    saba.4s v16, v7, v5
-; CHECK-NEXT:    saba.4s v4, v6, v19
-; CHECK-NEXT:    add.4s v4, v4, v16
-; CHECK-NEXT:    addv.4s s4, v4
-; CHECK-NEXT:    fmov w10, s4
+; CHECK-NEXT:    ldr q1, [x1, x9]
+; CHECK-NEXT:    uabdl.8h v2, v0, v1
+; CHECK-NEXT:    uabal2.8h v2, v0, v1
+; CHECK-NEXT:    uaddlv.8h s0, v2
+; CHECK-NEXT:    fmov w10, s0
 ; CHECK-NEXT:    add w0, w10, w0
 ; CHECK-NEXT:    b.ne LBB23_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:    .loh AdrpLdr Lloh41, Lloh45
-; CHECK-NEXT:    .loh AdrpLdr Lloh40, Lloh44
-; CHECK-NEXT:    .loh AdrpLdr Lloh39, Lloh43
-; CHECK-NEXT:    .loh AdrpLdr Lloh38, Lloh42
 ;
 ; CHECK-BE-LABEL: test_pr62620_widening_instr:
 ; CHECK-BE:       // %bb.0: // %entry
-; CHECK-BE-NEXT:    adrp x10, .LCPI23_0
-; CHECK-BE-NEXT:    add x10, x10, :lo12:.LCPI23_0
 ; CHECK-BE-NEXT:    mov x8, x0
 ; CHECK-BE-NEXT:    lsl x9, x2, #4
 ; CHECK-BE-NEXT:    mov w0, wzr
 ; CHECK-BE-NEXT:    add x8, x8, x9
-; CHECK-BE-NEXT:    ld1 { v0.16b }, [x10]
-; CHECK-BE-NEXT:    adrp x10, .LCPI23_1
-; CHECK-BE-NEXT:    add x10, x10, :lo12:.LCPI23_1
 ; CHECK-BE-NEXT:    add x9, x1, x9
-; CHECK-BE-NEXT:    ld1 { v1.16b }, [x10]
-; CHECK-BE-NEXT:    adrp x10, .LCPI23_2
-; CHECK-BE-NEXT:    add x10, x10, :lo12:.LCPI23_2
-; CHECK-BE-NEXT:    ld1 { v2.16b }, [x10]
-; CHECK-BE-NEXT:    adrp x10, .LCPI23_3
-; CHECK-BE-NEXT:    add x10, x10, :lo12:.LCPI23_3
-; CHECK-BE-NEXT:    ld1 { v3.16b }, [x10]
 ; CHECK-BE-NEXT:  .LBB23_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    ld1 { v4.16b }, [x8]
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x8]
 ; CHECK-BE-NEXT:    subs w3, w3, #1
-; CHECK-BE-NEXT:    ld1 { v5.16b }, [x9]
-; CHECK-BE-NEXT:    tbl v6.16b, { v4.16b }, v0.16b
-; CHECK-BE-NEXT:    tbl v7.16b, { v4.16b }, v1.16b
-; CHECK-BE-NEXT:    tbl v17.16b, { v5.16b }, v0.16b
-; CHECK-BE-NEXT:    tbl v18.16b, { v5.16b }, v1.16b
-; CHECK-BE-NEXT:    tbl v16.16b, { v4.16b }, v3.16b
-; CHECK-BE-NEXT:    tbl v4.16b, { v4.16b }, v2.16b
-; CHECK-BE-NEXT:    tbl v19.16b, { v5.16b }, v3.16b
-; CHECK-BE-NEXT:    tbl v5.16b, { v5.16b }, v2.16b
-; CHECK-BE-NEXT:    rev32 v7.16b, v7.16b
-; CHECK-BE-NEXT:    rev32 v6.16b, v6.16b
-; CHECK-BE-NEXT:    rev32 v18.16b, v18.16b
-; CHECK-BE-NEXT:    rev32 v17.16b, v17.16b
-; CHECK-BE-NEXT:    rev32 v16.16b, v16.16b
-; CHECK-BE-NEXT:    rev32 v4.16b, v4.16b
-; CHECK-BE-NEXT:    rev32 v19.16b, v19.16b
-; CHECK-BE-NEXT:    rev32 v5.16b, v5.16b
-; CHECK-BE-NEXT:    sabd v7.4s, v7.4s, v18.4s
-; CHECK-BE-NEXT:    sabd v6.4s, v6.4s, v17.4s
-; CHECK-BE-NEXT:    saba v7.4s, v4.4s, v5.4s
-; CHECK-BE-NEXT:    saba v6.4s, v16.4s, v19.4s
-; CHECK-BE-NEXT:    add v4.4s, v6.4s, v7.4s
-; CHECK-BE-NEXT:    addv s4, v4.4s
-; CHECK-BE-NEXT:    fmov w10, s4
+; CHECK-BE-NEXT:    ld1 { v1.16b }, [x9]
+; CHECK-BE-NEXT:    uabdl v2.8h, v0.8b, v1.8b
+; CHECK-BE-NEXT:    uabal2 v2.8h, v0.16b, v1.16b
+; CHECK-BE-NEXT:    uaddlv s0, v2.8h
+; CHECK-BE-NEXT:    fmov w10, s0
 ; CHECK-BE-NEXT:    add w0, w10, w0
 ; CHECK-BE-NEXT:    b.ne .LBB23_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit
 ; CHECK-BE-NEXT:    ret
 entry:
   br label %loop
 
 loop: