[AArch64] Support more types for ZEXT/SEXT with Global Isel

[arsenm]

llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h

1046–1070 ↗

(On Diff #547226)

Is it possible to separate this from the lowerEXT parts?

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

5951

This shouldn't need to consider the legality of intermediate ops and fail, just unconditionally do this and let the new artifacts legalize

[chuongg3]

Split combiner into a seperate commit.
Unconditionally lowers Extends, independent of the intermediate instruction

[dmgreen]

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

5952

Can reduce the indenting now.

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

523

Can you add a comment here explaining that this is trying to convert larger extends into two smaller ones.

525

(Query.Types[0].isVector()) -> Query.Types[0].isVector()

llvm/test/CodeGen/AArch64/GlobalISel/legalize-build-vector.mir

144–145

Is this with update_mir_test_checks? Maybe just replace %ext with %3 in the code below.

[aemerson]

A few comments but otherwise LGTM, thanks for the new tests.

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

5951–5952

No need for MidRegister, this can be simplified to:

auto NewExt = MIRBuilder.buildInstr(MI.getOpcode(), {MidTy}, {Src});
...
auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, NewExt);
...`

5958–5966

Same here, it's less verbose to just create the LLTs and pass those to buildInstr() instead of passing Registers.

// ZExt the vectors
Register ZExtResTy = DstTy.changeElementCount(DstTy.getElementCount().divideCoefficientBy(2));
auto ZExtRes1 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, {UnmergeSrc.getReg(0)});
auto ZExtRes2 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, {UnmergeSrc.getReg(1)});

[chuongg3]

Rectified comments made by aemerson
Instructions are built using register types instead of creating a register for the instruction to be built

[chuongg3]

Formatted the patch

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

public:
   LegalizeResult lowerFConstant(MachineInstr &MI);
   LegalizeResult lowerBitcast(MachineInstr &MI);
   LegalizeResult lowerLoad(GAnyLoad &MI);
   LegalizeResult lowerStore(GStore &MI);
   LegalizeResult lowerBitCount(MachineInstr &MI);
   LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI);
   LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI);
   LegalizeResult lowerFunnelShift(MachineInstr &MI);
+  LegalizeResult lowerEXT(MachineInstr &MI);
   LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI);
   LegalizeResult lowerRotate(MachineInstr &MI);
 
   LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI);
   LegalizeResult lowerUITOFP(MachineInstr &MI);
   LegalizeResult lowerSITOFP(MachineInstr &MI);
   LegalizeResult lowerFPTOUI(MachineInstr &MI);
   LegalizeResult lowerFPTOSI(MachineInstr &MI);

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
   case G_ROTR:
     return lowerRotate(MI);
   case G_MEMSET:
   case G_MEMCPY:
   case G_MEMMOVE:
     return lowerMemCpyFamily(MI);
   case G_MEMCPY_INLINE:
     return lowerMemcpyInline(MI);
+  case G_ZEXT:
+  case G_SEXT:
+  case G_ANYEXT:
+    return lowerEXT(MI);
     GISEL_VECREDUCE_CASES_NONSEQ
     return lowerVectorReduction(MI);
   }
 }
 
 Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
                                                   Align MinAlign) const {
   // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
   // datalayout for the preferred alignment. Also there should be a target hook
LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
 
   // This only works for powers of 2, fallback to shifts if it fails.
   LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
   if (Result == UnableToLegalize)
     return lowerFunnelShiftAsShifts(MI);
   return Result;
 }
 
+LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
+  auto [Dst, Src] = MI.getFirst2Regs();
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+
+  uint32_t DstTySize = DstTy.getSizeInBits();
+  uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits();
+  uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits();
+
+  if (!isPowerOf2_32(DstTySize) || !isPowerOf2_32(DstTyScalarSize) ||
+      !isPowerOf2_32(SrcTyScalarSize))
+    return UnableToLegalize;
+
+  // The step between extend is too large, split it by creating an intermediate
+  // extend instruction
+  if (SrcTyScalarSize * 2 < DstTyScalarSize) {
+    LLT MidTy = SrcTy.changeElementSize(SrcTyScalarSize * 2);
+    // If the destination type is illegal, split it into multiple statements
+    // zext x -> zext(merge(zext(unmerge), zext(unmerge)))
+    Register MidRegister = MRI.createGenericVirtualRegister(MidTy);
+    MIRBuilder.buildInstr(MI.getOpcode(), {MidRegister}, {Src});
+    // Unmerge the vector
+    LLT EltTy = MidTy.changeElementCount(
+        MidTy.getElementCount().divideCoefficientBy(2));
+    auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, MidRegister);
+
+    // ZExt the vectors
+    Register ZExtRes1 =
+        MRI.createGenericVirtualRegister(DstTy.changeElementCount(
+            DstTy.getElementCount().divideCoefficientBy(2)));
+    Register ZExtRes2 =
+        MRI.createGenericVirtualRegister(MRI.getType(ZExtRes1));
+
+    MIRBuilder.buildInstr(MI.getOpcode(), {ZExtRes1}, {UnmergeSrc.getReg(0)});
+    MIRBuilder.buildInstr(MI.getOpcode(), {ZExtRes2}, {UnmergeSrc.getReg(1)});
+
+    // Merge the ending vectors
+    MIRBuilder.buildMergeLikeInstr(Dst, {ZExtRes1, ZExtRes2});
+
+    MI.eraseFromParent();
+    return Legalized;
+  }
+  return UnableToLegalize;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
   auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
   auto Zero = MIRBuilder.buildConstant(AmtTy, 0);
   bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
   unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
   auto Neg = MIRBuilder.buildSub(AmtTy, Zero, Amt);
   MIRBuilder.buildInstr(RevRot, {Dst}, {Src, Neg});

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_FCMP)
           0, 1)
       .clampNumElements(0, v2s32, v4s32)
       .clampMaxNumElements(1, s64, 2);
 
   // Extensions
   auto ExtLegalFunc = [=](const LegalityQuery &Query) {
     unsigned DstSize = Query.Types[0].getSizeInBits();
 
-    if (DstSize == 128 && !Query.Types[0].isVector())
-      return false; // Extending to a scalar s128 needs narrowing.
-
-    // Make sure that we have something that will fit in a register, and
-    // make sure it's a power of 2.
-    if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize))
+    // Handle legal vectors using legalFor
+    if (Query.Types[0].isVector())
       return false;
 
+    if (DstSize < 8 || DstSize >= 128 || !isPowerOf2_32(DstSize))
+      return false; // Extending to a scalar s128 needs narrowing.
+
     const LLT &SrcTy = Query.Types[1];
 
     // Make sure we fit in a register otherwise. Don't bother checking that
     // the source type is below 128 bits. We shouldn't be allowing anything
     // through which is wider than the destination in the first place.
     unsigned SrcSize = SrcTy.getSizeInBits();
     if (SrcSize < 8 || !isPowerOf2_32(SrcSize))
       return false;
 
     return true;
   };
   getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
       .legalIf(ExtLegalFunc)
-      .clampScalar(0, s64, s64); // Just for s128, others are handled above.
+      .legalFor({{v2s64, v2s32}, {v4s32, v4s16}, {v8s16, v8s8}})
+      .clampScalar(0, s64, s64) // Just for s128, others are handled above.
+      .moreElementsToNextPow2(1)
+      .clampMaxNumElements(1, s8, 8)
+      .clampMaxNumElements(1, s16, 4)
+      .clampMaxNumElements(1, s32, 2)
+      // Tries to convert a large EXTEND into two smaller EXTENDs
+      .lowerIf([=](const LegalityQuery &Query) {
+        return (Query.Types[0].getScalarSizeInBits() >
+                Query.Types[1].getScalarSizeInBits() * 2) &&
+                Query.Types[0].isVector() && 
+               (Query.Types[1].getScalarSizeInBits() == 8 ||
+                Query.Types[1].getScalarSizeInBits() == 16);
+      });
 
   getActionDefinitionsBuilder(G_TRUNC)
       .minScalarOrEltIf(
           [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
           0, s8)
       .customIf([=](const LegalityQuery &Query) {
         LLT DstTy = Query.Types[0];
         LLT SrcTy = Query.Types[1];

llvm/test/CodeGen/AArch64/GlobalISel/legalize-and.mir

bb.1:
     liveins: $b0, $b1, $b2
 
     ; CHECK-LABEL: name: and_v3s1
     ; CHECK: liveins: $b0, $b1, $b2
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s8) = COPY $b0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s8) = COPY $b1
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s8) = COPY $b2
-    ; CHECK-NEXT: [[ANYEXT0:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY]](s8)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY]](s8)
     ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY1]](s8)
     ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY2]](s8)
-    ; CHECK-NEXT: [[IMPLICIT_DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[ANYEXT0]](s16), [[ANYEXT1]](s16), [[ANYEXT2]](s16), [[IMPLICIT_DEF]](s16)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[ANYEXT]](s16), [[ANYEXT1]](s16), [[ANYEXT2]](s16), [[DEF]](s16)
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[BUILD_VECTOR]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[VAL0:%[0-9]+]]:_(s16), [[VAL1:%[0-9]+]]:_(s16), [[VAL2:%[0-9]+]]:_(s16), [[VAL3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[AND]](<4 x s16>)
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[VAL0]](s16)
-    ; CHECK-NEXT: $b0 = COPY [[TRUNC3]](s8)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[AND]](<4 x s16>)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV]](s16)
+    ; CHECK-NEXT: $b0 = COPY [[TRUNC]](s8)
     ; CHECK-NEXT: RET_ReallyLR implicit $b0
     %1:_(s8) = COPY $b0
     %2:_(s8) = COPY $b1
     %3:_(s8) = COPY $b2
     %4:_(<3 x s8>) = G_BUILD_VECTOR %1(s8), %2(s8), %3(s8)
     %0:_(<3 x s1>) = G_TRUNC %4(<3 x s8>)
     %5:_(<3 x s1>) = G_AND %0, %0
     %7:_(<3 x s8>) = G_ANYEXT %5(<3 x s1>)

llvm/test/CodeGen/AArch64/GlobalISel/legalize-build-vector.mir

bb.0:
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY]](s16)
     ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY1]](s16)
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; CHECK-NEXT: RET_ReallyLR
     %0:_(s16) = COPY $h0
     %1:_(s16) = COPY $h1
     %2:_(<2 x s16>) = G_BUILD_VECTOR %0(s16), %1(s16)
-    %ext:_(<2 x s32>) = G_ANYEXT %2(<2 x s16>)
-    $d0 = COPY %ext(<2 x s32>)
+    %3:_(<2 x s32>) = G_ANYEXT %2(<2 x s16>)
+    $d0 = COPY %3(<2 x s32>)
     RET_ReallyLR
 ...
 
 ---
 name:            widen_v2s8
 body: |
   bb.0:
     ; CHECK-LABEL: name: widen_v2s8
     ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[DEF]](s32)
-    ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; CHECK-NEXT: %3:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[DEF]](s32)
+    ; CHECK-NEXT: $d0 = COPY %3(<2 x s32>)
     ; CHECK-NEXT: RET_ReallyLR
     %0:_(s8) = G_IMPLICIT_DEF
     %1:_(s8) = G_IMPLICIT_DEF
     %2:_(<2 x s8>) = G_BUILD_VECTOR %0(s8), %1(s8)
-    %ext:_(<2 x s32>) = G_ANYEXT %2(<2 x s8>)
-    $d0 = COPY %ext(<2 x s32>)
+    %3:_(<2 x s32>) = G_ANYEXT %2(<2 x s8>)
+    $d0 = COPY %3(<2 x s32>)
     RET_ReallyLR
 ...
 
 ---
 name:            widen_v4s8
 body: |
   bb.0:
     ; CHECK-LABEL: name: widen_v4s8
     ; CHECK: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY [[DEF]](s16)
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s16) = COPY [[DEF]](s16)
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[DEF]](s16)
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[COPY]](s16), [[COPY1]](s16), [[COPY2]](s16), [[DEF]](s16)
     ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<4 x s16>)
     ; CHECK-NEXT: RET_ReallyLR
     %0:_(s8) = G_IMPLICIT_DEF
     %1:_(s8) = G_IMPLICIT_DEF
     %2:_(s8) = G_IMPLICIT_DEF
     %3:_(s8) = G_IMPLICIT_DEF
     %4:_(<4 x s8>) = G_BUILD_VECTOR %0(s8), %1(s8), %2(s8), %3(s8)
-    %ext:_(<4 x s16>) = G_ANYEXT %4(<4 x s8>)
-    $d0 = COPY %ext(<4 x s16>)
+    %5:_(<4 x s16>) = G_ANYEXT %4(<4 x s8>)
+    $d0 = COPY %5(<4 x s16>)
     RET_ReallyLR
 ...

llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir

 ---
 name:            test_uitofp_v2s64_v2i1
 body: |
   bb.0:
     liveins: $q0
     ; CHECK-LABEL: name: test_uitofp_v2s64_v2i1
     ; CHECK: liveins: $q0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[DEF]](s32)
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<2 x s64>) = G_ANYEXT [[BUILD_VECTOR]](<2 x s32>)
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[ANYEXT]], [[BUILD_VECTOR1]]
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[DEF]](s64)
+    ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[DEF]](s64), [[COPY]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[BUILD_VECTOR1]], [[BUILD_VECTOR]]
     ; CHECK-NEXT: [[UITOFP:%[0-9]+]]:_(<2 x s64>) = G_UITOFP [[AND]](<2 x s64>)
     ; CHECK-NEXT: $q0 = COPY [[UITOFP]](<2 x s64>)
     %0:_(<2 x s1>) = G_IMPLICIT_DEF
     %1:_(<2 x s64>) = G_UITOFP %0
     $q0 = COPY %1
 ...
 
 ---
 name:            test_sitofp_v2s64_v2i1
 body: |
   bb.0:
     liveins: $q0
     ; CHECK-LABEL: name: test_sitofp_v2s64_v2i1
     ; CHECK: liveins: $q0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[DEF]](s32)
-    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<2 x s64>) = G_ANYEXT [[BUILD_VECTOR]](<2 x s32>)
-    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ANYEXT]], 1
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[DEF]](s64)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[DEF]](s64), [[COPY]](s64)
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[BUILD_VECTOR]], 1
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(<2 x s64>) = G_SITOFP [[SEXT_INREG]](<2 x s64>)
     ; CHECK-NEXT: $q0 = COPY [[SITOFP]](<2 x s64>)
     %0:_(<2 x s1>) = G_IMPLICIT_DEF
     %1:_(<2 x s64>) = G_SITOFP %0
     $q0 = COPY %1
 ...
 
 ---

llvm/test/CodeGen/AArch64/aarch64-addv.ll

; CHECK-NEXT:    ret
   %bin.rdx = load <2 x i64>, ptr %arr
   %r = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %bin.rdx)
   ret i64 %r
 }
 
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 
 define i32 @oversized_ADDV_256(ptr noalias nocapture readonly %arg1, ptr noalias nocapture readonly %arg2) {
-; CHECK-LABEL: oversized_ADDV_256:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    uabdl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    uaddlv s0, v0.8h
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: oversized_ADDV_256:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    ldr d0, [x0]
+; SDAG-NEXT:    ldr d1, [x1]
+; SDAG-NEXT:    uabdl v0.8h, v0.8b, v1.8b
+; SDAG-NEXT:    uaddlv s0, v0.8h
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: oversized_ADDV_256:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    ldr d1, [x0]
+; GISEL-NEXT:    ldr d2, [x1]
+; GISEL-NEXT:    movi v0.2d, #0000000000000000
+; GISEL-NEXT:    ushll v1.8h, v1.8b, #0
+; GISEL-NEXT:    ushll v2.8h, v2.8b, #0
+; GISEL-NEXT:    mov d3, v1.d[1]
+; GISEL-NEXT:    mov d4, v2.d[1]
+; GISEL-NEXT:    usubl v1.4s, v1.4h, v2.4h
+; GISEL-NEXT:    usubl v2.4s, v3.4h, v4.4h
+; GISEL-NEXT:    cmgt v3.4s, v0.4s, v1.4s
+; GISEL-NEXT:    neg v4.4s, v1.4s
+; GISEL-NEXT:    cmgt v0.4s, v0.4s, v2.4s
+; GISEL-NEXT:    shl v3.4s, v3.4s, #31
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    neg v5.4s, v2.4s
+; GISEL-NEXT:    sshr v3.4s, v3.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    bit v1.16b, v4.16b, v3.16b
+; GISEL-NEXT:    bsl v0.16b, v5.16b, v2.16b
+; GISEL-NEXT:    add v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    addv s0, v0.4s
+; GISEL-NEXT:    fmov w0, s0
+; GISEL-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %arg1, align 1
   %1 = zext <8 x i8> %0 to <8 x i32>
   %2 = load <8 x i8>, ptr %arg2, align 1
   %3 = zext <8 x i8> %2 to <8 x i32>
   %4 = sub nsw <8 x i32> %1, %3
   %5 = icmp slt <8 x i32> %4, zeroinitializer
   %6 = sub nsw <8 x i32> zeroinitializer, %4
   %7 = select <8 x i1> %5, <8 x i32> %6, <8 x i32> %4
   %r = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %7)
   ret i32 %r
 }
 
 declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
 
 define i32 @oversized_ADDV_512(ptr %arr)  {
-; SDAG-LABEL:  oversized_ADDV_512:
-; SDAG:        // %bb.0:
-; SDAG-NEXT:     ldp q0, q1, [x0, #32]
-; SDAG-NEXT:     ldp q3, q2, [x0]
-; SDAG-NEXT:     add v0.4s, v3.4s, v0.4s
-; SDAG-NEXT:     add v1.4s, v2.4s, v1.4s
-; SDAG-NEXT:     add v0.4s, v0.4s, v1.4s
-; SDAG-NEXT:     addv s0, v0.4s
-; SDAG-NEXT:     fmov w0, s0
-; SDAG-NEXT:     ret
+; SDAG-LABEL: oversized_ADDV_512:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    ldp q0, q1, [x0, #32]
+; SDAG-NEXT:    ldp q3, q2, [x0]
+; SDAG-NEXT:    add v0.4s, v3.4s, v0.4s
+; SDAG-NEXT:    add v1.4s, v2.4s, v1.4s
+; SDAG-NEXT:    add v0.4s, v0.4s, v1.4s
+; SDAG-NEXT:    addv s0, v0.4s
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: oversized_ADDV_512:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldp q0, q1, [x0]
 ; GISEL-NEXT:    ldp q2, q3, [x0, #32]
 ; GISEL-NEXT:    add v0.4s, v0.4s, v1.4s
 ; GISEL-NEXT:    add v1.4s, v2.4s, v3.4s
 ; GISEL-NEXT:    add v0.4s, v0.4s, v1.4s

 entry:
   %rdx.1 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a1)
   %rdx.2 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a2)
   %r = add i16 %rdx.1, %rdx.2
   ret i16 %r
 }
 
 define i32 @addv_combine_i32(<4 x i32> %a1, <4 x i32> %a2) {
-; SDAG-LABEL:  addv_combine_i32:
-; SDAG:        // %bb.0: // %entry
-; SDAG-NEXT:     add v0.4s, v0.4s, v1.4s
-; SDAG-NEXT:     addv s0, v0.4s
-; SDAG-NEXT:     fmov w0, s0
-; SDAG-NEXT:     ret
+; SDAG-LABEL: addv_combine_i32:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    add v0.4s, v0.4s, v1.4s
+; SDAG-NEXT:    addv s0, v0.4s
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: addv_combine_i32:
 ; GISEL:       // %bb.0: // %entry
-; GISEL-NEXT:    addv  s0, v0.4s
-; GISEL-NEXT:    addv  s1, v1.4s
-; GISEL-NEXT:    fmov  w8, s0
-; GISEL-NEXT:    fmov  w9, s1
+; GISEL-NEXT:    addv s0, v0.4s
+; GISEL-NEXT:    addv s1, v1.4s
+; GISEL-NEXT:    fmov w8, s0
+; GISEL-NEXT:    fmov w9, s1
 ; GISEL-NEXT:    add w0, w8, w9
 ; GISEL-NEXT:    ret
 entry:
   %rdx.1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a1)
   %rdx.2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a2)
   %r = add i32 %rdx.1, %rdx.2
   ret i32 %r
 }
 
 define i64 @addv_combine_i64(<2 x i64> %a1, <2 x i64> %a2) {
-; SDAG-LABEL:  addv_combine_i64:
-; SDAG:        // %bb.0: // %entry
-; SDAG-NEXT:     add v0.2d, v0.2d, v1.2d
-; SDAG-NEXT:     addp d0, v0.2d
-; SDAG-NEXT:     fmov x0, d0
-; SDAG-NEXT:     ret
+; SDAG-LABEL: addv_combine_i64:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    add v0.2d, v0.2d, v1.2d
+; SDAG-NEXT:    addp d0, v0.2d
+; SDAG-NEXT:    fmov x0, d0
+; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: addv_combine_i64:
 ; GISEL:       // %bb.0: // %entry
-; GISEL-NEXT:    addp  d0, v0.2d
-; GISEL-NEXT:    addp  d1, v1.2d
-; GISEL-NEXT:    fmov  x8, d0
-; GISEL-NEXT:    fmov  x9, d1
+; GISEL-NEXT:    addp d0, v0.2d
+; GISEL-NEXT:    addp d1, v1.2d
+; GISEL-NEXT:    fmov x8, d0
+; GISEL-NEXT:    fmov x9, d1
 ; GISEL-NEXT:    add x0, x8, x9
 ; GISEL-NEXT:    ret
 entry:
   %rdx.1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1)
   %rdx.2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a2)
   %r = add i64 %rdx.1, %rdx.2
   ret i64 %r
 }

llvm/test/CodeGen/AArch64/arm64-subvector-extend.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; Test efficient codegen of vector extends up from legal type to 128 bit
 ; and 256 bit vector types.
 
-; CHECK-GI:       warning: Instruction selection used fallback path for func3
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for func4
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for afunc3
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for afunc4
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bfunc1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bfunc2
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zfunc1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zfunc2
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bfunc3
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for cfunc4
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v4i8_to_v4i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v4i8_to_v4i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v8i8_to_v8i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v8i8_to_v8i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v32i1
+; CHECK-GI:        warning: Instruction selection used fallback path for zext_v32i1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v32i1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v64i1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v64i1
 
 ;-----
 ; Vectors of i16.
 ;-----
 

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sshll.8h v0, v0, #0
 ; CHECK-NEXT:    ret
   %r = sext <8 x i8> %v0 to <8 x i16>
   ret <8 x i16> %r
 }
 
 define <16 x i16> @func3(<16 x i8> %v0) nounwind {
-; CHECK-LABEL: func3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.8h v1, v0, #0
-; CHECK-NEXT:    ushll.8h v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: func3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll2.8h v1, v0, #0
+; CHECK-SD-NEXT:    ushll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: func3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    ushll.8h v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <16 x i8> %v0 to <16 x i16>
   ret <16 x i16> %r
 }
 
 define <16 x i16> @func4(<16 x i8> %v0) nounwind {
-; CHECK-LABEL: func4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.8h v1, v0, #0
-; CHECK-NEXT:    sshll.8h v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: func4:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll2.8h v1, v0, #0
+; CHECK-SD-NEXT:    sshll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: func4:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    sshll.8h v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <16 x i8> %v0 to <16 x i16>
   ret <16 x i16> %r
 }
 
 ;-----
 ; Vectors of i32.
 ;-----
 

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
 ; CHECK-NEXT:    ret
   %r = sext <4 x i16> %v0 to <4 x i32>
   ret <4 x i32> %r
 }
 
 define <8 x i32> @afunc3(<8 x i16> %v0) nounwind {
-; CHECK-LABEL: afunc3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.4s v1, v0, #0
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: afunc3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: afunc3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    ushll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <8 x i16> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @afunc4(<8 x i16> %v0) nounwind {
-; CHECK-LABEL: afunc4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.4s v1, v0, #0
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: afunc4:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: afunc4:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    sshll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <8 x i16> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @bfunc1(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: bfunc1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.8h v0, v0, #0
-; CHECK-NEXT:    ushll2.4s v1, v0, #0
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bfunc1:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bfunc1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    ushll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <8 x i8> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @bfunc2(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: bfunc2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll.8h v0, v0, #0
-; CHECK-NEXT:    sshll2.4s v1, v0, #0
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bfunc2:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll.8h v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bfunc2:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    sshll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <8 x i8> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 ;-----
 ; Vectors of i64.
 ;-----
 
 define <4 x i64> @zfunc1(<4 x i32> %v0) nounwind {
-; CHECK-LABEL: zfunc1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zfunc1:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zfunc1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <4 x i32> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @zfunc2(<4 x i32> %v0) nounwind {
-; CHECK-LABEL: zfunc2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.2d v1, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zfunc2:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zfunc2:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.2d v0, v0, #0
+; CHECK-GI-NEXT:    sshll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <4 x i32> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @bfunc3(<4 x i16> %v0) nounwind {
-; CHECK-LABEL: bfunc3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bfunc3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bfunc3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <4 x i16> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @cfunc4(<4 x i16> %v0) nounwind {
-; CHECK-LABEL: cfunc4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    sshll2.2d v1, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: cfunc4:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: cfunc4:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.2d v0, v0, #0
+; CHECK-GI-NEXT:    sshll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <4 x i16> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @zext_v4i8_to_v4i64(<4 x i8> %v0) nounwind {
-; CHECK-LABEL: zext_v4i8_to_v4i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    bic.4h v0, #255, lsl #8
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zext_v4i8_to_v4i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    bic.4h v0, #255, lsl #8
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_to_v4i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    adrp x8, .LCPI14_0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI14_0]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    and.16b v0, v0, v2
+; CHECK-GI-NEXT:    and.16b v1, v1, v2
+; CHECK-GI-NEXT:    ret
   %r = zext <4 x i8> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @sext_v4i8_to_v4i64(<4 x i8> %v0) nounwind {
-; CHECK-LABEL: sext_v4i8_to_v4i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll.2d v1, v0, #0
-; CHECK-NEXT:    ushll2.2d v0, v0, #0
-; CHECK-NEXT:    shl.2d v2, v1, #56
-; CHECK-NEXT:    shl.2d v0, v0, #56
-; CHECK-NEXT:    sshr.2d v1, v0, #56
-; CHECK-NEXT:    sshr.2d v0, v2, #56
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sext_v4i8_to_v4i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v0, v0, #0
+; CHECK-SD-NEXT:    shl.2d v2, v1, #56
+; CHECK-SD-NEXT:    shl.2d v0, v0, #56
+; CHECK-SD-NEXT:    sshr.2d v1, v0, #56
+; CHECK-SD-NEXT:    sshr.2d v0, v2, #56
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i8_to_v4i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    shl.2d v0, v0, #56
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    sshr.2d v0, v0, #56
+; CHECK-GI-NEXT:    shl.2d v1, v1, #56
+; CHECK-GI-NEXT:    sshr.2d v1, v1, #56
+; CHECK-GI-NEXT:    ret
   %r = sext <4 x i8> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <8 x i64> @zext_v8i8_to_v8i64(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: zext_v8i8_to_v8i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.8h v0, v0, #0
-; CHECK-NEXT:    ushll2.4s v2, v0, #0
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll2.2d v3, v2, #0
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ushll.2d v2, v2, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zext_v8i8_to_v8i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.4s v2, v0, #0
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v3, v2, #0
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v2, v2, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i8_to_v8i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d2, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v3, v1, #0
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    mov d4, v3[1]
+; CHECK-GI-NEXT:    ushll.2d v1, v2, #0
+; CHECK-GI-NEXT:    ushll.2d v2, v3, #0
+; CHECK-GI-NEXT:    ushll.2d v3, v4, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <8 x i8> %v0 to <8 x i64>
   ret <8 x i64> %r
 }
 
 define <8 x i64> @sext_v8i8_to_v8i64(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: sext_v8i8_to_v8i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll.8h v0, v0, #0
-; CHECK-NEXT:    sshll2.4s v2, v0, #0
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    sshll2.2d v3, v2, #0
-; CHECK-NEXT:    sshll2.2d v1, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    sshll.2d v2, v2, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sext_v8i8_to_v8i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll.8h v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.4s v2, v0, #0
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.2d v3, v2, #0
+; CHECK-SD-NEXT:    sshll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v0, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v2, v2, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i8_to_v8i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d2, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v3, v1, #0
+; CHECK-GI-NEXT:    sshll.2d v0, v0, #0
+; CHECK-GI-NEXT:    mov d4, v3[1]
+; CHECK-GI-NEXT:    sshll.2d v1, v2, #0
+; CHECK-GI-NEXT:    sshll.2d v2, v3, #0
+; CHECK-GI-NEXT:    sshll.2d v3, v4, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <8 x i8> %v0 to <8 x i64>
   ret <8 x i64> %r
 }
 
 ; Extends of vectors of i1.
 
 define <32 x i8> @zext_v32i1(<32 x i1> %arg) {
 ; CHECK-LABEL: zext_v32i1:

llvm/test/CodeGen/AArch64/arm64-vabs.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck -check-prefixes=CHECK,CHECK-SD %s
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; CHECK-GI:       warning: Instruction selection used fallback path for uabd16b_rdx
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for uabd16b_rdx_i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sabd16b_rdx_i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for uabd8h_rdx
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sabd8h_rdx
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for uabd4s_rdx
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sabd4s_rdx
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_8b
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_16b
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_4h
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_8h
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_2s
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_4s
; CHECK-NEXT:    ret
   %abcmp = icmp slt <16 x i16> %abdiff, zeroinitializer
   %ababs = sub nsw <16 x i16> zeroinitializer, %abdiff
   %absel = select <16 x i1> %abcmp, <16 x i16> %ababs, <16 x i16> %abdiff
   %reduced_v = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %absel)
   ret i16 %reduced_v
 }
 
 define i32 @uabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: uabd16b_rdx_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uabdl.8h v2, v0, v1
-; CHECK-NEXT:    uabal2.8h v2, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v2
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: uabd16b_rdx_i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    uabdl.8h v2, v0, v1
+; CHECK-SD-NEXT:    uabal2.8h v2, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v2
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uabd16b_rdx_i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d3, v0[1]
+; CHECK-GI-NEXT:    ushll.8h v4, v1, #0
+; CHECK-GI-NEXT:    mov d1, v1[1]
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d6, v4[1]
+; CHECK-GI-NEXT:    ushll.8h v3, v3, #0
+; CHECK-GI-NEXT:    mov d5, v0[1]
+; CHECK-GI-NEXT:    ushll.8h v1, v1, #0
+; CHECK-GI-NEXT:    mov d7, v3[1]
+; CHECK-GI-NEXT:    mov d16, v1[1]
+; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-GI-NEXT:    usubl.4s v0, v0, v4
+; CHECK-GI-NEXT:    usubl.4s v5, v5, v6
+; CHECK-GI-NEXT:    usubl.4s v1, v3, v1
+; CHECK-GI-NEXT:    usubl.4s v3, v7, v16
+; CHECK-GI-NEXT:    cmgt.4s v4, v2, v0
+; CHECK-GI-NEXT:    cmgt.4s v6, v2, v5
+; CHECK-GI-NEXT:    cmgt.4s v7, v2, v1
+; CHECK-GI-NEXT:    cmgt.4s v2, v2, v3
+; CHECK-GI-NEXT:    shl.4s v4, v4, #31
+; CHECK-GI-NEXT:    shl.4s v6, v6, #31
+; CHECK-GI-NEXT:    shl.4s v7, v7, #31
+; CHECK-GI-NEXT:    shl.4s v2, v2, #31
+; CHECK-GI-NEXT:    sshr.4s v4, v4, #31
+; CHECK-GI-NEXT:    neg.4s v17, v0
+; CHECK-GI-NEXT:    sshr.4s v6, v6, #31
+; CHECK-GI-NEXT:    neg.4s v16, v5
+; CHECK-GI-NEXT:    neg.4s v18, v1
+; CHECK-GI-NEXT:    neg.4s v19, v3
+; CHECK-GI-NEXT:    sshr.4s v7, v7, #31
+; CHECK-GI-NEXT:    sshr.4s v2, v2, #31
+; CHECK-GI-NEXT:    bit.16b v0, v17, v4
+; CHECK-GI-NEXT:    mov.16b v4, v6
+; CHECK-GI-NEXT:    bsl.16b v4, v16, v5
+; CHECK-GI-NEXT:    bit.16b v1, v18, v7
+; CHECK-GI-NEXT:    bsl.16b v2, v19, v3
+; CHECK-GI-NEXT:    add.4s v0, v0, v4
+; CHECK-GI-NEXT:    add.4s v1, v1, v2
+; CHECK-GI-NEXT:    add.4s v0, v0, v1
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aext = zext <16 x i8> %a to <16 x i32>
   %bext = zext <16 x i8> %b to <16 x i32>
   %abdiff = sub nsw <16 x i32> %aext, %bext
   %abcmp = icmp slt <16 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <16 x i32> zeroinitializer, %abdiff
   %absel = select <16 x i1> %abcmp, <16 x i32> %ababs, <16 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %absel)
   ret i32 %reduced_v
 }
 
 define i32 @sabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: sabd16b_rdx_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sabdl.8h v2, v0, v1
-; CHECK-NEXT:    sabal2.8h v2, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v2
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sabd16b_rdx_i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sabdl.8h v2, v0, v1
+; CHECK-SD-NEXT:    sabal2.8h v2, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v2
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sabd16b_rdx_i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d3, v0[1]
+; CHECK-GI-NEXT:    sshll.8h v4, v1, #0
+; CHECK-GI-NEXT:    mov d1, v1[1]
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d6, v4[1]
+; CHECK-GI-NEXT:    sshll.8h v3, v3, #0
+; CHECK-GI-NEXT:    mov d5, v0[1]
+; CHECK-GI-NEXT:    sshll.8h v1, v1, #0
+; CHECK-GI-NEXT:    mov d7, v3[1]
+; CHECK-GI-NEXT:    mov d16, v1[1]
+; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-GI-NEXT:    ssubl.4s v0, v0, v4
+; CHECK-GI-NEXT:    ssubl.4s v5, v5, v6
+; CHECK-GI-NEXT:    ssubl.4s v1, v3, v1
+; CHECK-GI-NEXT:    ssubl.4s v3, v7, v16
+; CHECK-GI-NEXT:    cmgt.4s v4, v2, v0
+; CHECK-GI-NEXT:    cmgt.4s v6, v2, v5
+; CHECK-GI-NEXT:    cmgt.4s v7, v2, v1
+; CHECK-GI-NEXT:    cmgt.4s v2, v2, v3
+; CHECK-GI-NEXT:    shl.4s v4, v4, #31
+; CHECK-GI-NEXT:    shl.4s v6, v6, #31
+; CHECK-GI-NEXT:    shl.4s v7, v7, #31
+; CHECK-GI-NEXT:    shl.4s v2, v2, #31
+; CHECK-GI-NEXT:    sshr.4s v4, v4, #31
+; CHECK-GI-NEXT:    neg.4s v17, v0
+; CHECK-GI-NEXT:    sshr.4s v6, v6, #31
+; CHECK-GI-NEXT:    neg.4s v16, v5
+; CHECK-GI-NEXT:    neg.4s v18, v1
+; CHECK-GI-NEXT:    neg.4s v19, v3
+; CHECK-GI-NEXT:    sshr.4s v7, v7, #31
+; CHECK-GI-NEXT:    sshr.4s v2, v2, #31
+; CHECK-GI-NEXT:    bit.16b v0, v17, v4
+; CHECK-GI-NEXT:    mov.16b v4, v6
+; CHECK-GI-NEXT:    bsl.16b v4, v16, v5
+; CHECK-GI-NEXT:    bit.16b v1, v18, v7
+; CHECK-GI-NEXT:    bsl.16b v2, v19, v3
+; CHECK-GI-NEXT:    add.4s v0, v0, v4
+; CHECK-GI-NEXT:    add.4s v1, v1, v2
+; CHECK-GI-NEXT:    add.4s v0, v0, v1
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aext = sext <16 x i8> %a to <16 x i32>
   %bext = sext <16 x i8> %b to <16 x i32>
   %abdiff = sub nsw <16 x i32> %aext, %bext
   %abcmp = icmp slt <16 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <16 x i32> zeroinitializer, %abdiff
   %absel = select <16 x i1> %abcmp, <16 x i32> %ababs, <16 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %absel)
   ret i32 %reduced_v
 }
 
 
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
 
 define i32 @uabd8h_rdx(ptr %a, ptr %b) {
-; CHECK-LABEL: uabd8h_rdx:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    ldr q1, [x1]
-; CHECK-NEXT:    uabd.8h v0, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v0
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: uabd8h_rdx:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr q0, [x0]
+; CHECK-SD-NEXT:    ldr q1, [x1]
+; CHECK-SD-NEXT:    uabd.8h v0, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v0
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uabd8h_rdx:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    ldr q2, [x1]
+; CHECK-GI-NEXT:    movi.2d v0, #0000000000000000
+; CHECK-GI-NEXT:    mov d3, v1[1]
+; CHECK-GI-NEXT:    mov d4, v2[1]
+; CHECK-GI-NEXT:    usubl.4s v1, v1, v2
+; CHECK-GI-NEXT:    usubl.4s v2, v3, v4
+; CHECK-GI-NEXT:    cmgt.4s v3, v0, v1
+; CHECK-GI-NEXT:    neg.4s v4, v1
+; CHECK-GI-NEXT:    cmgt.4s v0, v0, v2
+; CHECK-GI-NEXT:    shl.4s v3, v3, #31
+; CHECK-GI-NEXT:    shl.4s v0, v0, #31
+; CHECK-GI-NEXT:    neg.4s v5, v2
+; CHECK-GI-NEXT:    sshr.4s v3, v3, #31
+; CHECK-GI-NEXT:    sshr.4s v0, v0, #31
+; CHECK-GI-NEXT:    bit.16b v1, v4, v3
+; CHECK-GI-NEXT:    bsl.16b v0, v5, v2
+; CHECK-GI-NEXT:    add.4s v0, v1, v0
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aload = load <8 x i16>, ptr %a, align 1
   %bload = load <8 x i16>, ptr %b, align 1
   %aext = zext <8 x i16> %aload to <8 x i32>
   %bext = zext <8 x i16> %bload to <8 x i32>
   %abdiff = sub nsw <8 x i32> %aext, %bext
   %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
   %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
   ret i32 %reduced_v
 }
 
 define i32 @sabd8h_rdx(<8 x i16> %a, <8 x i16> %b) {
-; CHECK-LABEL: sabd8h_rdx:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sabd.8h v0, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v0
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sabd8h_rdx:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sabd.8h v0, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v0
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sabd8h_rdx:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d3, v0[1]
+; CHECK-GI-NEXT:    mov d4, v1[1]
+; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-GI-NEXT:    ssubl.4s v0, v0, v1
+; CHECK-GI-NEXT:    ssubl.4s v1, v3, v4
+; CHECK-GI-NEXT:    cmgt.4s v3, v2, v0
+; CHECK-GI-NEXT:    neg.4s v4, v0
+; CHECK-GI-NEXT:    cmgt.4s v2, v2, v1
+; CHECK-GI-NEXT:    shl.4s v3, v3, #31
+; CHECK-GI-NEXT:    shl.4s v2, v2, #31
+; CHECK-GI-NEXT:    neg.4s v5, v1
+; CHECK-GI-NEXT:    sshr.4s v3, v3, #31
+; CHECK-GI-NEXT:    sshr.4s v2, v2, #31
+; CHECK-GI-NEXT:    bit.16b v0, v4, v3
+; CHECK-GI-NEXT:    bit.16b v1, v5, v2
+; CHECK-GI-NEXT:    add.4s v0, v0, v1
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aext = sext <8 x i16> %a to <8 x i32>
   %bext = sext <8 x i16> %b to <8 x i32>
   %abdiff = sub nsw <8 x i32> %aext, %bext
   %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
   %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
   ret i32 %reduced_v

llvm/test/CodeGen/AArch64/sext.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:      warning: Instruction selection used fallback path for sext_v3i8_v3i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v3i8_v3i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v3i10_v3i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v3i10_v3i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v16i10_v16i16
+
+define i16 @sext_i8_to_i16(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i16
+  ret i16 %c
+}
+
+define i32 @sext_i8_to_i32(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i32
+  ret i32 %c
+}
+
+define i64 @sext_i8_to_i64(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtb x0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i64
+  ret i64 %c
+}
+
+define i10 @sext_i8_to_i10(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i10
+  ret i10 %c
+}
+
+define i32 @sext_i16_to_i32(i16 %a) {
+; CHECK-LABEL: sext_i16_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxth w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i16 %a to i32
+  ret i32 %c
+}
+
+define i64 @sext_i16_to_i64(i16 %a) {
+; CHECK-LABEL: sext_i16_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxth x0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i16 %a to i64
+  ret i64 %c
+}
+
+define i64 @sext_i32_to_i64(i32 %a) {
+; CHECK-LABEL: sext_i32_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i32 %a to i64
+  ret i64 %c
+}
+
+define i16 @sext_i10_to_i16(i10 %a) {
+; CHECK-LABEL: sext_i10_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sbfx w0, w0, #0, #10
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i10 %a to i16
+  ret i16 %c
+}
+
+define i32 @sext_i10_to_i32(i10 %a) {
+; CHECK-LABEL: sext_i10_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sbfx w0, w0, #0, #10
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i10 %a to i32
+  ret i32 %c
+}
+
+define i64 @sext_i10_to_i64(i10 %a) {
+; CHECK-LABEL: sext_i10_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x0, x0, #0, #10
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i10 %a to i64
+  ret i64 %c
+}
+
+define <2 x i16> @sext_v2i8_v2i16(<2 x i8> %a) {
+; CHECK-LABEL: sext_v2i8_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i8> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @sext_v2i8_v2i32(<2 x i8> %a) {
+; CHECK-LABEL: sext_v2i8_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i8> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @sext_v2i8_v2i64(<2 x i8> %a) {
+; CHECK-LABEL: sext_v2i8_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i8> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i32> @sext_v2i16_v2i32(<2 x i16> %a) {
+; CHECK-LABEL: sext_v2i16_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i16> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @sext_v2i16_v2i64(<2 x i16> %a) {
+; CHECK-LABEL: sext_v2i16_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    shl v0.2d, v0.2d, #48
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i16> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @sext_v2i32_v2i64(<2 x i32> %a) {
+; CHECK-LABEL: sext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i16> @sext_v2i10_v2i16(<2 x i10> %a) {
+; CHECK-LABEL: sext_v2i10_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #22
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i10> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @sext_v2i10_v2i32(<2 x i10> %a) {
+; CHECK-LABEL: sext_v2i10_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #22
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i10> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @sext_v2i10_v2i64(<2 x i10> %a) {
+; CHECK-LABEL: sext_v2i10_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i10> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <3 x i16> @sext_v3i8_v3i16(<3 x i8> %a) {
+; CHECK-LABEL: sext_v3i8_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i8> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) {
+; CHECK-LABEL: sext_v3i8_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i8> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @sext_v3i8_v3i64(<3 x i8> %a) {
+; CHECK-SD-LABEL: sext_v3i8_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s0, w2
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v0.2d, #56
+; CHECK-SD-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-SD-NEXT:    sshr v2.2d, v2.2d, #56
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i8_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    lsl x8, x2, #56
+; CHECK-GI-NEXT:    asr x8, x8, #56
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i8> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i32> @sext_v3i16_v3i32(<3 x i16> %a) {
+; CHECK-SD-LABEL: sext_v3i16_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i16_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov w8, s2
+; CHECK-GI-NEXT:    sxth w9, w9
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i16> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @sext_v3i16_v3i64(<3 x i16> %a) {
+; CHECK-SD-LABEL: sext_v3i16_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i16_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    sxth x8, w8
+; CHECK-GI-NEXT:    sxth x9, w9
+; CHECK-GI-NEXT:    sxth x10, w10
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i16> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @sext_v3i32_v3i64(<3 x i32> %a) {
+; CHECK-SD-LABEL: sext_v3i32_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v3.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll2 v2.2d, v0.4s, #0
+; CHECK-SD-NEXT:    fmov d0, d3
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i32_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    sxtw x8, w8
+; CHECK-GI-NEXT:    sxtw x9, w9
+; CHECK-GI-NEXT:    sxtw x10, w10
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i32> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i16> @sext_v3i10_v3i16(<3 x i10> %a) {
+; CHECK-LABEL: sext_v3i10_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    shl v0.4h, v0.4h, #6
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i10> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) {
+; CHECK-LABEL: sext_v3i10_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i10> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @sext_v3i10_v3i64(<3 x i10> %a) {
+; CHECK-SD-LABEL: sext_v3i10_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s0, w2
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v0.2d, #54
+; CHECK-SD-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-SD-NEXT:    sshr v2.2d, v2.2d, #54
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i10_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    lsl x8, x2, #54
+; CHECK-GI-NEXT:    asr x8, x8, #54
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i10> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <4 x i16> @sext_v4i8_v4i16(<4 x i8> %a) {
+; CHECK-LABEL: sext_v4i8_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i8> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @sext_v4i8_v4i32(<4 x i8> %a) {
+; CHECK-LABEL: sext_v4i8_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i8> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @sext_v4i8_v4i64(<4 x i8> %a) {
+; CHECK-SD-LABEL: sext_v4i8_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v1.2d, #56
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-SD-NEXT:    sshr v1.2d, v0.2d, #56
+; CHECK-SD-NEXT:    sshr v0.2d, v2.2d, #56
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i8_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #56
+; CHECK-GI-NEXT:    sshr v1.2d, v1.2d, #56
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i8> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i32> @sext_v4i16_v4i32(<4 x i16> %a) {
+; CHECK-LABEL: sext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i16> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @sext_v4i16_v4i64(<4 x i16> %a) {
+; CHECK-SD-LABEL: sext_v4i16_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i16_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i16> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @sext_v4i32_v4i64(<4 x i32> %a) {
+; CHECK-SD-LABEL: sext_v4i32_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i32_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i32> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i16> @sext_v4i10_v4i16(<4 x i10> %a) {
+; CHECK-LABEL: sext_v4i10_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.4h, v0.4h, #6
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i10> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @sext_v4i10_v4i32(<4 x i10> %a) {
+; CHECK-LABEL: sext_v4i10_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i10> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @sext_v4i10_v4i64(<4 x i10> %a) {
+; CHECK-SD-LABEL: sext_v4i10_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v1.2d, #54
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    sshr v1.2d, v0.2d, #54
+; CHECK-SD-NEXT:    sshr v0.2d, v2.2d, #54
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i10_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #54
+; CHECK-GI-NEXT:    sshr v1.2d, v1.2d, #54
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i10> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <8 x i16> @sext_v8i8_v8i16(<8 x i8> %a) {
+; CHECK-LABEL: sext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <8 x i8> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @sext_v8i8_v8i32(<8 x i8> %a) {
+; CHECK-SD-LABEL: sext_v8i8_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i8_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i8> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @sext_v8i8_v8i64(<8 x i8> %a) {
+; CHECK-SD-LABEL: sext_v8i8_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i8_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i8> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i32> @sext_v8i16_v8i32(<8 x i16> %a) {
+; CHECK-SD-LABEL: sext_v8i16_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i16_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i16> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @sext_v8i16_v8i64(<8 x i16> %a) {
+; CHECK-SD-LABEL: sext_v8i16_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i16_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i16> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @sext_v8i32_v8i64(<8 x i32> %a) {
+; CHECK-SD-LABEL: sext_v8i32_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v4.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v1.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i32_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i32> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i16> @sext_v8i10_v8i16(<8 x i10> %a) {
+; CHECK-LABEL: sext_v8i10_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.8h, v0.8h, #6
+; CHECK-NEXT:    sshr v0.8h, v0.8h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <8 x i10> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @sext_v8i10_v8i32(<8 x i10> %a) {
+; CHECK-SD-LABEL: sext_v8i10_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-SD-NEXT:    shl v2.4s, v1.4s, #22
+; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-SD-NEXT:    sshr v1.4s, v0.4s, #22
+; CHECK-SD-NEXT:    sshr v0.4s, v2.4s, #22
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i10_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #22
+; CHECK-GI-NEXT:    sshr v1.4s, v1.4s, #22
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i10> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @sext_v8i10_v8i64(<8 x i10> %a) {
+; CHECK-SD-LABEL: sext_v8i10_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-SD-NEXT:    shl v1.2d, v1.2d, #54
+; CHECK-SD-NEXT:    shl v2.2d, v2.2d, #54
+; CHECK-SD-NEXT:    shl v4.2d, v0.2d, #54
+; CHECK-SD-NEXT:    shl v5.2d, v3.2d, #54
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #54
+; CHECK-SD-NEXT:    sshr v0.2d, v2.2d, #54
+; CHECK-SD-NEXT:    sshr v3.2d, v4.2d, #54
+; CHECK-SD-NEXT:    sshr v2.2d, v5.2d, #54
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i10_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    shl v2.2d, v2.2d, #54
+; CHECK-GI-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-GI-NEXT:    shl v4.2d, v1.2d, #54
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v3.2d, v3.2d, #54
+; CHECK-GI-NEXT:    sshr v1.2d, v2.2d, #54
+; CHECK-GI-NEXT:    sshr v2.2d, v4.2d, #54
+; CHECK-GI-NEXT:    sshr v3.2d, v3.2d, #54
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i10> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <16 x i16> @sext_v16i8_v16i16(<16 x i8> %a) {
+; CHECK-SD-LABEL: sext_v16i8_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i8_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i8> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @sext_v16i8_v16i32(<16 x i8> %a) {
+; CHECK-SD-LABEL: sext_v16i8_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v2.8h, v0.16b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v2.8h, #0
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i8_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i8> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @sext_v16i8_v16i64(<16 x i8> %a) {
+; CHECK-SD-LABEL: sext_v16i8_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v2.4s, v1.8h, #0
+; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v16.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v7.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v4.4s, #0
+; CHECK-SD-NEXT:    sshll2 v5.2d, v16.4s, #0
+; CHECK-SD-NEXT:    sshll v6.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v4.2s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v4.2d, v16.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i8_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v1.d[1]
+; CHECK-GI-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d5, v2.d[1]
+; CHECK-GI-NEXT:    sshll v4.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v6.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v16.4s, v5.4h, #0
+; CHECK-GI-NEXT:    mov d3, v4.d[1]
+; CHECK-GI-NEXT:    mov d7, v6.d[1]
+; CHECK-GI-NEXT:    mov d17, v16.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v4.2s, #0
+; CHECK-GI-NEXT:    sshll v4.2d, v6.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v5.2d, v7.2s, #0
+; CHECK-GI-NEXT:    sshll v6.2d, v16.2s, #0
+; CHECK-GI-NEXT:    sshll v7.2d, v17.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i8> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i32> @sext_v16i16_v16i32(<16 x i16> %a) {
+; CHECK-SD-LABEL: sext_v16i16_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i16_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i16> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @sext_v16i16_v16i64(<16 x i16> %a) {
+; CHECK-SD-LABEL: sext_v16i16_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v4.4s, v1.8h, #0
+; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll2 v16.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v7.2d, v4.4s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll2 v5.2d, v1.4s, #0
+; CHECK-SD-NEXT:    sshll v6.2d, v4.2s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v4.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v16.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i16_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v5.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v2.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d6, v5.d[1]
+; CHECK-GI-NEXT:    sshll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    mov d7, v2.d[1]
+; CHECK-GI-NEXT:    mov d16, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    sshll v4.2d, v5.2s, #0
+; CHECK-GI-NEXT:    sshll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    sshll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v7.2s, #0
+; CHECK-GI-NEXT:    sshll v7.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i16> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i64> @sext_v16i32_v16i64(<16 x i32> %a) {
+; CHECK-SD-LABEL: sext_v16i32_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v17.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll2 v18.2d, v1.4s, #0
+; CHECK-SD-NEXT:    sshll v16.2d, v1.2s, #0
+; CHECK-SD-NEXT:    sshll2 v5.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v7.2d, v3.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll v6.2d, v3.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v17.16b
+; CHECK-SD-NEXT:    mov v2.16b, v16.16b
+; CHECK-SD-NEXT:    mov v3.16b, v18.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i32_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d5, v1.d[1]
+; CHECK-GI-NEXT:    mov d6, v2.d[1]
+; CHECK-GI-NEXT:    sshll v16.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d2, v3.d[1]
+; CHECK-GI-NEXT:    sshll v17.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll v18.2d, v5.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    sshll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v7.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov v0.16b, v16.16b
+; CHECK-GI-NEXT:    mov v2.16b, v17.16b
+; CHECK-GI-NEXT:    mov v3.16b, v18.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i32> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i16> @sext_v16i10_v16i16(<16 x i10> %a) {
+; CHECK-LABEL: sext_v16i10_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [sp]
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    ldr w9, [sp, #16]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    ldr w8, [sp, #8]
+; CHECK-NEXT:    mov v1.h[1], w1
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    ldr w8, [sp, #24]
+; CHECK-NEXT:    mov v1.h[2], w2
+; CHECK-NEXT:    mov v0.h[2], w9
+; CHECK-NEXT:    ldr w9, [sp, #32]
+; CHECK-NEXT:    mov v1.h[3], w3
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    ldr w8, [sp, #40]
+; CHECK-NEXT:    mov v1.h[4], w4
+; CHECK-NEXT:    mov v0.h[4], w9
+; CHECK-NEXT:    ldr w9, [sp, #48]
+; CHECK-NEXT:    mov v1.h[5], w5
+; CHECK-NEXT:    mov v0.h[5], w8
+; CHECK-NEXT:    ldr w8, [sp, #56]
+; CHECK-NEXT:    mov v1.h[6], w6
+; CHECK-NEXT:    mov v0.h[6], w9
+; CHECK-NEXT:    mov v1.h[7], w7
+; CHECK-NEXT:    mov v0.h[7], w8
+; CHECK-NEXT:    shl v1.8h, v1.8h, #6
+; CHECK-NEXT:    shl v2.8h, v0.8h, #6
+; CHECK-NEXT:    sshr v0.8h, v1.8h, #6
+; CHECK-NEXT:    sshr v1.8h, v2.8h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <16 x i10> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @sext_v16i10_v16i32(<16 x i10> %a) {
+; CHECK-SD-LABEL: sext_v16i10_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w11, [sp, #32]
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr w12, [sp]
+; CHECK-SD-NEXT:    fmov s1, w4
+; CHECK-SD-NEXT:    ldr w10, [sp, #40]
+; CHECK-SD-NEXT:    ldr w15, [sp, #8]
+; CHECK-SD-NEXT:    fmov s3, w11
+; CHECK-SD-NEXT:    fmov s2, w12
+; CHECK-SD-NEXT:    ldr w9, [sp, #48]
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    ldr w14, [sp, #16]
+; CHECK-SD-NEXT:    mov v1.h[1], w5
+; CHECK-SD-NEXT:    ldr w8, [sp, #56]
+; CHECK-SD-NEXT:    mov v2.h[1], w15
+; CHECK-SD-NEXT:    ldr w13, [sp, #24]
+; CHECK-SD-NEXT:    mov v3.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    mov v1.h[2], w6
+; CHECK-SD-NEXT:    mov v2.h[2], w14
+; CHECK-SD-NEXT:    mov v3.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w3
+; CHECK-SD-NEXT:    mov v1.h[3], w7
+; CHECK-SD-NEXT:    mov v2.h[3], w13
+; CHECK-SD-NEXT:    mov v3.h[3], w8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #22
+; CHECK-SD-NEXT:    shl v2.4s, v2.4s, #22
+; CHECK-SD-NEXT:    shl v3.4s, v3.4s, #22
+; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #22
+; CHECK-SD-NEXT:    sshr v2.4s, v2.4s, #22
+; CHECK-SD-NEXT:    sshr v3.4s, v3.4s, #22
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i10_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s7, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s17, w4
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    mov v7.s[1], w1
+; CHECK-GI-NEXT:    ldr s6, [sp, #48]
+; CHECK-GI-NEXT:    mov v17.s[1], w5
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ldr s16, [sp, #56]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], w2
+; CHECK-GI-NEXT:    mov v17.s[2], w6
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], w3
+; CHECK-GI-NEXT:    mov v17.s[3], w7
+; CHECK-GI-NEXT:    mov v4.s[3], v16.s[0]
+; CHECK-GI-NEXT:    shl v3.4s, v0.4s, #22
+; CHECK-GI-NEXT:    shl v1.4s, v7.4s, #22
+; CHECK-GI-NEXT:    shl v2.4s, v17.4s, #22
+; CHECK-GI-NEXT:    shl v4.4s, v4.4s, #22
+; CHECK-GI-NEXT:    sshr v0.4s, v1.4s, #22
+; CHECK-GI-NEXT:    sshr v1.4s, v2.4s, #22
+; CHECK-GI-NEXT:    sshr v2.4s, v3.4s, #22
+; CHECK-GI-NEXT:    sshr v3.4s, v4.4s, #22
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i10> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @sext_v16i10_v16i64(<16 x i10> %a) {
+; CHECK-SD-LABEL: sext_v16i10_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s0, [sp]
+; CHECK-SD-NEXT:    add x8, sp, #8
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s2, w2
+; CHECK-SD-NEXT:    fmov s3, w4
+; CHECK-SD-NEXT:    fmov s4, w6
+; CHECK-SD-NEXT:    ld1 { v0.s }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
+; CHECK-SD-NEXT:    ldr s5, [sp, #16]
+; CHECK-SD-NEXT:    add x9, sp, #40
+; CHECK-SD-NEXT:    ldr s6, [sp, #32]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    ldr s7, [sp, #48]
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ld1 { v5.s }[1], [x8]
+; CHECK-SD-NEXT:    mov v2.s[1], w3
+; CHECK-SD-NEXT:    ld1 { v6.s }[1], [x9]
+; CHECK-SD-NEXT:    mov v3.s[1], w5
+; CHECK-SD-NEXT:    ld1 { v7.s }[1], [x10]
+; CHECK-SD-NEXT:    mov v4.s[1], w7
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v16.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v5.2d, v5.2s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-SD-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-SD-NEXT:    shl v0.2d, v1.2d, #54
+; CHECK-SD-NEXT:    shl v1.2d, v2.2d, #54
+; CHECK-SD-NEXT:    shl v2.2d, v3.2d, #54
+; CHECK-SD-NEXT:    shl v3.2d, v4.2d, #54
+; CHECK-SD-NEXT:    shl v4.2d, v16.2d, #54
+; CHECK-SD-NEXT:    shl v5.2d, v5.2d, #54
+; CHECK-SD-NEXT:    shl v6.2d, v6.2d, #54
+; CHECK-SD-NEXT:    shl v7.2d, v7.2d, #54
+; CHECK-SD-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #54
+; CHECK-SD-NEXT:    sshr v2.2d, v2.2d, #54
+; CHECK-SD-NEXT:    sshr v3.2d, v3.2d, #54
+; CHECK-SD-NEXT:    sshr v4.2d, v4.2d, #54
+; CHECK-SD-NEXT:    sshr v5.2d, v5.2d, #54
+; CHECK-SD-NEXT:    sshr v6.2d, v6.2d, #54
+; CHECK-SD-NEXT:    sshr v7.2d, v7.2d, #54
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i10_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s6, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s16, w2
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    fmov s18, w4
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    fmov s19, w6
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    ldr s7, [sp, #48]
+; CHECK-GI-NEXT:    ldr s17, [sp, #56]
+; CHECK-GI-NEXT:    mov v6.s[1], w1
+; CHECK-GI-NEXT:    mov v16.s[1], w3
+; CHECK-GI-NEXT:    mov v18.s[1], w5
+; CHECK-GI-NEXT:    mov v19.s[1], w7
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    mov v7.s[1], v17.s[0]
+; CHECK-GI-NEXT:    ushll v1.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v19.2s, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #54
+; CHECK-GI-NEXT:    shl v3.2d, v3.2d, #54
+; CHECK-GI-NEXT:    shl v5.2d, v5.2d, #54
+; CHECK-GI-NEXT:    shl v6.2d, v6.2d, #54
+; CHECK-GI-NEXT:    shl v16.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v17.2d, v2.2d, #54
+; CHECK-GI-NEXT:    shl v18.2d, v4.2d, #54
+; CHECK-GI-NEXT:    shl v7.2d, v7.2d, #54
+; CHECK-GI-NEXT:    sshr v0.2d, v1.2d, #54
+; CHECK-GI-NEXT:    sshr v1.2d, v3.2d, #54
+; CHECK-GI-NEXT:    sshr v2.2d, v5.2d, #54
+; CHECK-GI-NEXT:    sshr v3.2d, v6.2d, #54
+; CHECK-GI-NEXT:    sshr v4.2d, v16.2d, #54
+; CHECK-GI-NEXT:    sshr v5.2d, v17.2d, #54
+; CHECK-GI-NEXT:    sshr v6.2d, v18.2d, #54
+; CHECK-GI-NEXT:    sshr v7.2d, v7.2d, #54
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i10> %a to <16 x i64>
+  ret <16 x i64> %c
+}

llvm/test/CodeGen/AArch64/zext.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:      warning: Instruction selection used fallback path for zext_v16i10_v16i16
+
+define i16 @zext_i8_to_i16(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i16
+  ret i16 %c
+}
+
+define i32 @zext_i8_to_i32(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i32
+  ret i32 %c
+}
+
+define i64 @zext_i8_to_i64(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x0, x0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i64
+  ret i64 %c
+}
+
+define i10 @zext_i8_to_i10(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i10
+  ret i10 %c
+}
+
+define i32 @zext_i16_to_i32(i16 %a) {
+; CHECK-LABEL: zext_i16_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i16 %a to i32
+  ret i32 %c
+}
+
+define i64 @zext_i16_to_i64(i16 %a) {
+; CHECK-LABEL: zext_i16_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x0, x0, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i16 %a to i64
+  ret i64 %c
+}
+
+define i64 @zext_i32_to_i64(i32 %a) {
+; CHECK-LABEL: zext_i32_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i32 %a to i64
+  ret i64 %c
+}
+
+define i16 @zext_i10_to_i16(i10 %a) {
+; CHECK-LABEL: zext_i10_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0x3ff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i10 %a to i16
+  ret i16 %c
+}
+
+define i32 @zext_i10_to_i32(i10 %a) {
+; CHECK-LABEL: zext_i10_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0x3ff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i10 %a to i32
+  ret i32 %c
+}
+
+define i64 @zext_i10_to_i64(i10 %a) {
+; CHECK-LABEL: zext_i10_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x0, x0, #0x3ff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i10 %a to i64
+  ret i64 %c
+}
+
+define <2 x i16> @zext_v2i8_v2i16(<2 x i8> %a) {
+; CHECK-SD-LABEL: zext_v2i8_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i8_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI10_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI10_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i8> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @zext_v2i8_v2i32(<2 x i8> %a) {
+; CHECK-SD-LABEL: zext_v2i8_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i8_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI11_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI11_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i8> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @zext_v2i8_v2i64(<2 x i8> %a) {
+; CHECK-SD-LABEL: zext_v2i8_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i8_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI12_0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI12_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i8> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i32> @zext_v2i16_v2i32(<2 x i16> %a) {
+; CHECK-SD-LABEL: zext_v2i16_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i16_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI13_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI13_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i16> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @zext_v2i16_v2i64(<2 x i16> %a) {
+; CHECK-SD-LABEL: zext_v2i16_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i16_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI14_0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI14_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i16> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @zext_v2i32_v2i64(<2 x i32> %a) {
+; CHECK-LABEL: zext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i16> @zext_v2i10_v2i16(<2 x i10> %a) {
+; CHECK-SD-LABEL: zext_v2i10_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #3, msl #8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i10_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI16_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI16_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i10> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @zext_v2i10_v2i32(<2 x i10> %a) {
+; CHECK-SD-LABEL: zext_v2i10_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #3, msl #8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i10_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI17_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI17_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i10> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @zext_v2i10_v2i64(<2 x i10> %a) {
+; CHECK-SD-LABEL: zext_v2i10_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #3, msl #8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i10_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI18_0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI18_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i10> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <3 x i16> @zext_v3i8_v3i16(<3 x i8> %a) {
+; CHECK-SD-LABEL: zext_v3i8_v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i8_v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #255 // =0xff
+; CHECK-GI-NEXT:    fmov s1, w0
+; CHECK-GI-NEXT:    fmov s2, w1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-GI-NEXT:    fmov s2, w2
+; CHECK-GI-NEXT:    mov v3.16b, v0.16b
+; CHECK-GI-NEXT:    mov v3.h[1], v0.h[0]
+; CHECK-GI-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-GI-NEXT:    mov v3.h[2], v0.h[0]
+; CHECK-GI-NEXT:    mov v1.h[3], v0.h[0]
+; CHECK-GI-NEXT:    mov v3.h[3], v0.h[0]
+; CHECK-GI-NEXT:    and v0.8b, v1.8b, v3.8b
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-GI-NEXT:    mov v0.h[3], v0.h[0]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i8> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @zext_v3i8_v3i32(<3 x i8> %a) {
+; CHECK-SD-LABEL: zext_v3i8_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i8_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #255 // =0xff
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mov v0.s[2], w2
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i8> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @zext_v3i8_v3i64(<3 x i8> %a) {
+; CHECK-SD-LABEL: zext_v3i8_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s3, w2
+; CHECK-SD-NEXT:    movi v0.2d, #0x000000000000ff
+; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    mov v2.b[0], v3.b[0]
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i8_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    adrp x8, .LCPI21_0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI21_0]
+; CHECK-GI-NEXT:    and x8, x2, #0xff
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i8> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i32> @zext_v3i16_v3i32(<3 x i16> %a) {
+; CHECK-SD-LABEL: zext_v3i16_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i16_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    uxth w8, w8
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov w8, s2
+; CHECK-GI-NEXT:    uxth w9, w9
+; CHECK-GI-NEXT:    uxth w8, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i16> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @zext_v3i16_v3i64(<3 x i16> %a) {
+; CHECK-SD-LABEL: zext_v3i16_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i16_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    ubfx x8, x8, #0, #16
+; CHECK-GI-NEXT:    ubfx x9, x9, #0, #16
+; CHECK-GI-NEXT:    ubfx x10, x10, #0, #16
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i16> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @zext_v3i32_v3i64(<3 x i32> %a) {
+; CHECK-SD-LABEL: zext_v3i32_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v2.2d, v0.4s, #0
+; CHECK-SD-NEXT:    fmov d0, d3
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i32_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i32> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i16> @zext_v3i10_v3i16(<3 x i10> %a) {
+; CHECK-SD-LABEL: zext_v3i10_v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i10_v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #1023 // =0x3ff
+; CHECK-GI-NEXT:    fmov s1, w0
+; CHECK-GI-NEXT:    fmov s2, w1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-GI-NEXT:    fmov s2, w2
+; CHECK-GI-NEXT:    mov v3.16b, v0.16b
+; CHECK-GI-NEXT:    mov v3.h[1], v0.h[0]
+; CHECK-GI-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-GI-NEXT:    mov v3.h[2], v0.h[0]
+; CHECK-GI-NEXT:    mov v1.h[3], v0.h[0]
+; CHECK-GI-NEXT:    mov v3.h[3], v0.h[0]
+; CHECK-GI-NEXT:    and v0.8b, v1.8b, v3.8b
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-GI-NEXT:    mov v0.h[3], v0.h[0]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i10> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @zext_v3i10_v3i32(<3 x i10> %a) {
+; CHECK-SD-LABEL: zext_v3i10_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    movi v1.4s, #3, msl #8
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i10_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #1023 // =0x3ff
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mov v0.s[2], w2
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i10> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @zext_v3i10_v3i64(<3 x i10> %a) {
+; CHECK-SD-LABEL: zext_v3i10_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov w8, #1023 // =0x3ff
+; CHECK-SD-NEXT:    fmov s3, w2
+; CHECK-SD-NEXT:    mov v0.s[1], w1
+; CHECK-SD-NEXT:    dup v2.2d, x8
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    and v2.8b, v3.8b, v2.8b
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i10_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    adrp x8, .LCPI27_0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI27_0]
+; CHECK-GI-NEXT:    and x8, x2, #0x3ff
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i10> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <4 x i16> @zext_v4i8_v4i16(<4 x i8> %a) {
+; CHECK-SD-LABEL: zext_v4i8_v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI28_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI28_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i8> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @zext_v4i8_v4i32(<4 x i8> %a) {
+; CHECK-SD-LABEL: zext_v4i8_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI29_0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI29_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i8> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @zext_v4i8_v4i64(<4 x i8> %a) {
+; CHECK-SD-LABEL: zext_v4i8_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    adrp x8, .LCPI30_0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI30_0]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i8> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i32> @zext_v4i16_v4i32(<4 x i16> %a) {
+; CHECK-LABEL: zext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <4 x i16> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @zext_v4i16_v4i64(<4 x i16> %a) {
+; CHECK-SD-LABEL: zext_v4i16_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i16_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i16> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @zext_v4i32_v4i64(<4 x i32> %a) {
+; CHECK-SD-LABEL: zext_v4i32_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i32_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i32> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i16> @zext_v4i10_v4i16(<4 x i10> %a) {
+; CHECK-SD-LABEL: zext_v4i10_v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i10_v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI34_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI34_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i10> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @zext_v4i10_v4i32(<4 x i10> %a) {
+; CHECK-SD-LABEL: zext_v4i10_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i10_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI35_0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI35_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i10> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @zext_v4i10_v4i64(<4 x i10> %a) {
+; CHECK-SD-LABEL: zext_v4i10_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i10_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    adrp x8, .LCPI36_0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI36_0]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i10> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <8 x i16> @zext_v8i8_v8i16(<8 x i8> %a) {
+; CHECK-LABEL: zext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <8 x i8> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @zext_v8i8_v8i32(<8 x i8> %a) {
+; CHECK-SD-LABEL: zext_v8i8_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i8_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i8> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @zext_v8i8_v8i64(<8 x i8> %a) {
+; CHECK-SD-LABEL: zext_v8i8_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i8_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i8> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i32> @zext_v8i16_v8i32(<8 x i16> %a) {
+; CHECK-SD-LABEL: zext_v8i16_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i16_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i16> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @zext_v8i16_v8i64(<8 x i16> %a) {
+; CHECK-SD-LABEL: zext_v8i16_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i16_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i16> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @zext_v8i32_v8i64(<8 x i32> %a) {
+; CHECK-SD-LABEL: zext_v8i32_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v4.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i32_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i32> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i16> @zext_v8i10_v8i16(<8 x i10> %a) {
+; CHECK-SD-LABEL: zext_v8i10_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i10_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI43_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI43_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i10> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @zext_v8i10_v8i32(<8 x i10> %a) {
+; CHECK-SD-LABEL: zext_v8i10_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i10_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI44_0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI44_0]
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i10> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @zext_v8i10_v8i64(<8 x i10> %a) {
+; CHECK-SD-LABEL: zext_v8i10_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i10_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    adrp x8, .LCPI45_0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI45_0]
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v1.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-GI-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-GI-NEXT:    and v1.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT:    and v2.16b, v5.16b, v3.16b
+; CHECK-GI-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i10> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <16 x i16> @zext_v16i8_v16i16(<16 x i8> %a) {
+; CHECK-SD-LABEL: zext_v16i8_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i8_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i8> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @zext_v16i8_v16i32(<16 x i8> %a) {
+; CHECK-SD-LABEL: zext_v16i8_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v2.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v3.4s, v2.8h, #0
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i8_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i8> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @zext_v16i8_v16i64(<16 x i8> %a) {
+; CHECK-SD-LABEL: zext_v16i8_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v2.4s, v1.8h, #0
+; CHECK-SD-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v16.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v7.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v4.4s, #0
+; CHECK-SD-NEXT:    ushll2 v5.2d, v16.4s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v16.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i8_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d5, v2.d[1]
+; CHECK-GI-NEXT:    ushll v4.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v6.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v16.4s, v5.4h, #0
+; CHECK-GI-NEXT:    mov d3, v4.d[1]
+; CHECK-GI-NEXT:    mov d7, v6.d[1]
+; CHECK-GI-NEXT:    mov d17, v16.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ushll v4.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v7.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v17.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i8> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i32> @zext_v16i16_v16i32(<16 x i16> %a) {
+; CHECK-SD-LABEL: zext_v16i16_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i16_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i16> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @zext_v16i16_v16i64(<16 x i16> %a) {
+; CHECK-SD-LABEL: zext_v16i16_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v4.4s, v1.8h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll2 v16.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v7.2d, v4.4s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll2 v5.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v16.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i16_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v5.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v2.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d6, v5.d[1]
+; CHECK-GI-NEXT:    ushll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    mov d7, v2.d[1]
+; CHECK-GI-NEXT:    mov d16, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v4.2d, v5.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v7.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i16> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i64> @zext_v16i32_v16i64(<16 x i32> %a) {
+; CHECK-SD-LABEL: zext_v16i32_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v17.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll2 v18.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll v16.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll2 v5.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v7.2d, v3.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v3.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v17.16b
+; CHECK-SD-NEXT:    mov v2.16b, v16.16b
+; CHECK-SD-NEXT:    mov v3.16b, v18.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i32_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d5, v1.d[1]
+; CHECK-GI-NEXT:    mov d6, v2.d[1]
+; CHECK-GI-NEXT:    ushll v16.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d2, v3.d[1]
+; CHECK-GI-NEXT:    ushll v17.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ushll v18.2d, v5.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov v0.16b, v16.16b
+; CHECK-GI-NEXT:    mov v2.16b, v17.16b
+; CHECK-GI-NEXT:    mov v3.16b, v18.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i32> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i16> @zext_v16i10_v16i16(<16 x i10> %a) {
+; CHECK-LABEL: zext_v16i10_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [sp]
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    ldr w8, [sp, #8]
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v1.h[1], w8
+; CHECK-NEXT:    ldr w8, [sp, #16]
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    mov v1.h[2], w8
+; CHECK-NEXT:    ldr w8, [sp, #24]
+; CHECK-NEXT:    mov v0.h[3], w3
+; CHECK-NEXT:    mov v1.h[3], w8
+; CHECK-NEXT:    ldr w8, [sp, #32]
+; CHECK-NEXT:    mov v0.h[4], w4
+; CHECK-NEXT:    mov v1.h[4], w8
+; CHECK-NEXT:    ldr w8, [sp, #40]
+; CHECK-NEXT:    mov v0.h[5], w5
+; CHECK-NEXT:    mov v1.h[5], w8
+; CHECK-NEXT:    ldr w8, [sp, #48]
+; CHECK-NEXT:    mov v0.h[6], w6
+; CHECK-NEXT:    mov v1.h[6], w8
+; CHECK-NEXT:    ldr w8, [sp, #56]
+; CHECK-NEXT:    mov v0.h[7], w7
+; CHECK-NEXT:    mov v1.h[7], w8
+; CHECK-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-NEXT:    bic v1.8h, #252, lsl #8
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <16 x i10> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @zext_v16i10_v16i32(<16 x i10> %a) {
+; CHECK-SD-LABEL: zext_v16i10_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w11, [sp, #32]
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr w13, [sp]
+; CHECK-SD-NEXT:    fmov s1, w4
+; CHECK-SD-NEXT:    ldr w10, [sp, #40]
+; CHECK-SD-NEXT:    ldr w15, [sp, #8]
+; CHECK-SD-NEXT:    fmov s3, w11
+; CHECK-SD-NEXT:    fmov s2, w13
+; CHECK-SD-NEXT:    ldr w9, [sp, #48]
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    ldr w14, [sp, #16]
+; CHECK-SD-NEXT:    mov v1.h[1], w5
+; CHECK-SD-NEXT:    ldr w8, [sp, #56]
+; CHECK-SD-NEXT:    mov v2.h[1], w15
+; CHECK-SD-NEXT:    ldr w12, [sp, #24]
+; CHECK-SD-NEXT:    mov v3.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    mov v1.h[2], w6
+; CHECK-SD-NEXT:    mov v2.h[2], w14
+; CHECK-SD-NEXT:    mov v3.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w3
+; CHECK-SD-NEXT:    mov v1.h[3], w7
+; CHECK-SD-NEXT:    mov v2.h[3], w12
+; CHECK-SD-NEXT:    mov v3.h[3], w8
+; CHECK-SD-NEXT:    movi v4.4s, #3, msl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v4.16b
+; CHECK-SD-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-SD-NEXT:    and v2.16b, v2.16b, v4.16b
+; CHECK-SD-NEXT:    and v3.16b, v3.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i10_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s16, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s17, w4
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    adrp x8, .LCPI53_0
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    mov v16.s[1], w1
+; CHECK-GI-NEXT:    ldr s6, [sp, #48]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    mov v17.s[1], w5
+; CHECK-GI-NEXT:    ldr s7, [sp, #56]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI53_0]
+; CHECK-GI-NEXT:    mov v16.s[2], w2
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v17.s[2], w6
+; CHECK-GI-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-GI-NEXT:    mov v16.s[3], w3
+; CHECK-GI-NEXT:    mov v4.s[3], v7.s[0]
+; CHECK-GI-NEXT:    mov v17.s[3], w7
+; CHECK-GI-NEXT:    and v2.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    and v0.16b, v16.16b, v1.16b
+; CHECK-GI-NEXT:    and v3.16b, v4.16b, v1.16b
+; CHECK-GI-NEXT:    and v1.16b, v17.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i10> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @zext_v16i10_v16i64(<16 x i10> %a) {
+; CHECK-SD-LABEL: zext_v16i10_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1023 // =0x3ff
+; CHECK-SD-NEXT:    ldr s4, [sp]
+; CHECK-SD-NEXT:    ldr s5, [sp, #16]
+; CHECK-SD-NEXT:    add x9, sp, #24
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    fmov s1, w2
+; CHECK-SD-NEXT:    dup v7.2d, x8
+; CHECK-SD-NEXT:    add x8, sp, #8
+; CHECK-SD-NEXT:    fmov s2, w4
+; CHECK-SD-NEXT:    fmov s3, w6
+; CHECK-SD-NEXT:    ld1 { v5.s }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #56
+; CHECK-SD-NEXT:    ld1 { v4.s }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    ldr s6, [sp, #32]
+; CHECK-SD-NEXT:    ldr s16, [sp, #48]
+; CHECK-SD-NEXT:    mov v0.s[1], w1
+; CHECK-SD-NEXT:    mov v1.s[1], w3
+; CHECK-SD-NEXT:    ld1 { v6.s }[1], [x8]
+; CHECK-SD-NEXT:    mov v2.s[1], w5
+; CHECK-SD-NEXT:    ld1 { v16.s }[1], [x9]
+; CHECK-SD-NEXT:    mov v3.s[1], w7
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v5.2d, v5.2s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-SD-NEXT:    ushll v16.2d, v16.2s, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v7.16b
+; CHECK-SD-NEXT:    and v1.16b, v1.16b, v7.16b
+; CHECK-SD-NEXT:    and v2.16b, v2.16b, v7.16b
+; CHECK-SD-NEXT:    and v3.16b, v3.16b, v7.16b
+; CHECK-SD-NEXT:    and v4.16b, v4.16b, v7.16b
+; CHECK-SD-NEXT:    and v5.16b, v5.16b, v7.16b
+; CHECK-SD-NEXT:    and v6.16b, v6.16b, v7.16b
+; CHECK-SD-NEXT:    and v7.16b, v16.16b, v7.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i10_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s6, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s16, w2
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    fmov s18, w4
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    fmov s19, w6
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    adrp x8, .LCPI54_0
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    ldr s7, [sp, #48]
+; CHECK-GI-NEXT:    ldr s17, [sp, #56]
+; CHECK-GI-NEXT:    mov v6.s[1], w1
+; CHECK-GI-NEXT:    mov v16.s[1], w3
+; CHECK-GI-NEXT:    mov v18.s[1], w5
+; CHECK-GI-NEXT:    mov v19.s[1], w7
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    mov v7.s[1], v17.s[0]
+; CHECK-GI-NEXT:    ldr q17, [x8, :lo12:.LCPI54_0]
+; CHECK-GI-NEXT:    ushll v1.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v19.2s, #0
+; CHECK-GI-NEXT:    ushll v16.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v18.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v19.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v1.16b, v17.16b
+; CHECK-GI-NEXT:    and v1.16b, v3.16b, v17.16b
+; CHECK-GI-NEXT:    and v2.16b, v5.16b, v17.16b
+; CHECK-GI-NEXT:    and v3.16b, v6.16b, v17.16b
+; CHECK-GI-NEXT:    and v4.16b, v16.16b, v17.16b
+; CHECK-GI-NEXT:    and v5.16b, v18.16b, v17.16b
+; CHECK-GI-NEXT:    and v6.16b, v19.16b, v17.16b
+; CHECK-GI-NEXT:    and v7.16b, v7.16b, v17.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i10> %a to <16 x i64>
+  ret <16 x i64> %c
+}