[AArch64] Support more types for ZEXT/SEXT with Global Isel

[arsenm]

llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h

1046–1070

Is it possible to separate this from the lowerEXT parts?

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

5981

This shouldn't need to consider the legality of intermediate ops and fail, just unconditionally do this and let the new artifacts legalize

[chuongg3]

Split combiner into a seperate commit.
Unconditionally lowers Extends, independent of the intermediate instruction

[dmgreen]

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

5982

Can reduce the indenting now.

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

522

Can you add a comment here explaining that this is trying to convert larger extends into two smaller ones.

524

(Query.Types[0].isVector()) -> Query.Types[0].isVector()

llvm/test/CodeGen/AArch64/GlobalISel/legalize-build-vector.mir

144 ↗

(On Diff #549353)

Is this with update_mir_test_checks? Maybe just replace %ext with %3 in the code below.

[aemerson]

A few comments but otherwise LGTM, thanks for the new tests.

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

5981–5982

No need for MidRegister, this can be simplified to:

auto NewExt = MIRBuilder.buildInstr(MI.getOpcode(), {MidTy}, {Src});
...
auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, NewExt);
...`

5988–5996

Same here, it's less verbose to just create the LLTs and pass those to buildInstr() instead of passing Registers.

// ZExt the vectors
Register ZExtResTy = DstTy.changeElementCount(DstTy.getElementCount().divideCoefficientBy(2));
auto ZExtRes1 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, {UnmergeSrc.getReg(0)});
auto ZExtRes2 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, {UnmergeSrc.getReg(1)});

[chuongg3]

Rectified comments made by aemerson
Instructions are built using register types instead of creating a register for the instruction to be built

[chuongg3]

Formatted the patch

llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h

if (NumMergeRegs < NumDefs) {
 
       const unsigned NewNumDefs = NumDefs / NumMergeRegs;
       for (unsigned Idx = 0; Idx < NumMergeRegs; ++Idx) {
         SmallVector<Register, 8> DstRegs;
         for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs;
              ++j, ++DefIdx)
           DstRegs.push_back(MI.getReg(DefIdx));
 
         if (ConvertOp) {
-          LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg());
+          LLT MergeDstTy = MRI.getType(SrcDef->getOperand(0).getReg());
 
           // This is a vector that is being split and casted. Extract to the
           // element type, and do the conversion on the scalars (or smaller
           // vectors).
-          LLT MergeEltTy = MergeSrcTy.divide(NewNumDefs);
+          LLT MergeEltTy = MergeDstTy.divide(NumMergeRegs);
 
           // Handle split to smaller vectors, with conversions.
           // %2(<8 x s8>) = G_CONCAT_VECTORS %0(<4 x s8>), %1(<4 x s8>)
           // %3(<8 x s16>) = G_SEXT %2
-          // %4(<2 x s16>), %5(<2 x s16>), %6(<2 x s16>), %7(<2 x s16>) = G_UNMERGE_VALUES %3
+          // %4(<2 x s16>), %5(<2 x s16>), %6(<2 x s16>), %7(<2 x s16>) =
+          // G_UNMERGE_VALUES %3
           //
           // =>
           //
-          // %8(<2 x s8>), %9(<2 x s8>) = G_UNMERGE_VALUES %0
-          // %10(<2 x s8>), %11(<2 x s8>) = G_UNMERGE_VALUES %1
-          // %4(<2 x s16>) = G_SEXT %8
-          // %5(<2 x s16>) = G_SEXT %9
-          // %6(<2 x s16>) = G_SEXT %10
-          // %7(<2 x s16>)= G_SEXT %11
-
-          SmallVector<Register, 4> TmpRegs(NewNumDefs);
-          for (unsigned k = 0; k < NewNumDefs; ++k)
-            TmpRegs[k] = MRI.createGenericVirtualRegister(MergeEltTy);
-
-          Builder.buildUnmerge(TmpRegs, MergeI->getOperand(Idx + 1).getReg());
-
-          for (unsigned k = 0; k < NewNumDefs; ++k)
-            Builder.buildInstr(ConvertOp, {DstRegs[k]}, {TmpRegs[k]});
+          // %8(<4 x s16>) = G_SEXT %0
+          // %9(<4 x s16>) = G_SEXT %1
+          // %4(<2 x s16>), %5(<2 x s16>) = G_UNMERGE_VALUES %8
+          // %7(<2 x s16>), %7(<2 x s16>) = G_UNMERGE_VALUES %9
+
+          Register TmpReg = MRI.createGenericVirtualRegister(MergeEltTy);
+          Builder.buildInstr(ConvertOp, {TmpReg},
+                             {MergeI->getOperand(Idx + 1).getReg()});
+          Builder.buildUnmerge(DstRegs, TmpReg);
         } else {
           Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg());
         }
         UpdatedDefs.append(DstRegs.begin(), DstRegs.end());
       }
 
     } else if (NumMergeRegs > NumDefs) {
       if (ConvertOp != 0 || NumMergeRegs % NumDefs != 0)

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

public:
   LegalizeResult lowerFConstant(MachineInstr &MI);
   LegalizeResult lowerBitcast(MachineInstr &MI);
   LegalizeResult lowerLoad(GAnyLoad &MI);
   LegalizeResult lowerStore(GStore &MI);
   LegalizeResult lowerBitCount(MachineInstr &MI);
   LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI);
   LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI);
   LegalizeResult lowerFunnelShift(MachineInstr &MI);
+  LegalizeResult lowerEXT(MachineInstr &MI);
   LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI);
   LegalizeResult lowerRotate(MachineInstr &MI);
 
   LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI);
   LegalizeResult lowerUITOFP(MachineInstr &MI);
   LegalizeResult lowerSITOFP(MachineInstr &MI);
   LegalizeResult lowerFPTOUI(MachineInstr &MI);
   LegalizeResult lowerFPTOSI(MachineInstr &MI);

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
   case G_ROTR:
     return lowerRotate(MI);
   case G_MEMSET:
   case G_MEMCPY:
   case G_MEMMOVE:
     return lowerMemCpyFamily(MI);
   case G_MEMCPY_INLINE:
     return lowerMemcpyInline(MI);
+  case G_ZEXT:
+  case G_SEXT:
+  case G_ANYEXT:
+    return lowerEXT(MI);
     GISEL_VECREDUCE_CASES_NONSEQ
     return lowerVectorReduction(MI);
   }
 }
 
 Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
                                                   Align MinAlign) const {
   // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
   // datalayout for the preferred alignment. Also there should be a target hook
LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
 
   // This only works for powers of 2, fallback to shifts if it fails.
   LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
   if (Result == UnableToLegalize)
     return lowerFunnelShiftAsShifts(MI);
   return Result;
 }
 
+LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
+  auto [Dst, Src] = MI.getFirst2Regs();
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+
+  uint32_t DstTySize = DstTy.getSizeInBits();
+  uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits();
+  uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits();
+
+  if (!isPowerOf2_32(DstTySize) || !isPowerOf2_32(DstTyScalarSize) ||
+      !isPowerOf2_32(SrcTyScalarSize))
+    return UnableToLegalize;
+
+  // The step between extend is too large, split it by creating an intermediate
+  // extend instruction
+  if (SrcTyScalarSize * 2 < DstTyScalarSize) {
+    LLT MidTy = SrcTy.changeElementSize(SrcTyScalarSize * 2);
+    // If the destination type is illegal, split it into multiple statements
+    // zext x -> zext(merge(zext(unmerge), zext(unmerge)))
+    if (LI.getAction({MI.getOpcode(), {MidTy, SrcTy}}).Action == Legal) {
+      Register MidRegister = MRI.createGenericVirtualRegister(MidTy);
+      MIRBuilder.buildInstr(MI.getOpcode(), {MidRegister}, {Src});
+      // Unmerge the vector
+      LLT EltTy = MidTy.changeElementCount(
+          MidTy.getElementCount().divideCoefficientBy(2));
+      auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, MidRegister);
+
+      // ZExt the vectors
+      Register ZExtRes1 =
+          MRI.createGenericVirtualRegister(DstTy.changeElementCount(
+              DstTy.getElementCount().divideCoefficientBy(2)));
+      Register ZExtRes2 =
+          MRI.createGenericVirtualRegister(MRI.getType(ZExtRes1));
+
+      MIRBuilder.buildInstr(MI.getOpcode(), {ZExtRes1}, {UnmergeSrc.getReg(0)});
+      MIRBuilder.buildInstr(MI.getOpcode(), {ZExtRes2}, {UnmergeSrc.getReg(1)});
+
+      // Merge the ending vectors
+      MIRBuilder.buildMergeLikeInstr(Dst, {ZExtRes1, ZExtRes2});
+
+      MI.eraseFromParent();
+      return Legalized;
+    }
+  }
+  return UnableToLegalize;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
   auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
   auto Zero = MIRBuilder.buildConstant(AmtTy, 0);
   bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
   unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
   auto Neg = MIRBuilder.buildSub(AmtTy, Zero, Amt);
   MIRBuilder.buildInstr(RevRot, {Dst}, {Src, Neg});

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_FCMP)
           },
           0, 1)
       .clampNumElements(0, v2s32, v4s32);
 
   // Extensions
   auto ExtLegalFunc = [=](const LegalityQuery &Query) {
     unsigned DstSize = Query.Types[0].getSizeInBits();
 
-    if (DstSize == 128 && !Query.Types[0].isVector())
-      return false; // Extending to a scalar s128 needs narrowing.
-
-    // Make sure that we have something that will fit in a register, and
-    // make sure it's a power of 2.
-    if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize))
+    // Handle legal vectors using legalFor
+    if (Query.Types[0].isVector())
       return false;
 
+    if (DstSize < 8 || DstSize >= 128 || !isPowerOf2_32(DstSize))
+      return false; // Extending to a scalar s128 needs narrowing.
+
     const LLT &SrcTy = Query.Types[1];
 
     // Make sure we fit in a register otherwise. Don't bother checking that
     // the source type is below 128 bits. We shouldn't be allowing anything
     // through which is wider than the destination in the first place.
     unsigned SrcSize = SrcTy.getSizeInBits();
     if (SrcSize < 8 || !isPowerOf2_32(SrcSize))
       return false;
 
     return true;
   };
   getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
       .legalIf(ExtLegalFunc)
-      .clampScalar(0, s64, s64); // Just for s128, others are handled above.
+      .legalFor({{v2s64, v2s32}, {v4s32, v4s16}, {v8s16, v8s8}})
+      .clampScalar(0, s64, s64) // Just for s128, others are handled above.
+      .moreElementsToNextPow2(1)
+      .clampMaxNumElements(1, s8, 8)
+      .clampMaxNumElements(1, s16, 4)
+      .clampMaxNumElements(1, s32, 2)
+      .lowerIf([=](const LegalityQuery &Query) {
+        return (Query.Types[0].getScalarSizeInBits() >
+                Query.Types[1].getScalarSizeInBits() * 2) &&
+               (Query.Types[0].isVector());
+      });
 
   getActionDefinitionsBuilder(G_TRUNC)
       .minScalarOrEltIf(
           [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
           0, s8)
       .customIf([=](const LegalityQuery &Query) {
         LLT DstTy = Query.Types[0];
         LLT SrcTy = Query.Types[1];

llvm/test/CodeGen/AArch64/GlobalISel/legalize-and.mir

bb.1:
     liveins: $b0, $b1, $b2
 
     ; CHECK-LABEL: name: and_v3s1
     ; CHECK: liveins: $b0, $b1, $b2
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s8) = COPY $b0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s8) = COPY $b1
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s8) = COPY $b2
-    ; CHECK-NEXT: [[ANYEXT0:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY]](s8)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY]](s8)
     ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY1]](s8)
     ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[COPY2]](s8)
-    ; CHECK-NEXT: [[IMPLICIT_DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[ANYEXT0]](s16), [[ANYEXT1]](s16), [[ANYEXT2]](s16), [[IMPLICIT_DEF]](s16)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[ANYEXT]](s16), [[ANYEXT1]](s16), [[ANYEXT2]](s16), [[DEF]](s16)
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[BUILD_VECTOR]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[VAL0:%[0-9]+]]:_(s16), [[VAL1:%[0-9]+]]:_(s16), [[VAL2:%[0-9]+]]:_(s16), [[VAL3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[AND]](<4 x s16>)
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[VAL0]](s16)
-    ; CHECK-NEXT: $b0 = COPY [[TRUNC3]](s8)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[AND]](<4 x s16>)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV]](s16)
+    ; CHECK-NEXT: $b0 = COPY [[TRUNC]](s8)
     ; CHECK-NEXT: RET_ReallyLR implicit $b0
     %1:_(s8) = COPY $b0
     %2:_(s8) = COPY $b1
     %3:_(s8) = COPY $b2
     %4:_(<3 x s8>) = G_BUILD_VECTOR %1(s8), %2(s8), %3(s8)
     %0:_(<3 x s1>) = G_TRUNC %4(<3 x s8>)
     %5:_(<3 x s1>) = G_AND %0, %0
     %7:_(<3 x s8>) = G_ANYEXT %5(<3 x s1>)

llvm/test/CodeGen/AArch64/GlobalISel/retry-artifact-combine.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -o - -mtriple=aarch64 -run-pass=legalizer %s | FileCheck %s
 ---
 name:            combine_anyext_s2
 body:             |
   bb.1:
     ; CHECK-LABEL: name: combine_anyext_s2
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
-    ; CHECK: [[FCMP:%[0-9]+]]:_(s32) = G_FCMP floatpred(ogt), [[COPY]](s32), [[COPY1]]
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[FCMP]], [[C]]
-    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
-    ; CHECK: $w0 = COPY [[AND1]](s32)
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK-NEXT: [[FCMP:%[0-9]+]]:_(s32) = G_FCMP floatpred(ogt), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[FCMP]], [[C]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; CHECK-NEXT: $w0 = COPY [[AND1]](s32)
     %0:_(s32) = COPY $w0
     %1:_(s32) = COPY $w1
     %2:_(s1) = G_FCMP floatpred(ogt), %0(s32), %1
     %3:_(s2) = G_ZEXT %2(s1)
     %4:_(s2) = G_CONSTANT i2 1
     %5:_(s2) = G_AND %3, %4
     %6:_(s32) = G_ANYEXT %5(s2)
     $w0 = COPY %6(s32)
 ...
 

llvm/test/CodeGen/AArch64/aarch64-addv.ll

; CHECK-NEXT:    ret
   %bin.rdx = load <2 x i64>, ptr %arr
   %r = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %bin.rdx)
   ret i64 %r
 }
 
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 
 define i32 @oversized_ADDV_256(ptr noalias nocapture readonly %arg1, ptr noalias nocapture readonly %arg2) {
-; CHECK-LABEL: oversized_ADDV_256:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    uabdl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    uaddlv s0, v0.8h
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: oversized_ADDV_256:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    ldr d0, [x0]
+; SDAG-NEXT:    ldr d1, [x1]
+; SDAG-NEXT:    uabdl v0.8h, v0.8b, v1.8b
+; SDAG-NEXT:    uaddlv s0, v0.8h
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: oversized_ADDV_256:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    ldr d1, [x0]
+; GISEL-NEXT:    ldr d2, [x1]
+; GISEL-NEXT:    movi v0.2d, #0000000000000000
+; GISEL-NEXT:    ushll v1.8h, v1.8b, #0
+; GISEL-NEXT:    ushll v2.8h, v2.8b, #0
+; GISEL-NEXT:    mov d3, v1.d[1]
+; GISEL-NEXT:    mov d4, v2.d[1]
+; GISEL-NEXT:    usubl v1.4s, v1.4h, v2.4h
+; GISEL-NEXT:    usubl v2.4s, v3.4h, v4.4h
+; GISEL-NEXT:    cmgt v3.4s, v0.4s, v1.4s
+; GISEL-NEXT:    neg v4.4s, v1.4s
+; GISEL-NEXT:    cmgt v0.4s, v0.4s, v2.4s
+; GISEL-NEXT:    shl v3.4s, v3.4s, #31
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    neg v5.4s, v2.4s
+; GISEL-NEXT:    sshr v3.4s, v3.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    bit v1.16b, v4.16b, v3.16b
+; GISEL-NEXT:    bsl v0.16b, v5.16b, v2.16b
+; GISEL-NEXT:    add v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    addv s0, v0.4s
+; GISEL-NEXT:    fmov w0, s0
+; GISEL-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %arg1, align 1
   %1 = zext <8 x i8> %0 to <8 x i32>
   %2 = load <8 x i8>, ptr %arg2, align 1
   %3 = zext <8 x i8> %2 to <8 x i32>
   %4 = sub nsw <8 x i32> %1, %3
   %5 = icmp slt <8 x i32> %4, zeroinitializer
   %6 = sub nsw <8 x i32> zeroinitializer, %4
   %7 = select <8 x i1> %5, <8 x i32> %6, <8 x i32> %4
   %r = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %7)
   ret i32 %r
 }
 
 declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
 
 define i32 @oversized_ADDV_512(ptr %arr)  {
-; SDAG-LABEL:  oversized_ADDV_512:
-; SDAG:        // %bb.0:
-; SDAG-NEXT:     ldp q0, q1, [x0, #32]
-; SDAG-NEXT:     ldp q3, q2, [x0]
-; SDAG-NEXT:     add v0.4s, v3.4s, v0.4s
-; SDAG-NEXT:     add v1.4s, v2.4s, v1.4s
-; SDAG-NEXT:     add v0.4s, v0.4s, v1.4s
-; SDAG-NEXT:     addv s0, v0.4s
-; SDAG-NEXT:     fmov w0, s0
-; SDAG-NEXT:     ret
+; SDAG-LABEL: oversized_ADDV_512:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    ldp q0, q1, [x0, #32]
+; SDAG-NEXT:    ldp q3, q2, [x0]
+; SDAG-NEXT:    add v0.4s, v3.4s, v0.4s
+; SDAG-NEXT:    add v1.4s, v2.4s, v1.4s
+; SDAG-NEXT:    add v0.4s, v0.4s, v1.4s
+; SDAG-NEXT:    addv s0, v0.4s
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: oversized_ADDV_512:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldp q0, q1, [x0]
 ; GISEL-NEXT:    ldp q2, q3, [x0, #32]
 ; GISEL-NEXT:    add v0.4s, v0.4s, v1.4s
 ; GISEL-NEXT:    add v1.4s, v2.4s, v3.4s
 ; GISEL-NEXT:    add v0.4s, v0.4s, v1.4s

 entry:
   %rdx.1 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a1)
   %rdx.2 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a2)
   %r = add i16 %rdx.1, %rdx.2
   ret i16 %r
 }
 
 define i32 @addv_combine_i32(<4 x i32> %a1, <4 x i32> %a2) {
-; SDAG-LABEL:  addv_combine_i32:
-; SDAG:        // %bb.0: // %entry
-; SDAG-NEXT:     add v0.4s, v0.4s, v1.4s
-; SDAG-NEXT:     addv s0, v0.4s
-; SDAG-NEXT:     fmov w0, s0
-; SDAG-NEXT:     ret
+; SDAG-LABEL: addv_combine_i32:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    add v0.4s, v0.4s, v1.4s
+; SDAG-NEXT:    addv s0, v0.4s
+; SDAG-NEXT:    fmov w0, s0
+; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: addv_combine_i32:
 ; GISEL:       // %bb.0: // %entry
-; GISEL-NEXT:    addv  s0, v0.4s
-; GISEL-NEXT:    addv  s1, v1.4s
-; GISEL-NEXT:    fmov  w8, s0
-; GISEL-NEXT:    fmov  w9, s1
+; GISEL-NEXT:    addv s0, v0.4s
+; GISEL-NEXT:    addv s1, v1.4s
+; GISEL-NEXT:    fmov w8, s0
+; GISEL-NEXT:    fmov w9, s1
 ; GISEL-NEXT:    add w0, w8, w9
 ; GISEL-NEXT:    ret
 entry:
   %rdx.1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a1)
   %rdx.2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a2)
   %r = add i32 %rdx.1, %rdx.2
   ret i32 %r
 }
 
 define i64 @addv_combine_i64(<2 x i64> %a1, <2 x i64> %a2) {
-; SDAG-LABEL:  addv_combine_i64:
-; SDAG:        // %bb.0: // %entry
-; SDAG-NEXT:     add v0.2d, v0.2d, v1.2d
-; SDAG-NEXT:     addp d0, v0.2d
-; SDAG-NEXT:     fmov x0, d0
-; SDAG-NEXT:     ret
+; SDAG-LABEL: addv_combine_i64:
+; SDAG:       // %bb.0: // %entry
+; SDAG-NEXT:    add v0.2d, v0.2d, v1.2d
+; SDAG-NEXT:    addp d0, v0.2d
+; SDAG-NEXT:    fmov x0, d0
+; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: addv_combine_i64:
 ; GISEL:       // %bb.0: // %entry
-; GISEL-NEXT:    addp  d0, v0.2d
-; GISEL-NEXT:    addp  d1, v1.2d
-; GISEL-NEXT:    fmov  x8, d0
-; GISEL-NEXT:    fmov  x9, d1
+; GISEL-NEXT:    addp d0, v0.2d
+; GISEL-NEXT:    addp d1, v1.2d
+; GISEL-NEXT:    fmov x8, d0
+; GISEL-NEXT:    fmov x9, d1
 ; GISEL-NEXT:    add x0, x8, x9
 ; GISEL-NEXT:    ret
 entry:
   %rdx.1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1)
   %rdx.2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a2)
   %r = add i64 %rdx.1, %rdx.2
   ret i64 %r
 }

llvm/test/CodeGen/AArch64/arm64-subvector-extend.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; Test efficient codegen of vector extends up from legal type to 128 bit
 ; and 256 bit vector types.
 
-; CHECK-GI:       warning: Instruction selection used fallback path for func3
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for func4
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for afunc3
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for afunc4
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bfunc1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bfunc2
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zfunc1
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zfunc2
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bfunc3
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for cfunc4
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v4i8_to_v4i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v4i8_to_v4i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v8i8_to_v8i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v8i8_to_v8i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v32i1
+; CHECK-GI:        warning: Instruction selection used fallback path for zext_v32i1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v32i1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for zext_v64i1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sext_v64i1
 
 ;-----
 ; Vectors of i16.
 ;-----
 

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sshll.8h v0, v0, #0
 ; CHECK-NEXT:    ret
   %r = sext <8 x i8> %v0 to <8 x i16>
   ret <8 x i16> %r
 }
 
 define <16 x i16> @func3(<16 x i8> %v0) nounwind {
-; CHECK-LABEL: func3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.8h v1, v0, #0
-; CHECK-NEXT:    ushll.8h v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: func3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll2.8h v1, v0, #0
+; CHECK-SD-NEXT:    ushll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: func3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    ushll.8h v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <16 x i8> %v0 to <16 x i16>
   ret <16 x i16> %r
 }
 
 define <16 x i16> @func4(<16 x i8> %v0) nounwind {
-; CHECK-LABEL: func4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.8h v1, v0, #0
-; CHECK-NEXT:    sshll.8h v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: func4:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll2.8h v1, v0, #0
+; CHECK-SD-NEXT:    sshll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: func4:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    sshll.8h v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <16 x i8> %v0 to <16 x i16>
   ret <16 x i16> %r
 }
 
 ;-----
 ; Vectors of i32.
 ;-----
 

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
 ; CHECK-NEXT:    ret
   %r = sext <4 x i16> %v0 to <4 x i32>
   ret <4 x i32> %r
 }
 
 define <8 x i32> @afunc3(<8 x i16> %v0) nounwind {
-; CHECK-LABEL: afunc3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.4s v1, v0, #0
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: afunc3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: afunc3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    ushll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <8 x i16> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @afunc4(<8 x i16> %v0) nounwind {
-; CHECK-LABEL: afunc4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.4s v1, v0, #0
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: afunc4:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: afunc4:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    sshll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <8 x i16> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @bfunc1(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: bfunc1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.8h v0, v0, #0
-; CHECK-NEXT:    ushll2.4s v1, v0, #0
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bfunc1:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bfunc1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    ushll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <8 x i8> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @bfunc2(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: bfunc2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll.8h v0, v0, #0
-; CHECK-NEXT:    sshll2.4s v1, v0, #0
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bfunc2:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll.8h v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.4s v1, v0, #0
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bfunc2:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    sshll.4s v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <8 x i8> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 ;-----
 ; Vectors of i64.
 ;-----
 
 define <4 x i64> @zfunc1(<4 x i32> %v0) nounwind {
-; CHECK-LABEL: zfunc1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zfunc1:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zfunc1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <4 x i32> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @zfunc2(<4 x i32> %v0) nounwind {
-; CHECK-LABEL: zfunc2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.2d v1, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zfunc2:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zfunc2:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.2d v0, v0, #0
+; CHECK-GI-NEXT:    sshll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <4 x i32> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @bfunc3(<4 x i16> %v0) nounwind {
-; CHECK-LABEL: bfunc3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bfunc3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bfunc3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <4 x i16> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @cfunc4(<4 x i16> %v0) nounwind {
-; CHECK-LABEL: cfunc4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    sshll2.2d v1, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: cfunc4:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: cfunc4:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.2d v0, v0, #0
+; CHECK-GI-NEXT:    sshll.2d v1, v1, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <4 x i16> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @zext_v4i8_to_v4i64(<4 x i8> %v0) nounwind {
-; CHECK-LABEL: zext_v4i8_to_v4i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    bic.4h v0, #255, lsl #8
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zext_v4i8_to_v4i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    bic.4h v0, #255, lsl #8
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_to_v4i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    adrp x8, .LCPI14_0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI14_0]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    and.16b v0, v0, v2
+; CHECK-GI-NEXT:    and.16b v1, v1, v2
+; CHECK-GI-NEXT:    ret
   %r = zext <4 x i8> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @sext_v4i8_to_v4i64(<4 x i8> %v0) nounwind {
-; CHECK-LABEL: sext_v4i8_to_v4i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll.2d v1, v0, #0
-; CHECK-NEXT:    ushll2.2d v0, v0, #0
-; CHECK-NEXT:    shl.2d v2, v1, #56
-; CHECK-NEXT:    shl.2d v0, v0, #56
-; CHECK-NEXT:    sshr.2d v1, v0, #56
-; CHECK-NEXT:    sshr.2d v0, v2, #56
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sext_v4i8_to_v4i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v0, v0, #0
+; CHECK-SD-NEXT:    shl.2d v2, v1, #56
+; CHECK-SD-NEXT:    shl.2d v0, v0, #56
+; CHECK-SD-NEXT:    sshr.2d v1, v0, #56
+; CHECK-SD-NEXT:    sshr.2d v0, v2, #56
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i8_to_v4i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    shl.2d v0, v0, #56
+; CHECK-GI-NEXT:    ushll.2d v1, v1, #0
+; CHECK-GI-NEXT:    sshr.2d v0, v0, #56
+; CHECK-GI-NEXT:    shl.2d v1, v1, #56
+; CHECK-GI-NEXT:    sshr.2d v1, v1, #56
+; CHECK-GI-NEXT:    ret
   %r = sext <4 x i8> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <8 x i64> @zext_v8i8_to_v8i64(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: zext_v8i8_to_v8i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.8h v0, v0, #0
-; CHECK-NEXT:    ushll2.4s v2, v0, #0
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushll2.2d v3, v2, #0
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ushll.2d v2, v2, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: zext_v8i8_to_v8i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ushll.8h v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.4s v2, v0, #0
+; CHECK-SD-NEXT:    ushll.4s v0, v0, #0
+; CHECK-SD-NEXT:    ushll2.2d v3, v2, #0
+; CHECK-SD-NEXT:    ushll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v0, v0, #0
+; CHECK-SD-NEXT:    ushll.2d v2, v2, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i8_to_v8i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d2, v0[1]
+; CHECK-GI-NEXT:    ushll.4s v3, v1, #0
+; CHECK-GI-NEXT:    ushll.2d v0, v0, #0
+; CHECK-GI-NEXT:    mov d4, v3[1]
+; CHECK-GI-NEXT:    ushll.2d v1, v2, #0
+; CHECK-GI-NEXT:    ushll.2d v2, v3, #0
+; CHECK-GI-NEXT:    ushll.2d v3, v4, #0
+; CHECK-GI-NEXT:    ret
   %r = zext <8 x i8> %v0 to <8 x i64>
   ret <8 x i64> %r
 }
 
 define <8 x i64> @sext_v8i8_to_v8i64(<8 x i8> %v0) nounwind {
-; CHECK-LABEL: sext_v8i8_to_v8i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll.8h v0, v0, #0
-; CHECK-NEXT:    sshll2.4s v2, v0, #0
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    sshll2.2d v3, v2, #0
-; CHECK-NEXT:    sshll2.2d v1, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    sshll.2d v2, v2, #0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sext_v8i8_to_v8i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sshll.8h v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.4s v2, v0, #0
+; CHECK-SD-NEXT:    sshll.4s v0, v0, #0
+; CHECK-SD-NEXT:    sshll2.2d v3, v2, #0
+; CHECK-SD-NEXT:    sshll2.2d v1, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v0, v0, #0
+; CHECK-SD-NEXT:    sshll.2d v2, v2, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i8_to_v8i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d1, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v0, v0, #0
+; CHECK-GI-NEXT:    mov d2, v0[1]
+; CHECK-GI-NEXT:    sshll.4s v3, v1, #0
+; CHECK-GI-NEXT:    sshll.2d v0, v0, #0
+; CHECK-GI-NEXT:    mov d4, v3[1]
+; CHECK-GI-NEXT:    sshll.2d v1, v2, #0
+; CHECK-GI-NEXT:    sshll.2d v2, v3, #0
+; CHECK-GI-NEXT:    sshll.2d v3, v4, #0
+; CHECK-GI-NEXT:    ret
   %r = sext <8 x i8> %v0 to <8 x i64>
   ret <8 x i64> %r
 }
 
 ; Extends of vectors of i1.
 
 define <32 x i8> @zext_v32i1(<32 x i1> %arg) {
 ; CHECK-LABEL: zext_v32i1:

llvm/test/CodeGen/AArch64/arm64-vabs.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck -check-prefixes=CHECK,CHECK-SD %s
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; CHECK-GI:       warning: Instruction selection used fallback path for uabd16b_rdx
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for uabd16b_rdx_i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sabd16b_rdx_i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for uabd8h_rdx
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sabd8h_rdx
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for uabd4s_rdx
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sabd4s_rdx
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_8b
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_16b
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_4h
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_8h
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_2s
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for abs_4s
; CHECK-NEXT:    ret
   %abcmp = icmp slt <16 x i16> %abdiff, zeroinitializer
   %ababs = sub nsw <16 x i16> zeroinitializer, %abdiff
   %absel = select <16 x i1> %abcmp, <16 x i16> %ababs, <16 x i16> %abdiff
   %reduced_v = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %absel)
   ret i16 %reduced_v
 }
 
 define i32 @uabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: uabd16b_rdx_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uabdl.8h v2, v0, v1
-; CHECK-NEXT:    uabal2.8h v2, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v2
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: uabd16b_rdx_i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    uabdl.8h v2, v0, v1
+; CHECK-SD-NEXT:    uabal2.8h v2, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v2
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uabd16b_rdx_i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d3, v0[1]
+; CHECK-GI-NEXT:    ushll.8h v4, v1, #0
+; CHECK-GI-NEXT:    mov d1, v1[1]
+; CHECK-GI-NEXT:    ushll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d6, v4[1]
+; CHECK-GI-NEXT:    ushll.8h v3, v3, #0
+; CHECK-GI-NEXT:    mov d5, v0[1]
+; CHECK-GI-NEXT:    ushll.8h v1, v1, #0
+; CHECK-GI-NEXT:    mov d7, v3[1]
+; CHECK-GI-NEXT:    mov d16, v1[1]
+; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-GI-NEXT:    usubl.4s v0, v0, v4
+; CHECK-GI-NEXT:    usubl.4s v5, v5, v6
+; CHECK-GI-NEXT:    usubl.4s v1, v3, v1
+; CHECK-GI-NEXT:    usubl.4s v3, v7, v16
+; CHECK-GI-NEXT:    cmgt.4s v4, v2, v0
+; CHECK-GI-NEXT:    cmgt.4s v6, v2, v5
+; CHECK-GI-NEXT:    cmgt.4s v7, v2, v1
+; CHECK-GI-NEXT:    cmgt.4s v2, v2, v3
+; CHECK-GI-NEXT:    shl.4s v4, v4, #31
+; CHECK-GI-NEXT:    shl.4s v6, v6, #31
+; CHECK-GI-NEXT:    shl.4s v7, v7, #31
+; CHECK-GI-NEXT:    shl.4s v2, v2, #31
+; CHECK-GI-NEXT:    sshr.4s v4, v4, #31
+; CHECK-GI-NEXT:    neg.4s v17, v0
+; CHECK-GI-NEXT:    sshr.4s v6, v6, #31
+; CHECK-GI-NEXT:    neg.4s v16, v5
+; CHECK-GI-NEXT:    neg.4s v18, v1
+; CHECK-GI-NEXT:    neg.4s v19, v3
+; CHECK-GI-NEXT:    sshr.4s v7, v7, #31
+; CHECK-GI-NEXT:    sshr.4s v2, v2, #31
+; CHECK-GI-NEXT:    bit.16b v0, v17, v4
+; CHECK-GI-NEXT:    mov.16b v4, v6
+; CHECK-GI-NEXT:    bsl.16b v4, v16, v5
+; CHECK-GI-NEXT:    bit.16b v1, v18, v7
+; CHECK-GI-NEXT:    bsl.16b v2, v19, v3
+; CHECK-GI-NEXT:    add.4s v0, v0, v4
+; CHECK-GI-NEXT:    add.4s v1, v1, v2
+; CHECK-GI-NEXT:    add.4s v0, v0, v1
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aext = zext <16 x i8> %a to <16 x i32>
   %bext = zext <16 x i8> %b to <16 x i32>
   %abdiff = sub nsw <16 x i32> %aext, %bext
   %abcmp = icmp slt <16 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <16 x i32> zeroinitializer, %abdiff
   %absel = select <16 x i1> %abcmp, <16 x i32> %ababs, <16 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %absel)
   ret i32 %reduced_v
 }
 
 define i32 @sabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: sabd16b_rdx_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sabdl.8h v2, v0, v1
-; CHECK-NEXT:    sabal2.8h v2, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v2
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sabd16b_rdx_i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sabdl.8h v2, v0, v1
+; CHECK-SD-NEXT:    sabal2.8h v2, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v2
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sabd16b_rdx_i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d3, v0[1]
+; CHECK-GI-NEXT:    sshll.8h v4, v1, #0
+; CHECK-GI-NEXT:    mov d1, v1[1]
+; CHECK-GI-NEXT:    sshll.8h v0, v0, #0
+; CHECK-GI-NEXT:    mov d6, v4[1]
+; CHECK-GI-NEXT:    sshll.8h v3, v3, #0
+; CHECK-GI-NEXT:    mov d5, v0[1]
+; CHECK-GI-NEXT:    sshll.8h v1, v1, #0
+; CHECK-GI-NEXT:    mov d7, v3[1]
+; CHECK-GI-NEXT:    mov d16, v1[1]
+; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-GI-NEXT:    ssubl.4s v0, v0, v4
+; CHECK-GI-NEXT:    ssubl.4s v5, v5, v6
+; CHECK-GI-NEXT:    ssubl.4s v1, v3, v1
+; CHECK-GI-NEXT:    ssubl.4s v3, v7, v16
+; CHECK-GI-NEXT:    cmgt.4s v4, v2, v0
+; CHECK-GI-NEXT:    cmgt.4s v6, v2, v5
+; CHECK-GI-NEXT:    cmgt.4s v7, v2, v1
+; CHECK-GI-NEXT:    cmgt.4s v2, v2, v3
+; CHECK-GI-NEXT:    shl.4s v4, v4, #31
+; CHECK-GI-NEXT:    shl.4s v6, v6, #31
+; CHECK-GI-NEXT:    shl.4s v7, v7, #31
+; CHECK-GI-NEXT:    shl.4s v2, v2, #31
+; CHECK-GI-NEXT:    sshr.4s v4, v4, #31
+; CHECK-GI-NEXT:    neg.4s v17, v0
+; CHECK-GI-NEXT:    sshr.4s v6, v6, #31
+; CHECK-GI-NEXT:    neg.4s v16, v5
+; CHECK-GI-NEXT:    neg.4s v18, v1
+; CHECK-GI-NEXT:    neg.4s v19, v3
+; CHECK-GI-NEXT:    sshr.4s v7, v7, #31
+; CHECK-GI-NEXT:    sshr.4s v2, v2, #31
+; CHECK-GI-NEXT:    bit.16b v0, v17, v4
+; CHECK-GI-NEXT:    mov.16b v4, v6
+; CHECK-GI-NEXT:    bsl.16b v4, v16, v5
+; CHECK-GI-NEXT:    bit.16b v1, v18, v7
+; CHECK-GI-NEXT:    bsl.16b v2, v19, v3
+; CHECK-GI-NEXT:    add.4s v0, v0, v4
+; CHECK-GI-NEXT:    add.4s v1, v1, v2
+; CHECK-GI-NEXT:    add.4s v0, v0, v1
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aext = sext <16 x i8> %a to <16 x i32>
   %bext = sext <16 x i8> %b to <16 x i32>
   %abdiff = sub nsw <16 x i32> %aext, %bext
   %abcmp = icmp slt <16 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <16 x i32> zeroinitializer, %abdiff
   %absel = select <16 x i1> %abcmp, <16 x i32> %ababs, <16 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %absel)
   ret i32 %reduced_v
 }
 
 
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
 
 define i32 @uabd8h_rdx(ptr %a, ptr %b) {
-; CHECK-LABEL: uabd8h_rdx:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    ldr q1, [x1]
-; CHECK-NEXT:    uabd.8h v0, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v0
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: uabd8h_rdx:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr q0, [x0]
+; CHECK-SD-NEXT:    ldr q1, [x1]
+; CHECK-SD-NEXT:    uabd.8h v0, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v0
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uabd8h_rdx:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    ldr q2, [x1]
+; CHECK-GI-NEXT:    movi.2d v0, #0000000000000000
+; CHECK-GI-NEXT:    mov d3, v1[1]
+; CHECK-GI-NEXT:    mov d4, v2[1]
+; CHECK-GI-NEXT:    usubl.4s v1, v1, v2
+; CHECK-GI-NEXT:    usubl.4s v2, v3, v4
+; CHECK-GI-NEXT:    cmgt.4s v3, v0, v1
+; CHECK-GI-NEXT:    neg.4s v4, v1
+; CHECK-GI-NEXT:    cmgt.4s v0, v0, v2
+; CHECK-GI-NEXT:    shl.4s v3, v3, #31
+; CHECK-GI-NEXT:    shl.4s v0, v0, #31
+; CHECK-GI-NEXT:    neg.4s v5, v2
+; CHECK-GI-NEXT:    sshr.4s v3, v3, #31
+; CHECK-GI-NEXT:    sshr.4s v0, v0, #31
+; CHECK-GI-NEXT:    bit.16b v1, v4, v3
+; CHECK-GI-NEXT:    bsl.16b v0, v5, v2
+; CHECK-GI-NEXT:    add.4s v0, v1, v0
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aload = load <8 x i16>, ptr %a, align 1
   %bload = load <8 x i16>, ptr %b, align 1
   %aext = zext <8 x i16> %aload to <8 x i32>
   %bext = zext <8 x i16> %bload to <8 x i32>
   %abdiff = sub nsw <8 x i32> %aext, %bext
   %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
   %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
   ret i32 %reduced_v
 }
 
 define i32 @sabd8h_rdx(<8 x i16> %a, <8 x i16> %b) {
-; CHECK-LABEL: sabd8h_rdx:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sabd.8h v0, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v0
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sabd8h_rdx:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sabd.8h v0, v0, v1
+; CHECK-SD-NEXT:    uaddlv.8h s0, v0
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sabd8h_rdx:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d3, v0[1]
+; CHECK-GI-NEXT:    mov d4, v1[1]
+; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-GI-NEXT:    ssubl.4s v0, v0, v1
+; CHECK-GI-NEXT:    ssubl.4s v1, v3, v4
+; CHECK-GI-NEXT:    cmgt.4s v3, v2, v0
+; CHECK-GI-NEXT:    neg.4s v4, v0
+; CHECK-GI-NEXT:    cmgt.4s v2, v2, v1
+; CHECK-GI-NEXT:    shl.4s v3, v3, #31
+; CHECK-GI-NEXT:    shl.4s v2, v2, #31
+; CHECK-GI-NEXT:    neg.4s v5, v1
+; CHECK-GI-NEXT:    sshr.4s v3, v3, #31
+; CHECK-GI-NEXT:    sshr.4s v2, v2, #31
+; CHECK-GI-NEXT:    bit.16b v0, v4, v3
+; CHECK-GI-NEXT:    bit.16b v1, v5, v2
+; CHECK-GI-NEXT:    add.4s v0, v0, v1
+; CHECK-GI-NEXT:    addv.4s s0, v0
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
   %aext = sext <8 x i16> %a to <8 x i32>
   %bext = sext <8 x i16> %b to <8 x i32>
   %abdiff = sub nsw <8 x i32> %aext, %bext
   %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
   %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
   ret i32 %reduced_v

llvm/test/CodeGen/AArch64/sext.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:      warning: Instruction selection used fallback path for sext_v3i8_v3i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v3i8_v3i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v3i10_v3i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v3i10_v3i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sext_v16i10_v16i16
+
+define i16 @sext_i8_to_i16(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i16
+  ret i16 %c
+}
+
+define i32 @sext_i8_to_i32(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i32
+  ret i32 %c
+}
+
+define i64 @sext_i8_to_i64(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtb x0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i64
+  ret i64 %c
+}
+
+define i10 @sext_i8_to_i10(i8 %a) {
+; CHECK-LABEL: sext_i8_to_i10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i8 %a to i10
+  ret i10 %c
+}
+
+define i32 @sext_i16_to_i32(i16 %a) {
+; CHECK-LABEL: sext_i16_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxth w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i16 %a to i32
+  ret i32 %c
+}
+
+define i64 @sext_i16_to_i64(i16 %a) {
+; CHECK-LABEL: sext_i16_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxth x0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i16 %a to i64
+  ret i64 %c
+}
+
+define i64 @sext_i32_to_i64(i32 %a) {
+; CHECK-LABEL: sext_i32_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i32 %a to i64
+  ret i64 %c
+}
+
+define i16 @sext_i10_to_i16(i10 %a) {
+; CHECK-LABEL: sext_i10_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sbfx w0, w0, #0, #10
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i10 %a to i16
+  ret i16 %c
+}
+
+define i32 @sext_i10_to_i32(i10 %a) {
+; CHECK-LABEL: sext_i10_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sbfx w0, w0, #0, #10
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i10 %a to i32
+  ret i32 %c
+}
+
+define i64 @sext_i10_to_i64(i10 %a) {
+; CHECK-LABEL: sext_i10_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x0, x0, #0, #10
+; CHECK-NEXT:    ret
+entry:
+  %c = sext i10 %a to i64
+  ret i64 %c
+}
+
+define <2 x i16> @sext_v2i8_v2i16(<2 x i8> %a) {
+; CHECK-LABEL: sext_v2i8_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i8> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @sext_v2i8_v2i32(<2 x i8> %a) {
+; CHECK-LABEL: sext_v2i8_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i8> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @sext_v2i8_v2i64(<2 x i8> %a) {
+; CHECK-LABEL: sext_v2i8_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i8> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i32> @sext_v2i16_v2i32(<2 x i16> %a) {
+; CHECK-LABEL: sext_v2i16_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i16> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @sext_v2i16_v2i64(<2 x i16> %a) {
+; CHECK-LABEL: sext_v2i16_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    shl v0.2d, v0.2d, #48
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i16> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @sext_v2i32_v2i64(<2 x i32> %a) {
+; CHECK-LABEL: sext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i16> @sext_v2i10_v2i16(<2 x i10> %a) {
+; CHECK-LABEL: sext_v2i10_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #22
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i10> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @sext_v2i10_v2i32(<2 x i10> %a) {
+; CHECK-LABEL: sext_v2i10_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.2s, v0.2s, #22
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i10> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @sext_v2i10_v2i64(<2 x i10> %a) {
+; CHECK-LABEL: sext_v2i10_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <2 x i10> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <3 x i16> @sext_v3i8_v3i16(<3 x i8> %a) {
+; CHECK-LABEL: sext_v3i8_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i8> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) {
+; CHECK-LABEL: sext_v3i8_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i8> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @sext_v3i8_v3i64(<3 x i8> %a) {
+; CHECK-SD-LABEL: sext_v3i8_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s0, w2
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v0.2d, #56
+; CHECK-SD-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-SD-NEXT:    sshr v2.2d, v2.2d, #56
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i8_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    lsl x8, x2, #56
+; CHECK-GI-NEXT:    asr x8, x8, #56
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i8> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i32> @sext_v3i16_v3i32(<3 x i16> %a) {
+; CHECK-SD-LABEL: sext_v3i16_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i16_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov w8, s2
+; CHECK-GI-NEXT:    sxth w9, w9
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i16> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @sext_v3i16_v3i64(<3 x i16> %a) {
+; CHECK-SD-LABEL: sext_v3i16_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i16_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    sxth x8, w8
+; CHECK-GI-NEXT:    sxth x9, w9
+; CHECK-GI-NEXT:    sxth x10, w10
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i16> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @sext_v3i32_v3i64(<3 x i32> %a) {
+; CHECK-SD-LABEL: sext_v3i32_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v3.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll2 v2.2d, v0.4s, #0
+; CHECK-SD-NEXT:    fmov d0, d3
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i32_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    sxtw x8, w8
+; CHECK-GI-NEXT:    sxtw x9, w9
+; CHECK-GI-NEXT:    sxtw x10, w10
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i32> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i16> @sext_v3i10_v3i16(<3 x i10> %a) {
+; CHECK-LABEL: sext_v3i10_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    shl v0.4h, v0.4h, #6
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i10> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) {
+; CHECK-LABEL: sext_v3i10_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <3 x i10> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @sext_v3i10_v3i64(<3 x i10> %a) {
+; CHECK-SD-LABEL: sext_v3i10_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s0, w2
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v0.2d, #54
+; CHECK-SD-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-SD-NEXT:    sshr v2.2d, v2.2d, #54
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v3i10_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    lsl x8, x2, #54
+; CHECK-GI-NEXT:    asr x8, x8, #54
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <3 x i10> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <4 x i16> @sext_v4i8_v4i16(<4 x i8> %a) {
+; CHECK-LABEL: sext_v4i8_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i8> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @sext_v4i8_v4i32(<4 x i8> %a) {
+; CHECK-LABEL: sext_v4i8_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i8> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @sext_v4i8_v4i64(<4 x i8> %a) {
+; CHECK-SD-LABEL: sext_v4i8_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v1.2d, #56
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-SD-NEXT:    sshr v1.2d, v0.2d, #56
+; CHECK-SD-NEXT:    sshr v0.2d, v2.2d, #56
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i8_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #56
+; CHECK-GI-NEXT:    sshr v1.2d, v1.2d, #56
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i8> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i32> @sext_v4i16_v4i32(<4 x i16> %a) {
+; CHECK-LABEL: sext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i16> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @sext_v4i16_v4i64(<4 x i16> %a) {
+; CHECK-SD-LABEL: sext_v4i16_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i16_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i16> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @sext_v4i32_v4i64(<4 x i32> %a) {
+; CHECK-SD-LABEL: sext_v4i32_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i32_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i32> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i16> @sext_v4i10_v4i16(<4 x i10> %a) {
+; CHECK-LABEL: sext_v4i10_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.4h, v0.4h, #6
+; CHECK-NEXT:    sshr v0.4h, v0.4h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i10> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @sext_v4i10_v4i32(<4 x i10> %a) {
+; CHECK-LABEL: sext_v4i10_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <4 x i10> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @sext_v4i10_v4i64(<4 x i10> %a) {
+; CHECK-SD-LABEL: sext_v4i10_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-SD-NEXT:    shl v2.2d, v1.2d, #54
+; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    sshr v1.2d, v0.2d, #54
+; CHECK-SD-NEXT:    sshr v0.2d, v2.2d, #54
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v4i10_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #54
+; CHECK-GI-NEXT:    sshr v1.2d, v1.2d, #54
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <4 x i10> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <8 x i16> @sext_v8i8_v8i16(<8 x i8> %a) {
+; CHECK-LABEL: sext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <8 x i8> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @sext_v8i8_v8i32(<8 x i8> %a) {
+; CHECK-SD-LABEL: sext_v8i8_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i8_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i8> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @sext_v8i8_v8i64(<8 x i8> %a) {
+; CHECK-SD-LABEL: sext_v8i8_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i8_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i8> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i32> @sext_v8i16_v8i32(<8 x i16> %a) {
+; CHECK-SD-LABEL: sext_v8i16_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i16_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i16> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @sext_v8i16_v8i64(<8 x i16> %a) {
+; CHECK-SD-LABEL: sext_v8i16_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i16_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i16> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @sext_v8i32_v8i64(<8 x i32> %a) {
+; CHECK-SD-LABEL: sext_v8i32_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v4.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v1.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i32_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i32> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i16> @sext_v8i10_v8i16(<8 x i10> %a) {
+; CHECK-LABEL: sext_v8i10_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    shl v0.8h, v0.8h, #6
+; CHECK-NEXT:    sshr v0.8h, v0.8h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <8 x i10> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @sext_v8i10_v8i32(<8 x i10> %a) {
+; CHECK-SD-LABEL: sext_v8i10_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-SD-NEXT:    shl v2.4s, v1.4s, #22
+; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-SD-NEXT:    sshr v1.4s, v0.4s, #22
+; CHECK-SD-NEXT:    sshr v0.4s, v2.4s, #22
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i10_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #22
+; CHECK-GI-NEXT:    sshr v1.4s, v1.4s, #22
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i10> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @sext_v8i10_v8i64(<8 x i10> %a) {
+; CHECK-SD-LABEL: sext_v8i10_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-SD-NEXT:    shl v1.2d, v1.2d, #54
+; CHECK-SD-NEXT:    shl v2.2d, v2.2d, #54
+; CHECK-SD-NEXT:    shl v4.2d, v0.2d, #54
+; CHECK-SD-NEXT:    shl v5.2d, v3.2d, #54
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #54
+; CHECK-SD-NEXT:    sshr v0.2d, v2.2d, #54
+; CHECK-SD-NEXT:    sshr v3.2d, v4.2d, #54
+; CHECK-SD-NEXT:    sshr v2.2d, v5.2d, #54
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v8i10_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    shl v2.2d, v2.2d, #54
+; CHECK-GI-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-GI-NEXT:    shl v4.2d, v1.2d, #54
+; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v3.2d, v3.2d, #54
+; CHECK-GI-NEXT:    sshr v1.2d, v2.2d, #54
+; CHECK-GI-NEXT:    sshr v2.2d, v4.2d, #54
+; CHECK-GI-NEXT:    sshr v3.2d, v3.2d, #54
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <8 x i10> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <16 x i16> @sext_v16i8_v16i16(<16 x i8> %a) {
+; CHECK-SD-LABEL: sext_v16i8_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i8_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i8> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @sext_v16i8_v16i32(<16 x i8> %a) {
+; CHECK-SD-LABEL: sext_v16i8_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v2.8h, v0.16b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v2.8h, #0
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i8_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i8> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @sext_v16i8_v16i64(<16 x i8> %a) {
+; CHECK-SD-LABEL: sext_v16i8_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll2 v2.4s, v1.8h, #0
+; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v16.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v7.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v4.4s, #0
+; CHECK-SD-NEXT:    sshll2 v5.2d, v16.4s, #0
+; CHECK-SD-NEXT:    sshll v6.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v4.2s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v4.2d, v16.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i8_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v1.d[1]
+; CHECK-GI-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d5, v2.d[1]
+; CHECK-GI-NEXT:    sshll v4.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v6.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v16.4s, v5.4h, #0
+; CHECK-GI-NEXT:    mov d3, v4.d[1]
+; CHECK-GI-NEXT:    mov d7, v6.d[1]
+; CHECK-GI-NEXT:    mov d17, v16.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v4.2s, #0
+; CHECK-GI-NEXT:    sshll v4.2d, v6.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v5.2d, v7.2s, #0
+; CHECK-GI-NEXT:    sshll v6.2d, v16.2s, #0
+; CHECK-GI-NEXT:    sshll v7.2d, v17.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i8> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i32> @sext_v16i16_v16i32(<16 x i16> %a) {
+; CHECK-SD-LABEL: sext_v16i16_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i16_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i16> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @sext_v16i16_v16i64(<16 x i16> %a) {
+; CHECK-SD-LABEL: sext_v16i16_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    sshll2 v4.4s, v1.8h, #0
+; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll2 v16.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v7.2d, v4.4s, #0
+; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll2 v5.2d, v1.4s, #0
+; CHECK-SD-NEXT:    sshll v6.2d, v4.2s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v4.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v16.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i16_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v5.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v2.d[1]
+; CHECK-GI-NEXT:    sshll v0.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d6, v5.d[1]
+; CHECK-GI-NEXT:    sshll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    mov d7, v2.d[1]
+; CHECK-GI-NEXT:    mov d16, v3.d[1]
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    sshll v4.2d, v5.2s, #0
+; CHECK-GI-NEXT:    sshll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    sshll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v3.2d, v7.2s, #0
+; CHECK-GI-NEXT:    sshll v7.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i16> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i64> @sext_v16i32_v16i64(<16 x i32> %a) {
+; CHECK-SD-LABEL: sext_v16i32_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll2 v17.2d, v0.4s, #0
+; CHECK-SD-NEXT:    sshll2 v18.2d, v1.4s, #0
+; CHECK-SD-NEXT:    sshll v16.2d, v1.2s, #0
+; CHECK-SD-NEXT:    sshll2 v5.2d, v2.4s, #0
+; CHECK-SD-NEXT:    sshll2 v7.2d, v3.4s, #0
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-SD-NEXT:    sshll v6.2d, v3.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v17.16b
+; CHECK-SD-NEXT:    mov v2.16b, v16.16b
+; CHECK-SD-NEXT:    mov v3.16b, v18.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i32_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d5, v1.d[1]
+; CHECK-GI-NEXT:    mov d6, v2.d[1]
+; CHECK-GI-NEXT:    sshll v16.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d2, v3.d[1]
+; CHECK-GI-NEXT:    sshll v17.2d, v1.2s, #0
+; CHECK-GI-NEXT:    sshll v18.2d, v5.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    sshll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    sshll v7.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov v0.16b, v16.16b
+; CHECK-GI-NEXT:    mov v2.16b, v17.16b
+; CHECK-GI-NEXT:    mov v3.16b, v18.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i32> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i16> @sext_v16i10_v16i16(<16 x i10> %a) {
+; CHECK-LABEL: sext_v16i10_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [sp]
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    ldr w9, [sp, #16]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    ldr w8, [sp, #8]
+; CHECK-NEXT:    mov v1.h[1], w1
+; CHECK-NEXT:    mov v0.h[1], w8
+; CHECK-NEXT:    ldr w8, [sp, #24]
+; CHECK-NEXT:    mov v1.h[2], w2
+; CHECK-NEXT:    mov v0.h[2], w9
+; CHECK-NEXT:    ldr w9, [sp, #32]
+; CHECK-NEXT:    mov v1.h[3], w3
+; CHECK-NEXT:    mov v0.h[3], w8
+; CHECK-NEXT:    ldr w8, [sp, #40]
+; CHECK-NEXT:    mov v1.h[4], w4
+; CHECK-NEXT:    mov v0.h[4], w9
+; CHECK-NEXT:    ldr w9, [sp, #48]
+; CHECK-NEXT:    mov v1.h[5], w5
+; CHECK-NEXT:    mov v0.h[5], w8
+; CHECK-NEXT:    ldr w8, [sp, #56]
+; CHECK-NEXT:    mov v1.h[6], w6
+; CHECK-NEXT:    mov v0.h[6], w9
+; CHECK-NEXT:    mov v1.h[7], w7
+; CHECK-NEXT:    mov v0.h[7], w8
+; CHECK-NEXT:    shl v1.8h, v1.8h, #6
+; CHECK-NEXT:    shl v2.8h, v0.8h, #6
+; CHECK-NEXT:    sshr v0.8h, v1.8h, #6
+; CHECK-NEXT:    sshr v1.8h, v2.8h, #6
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <16 x i10> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @sext_v16i10_v16i32(<16 x i10> %a) {
+; CHECK-SD-LABEL: sext_v16i10_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w11, [sp, #32]
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr w12, [sp]
+; CHECK-SD-NEXT:    fmov s1, w4
+; CHECK-SD-NEXT:    ldr w10, [sp, #40]
+; CHECK-SD-NEXT:    ldr w15, [sp, #8]
+; CHECK-SD-NEXT:    fmov s3, w11
+; CHECK-SD-NEXT:    fmov s2, w12
+; CHECK-SD-NEXT:    ldr w9, [sp, #48]
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    ldr w14, [sp, #16]
+; CHECK-SD-NEXT:    mov v1.h[1], w5
+; CHECK-SD-NEXT:    ldr w8, [sp, #56]
+; CHECK-SD-NEXT:    mov v2.h[1], w15
+; CHECK-SD-NEXT:    ldr w13, [sp, #24]
+; CHECK-SD-NEXT:    mov v3.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    mov v1.h[2], w6
+; CHECK-SD-NEXT:    mov v2.h[2], w14
+; CHECK-SD-NEXT:    mov v3.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w3
+; CHECK-SD-NEXT:    mov v1.h[3], w7
+; CHECK-SD-NEXT:    mov v2.h[3], w13
+; CHECK-SD-NEXT:    mov v3.h[3], w8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #22
+; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #22
+; CHECK-SD-NEXT:    shl v2.4s, v2.4s, #22
+; CHECK-SD-NEXT:    shl v3.4s, v3.4s, #22
+; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #22
+; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #22
+; CHECK-SD-NEXT:    sshr v2.4s, v2.4s, #22
+; CHECK-SD-NEXT:    sshr v3.4s, v3.4s, #22
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i10_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s7, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s17, w4
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    mov v7.s[1], w1
+; CHECK-GI-NEXT:    ldr s6, [sp, #48]
+; CHECK-GI-NEXT:    mov v17.s[1], w5
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ldr s16, [sp, #56]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], w2
+; CHECK-GI-NEXT:    mov v17.s[2], w6
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], w3
+; CHECK-GI-NEXT:    mov v17.s[3], w7
+; CHECK-GI-NEXT:    mov v4.s[3], v16.s[0]
+; CHECK-GI-NEXT:    shl v3.4s, v0.4s, #22
+; CHECK-GI-NEXT:    shl v1.4s, v7.4s, #22
+; CHECK-GI-NEXT:    shl v2.4s, v17.4s, #22
+; CHECK-GI-NEXT:    shl v4.4s, v4.4s, #22
+; CHECK-GI-NEXT:    sshr v0.4s, v1.4s, #22
+; CHECK-GI-NEXT:    sshr v1.4s, v2.4s, #22
+; CHECK-GI-NEXT:    sshr v2.4s, v3.4s, #22
+; CHECK-GI-NEXT:    sshr v3.4s, v4.4s, #22
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i10> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @sext_v16i10_v16i64(<16 x i10> %a) {
+; CHECK-SD-LABEL: sext_v16i10_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s0, [sp]
+; CHECK-SD-NEXT:    add x8, sp, #8
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s2, w2
+; CHECK-SD-NEXT:    fmov s3, w4
+; CHECK-SD-NEXT:    fmov s4, w6
+; CHECK-SD-NEXT:    ld1 { v0.s }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
+; CHECK-SD-NEXT:    ldr s5, [sp, #16]
+; CHECK-SD-NEXT:    add x9, sp, #40
+; CHECK-SD-NEXT:    ldr s6, [sp, #32]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    ldr s7, [sp, #48]
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ld1 { v5.s }[1], [x8]
+; CHECK-SD-NEXT:    mov v2.s[1], w3
+; CHECK-SD-NEXT:    ld1 { v6.s }[1], [x9]
+; CHECK-SD-NEXT:    mov v3.s[1], w5
+; CHECK-SD-NEXT:    ld1 { v7.s }[1], [x10]
+; CHECK-SD-NEXT:    mov v4.s[1], w7
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v16.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v5.2d, v5.2s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-SD-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-SD-NEXT:    shl v0.2d, v1.2d, #54
+; CHECK-SD-NEXT:    shl v1.2d, v2.2d, #54
+; CHECK-SD-NEXT:    shl v2.2d, v3.2d, #54
+; CHECK-SD-NEXT:    shl v3.2d, v4.2d, #54
+; CHECK-SD-NEXT:    shl v4.2d, v16.2d, #54
+; CHECK-SD-NEXT:    shl v5.2d, v5.2d, #54
+; CHECK-SD-NEXT:    shl v6.2d, v6.2d, #54
+; CHECK-SD-NEXT:    shl v7.2d, v7.2d, #54
+; CHECK-SD-NEXT:    sshr v0.2d, v0.2d, #54
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #54
+; CHECK-SD-NEXT:    sshr v2.2d, v2.2d, #54
+; CHECK-SD-NEXT:    sshr v3.2d, v3.2d, #54
+; CHECK-SD-NEXT:    sshr v4.2d, v4.2d, #54
+; CHECK-SD-NEXT:    sshr v5.2d, v5.2d, #54
+; CHECK-SD-NEXT:    sshr v6.2d, v6.2d, #54
+; CHECK-SD-NEXT:    sshr v7.2d, v7.2d, #54
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sext_v16i10_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s6, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s16, w2
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    fmov s18, w4
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    fmov s19, w6
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    ldr s7, [sp, #48]
+; CHECK-GI-NEXT:    ldr s17, [sp, #56]
+; CHECK-GI-NEXT:    mov v6.s[1], w1
+; CHECK-GI-NEXT:    mov v16.s[1], w3
+; CHECK-GI-NEXT:    mov v18.s[1], w5
+; CHECK-GI-NEXT:    mov v19.s[1], w7
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    mov v7.s[1], v17.s[0]
+; CHECK-GI-NEXT:    ushll v1.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v19.2s, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #54
+; CHECK-GI-NEXT:    shl v3.2d, v3.2d, #54
+; CHECK-GI-NEXT:    shl v5.2d, v5.2d, #54
+; CHECK-GI-NEXT:    shl v6.2d, v6.2d, #54
+; CHECK-GI-NEXT:    shl v16.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v17.2d, v2.2d, #54
+; CHECK-GI-NEXT:    shl v18.2d, v4.2d, #54
+; CHECK-GI-NEXT:    shl v7.2d, v7.2d, #54
+; CHECK-GI-NEXT:    sshr v0.2d, v1.2d, #54
+; CHECK-GI-NEXT:    sshr v1.2d, v3.2d, #54
+; CHECK-GI-NEXT:    sshr v2.2d, v5.2d, #54
+; CHECK-GI-NEXT:    sshr v3.2d, v6.2d, #54
+; CHECK-GI-NEXT:    sshr v4.2d, v16.2d, #54
+; CHECK-GI-NEXT:    sshr v5.2d, v17.2d, #54
+; CHECK-GI-NEXT:    sshr v6.2d, v18.2d, #54
+; CHECK-GI-NEXT:    sshr v7.2d, v7.2d, #54
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = sext <16 x i10> %a to <16 x i64>
+  ret <16 x i64> %c
+}

llvm/test/CodeGen/AArch64/zext.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:      warning: Instruction selection used fallback path for zext_v3i8_v3i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for zext_v3i10_v3i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for zext_v16i10_v16i1
+
+define i16 @zext_i8_to_i16(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i16
+  ret i16 %c
+}
+
+define i32 @zext_i8_to_i32(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i32
+  ret i32 %c
+}
+
+define i64 @zext_i8_to_i64(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x0, x0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i64
+  ret i64 %c
+}
+
+define i10 @zext_i8_to_i10(i8 %a) {
+; CHECK-LABEL: zext_i8_to_i10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i8 %a to i10
+  ret i10 %c
+}
+
+define i32 @zext_i16_to_i32(i16 %a) {
+; CHECK-LABEL: zext_i16_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i16 %a to i32
+  ret i32 %c
+}
+
+define i64 @zext_i16_to_i64(i16 %a) {
+; CHECK-LABEL: zext_i16_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x0, x0, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i16 %a to i64
+  ret i64 %c
+}
+
+define i64 @zext_i32_to_i64(i32 %a) {
+; CHECK-LABEL: zext_i32_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w0, w0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i32 %a to i64
+  ret i64 %c
+}
+
+define i16 @zext_i10_to_i16(i10 %a) {
+; CHECK-LABEL: zext_i10_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0x3ff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i10 %a to i16
+  ret i16 %c
+}
+
+define i32 @zext_i10_to_i32(i10 %a) {
+; CHECK-LABEL: zext_i10_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w0, w0, #0x3ff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i10 %a to i32
+  ret i32 %c
+}
+
+define i64 @zext_i10_to_i64(i10 %a) {
+; CHECK-LABEL: zext_i10_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x0, x0, #0x3ff
+; CHECK-NEXT:    ret
+entry:
+  %c = zext i10 %a to i64
+  ret i64 %c
+}
+
+define <2 x i16> @zext_v2i8_v2i16(<2 x i8> %a) {
+; CHECK-SD-LABEL: zext_v2i8_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i8_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI10_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI10_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i8> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @zext_v2i8_v2i32(<2 x i8> %a) {
+; CHECK-SD-LABEL: zext_v2i8_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i8_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI11_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI11_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i8> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @zext_v2i8_v2i64(<2 x i8> %a) {
+; CHECK-SD-LABEL: zext_v2i8_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i8_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI12_0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI12_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i8> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i32> @zext_v2i16_v2i32(<2 x i16> %a) {
+; CHECK-SD-LABEL: zext_v2i16_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i16_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI13_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI13_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i16> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @zext_v2i16_v2i64(<2 x i16> %a) {
+; CHECK-SD-LABEL: zext_v2i16_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i16_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI14_0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI14_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i16> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @zext_v2i32_v2i64(<2 x i32> %a) {
+; CHECK-LABEL: zext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i16> @zext_v2i10_v2i16(<2 x i10> %a) {
+; CHECK-SD-LABEL: zext_v2i10_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #3, msl #8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i10_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI16_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI16_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i10> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i32> @zext_v2i10_v2i32(<2 x i10> %a) {
+; CHECK-SD-LABEL: zext_v2i10_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #3, msl #8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i10_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI17_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI17_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i10> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i64> @zext_v2i10_v2i64(<2 x i10> %a) {
+; CHECK-SD-LABEL: zext_v2i10_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #3, msl #8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v2i10_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI18_0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI18_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <2 x i10> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <3 x i16> @zext_v3i8_v3i16(<3 x i8> %a) {
+; CHECK-LABEL: zext_v3i8_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <3 x i8> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @zext_v3i8_v3i32(<3 x i8> %a) {
+; CHECK-SD-LABEL: zext_v3i8_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i8_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #255 // =0xff
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mov v0.s[2], w2
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i8> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @zext_v3i8_v3i64(<3 x i8> %a) {
+; CHECK-SD-LABEL: zext_v3i8_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s3, w2
+; CHECK-SD-NEXT:    movi v0.2d, #0x000000000000ff
+; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    mov v2.b[0], v3.b[0]
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i8_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    adrp x8, .LCPI21_0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI21_0]
+; CHECK-GI-NEXT:    and x8, x2, #0xff
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i8> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i32> @zext_v3i16_v3i32(<3 x i16> %a) {
+; CHECK-SD-LABEL: zext_v3i16_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i16_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    uxth w8, w8
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov w8, s2
+; CHECK-GI-NEXT:    uxth w9, w9
+; CHECK-GI-NEXT:    uxth w8, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i16> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @zext_v3i16_v3i64(<3 x i16> %a) {
+; CHECK-SD-LABEL: zext_v3i16_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i16_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    ubfx x8, x8, #0, #16
+; CHECK-GI-NEXT:    ubfx x9, x9, #0, #16
+; CHECK-GI-NEXT:    ubfx x10, x10, #0, #16
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i16> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @zext_v3i32_v3i64(<3 x i32> %a) {
+; CHECK-SD-LABEL: zext_v3i32_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll2 v2.2d, v0.4s, #0
+; CHECK-SD-NEXT:    fmov d0, d3
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i32_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i32> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i16> @zext_v3i10_v3i16(<3 x i10> %a) {
+; CHECK-LABEL: zext_v3i10_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <3 x i10> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i32> @zext_v3i10_v3i32(<3 x i10> %a) {
+; CHECK-SD-LABEL: zext_v3i10_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    movi v1.4s, #3, msl #8
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i10_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #1023 // =0x3ff
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mov v0.s[2], w2
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i10> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i64> @zext_v3i10_v3i64(<3 x i10> %a) {
+; CHECK-SD-LABEL: zext_v3i10_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov w8, #1023 // =0x3ff
+; CHECK-SD-NEXT:    fmov s3, w2
+; CHECK-SD-NEXT:    mov v0.s[1], w1
+; CHECK-SD-NEXT:    dup v2.2d, x8
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    and v2.8b, v3.8b, v2.8b
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v3i10_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    adrp x8, .LCPI27_0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI27_0]
+; CHECK-GI-NEXT:    and x8, x2, #0x3ff
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <3 x i10> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <4 x i16> @zext_v4i8_v4i16(<4 x i8> %a) {
+; CHECK-SD-LABEL: zext_v4i8_v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI28_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI28_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i8> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @zext_v4i8_v4i32(<4 x i8> %a) {
+; CHECK-SD-LABEL: zext_v4i8_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI29_0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI29_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i8> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @zext_v4i8_v4i64(<4 x i8> %a) {
+; CHECK-SD-LABEL: zext_v4i8_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i8_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    adrp x8, .LCPI30_0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI30_0]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i8> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i32> @zext_v4i16_v4i32(<4 x i16> %a) {
+; CHECK-LABEL: zext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <4 x i16> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @zext_v4i16_v4i64(<4 x i16> %a) {
+; CHECK-SD-LABEL: zext_v4i16_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i16_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i16> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @zext_v4i32_v4i64(<4 x i32> %a) {
+; CHECK-SD-LABEL: zext_v4i32_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i32_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i32> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i16> @zext_v4i10_v4i16(<4 x i10> %a) {
+; CHECK-SD-LABEL: zext_v4i10_v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i10_v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI34_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI34_0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i10> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i32> @zext_v4i10_v4i32(<4 x i10> %a) {
+; CHECK-SD-LABEL: zext_v4i10_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i10_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI35_0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI35_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i10> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i64> @zext_v4i10_v4i64(<4 x i10> %a) {
+; CHECK-SD-LABEL: zext_v4i10_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.4h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v4i10_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    adrp x8, .LCPI36_0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI36_0]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <4 x i10> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <8 x i16> @zext_v8i8_v8i16(<8 x i8> %a) {
+; CHECK-LABEL: zext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <8 x i8> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @zext_v8i8_v8i32(<8 x i8> %a) {
+; CHECK-SD-LABEL: zext_v8i8_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i8_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i8> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @zext_v8i8_v8i64(<8 x i8> %a) {
+; CHECK-SD-LABEL: zext_v8i8_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i8_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i8> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i32> @zext_v8i16_v8i32(<8 x i16> %a) {
+; CHECK-SD-LABEL: zext_v8i16_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i16_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i16> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @zext_v8i16_v8i64(<8 x i16> %a) {
+; CHECK-SD-LABEL: zext_v8i16_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i16_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i16> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @zext_v8i32_v8i64(<8 x i32> %a) {
+; CHECK-SD-LABEL: zext_v8i32_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v4.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i32_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i32> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i16> @zext_v8i10_v8i16(<8 x i10> %a) {
+; CHECK-SD-LABEL: zext_v8i10_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i10_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI43_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI43_0]
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i10> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i32> @zext_v8i10_v8i32(<8 x i10> %a) {
+; CHECK-SD-LABEL: zext_v8i10_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i10_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI44_0
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI44_0]
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i10> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i64> @zext_v8i10_v8i64(<8 x i10> %a) {
+; CHECK-SD-LABEL: zext_v8i10_v8i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v8i10_v8i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    adrp x8, .LCPI45_0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI45_0]
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v1.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-GI-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-GI-NEXT:    and v1.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT:    and v2.16b, v5.16b, v3.16b
+; CHECK-GI-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <8 x i10> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <16 x i16> @zext_v16i8_v16i16(<16 x i8> %a) {
+; CHECK-SD-LABEL: zext_v16i8_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i8_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i8> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @zext_v16i8_v16i32(<16 x i8> %a) {
+; CHECK-SD-LABEL: zext_v16i8_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v2.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v3.4s, v2.8h, #0
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i8_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov d4, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i8> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @zext_v16i8_v16i64(<16 x i8> %a) {
+; CHECK-SD-LABEL: zext_v16i8_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v1.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v2.4s, v1.8h, #0
+; CHECK-SD-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v16.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v7.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v4.4s, #0
+; CHECK-SD-NEXT:    ushll2 v5.2d, v16.4s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v16.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i8_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d5, v2.d[1]
+; CHECK-GI-NEXT:    ushll v4.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v6.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v16.4s, v5.4h, #0
+; CHECK-GI-NEXT:    mov d3, v4.d[1]
+; CHECK-GI-NEXT:    mov d7, v6.d[1]
+; CHECK-GI-NEXT:    mov d17, v16.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ushll v4.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v7.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v17.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i8> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i32> @zext_v16i16_v16i32(<16 x i16> %a) {
+; CHECK-SD-LABEL: zext_v16i16_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-SD-NEXT:    mov v1.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i16_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i16> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @zext_v16i16_v16i64(<16 x i16> %a) {
+; CHECK-SD-LABEL: zext_v16i16_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll2 v4.4s, v1.8h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll2 v16.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll2 v3.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v7.2d, v4.4s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll2 v5.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v1.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v16.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i16_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    mov d4, v1.d[1]
+; CHECK-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v5.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov d1, v2.d[1]
+; CHECK-GI-NEXT:    ushll v0.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d6, v5.d[1]
+; CHECK-GI-NEXT:    ushll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v4.4h, #0
+; CHECK-GI-NEXT:    mov d7, v2.d[1]
+; CHECK-GI-NEXT:    mov d16, v3.d[1]
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v4.2d, v5.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v7.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i16> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i64> @zext_v16i32_v16i64(<16 x i32> %a) {
+; CHECK-SD-LABEL: zext_v16i32_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll2 v17.2d, v0.4s, #0
+; CHECK-SD-NEXT:    ushll2 v18.2d, v1.4s, #0
+; CHECK-SD-NEXT:    ushll v16.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll2 v5.2d, v2.4s, #0
+; CHECK-SD-NEXT:    ushll2 v7.2d, v3.4s, #0
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v3.2s, #0
+; CHECK-SD-NEXT:    mov v1.16b, v17.16b
+; CHECK-SD-NEXT:    mov v2.16b, v16.16b
+; CHECK-SD-NEXT:    mov v3.16b, v18.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i32_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d5, v1.d[1]
+; CHECK-GI-NEXT:    mov d6, v2.d[1]
+; CHECK-GI-NEXT:    ushll v16.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov d2, v3.d[1]
+; CHECK-GI-NEXT:    ushll v17.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ushll v18.2d, v5.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v3.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v2.2s, #0
+; CHECK-GI-NEXT:    mov v0.16b, v16.16b
+; CHECK-GI-NEXT:    mov v2.16b, v17.16b
+; CHECK-GI-NEXT:    mov v3.16b, v18.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i32> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i16> @zext_v16i10_v16i16(<16 x i10> %a) {
+; CHECK-LABEL: zext_v16i10_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [sp]
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    ldr w8, [sp, #8]
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v1.h[1], w8
+; CHECK-NEXT:    ldr w8, [sp, #16]
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    mov v1.h[2], w8
+; CHECK-NEXT:    ldr w8, [sp, #24]
+; CHECK-NEXT:    mov v0.h[3], w3
+; CHECK-NEXT:    mov v1.h[3], w8
+; CHECK-NEXT:    ldr w8, [sp, #32]
+; CHECK-NEXT:    mov v0.h[4], w4
+; CHECK-NEXT:    mov v1.h[4], w8
+; CHECK-NEXT:    ldr w8, [sp, #40]
+; CHECK-NEXT:    mov v0.h[5], w5
+; CHECK-NEXT:    mov v1.h[5], w8
+; CHECK-NEXT:    ldr w8, [sp, #48]
+; CHECK-NEXT:    mov v0.h[6], w6
+; CHECK-NEXT:    mov v1.h[6], w8
+; CHECK-NEXT:    ldr w8, [sp, #56]
+; CHECK-NEXT:    mov v0.h[7], w7
+; CHECK-NEXT:    mov v1.h[7], w8
+; CHECK-NEXT:    bic v0.8h, #252, lsl #8
+; CHECK-NEXT:    bic v1.8h, #252, lsl #8
+; CHECK-NEXT:    ret
+entry:
+  %c = zext <16 x i10> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i32> @zext_v16i10_v16i32(<16 x i10> %a) {
+; CHECK-SD-LABEL: zext_v16i10_v16i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w11, [sp, #32]
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr w13, [sp]
+; CHECK-SD-NEXT:    fmov s1, w4
+; CHECK-SD-NEXT:    ldr w10, [sp, #40]
+; CHECK-SD-NEXT:    ldr w15, [sp, #8]
+; CHECK-SD-NEXT:    fmov s3, w11
+; CHECK-SD-NEXT:    fmov s2, w13
+; CHECK-SD-NEXT:    ldr w9, [sp, #48]
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    ldr w14, [sp, #16]
+; CHECK-SD-NEXT:    mov v1.h[1], w5
+; CHECK-SD-NEXT:    ldr w8, [sp, #56]
+; CHECK-SD-NEXT:    mov v2.h[1], w15
+; CHECK-SD-NEXT:    ldr w12, [sp, #24]
+; CHECK-SD-NEXT:    mov v3.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    mov v1.h[2], w6
+; CHECK-SD-NEXT:    mov v2.h[2], w14
+; CHECK-SD-NEXT:    mov v3.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w3
+; CHECK-SD-NEXT:    mov v1.h[3], w7
+; CHECK-SD-NEXT:    mov v2.h[3], w12
+; CHECK-SD-NEXT:    mov v3.h[3], w8
+; CHECK-SD-NEXT:    movi v4.4s, #3, msl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v4.16b
+; CHECK-SD-NEXT:    and v1.16b, v1.16b, v4.16b
+; CHECK-SD-NEXT:    and v2.16b, v2.16b, v4.16b
+; CHECK-SD-NEXT:    and v3.16b, v3.16b, v4.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i10_v16i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s16, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s17, w4
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    adrp x8, .LCPI53_0
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    mov v16.s[1], w1
+; CHECK-GI-NEXT:    ldr s6, [sp, #48]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    mov v17.s[1], w5
+; CHECK-GI-NEXT:    ldr s7, [sp, #56]
+; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI53_0]
+; CHECK-GI-NEXT:    mov v16.s[2], w2
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v17.s[2], w6
+; CHECK-GI-NEXT:    mov v0.s[3], v3.s[0]
+; CHECK-GI-NEXT:    mov v16.s[3], w3
+; CHECK-GI-NEXT:    mov v4.s[3], v7.s[0]
+; CHECK-GI-NEXT:    mov v17.s[3], w7
+; CHECK-GI-NEXT:    and v2.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    and v0.16b, v16.16b, v1.16b
+; CHECK-GI-NEXT:    and v3.16b, v4.16b, v1.16b
+; CHECK-GI-NEXT:    and v1.16b, v17.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i10> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i64> @zext_v16i10_v16i64(<16 x i10> %a) {
+; CHECK-SD-LABEL: zext_v16i10_v16i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1023 // =0x3ff
+; CHECK-SD-NEXT:    ldr s4, [sp]
+; CHECK-SD-NEXT:    ldr s5, [sp, #16]
+; CHECK-SD-NEXT:    add x9, sp, #24
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    fmov s1, w2
+; CHECK-SD-NEXT:    dup v7.2d, x8
+; CHECK-SD-NEXT:    add x8, sp, #8
+; CHECK-SD-NEXT:    fmov s2, w4
+; CHECK-SD-NEXT:    fmov s3, w6
+; CHECK-SD-NEXT:    ld1 { v5.s }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #56
+; CHECK-SD-NEXT:    ld1 { v4.s }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    ldr s6, [sp, #32]
+; CHECK-SD-NEXT:    ldr s16, [sp, #48]
+; CHECK-SD-NEXT:    mov v0.s[1], w1
+; CHECK-SD-NEXT:    mov v1.s[1], w3
+; CHECK-SD-NEXT:    ld1 { v6.s }[1], [x8]
+; CHECK-SD-NEXT:    mov v2.s[1], w5
+; CHECK-SD-NEXT:    ld1 { v16.s }[1], [x9]
+; CHECK-SD-NEXT:    mov v3.s[1], w7
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-SD-NEXT:    ushll v5.2d, v5.2s, #0
+; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-SD-NEXT:    ushll v16.2d, v16.2s, #0
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v7.16b
+; CHECK-SD-NEXT:    and v1.16b, v1.16b, v7.16b
+; CHECK-SD-NEXT:    and v2.16b, v2.16b, v7.16b
+; CHECK-SD-NEXT:    and v3.16b, v3.16b, v7.16b
+; CHECK-SD-NEXT:    and v4.16b, v4.16b, v7.16b
+; CHECK-SD-NEXT:    and v5.16b, v5.16b, v7.16b
+; CHECK-SD-NEXT:    and v6.16b, v6.16b, v7.16b
+; CHECK-SD-NEXT:    and v7.16b, v16.16b, v7.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: zext_v16i10_v16i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [sp]
+; CHECK-GI-NEXT:    fmov s6, w0
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    fmov s16, w2
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    fmov s18, w4
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    fmov s19, w6
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    adrp x8, .LCPI54_0
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    ldr s7, [sp, #48]
+; CHECK-GI-NEXT:    ldr s17, [sp, #56]
+; CHECK-GI-NEXT:    mov v6.s[1], w1
+; CHECK-GI-NEXT:    mov v16.s[1], w3
+; CHECK-GI-NEXT:    mov v18.s[1], w5
+; CHECK-GI-NEXT:    mov v19.s[1], w7
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    mov v7.s[1], v17.s[0]
+; CHECK-GI-NEXT:    ldr q17, [x8, :lo12:.LCPI54_0]
+; CHECK-GI-NEXT:    ushll v1.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT:    ushll v6.2d, v19.2s, #0
+; CHECK-GI-NEXT:    ushll v16.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v18.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v19.2d, v4.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-GI-NEXT:    and v0.16b, v1.16b, v17.16b
+; CHECK-GI-NEXT:    and v1.16b, v3.16b, v17.16b
+; CHECK-GI-NEXT:    and v2.16b, v5.16b, v17.16b
+; CHECK-GI-NEXT:    and v3.16b, v6.16b, v17.16b
+; CHECK-GI-NEXT:    and v4.16b, v16.16b, v17.16b
+; CHECK-GI-NEXT:    and v5.16b, v18.16b, v17.16b
+; CHECK-GI-NEXT:    and v6.16b, v19.16b, v17.16b
+; CHECK-GI-NEXT:    and v7.16b, v7.16b, v17.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = zext <16 x i10> %a to <16 x i64>
+  ret <16 x i64> %c
+}

llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir

bb.1:
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
     ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
     ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
     ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
     ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
     ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY]], 8
     ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY1]], 8
+    ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
+    ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
-    ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C]]
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C]]
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
     ; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; CHECK-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
+    ; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
     ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG4]], [[C]]
     ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG5]], [[C]]
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL2]]
     ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
-    ; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
     ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG6]], [[C]]
     ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG7]], [[C]]
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL3]]
     ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BITCAST]](<2 x s16>), implicit [[BITCAST1]](<2 x s16>), implicit [[BITCAST2]](<2 x s16>), implicit [[BITCAST3]](<2 x s16>)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
bb.1:
     ; CHECK-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
     ; CHECK-NEXT: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
     ; CHECK-NEXT: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
     ; CHECK-NEXT: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
     ; CHECK-NEXT: [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
     ; CHECK-NEXT: [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
     ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY]], 8
     ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY1]], 8
+    ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
+    ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
+    ; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; CHECK-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
+    ; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
-    ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C]]
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C]]
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
-    ; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
     ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG4]], [[C]]
     ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG5]], [[C]]
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL2]]
     ; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
-    ; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
     ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG6]], [[C]]
     ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG7]], [[C]]
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL3]]
     ; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](s32)
     ; CHECK-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY8]], 8
     ; CHECK-NEXT: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY9]], 8
+    ; CHECK-NEXT: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY10]], 8
+    ; CHECK-NEXT: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY11]], 8
+    ; CHECK-NEXT: [[SEXT_INREG12:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY12]], 8
+    ; CHECK-NEXT: [[SEXT_INREG13:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY13]], 8
+    ; CHECK-NEXT: [[SEXT_INREG14:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY14]], 8
+    ; CHECK-NEXT: [[SEXT_INREG15:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY15]], 8
     ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG8]], [[C]]
     ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG9]], [[C]]
     ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL4]]
     ; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY10]], 8
-    ; CHECK-NEXT: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY11]], 8
     ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG10]], [[C]]
     ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG11]], [[C]]
     ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C1]](s32)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[AND10]], [[SHL5]]
     ; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG12:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY12]], 8
-    ; CHECK-NEXT: [[SEXT_INREG13:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY13]], 8
     ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG12]], [[C]]
     ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG13]], [[C]]
     ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C1]](s32)
     ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[AND12]], [[SHL6]]
     ; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](s32)
-    ; CHECK-NEXT: [[SEXT_INREG14:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY14]], 8
-    ; CHECK-NEXT: [[SEXT_INREG15:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY15]], 8
     ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG14]], [[C]]
     ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG15]], [[C]]
     ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C1]](s32)
     ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND14]], [[SHL7]]
     ; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BITCAST]](<2 x s16>), implicit [[BITCAST1]](<2 x s16>), implicit [[BITCAST2]](<2 x s16>), implicit [[BITCAST3]](<2 x s16>), implicit [[BITCAST4]](<2 x s16>), implicit [[BITCAST5]](<2 x s16>), implicit [[BITCAST6]](<2 x s16>), implicit [[BITCAST7]](<2 x s16>)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1