[RISCV] Teach lowerVECTOR_SHUFFLE to recognize some shuffles as vnsrl.

[craig.topper]

Summary

Unary shuffles such as <0,2,4,6,8,10,12,14> or <1,3,5,7,9,11,13,15>
where half the elements are returned, can be lowered using vnsrl.

SelectionDAGBuilder lowers such shuffles as a build_vector of
extract_elements since the mask has less elements than the source.
To fix this, I've enable the extractSubvectorIsCheapHook to allow
DAGCombine to rebuild the shuffle using 2 extract_subvectors preceding
the shufffle.

I've gone very conservative on extractSubvectorIsCheapHook to minimize
test impact and match what we have test coverage for. This can be
improved in the future.

[reames]

LGTM

A couple thoughts for possible generalization.

I think we can extract any Nth element where N is a power of two, and N * sizeof(Element) is less than equal ELEN. So, every 8th element in an i8 vector for instance.

We should be able to handle a sequence with a non-zero/non-one base. This requires an extra slidedown to align the sequence.

As noted, we can handle undefs.

llvm/lib/Target/RISCV/RISCVISelLowering.h

enum NodeType : unsigned {
   VWADDU_VL,
   VWSUB_VL,
   VWSUBU_VL,
   VWADD_W_VL,
   VWADDU_W_VL,
   VWSUB_W_VL,
   VWSUBU_W_VL,
 
+  VNSRL_VL,
+
   // Vector compare producing a mask. Fourth operand is input mask. Fifth
   // operand is VL.
   SETCC_VL,
 
   // Vector select with an additional VL operand. This operation is unmasked.
   VSELECT_VL,
   // Vector select with operand #2 (the value when the condition is false) tied
   // to the destination and an additional VL operand. This operation is
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
       unsigned OldShiftOpcode, unsigned NewShiftOpcode,
       SelectionDAG &DAG) const override;
   bool shouldSinkOperands(Instruction *I,
                           SmallVectorImpl<Use *> &Ops) const override;
   bool shouldScalarizeBinop(SDValue VecOp) const override;
   bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
+  bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
+                               unsigned Index) const override;
 
   bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
 
   bool softPromoteHalfType() const override { return true; }
 
   /// Return the register type for a given MVT, ensuring vectors are treated
   /// as a series of gpr sized integers.
   MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
     return false;
   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
     return false;
   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
     return false;
   return Imm.isZero();
 }
 
+// TODO: This is very conservative.
+bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
+                                                  unsigned Index) const {
+  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
+    return false;
+
+  // Only support extracting a fixed from a fixed vector for now.
+  if (ResVT.isScalableVector() || SrcVT.isScalableVector())
+    return false;
+
+  unsigned ResElts = ResVT.getVectorNumElements();
+  unsigned SrcElts = SrcVT.getVectorNumElements();
+
+  // Convervatively only handle extracting half of a vector.
+  // TODO: Relax this.
+  if ((ResElts * 2) != SrcElts)
+    return false;
+
+  // The smallest type we can slide is i8.
+  // TODO: We can extract index 0 from a mask vector without a slide.
+  if (ResVT.getVectorElementType() == MVT::i1)
+    return false;
+
+  // Slide can support arbitrary index, but we only treat vslidedown.vi as
+  // cheap.
+  if (Index >= 32)
+    return false;
+
+  // TODO: We can do arbitrary slidedowns, but for now only support extracting
+  // the upper half of a vector until we have more test coverage.
+  return Index == 0 || Index == ResElts;
+}
+
 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
   return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
          (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
          (VT == MVT::f64 && Subtarget.hasStdExtD());
 }
 
 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                       CallingConv::ID CC,
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef<int> Mask) {
   // Check that we successfully analyzed the mask, and normalize the results.
   assert(Rotation != 0 && "Failed to locate a viable rotation!");
   assert((LoSrc >= 0 || HiSrc >= 0) &&
          "Failed to find a rotated input vector!");
 
   return Rotation;
 }
 
+// Lower the following shuffles to vnsrl.
+// t34: v8i8 = extract_subvector t11, Constant:i64<0>
+// t33: v8i8 = extract_subvector t11, Constant:i64<8>
+// a) t35: v8i8 = vector_shuffle<0,2,4,6,8,10,12,14> t34, t33
+// b) t35: v8i8 = vector_shuffle<1,3,5,7,9,11,13,15> t34, t33
+static SDValue lowerVECTOR_SHUFFLEAsVNSRL(const SDLoc &DL, MVT VT,
+                                          MVT ContainerVT, SDValue V1,
+                                          SDValue V2, SDValue TrueMask,
+                                          SDValue VL, ArrayRef<int> Mask,
+                                          const RISCVSubtarget &Subtarget,
+                                          SelectionDAG &DAG) {
+  // Need to be able to widen the vector.
+  if (VT.getScalarSizeInBits() >= Subtarget.getELEN())
+    return SDValue();
+
+  // Both input must be extracts.
+  if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+      V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
+    return SDValue();
+
+  // Extracting from the same source.
+  SDValue Src = V1.getOperand(0);
+  if (Src != V2.getOperand(0))
+    return SDValue();
+
+  // Src needs to have twice the number of elements.
+  if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
+    return SDValue();
+
+  // The extracts must extract the two halves of the source.
+  if (V1.getConstantOperandVal(1) != 0 ||
+      V2.getConstantOperandVal(1) != Mask.size())
+    return SDValue();
+
+  // First index must be the first even or odd element from V1.
+  if (Mask[0] != 0 && Mask[0] != 1)
+    return SDValue();
+
+  // The others must increase by 2 each time.
+  // TODO: Support undef elements?
+  for (unsigned i = 1; i != Mask.size(); ++i)
+    if (Mask[i] != Mask[i - 1] + 2)
+      return SDValue();
+
+  // Convert the source using a container type with twice the elements. Since
+  // source VT is legal and twice this VT, we know VT isn't LMUL=8 so it is
+  // safe to double.
+  MVT DoubleContainerVT =
+      MVT::getVectorVT(ContainerVT.getVectorElementType(),
+                       ContainerVT.getVectorElementCount() * 2);
+  Src = convertToScalableVector(DoubleContainerVT, Src, DAG, Subtarget);
+
+  // Convert the vector to a wider integer type with the original element
+  // count. This also converts FP to int.
+  unsigned EltBits = ContainerVT.getScalarSizeInBits();
+  MVT WideIntEltVT = MVT::getIntegerVT(EltBits * 2);
+  MVT WideIntContainerVT =
+      MVT::getVectorVT(WideIntEltVT, ContainerVT.getVectorElementCount());
+  Src = DAG.getBitcast(WideIntContainerVT, Src);
+
+  // Convert to the integer version of the container type.
+  MVT IntEltVT = MVT::getIntegerVT(EltBits);
+  MVT IntContainerVT =
+      MVT::getVectorVT(IntEltVT, ContainerVT.getVectorElementCount());
+
+  // If we want even elements, then the shift amount is 0. Otherwise, shift by
+  // the original element size.
+  unsigned Shift = Mask[0] == 0 ? 0 : EltBits;
+  SDValue SplatShift = DAG.getNode(
+      RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT),
+      DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL);
+  SDValue Res =
+      DAG.getNode(RISCVISD::VNSRL_VL, DL, IntContainerVT, Src, SplatShift,
+                  DAG.getUNDEF(IntContainerVT), TrueMask, VL);
+  // Cast back to FP if needed.
+  Res = DAG.getBitcast(ContainerVT, Res);
+
+  return convertFromScalableVector(VT, Res, DAG, Subtarget);
+}
+
 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   SDLoc DL(Op);
   MVT XLenVT = Subtarget.getXLenVT();
   MVT VT = Op.getSimpleValueType();
   unsigned NumElts = VT.getVectorNumElements();
if (Rotation > 0) {
     }
     if (LoV)
       Res = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Res, LoV,
                         DAG.getConstant(InvRotate, DL, XLenVT), TrueMask, VL);
 
     return convertFromScalableVector(VT, Res, DAG, Subtarget);
   }
 
+  if (SDValue V = lowerVECTOR_SHUFFLEAsVNSRL(
+          DL, VT, ContainerVT, V1, V2, TrueMask, VL, Mask, Subtarget, DAG))
+    return V;
+
   // Detect an interleave shuffle and lower to
   // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
   bool SwapSources;
   if (isInterleaveShuffle(Mask, VT, SwapSources, Subtarget)) {
     // Swap sources if needed.
     if (SwapSources)
       std::swap(V1, V2);
 
#define NODE_NAME_CASE(NODE)                                                   \
   NODE_NAME_CASE(VWADD_VL)
   NODE_NAME_CASE(VWADDU_VL)
   NODE_NAME_CASE(VWSUB_VL)
   NODE_NAME_CASE(VWSUBU_VL)
   NODE_NAME_CASE(VWADD_W_VL)
   NODE_NAME_CASE(VWADDU_W_VL)
   NODE_NAME_CASE(VWSUB_W_VL)
   NODE_NAME_CASE(VWSUBU_W_VL)
+  NODE_NAME_CASE(VNSRL_VL)
   NODE_NAME_CASE(SETCC_VL)
   NODE_NAME_CASE(VSELECT_VL)
   NODE_NAME_CASE(VP_MERGE_VL)
   NODE_NAME_CASE(VMAND_VL)
   NODE_NAME_CASE(VMOR_VL)
   NODE_NAME_CASE(VMXOR_VL)
   NODE_NAME_CASE(VMCLR_VL)
   NODE_NAME_CASE(VMSET_VL)

llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td

 def riscv_vwmul_vl  : SDNode<"RISCVISD::VWMUL_VL",  SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
 def riscv_vwmulu_vl : SDNode<"RISCVISD::VWMULU_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
 def riscv_vwmulsu_vl : SDNode<"RISCVISD::VWMULSU_VL", SDT_RISCVVWBinOp_VL>;
 def riscv_vwadd_vl :  SDNode<"RISCVISD::VWADD_VL",  SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
 def riscv_vwaddu_vl : SDNode<"RISCVISD::VWADDU_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
 def riscv_vwsub_vl :  SDNode<"RISCVISD::VWSUB_VL",  SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
 def riscv_vwsubu_vl : SDNode<"RISCVISD::VWSUBU_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
 
+def SDT_RISCVVNBinOp_VL : SDTypeProfile<1, 5, [SDTCisVec<0>,
+                                               SDTCisSameNumEltsAs<0, 1>,
+                                               SDTCisOpSmallerThanOp<0, 1>,
+                                               SDTCisSameAs<0, 2>,
+                                               SDTCisSameAs<0, 3>,
+                                               SDTCisSameNumEltsAs<0, 4>,
+                                               SDTCVecEltisVT<4, i1>,
+                                               SDTCisVT<5, XLenVT>]>;
+def riscv_vnsrl_vl : SDNode<"RISCVISD::VNSRL_VL", SDT_RISCVVNBinOp_VL>;
+
 def SDT_RISCVVWBinOpW_VL : SDTypeProfile<1, 5, [SDTCisVec<0>,
                                                 SDTCisSameAs<0, 1>,
                                                 SDTCisSameNumEltsAs<1, 2>,
                                                 SDTCisOpSmallerThanOp<2, 1>,
                                                 SDTCisSameAs<0, 3>,
                                                 SDTCisSameNumEltsAs<1, 4>,
                                                 SDTCVecEltisVT<4, i1>,
                                                 SDTCisVT<5, XLenVT>]>;
defm : VPatBinaryVL_V<vop_w, instruction_name, "WV",
                            vti.RegClass>;
     defm : VPatBinaryVL_XI<vop_w, instruction_name, "WX",
                            wti.Vector, wti.Vector, vti.Vector, vti.Mask,
                            vti.Log2SEW, vti.LMul, wti.RegClass, wti.RegClass,
                            SplatPat, GPR>;
   }
 }
 
+multiclass VPatBinaryNVL_WV_WX_WI<SDNode vop, string instruction_name> {
+  foreach VtiToWti = AllWidenableIntVectors in {
+    defvar vti = VtiToWti.Vti;
+    defvar wti = VtiToWti.Wti;
+    defm : VPatBinaryVL_V<vop, instruction_name, "WV",
+                           vti.Vector, wti.Vector, vti.Vector, vti.Mask,
+                           vti.Log2SEW, vti.LMul, vti.RegClass, wti.RegClass, vti.RegClass>;
+    defm : VPatBinaryVL_XI<vop, instruction_name, "WX",
+                           vti.Vector, wti.Vector, vti.Vector, vti.Mask,
+                           vti.Log2SEW, vti.LMul, vti.RegClass, wti.RegClass, SplatPat, GPR>;
+    defm : VPatBinaryVL_XI<vop, instruction_name, "WI",
+                           vti.Vector, wti.Vector, vti.Vector, vti.Mask,
+                           vti.Log2SEW, vti.LMul, vti.RegClass, wti.RegClass,
+                           !cast<ComplexPattern>(SplatPat#_#uimm5),
+                           uimm5>;
+  }
+}
+
 multiclass VPatBinaryVL_VF<SDNode vop,
                            string instruction_name,
                            ValueType result_type,
                            ValueType vop_type,
                            ValueType mask_type,
                            int sew,
                            LMULInfo vlmul,
                            VReg result_reg_class,

 
 defm : VPatNarrowShiftSplat_WX_WI<riscv_sra_vl, "PseudoVNSRA">;
 defm : VPatNarrowShiftSplat_WX_WI<riscv_srl_vl, "PseudoVNSRL">;
 defm : VPatNarrowShiftSplatExt_WX<riscv_sra_vl, riscv_sext_vl_oneuse, "PseudoVNSRA">;
 defm : VPatNarrowShiftSplatExt_WX<riscv_sra_vl, riscv_zext_vl_oneuse, "PseudoVNSRA">;
 defm : VPatNarrowShiftSplatExt_WX<riscv_srl_vl, riscv_sext_vl_oneuse, "PseudoVNSRL">;
 defm : VPatNarrowShiftSplatExt_WX<riscv_srl_vl, riscv_zext_vl_oneuse, "PseudoVNSRL">;
 
+defm : VPatBinaryNVL_WV_WX_WI<riscv_vnsrl_vl, "PseudoVNSRL">;
+
 foreach vtiTowti = AllWidenableIntVectors in {
   defvar vti = vtiTowti.Vti;
   defvar wti = vtiTowti.Wti;
   def : Pat<(vti.Vector (riscv_trunc_vector_vl (wti.Vector wti.RegClass:$rs1),
                                                (vti.Mask V0),
                                                VLOpFrag)),
             (!cast<Instruction>("PseudoVNSRL_WI_"#vti.LMul.MX#"_MASK")
                 (vti.Vector (IMPLICIT_DEF)), wti.RegClass:$rs1, 0,

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll

 ; LMULMAX1-NEXT:    vmv.s.x v8, a0
 ; LMULMAX1-NEXT:    vrgather.vi v9, v11, 3, v0.t
 ; LMULMAX1-NEXT:    vmv.v.v v0, v8
 ; LMULMAX1-NEXT:    vmerge.vvm v8, v9, v12, v0
 ; LMULMAX1-NEXT:    ret
 ;
 ; LMULMAX2-LABEL: hang_when_merging_stores_after_legalization:
 ; LMULMAX2:       # %bb.0:
-; LMULMAX2-NEXT:    addi sp, sp, -16
-; LMULMAX2-NEXT:    .cfi_def_cfa_offset 16
-; LMULMAX2-NEXT:    addi a0, sp, 8
-; LMULMAX2-NEXT:    vsetivli zero, 1, e32, m2, ta, mu
-; LMULMAX2-NEXT:    vse32.v v10, (a0)
-; LMULMAX2-NEXT:    mv a0, sp
-; LMULMAX2-NEXT:    vse32.v v8, (a0)
-; LMULMAX2-NEXT:    vslidedown.vi v10, v10, 7
-; LMULMAX2-NEXT:    addi a1, sp, 12
-; LMULMAX2-NEXT:    vse32.v v10, (a1)
-; LMULMAX2-NEXT:    vslidedown.vi v8, v8, 7
-; LMULMAX2-NEXT:    addi a1, sp, 4
-; LMULMAX2-NEXT:    vse32.v v8, (a1)
-; LMULMAX2-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; LMULMAX2-NEXT:    vle32.v v8, (a0)
-; LMULMAX2-NEXT:    addi sp, sp, 16
+; LMULMAX2-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
+; LMULMAX2-NEXT:    vid.v v12
+; LMULMAX2-NEXT:    li a0, 7
+; LMULMAX2-NEXT:    vmul.vx v14, v12, a0
+; LMULMAX2-NEXT:    vrgather.vv v12, v8, v14
+; LMULMAX2-NEXT:    li a0, 12
+; LMULMAX2-NEXT:    vmv.s.x v0, a0
+; LMULMAX2-NEXT:    vadd.vi v8, v14, -14
+; LMULMAX2-NEXT:    vrgather.vv v12, v10, v8, v0.t
+; LMULMAX2-NEXT:    vmv1r.v v8, v12
 ; LMULMAX2-NEXT:    ret
   %z = shufflevector <8 x float> %x, <8 x float> %y, <4 x i32> <i32 0, i32 7, i32 8, i32 15>
   ret <4 x float> %z
 }
 
 define void @buildvec_dominant0_v2f32(<2 x float>* %x) {
 ; CHECK-LABEL: buildvec_dominant0_v2f32:
 ; CHECK:       # %bb.0:

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zfh,+experimental-zvfh,+zvl256b \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,V
+; RUN: llc < %s -mtriple=riscv64 -mattr=+f,+zve32f,+zfh,+experimental-zvfh,+zvl256b \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,ZVE32F
+
+define void @vnsrl_0_i8(ptr %in, ptr %out) {
+; CHECK-LABEL: vnsrl_0_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e8, mf2, ta, mu
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf4, ta, mu
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vse8.v v8, (a1)
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i8>, ptr %in, align 1
+  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  store <8 x i8> %shuffle.i5, ptr %out, align 1
+  ret void
+}
+
+define void @vnsrl_8_i8(ptr %in, ptr %out) {
+; CHECK-LABEL: vnsrl_8_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e8, mf2, ta, mu
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf4, ta, mu
+; CHECK-NEXT:    vnsrl.wi v8, v8, 8
+; CHECK-NEXT:    vse8.v v8, (a1)
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i8>, ptr %in, align 1
+  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  store <8 x i8> %shuffle.i5, ptr %out, align 1
+  ret void
+}
+
+define void @vnsrl_0_i16(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_0_i16:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; V-NEXT:    vle16.v v8, (a0)
+; V-NEXT:    vsetivli zero, 4, e16, mf4, ta, mu
+; V-NEXT:    vnsrl.wi v8, v8, 0
+; V-NEXT:    vse16.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_i16:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vle16.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 4, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vnsrl.wi v8, v8, 0
+; ZVE32F-NEXT:    vse16.v v8, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <8 x i16>, ptr %in, align 2
+  %shuffle.i5 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  store <4 x i16> %shuffle.i5, ptr %out, align 2
+  ret void
+}
+
+define void @vnsrl_16_i16(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_16_i16:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; V-NEXT:    vle16.v v8, (a0)
+; V-NEXT:    vsetivli zero, 4, e16, mf4, ta, mu
+; V-NEXT:    vnsrl.wi v8, v8, 16
+; V-NEXT:    vse16.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_16_i16:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vle16.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 4, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vnsrl.wi v8, v8, 16
+; ZVE32F-NEXT:    vse16.v v8, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <8 x i16>, ptr %in, align 2
+  %shuffle.i5 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  store <4 x i16> %shuffle.i5, ptr %out, align 2
+  ret void
+}
+
+define void @vnsrl_0_half(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_0_half:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; V-NEXT:    vle16.v v8, (a0)
+; V-NEXT:    vsetivli zero, 4, e16, mf4, ta, mu
+; V-NEXT:    vnsrl.wi v8, v8, 0
+; V-NEXT:    vse16.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_half:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vle16.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 4, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vnsrl.wi v8, v8, 0
+; ZVE32F-NEXT:    vse16.v v8, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <8 x half>, ptr %in, align 2
+  %shuffle.i5 = shufflevector <8 x half> %0, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  store <4 x half> %shuffle.i5, ptr %out, align 2
+  ret void
+}
+
+define void @vnsrl_16_half(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_16_half:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; V-NEXT:    vle16.v v8, (a0)
+; V-NEXT:    vsetivli zero, 4, e16, mf4, ta, mu
+; V-NEXT:    vnsrl.wi v8, v8, 16
+; V-NEXT:    vse16.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_16_half:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 8, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vle16.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 4, e16, mf2, ta, mu
+; ZVE32F-NEXT:    vnsrl.wi v8, v8, 16
+; ZVE32F-NEXT:    vse16.v v8, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <8 x half>, ptr %in, align 2
+  %shuffle.i5 = shufflevector <8 x half> %0, <8 x half> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  store <4 x half> %shuffle.i5, ptr %out, align 2
+  ret void
+}
+
+define void @vnsrl_0_i32(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_0_i32:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e32, mf2, ta, mu
+; V-NEXT:    vle32.v v8, (a0)
+; V-NEXT:    vsetivli zero, 2, e32, mf2, ta, mu
+; V-NEXT:    vnsrl.wi v8, v8, 0
+; V-NEXT:    vse32.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_i32:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; ZVE32F-NEXT:    vle32.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
+; ZVE32F-NEXT:    li a0, 2
+; ZVE32F-NEXT:    vmv.s.x v0, a0
+; ZVE32F-NEXT:    vrgather.vi v10, v8, 0
+; ZVE32F-NEXT:    vrgather.vi v10, v9, 0, v0.t
+; ZVE32F-NEXT:    vse32.v v10, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x i32>, ptr %in, align 4
+  %shuffle.i5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+  store <2 x i32> %shuffle.i5, ptr %out, align 4
+  ret void
+}
+
+define void @vnsrl_32_i32(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_32_i32:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e32, mf2, ta, mu
+; V-NEXT:    vle32.v v8, (a0)
+; V-NEXT:    li a0, 32
+; V-NEXT:    vsetivli zero, 2, e32, mf2, ta, mu
+; V-NEXT:    vnsrl.wx v8, v8, a0
+; V-NEXT:    vse32.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_32_i32:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; ZVE32F-NEXT:    vle32.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
+; ZVE32F-NEXT:    li a0, 2
+; ZVE32F-NEXT:    vmv.s.x v0, a0
+; ZVE32F-NEXT:    vrgather.vi v10, v8, 1
+; ZVE32F-NEXT:    vrgather.vi v10, v9, 1, v0.t
+; ZVE32F-NEXT:    vse32.v v10, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x i32>, ptr %in, align 4
+  %shuffle.i5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+  store <2 x i32> %shuffle.i5, ptr %out, align 4
+  ret void
+}
+
+define void @vnsrl_0_float(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_0_float:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e32, mf2, ta, mu
+; V-NEXT:    vle32.v v8, (a0)
+; V-NEXT:    vsetivli zero, 2, e32, mf2, ta, mu
+; V-NEXT:    vnsrl.wi v8, v8, 0
+; V-NEXT:    vse32.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_float:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; ZVE32F-NEXT:    vle32.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
+; ZVE32F-NEXT:    li a0, 2
+; ZVE32F-NEXT:    vmv.s.x v0, a0
+; ZVE32F-NEXT:    vrgather.vi v10, v8, 0
+; ZVE32F-NEXT:    vrgather.vi v10, v9, 0, v0.t
+; ZVE32F-NEXT:    vse32.v v10, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x float>, ptr %in, align 4
+  %shuffle.i5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+  store <2 x float> %shuffle.i5, ptr %out, align 4
+  ret void
+}
+
+define void @vnsrl_32_float(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_32_float:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e32, mf2, ta, mu
+; V-NEXT:    vle32.v v8, (a0)
+; V-NEXT:    li a0, 32
+; V-NEXT:    vsetivli zero, 2, e32, mf2, ta, mu
+; V-NEXT:    vnsrl.wx v8, v8, a0
+; V-NEXT:    vse32.v v8, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_32_float:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; ZVE32F-NEXT:    vle32.v v8, (a0)
+; ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
+; ZVE32F-NEXT:    li a0, 2
+; ZVE32F-NEXT:    vmv.s.x v0, a0
+; ZVE32F-NEXT:    vrgather.vi v10, v8, 1
+; ZVE32F-NEXT:    vrgather.vi v10, v9, 1, v0.t
+; ZVE32F-NEXT:    vse32.v v10, (a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x float>, ptr %in, align 4
+  %shuffle.i5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+  store <2 x float> %shuffle.i5, ptr %out, align 4
+  ret void
+}
+
+define void @vnsrl_0_i64(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_0_i64:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vle64.v v8, (a0)
+; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT:    vslidedown.vi v9, v8, 2
+; V-NEXT:    li a0, 2
+; V-NEXT:    vmv.s.x v0, a0
+; V-NEXT:    vrgather.vi v10, v8, 0
+; V-NEXT:    vrgather.vi v10, v9, 0, v0.t
+; V-NEXT:    vse64.v v10, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_i64:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a2, 16(a0)
+; ZVE32F-NEXT:    ld a0, 0(a0)
+; ZVE32F-NEXT:    sd a2, 8(a1)
+; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x i64>, ptr %in, align 8
+  %shuffle.i5 = shufflevector <4 x i64> %0, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+  store <2 x i64> %shuffle.i5, ptr %out, align 8
+  ret void
+}
+
+define void @vnsrl_64_i64(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_64_i64:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vle64.v v8, (a0)
+; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT:    vslidedown.vi v9, v8, 2
+; V-NEXT:    li a0, 2
+; V-NEXT:    vmv.s.x v0, a0
+; V-NEXT:    vrgather.vi v10, v8, 1
+; V-NEXT:    vrgather.vi v10, v9, 1, v0.t
+; V-NEXT:    vse64.v v10, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_64_i64:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a2, 24(a0)
+; ZVE32F-NEXT:    ld a0, 8(a0)
+; ZVE32F-NEXT:    sd a2, 8(a1)
+; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x i64>, ptr %in, align 8
+  %shuffle.i5 = shufflevector <4 x i64> %0, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+  store <2 x i64> %shuffle.i5, ptr %out, align 8
+  ret void
+}
+
+define void @vnsrl_0_double(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_0_double:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vle64.v v8, (a0)
+; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT:    vslidedown.vi v9, v8, 2
+; V-NEXT:    li a0, 2
+; V-NEXT:    vmv.s.x v0, a0
+; V-NEXT:    vrgather.vi v10, v8, 0
+; V-NEXT:    vrgather.vi v10, v9, 0, v0.t
+; V-NEXT:    vse64.v v10, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_0_double:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a2, 16(a0)
+; ZVE32F-NEXT:    ld a0, 0(a0)
+; ZVE32F-NEXT:    sd a2, 8(a1)
+; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x double>, ptr %in, align 8
+  %shuffle.i5 = shufflevector <4 x double> %0, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  store <2 x double> %shuffle.i5, ptr %out, align 8
+  ret void
+}
+
+define void @vnsrl_64_double(ptr %in, ptr %out) {
+; V-LABEL: vnsrl_64_double:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vle64.v v8, (a0)
+; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT:    vslidedown.vi v9, v8, 2
+; V-NEXT:    li a0, 2
+; V-NEXT:    vmv.s.x v0, a0
+; V-NEXT:    vrgather.vi v10, v8, 1
+; V-NEXT:    vrgather.vi v10, v9, 1, v0.t
+; V-NEXT:    vse64.v v10, (a1)
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: vnsrl_64_double:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a2, 24(a0)
+; ZVE32F-NEXT:    ld a0, 8(a0)
+; ZVE32F-NEXT:    sd a2, 8(a1)
+; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ret
+entry:
+  %0 = load <4 x double>, ptr %in, align 8
+  %shuffle.i5 = shufflevector <4 x double> %0, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  store <2 x double> %shuffle.i5, ptr %out, align 8
+  ret void
+}