[RISCV] Support fixed-length vector sign/zero extension

[frasercrmck]

Summary

This patch adds support for the custom lowering sign- and zero-extension
of fixed-length vector types. It does so through custom nodes. Since the
source and destination types are (necessarily) of different sizes, it is
possible that the source type is legal whilst the larger destination
type isn't. In this case the legalization makes heavy use of
EXTRACT_SUBVECTOR.

[craig.topper]

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-exttrunc.ll

15

This is using LMUL 4 for a result that fits in 128 bits. Should we be getting the container from the destination and deriving the source lmul from it instead?

[frasercrmck]

• rebase
• address review feedback: grab container from extended type

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-exttrunc.ll

15

That was what I tried at first, but I was seeing incorrect codegen when it came to split legalized vectors and subvector extraction. It seems to have resolved itself now though so I was obviously wrong about where the bug was. I've fixed this up now, cheers.

[craig.topper]

LGTM

[frasercrmck]

• rebase
• regenerate test cases (vsetivli)

llvm/lib/Target/RISCV/RISCVISelLowering.h

enum NodeType : unsigned {
 
   // Set mask vector to all zeros or ones.
   VMCLR_VL,
   VMSET_VL,
 
   // Matches the semantics of vrgather.vx with an extra operand for VL.
   VRGATHER_VX_VL,
 
+  // Vector sign/zero extend with additional mask & VL operands.
+  VSEXT_VL,
+  VZEXT_VL,
+
   // Memory opcodes start here.
   VLE_VL = ISD::FIRST_TARGET_MEMORY_OPCODE,
   VSE_VL,
 
   // WARNING: Do not add anything in the end unless you want the node to
   // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all
   // opcodes will be thought as target memory ops!
 };
private:
   SDValue lowerFixedLengthVectorSetccToRVV(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFixedLengthVectorLogicOpToRVV(SDValue Op, SelectionDAG &DAG,
                                              unsigned MaskOpc,
                                              unsigned VecOpc) const;
   SDValue lowerFixedLengthVectorSelectToRVV(SDValue Op,
                                             SelectionDAG &DAG) const;
   SDValue lowerToScalableOp(SDValue Op, SelectionDAG &DAG, unsigned NewOpc,
                             bool HasMask = true) const;
+  SDValue lowerFixedLengthVectorExtendToRVV(SDValue Op, SelectionDAG &DAG,
+                                            unsigned ExtendOpc) const;
 
   bool isEligibleForTailCallOptimization(
       CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
       const SmallVector<CCValAssign, 16> &ArgLocs) const;
 
   /// Generate error diagnostics if any register used by CC has been marked
   /// reserved.
   void validateCCReservedRegs(

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

do {
       EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
       Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR, DL, ResultVT, Result);
     } while (SrcEltVT != DstEltVT);
 
     return Result;
   }
   case ISD::ANY_EXTEND:
   case ISD::ZERO_EXTEND:
+    if (Op.getOperand(0).getValueType().isVector() &&
+        Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
+    return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
   case ISD::SIGN_EXTEND:
+    if (Op.getOperand(0).getValueType().isVector() &&
+        Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
+    return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
   case ISD::SPLAT_VECTOR:
     return lowerSPLATVECTOR(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:
     return lowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT:
     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::VSCALE: {
     MVT VT = Op.getSimpleValueType();

 //   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
 // Note that any-extension is lowered identically to zero-extension.
 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
                                                 int64_t ExtTrueVal) const {
   SDLoc DL(Op);
   MVT VecVT = Op.getSimpleValueType();
   SDValue Src = Op.getOperand(0);
   // Only custom-lower extensions from mask types
-  if (!Src.getValueType().isVector() ||
-      Src.getValueType().getVectorElementType() != MVT::i1)
-    return Op;
+  assert(Src.getValueType().isVector() &&
+         Src.getValueType().getVectorElementType() == MVT::i1);
 
   MVT XLenVT = Subtarget.getXLenVT();
   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
   SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
 
   if (VecVT.isScalableVector()) {
     // Be careful not to introduce illegal scalar types at this stage, and be
     // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
   SplatTrueVal =
       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
   SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
                                SplatTrueVal, SplatZero, VL);
 
   return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
 }
 
+SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
+    SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
+  MVT ExtVT = Op.getSimpleValueType();
+  // Only custom-lower extensions from fixed-length vector types.
+  if (!ExtVT.isFixedLengthVector())
+    return Op;
+  MVT VT = Op.getOperand(0).getSimpleValueType();
+  MVT ContainerVT =
+      RISCVTargetLowering::getContainerForFixedLengthVector(DAG, VT, Subtarget);
+  // Get the extended container type manually to ensure the same number of
+  // vector elements between source and dest.
+  MVT ContainerExtVT = MVT::getVectorVT(ExtVT.getVectorElementType(),
+                                        ContainerVT.getVectorElementCount());
+
+  SDValue Op1 =
+      convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
+
+  SDLoc DL(Op);
+  SDValue Mask, VL;
+  std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
+
+  SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
+
+  return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
+}
+
 // Custom-lower truncations from vectors to mask vectors by using a mask and a
 // setcc operation:
 //   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
                                                   SelectionDAG &DAG) const {
   SDLoc DL(Op);
   EVT MaskVT = Op.getValueType();
   // Only expect to custom-lower truncations to mask types
#define NODE_NAME_CASE(NODE)                                                   \
   NODE_NAME_CASE(SETCC_VL)
   NODE_NAME_CASE(VSELECT_VL)
   NODE_NAME_CASE(VMAND_VL)
   NODE_NAME_CASE(VMOR_VL)
   NODE_NAME_CASE(VMXOR_VL)
   NODE_NAME_CASE(VMCLR_VL)
   NODE_NAME_CASE(VMSET_VL)
   NODE_NAME_CASE(VRGATHER_VX_VL)
+  NODE_NAME_CASE(VSEXT_VL)
+  NODE_NAME_CASE(VZEXT_VL)
   NODE_NAME_CASE(VLE_VL)
   NODE_NAME_CASE(VSE_VL)
   }
   // clang-format on
   return nullptr;
 #undef NODE_NAME_CASE
 }
 

llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td

 def riscv_vmclr_vl : SDNode<"RISCVISD::VMCLR_VL", SDT_RISCVVMSETCLR_VL>;
 def riscv_vmset_vl : SDNode<"RISCVISD::VMSET_VL", SDT_RISCVVMSETCLR_VL>;
 
 def true_mask : PatLeaf<(riscv_vmset_vl (XLenVT srcvalue))>;
 
 def riscv_vmnot_vl : PatFrag<(ops node:$rs, node:$vl),
                              (riscv_vmxor_vl node:$rs, true_mask, node:$vl)>;
 
+def SDT_RISCVVEXTEND_VL : SDTypeProfile<1, 3, [SDTCisVec<0>,
+                                               SDTCisSameNumEltsAs<0, 1>,
+                                               SDTCisSameNumEltsAs<1, 2>,
+                                               SDTCVecEltisVT<2, i1>,
+                                               SDTCisVT<3, XLenVT>]>;
+def riscv_sext_vl : SDNode<"RISCVISD::VSEXT_VL", SDT_RISCVVEXTEND_VL>;
+def riscv_zext_vl : SDNode<"RISCVISD::VZEXT_VL", SDT_RISCVVEXTEND_VL>;
+
 // Ignore the vl operand.
 def SplatFPOp : PatFrag<(ops node:$op),
                         (riscv_vfmv_v_f_vl node:$op, srcvalue)>;
 
 def sew8simm5  : ComplexPattern<XLenVT, 1, "selectRVVSimm5<8>",  []>;
 def sew16simm5 : ComplexPattern<XLenVT, 1, "selectRVVSimm5<16>", []>;
 def sew32simm5 : ComplexPattern<XLenVT, 1, "selectRVVSimm5<32>", []>;
 def sew64simm5 : ComplexPattern<XLenVT, 1, "selectRVVSimm5<64>", []>;
def : Pat<(fvti.Mask (riscv_setcc_vl (SplatFPOp fvti.ScalarRegClass:$rs2),
                                          (fvti.Mask true_mask),
                                          (XLenVT (VLOp GPR:$vl)))),
               (!cast<Instruction>(swapped_op_inst_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX)
                   fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2,
                   GPR:$vl, fvti.SEW)>;
   }
 }
 
+multiclass VPatExtendSDNode_V_VL<SDNode vop, string inst_name, string suffix,
+                                 list <VTypeInfoToFraction> fraction_list> {
+  foreach vtiTofti = fraction_list in {
+    defvar vti = vtiTofti.Vti;
+    defvar fti = vtiTofti.Fti;
+    def : Pat<(vti.Vector (vop (fti.Vector fti.RegClass:$rs2),
+                               true_mask, (XLenVT (VLOp GPR:$vl)))),
+              (!cast<Instruction>(inst_name#"_"#suffix#"_"#vti.LMul.MX)
+                  fti.RegClass:$rs2, GPR:$vl, vti.SEW)>;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Patterns.
 //===----------------------------------------------------------------------===//
 
 let Predicates = [HasStdExtV] in {
 
 // 7.4. Vector Unit-Stride Instructions
 foreach vti = AllVectors in {
def : Pat<(riscv_sub_vl (vti.Vector (SplatPat GPR:$rs2)),
                  vti.RegClass:$rs1, GPR:$rs2, GPR:$vl, vti.SEW)>;
   def : Pat<(riscv_sub_vl (vti.Vector (SplatPat_simm5 simm5:$rs2)),
                           (vti.Vector vti.RegClass:$rs1), (vti.Mask true_mask),
                           (XLenVT (VLOp GPR:$vl))),
             (!cast<Instruction>("PseudoVRSUB_VI_"# vti.LMul.MX)
                  vti.RegClass:$rs1, simm5:$rs2, GPR:$vl, vti.SEW)>;
 }
 
+// 12.3. Vector Integer Extension
+defm "" : VPatExtendSDNode_V_VL<riscv_zext_vl, "PseudoVZEXT", "VF2",
+                                AllFractionableVF2IntVectors>;
+defm "" : VPatExtendSDNode_V_VL<riscv_sext_vl, "PseudoVSEXT", "VF2",
+                                AllFractionableVF2IntVectors>;
+defm "" : VPatExtendSDNode_V_VL<riscv_zext_vl, "PseudoVZEXT", "VF4",
+                                AllFractionableVF4IntVectors>;
+defm "" : VPatExtendSDNode_V_VL<riscv_sext_vl, "PseudoVSEXT", "VF4",
+                                AllFractionableVF4IntVectors>;
+defm "" : VPatExtendSDNode_V_VL<riscv_zext_vl, "PseudoVZEXT", "VF8",
+                                AllFractionableVF8IntVectors>;
+defm "" : VPatExtendSDNode_V_VL<riscv_sext_vl, "PseudoVSEXT", "VF8",
+                                AllFractionableVF8IntVectors>;
+
 // 12.5. Vector Bitwise Logical Instructions
 defm "" : VPatBinaryVL_VV_VX_VI<riscv_and_vl, "PseudoVAND">;
 defm "" : VPatBinaryVL_VV_VX_VI<riscv_or_vl,  "PseudoVOR">;
 defm "" : VPatBinaryVL_VV_VX_VI<riscv_xor_vl, "PseudoVXOR">;
 
 // 12.6. Vector Single-Width Bit Shift Instructions
 defm "" : VPatBinaryVL_VV_VX_VI<riscv_shl_vl, "PseudoVSLL", uimm5>;
 defm "" : VPatBinaryVL_VV_VX_VI<riscv_srl_vl, "PseudoVSRL", uimm5>;

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-exttrunc.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
+
+define void @sext_v4i8_v4i32(<4 x i8>* %x, <4 x i32>* %z) {
+; CHECK-LABEL: sext_v4i8_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a2, zero, 4
+; CHECK-NEXT:    vsetvli a3, a2, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a0)
+; CHECK-NEXT:    vsetvli a0, a2, e32,m4,ta,mu
+; CHECK-NEXT:    vsext.vf4 v28, v25
+; CHECK-NEXT:    vsetvli a0, a2, e32,m1,ta,mu
+; CHECK-NEXT:    vse32.v v28, (a1)
+; CHECK-NEXT:    ret
+  %a = load <4 x i8>, <4 x i8>* %x
+  %b = sext <4 x i8> %a to <4 x i32>
+  store <4 x i32> %b, <4 x i32>* %z
+  ret void
+}
+
+define void @zext_v4i8_v4i32(<4 x i8>* %x, <4 x i32>* %z) {
+; CHECK-LABEL: zext_v4i8_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a2, zero, 4
+; CHECK-NEXT:    vsetvli a3, a2, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a0)
+; CHECK-NEXT:    vsetvli a0, a2, e32,m4,ta,mu
+; CHECK-NEXT:    vzext.vf4 v28, v25
+; CHECK-NEXT:    vsetvli a0, a2, e32,m1,ta,mu
+; CHECK-NEXT:    vse32.v v28, (a1)
+; CHECK-NEXT:    ret
+  %a = load <4 x i8>, <4 x i8>* %x
+  %b = zext <4 x i8> %a to <4 x i32>
+  store <4 x i32> %b, <4 x i32>* %z
+  ret void
+}
+
+define void @sext_v8i8_v8i32(<8 x i8>* %x, <8 x i32>* %z) {
+; LMULMAX8-LABEL: sext_v8i8_v8i32:
+; LMULMAX8:       # %bb.0:
+; LMULMAX8-NEXT:    addi a2, zero, 8
+; LMULMAX8-NEXT:    vsetvli a3, a2, e8,m1,ta,mu
+; LMULMAX8-NEXT:    vle8.v v25, (a0)
+; LMULMAX8-NEXT:    vsetvli a0, a2, e32,m4,ta,mu
+; LMULMAX8-NEXT:    vsext.vf4 v28, v25
+; LMULMAX8-NEXT:    vsetvli a0, a2, e32,m2,ta,mu
+; LMULMAX8-NEXT:    vse32.v v28, (a1)
+; LMULMAX8-NEXT:    ret
+;
+; LMULMAX2-LABEL: sext_v8i8_v8i32:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    addi a2, zero, 8
+; LMULMAX2-NEXT:    vsetvli a3, a2, e8,m1,ta,mu
+; LMULMAX2-NEXT:    vle8.v v25, (a0)
+; LMULMAX2-NEXT:    vsetvli a0, a2, e32,m4,ta,mu
+; LMULMAX2-NEXT:    vsext.vf4 v28, v25
+; LMULMAX2-NEXT:    vsetvli a0, a2, e32,m2,ta,mu
+; LMULMAX2-NEXT:    vse32.v v28, (a1)
+; LMULMAX2-NEXT:    ret
+;
+; LMULMAX1-LABEL: sext_v8i8_v8i32:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    addi a2, zero, 8
+; LMULMAX1-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vle8.v v25, (a0)
+; LMULMAX1-NEXT:    addi a0, zero, 4
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v28, v25
+; LMULMAX1-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vslidedown.vi v25, v25, 4
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v8, v25
+; LMULMAX1-NEXT:    addi a2, a1, 16
+; LMULMAX1-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vse32.v v8, (a2)
+; LMULMAX1-NEXT:    vse32.v v28, (a1)
+; LMULMAX1-NEXT:    ret
+  %a = load <8 x i8>, <8 x i8>* %x
+  %b = sext <8 x i8> %a to <8 x i32>
+  store <8 x i32> %b, <8 x i32>* %z
+  ret void
+}
+
+define void @sext_v32i8_v32i32(<32 x i8>* %x, <32 x i32>* %z) {
+; LMULMAX8-LABEL: sext_v32i8_v32i32:
+; LMULMAX8:       # %bb.0:
+; LMULMAX8-NEXT:    addi a2, zero, 32
+; LMULMAX8-NEXT:    vsetvli a3, a2, e8,m2,ta,mu
+; LMULMAX8-NEXT:    vle8.v v26, (a0)
+; LMULMAX8-NEXT:    vsetvli a0, a2, e32,m8,ta,mu
+; LMULMAX8-NEXT:    vsext.vf4 v8, v26
+; LMULMAX8-NEXT:    vse32.v v8, (a1)
+; LMULMAX8-NEXT:    ret
+;
+; LMULMAX2-LABEL: sext_v32i8_v32i32:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    addi a2, zero, 32
+; LMULMAX2-NEXT:    vsetvli a2, a2, e8,m2,ta,mu
+; LMULMAX2-NEXT:    vle8.v v26, (a0)
+; LMULMAX2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
+; LMULMAX2-NEXT:    vslidedown.vi v25, v27, 8
+; LMULMAX2-NEXT:    addi a0, zero, 8
+; LMULMAX2-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX2-NEXT:    vsext.vf4 v28, v25
+; LMULMAX2-NEXT:    vsext.vf4 v8, v26
+; LMULMAX2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX2-NEXT:    vslidedown.vi v26, v26, 8
+; LMULMAX2-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX2-NEXT:    vsext.vf4 v12, v26
+; LMULMAX2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX2-NEXT:    vslidedown.vi v25, v25, 8
+; LMULMAX2-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX2-NEXT:    vsext.vf4 v16, v25
+; LMULMAX2-NEXT:    addi a2, a1, 96
+; LMULMAX2-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
+; LMULMAX2-NEXT:    vse32.v v16, (a2)
+; LMULMAX2-NEXT:    addi a0, a1, 32
+; LMULMAX2-NEXT:    vse32.v v12, (a0)
+; LMULMAX2-NEXT:    vse32.v v8, (a1)
+; LMULMAX2-NEXT:    addi a0, a1, 64
+; LMULMAX2-NEXT:    vse32.v v28, (a0)
+; LMULMAX2-NEXT:    ret
+;
+; LMULMAX1-LABEL: sext_v32i8_v32i32:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    .cfi_def_cfa_offset 0
+; LMULMAX1-NEXT:    csrr a2, vlenb
+; LMULMAX1-NEXT:    slli a2, a2, 2
+; LMULMAX1-NEXT:    sub sp, sp, a2
+; LMULMAX1-NEXT:    addi a2, zero, 16
+; LMULMAX1-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
+; LMULMAX1-NEXT:    addi a2, a0, 16
+; LMULMAX1-NEXT:    vle8.v v25, (a2)
+; LMULMAX1-NEXT:    vle8.v v26, (a0)
+; LMULMAX1-NEXT:    addi a0, zero, 4
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v28, v25
+; LMULMAX1-NEXT:    vs4r.v v28, (sp) # Unknown-size Folded Spill
+; LMULMAX1-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vslidedown.vi v27, v25, 4
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v8, v27
+; LMULMAX1-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vslidedown.vi v0, v25, 8
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v12, v0
+; LMULMAX1-NEXT:    vsext.vf4 v16, v26
+; LMULMAX1-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vslidedown.vi v25, v26, 4
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v20, v25
+; LMULMAX1-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vslidedown.vi v1, v26, 8
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v24, v1
+; LMULMAX1-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vslidedown.vi v0, v0, 4
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v4, v0
+; LMULMAX1-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vslidedown.vi v0, v1, 4
+; LMULMAX1-NEXT:    vsetvli a2, a0, e32,m4,ta,mu
+; LMULMAX1-NEXT:    vsext.vf4 v28, v0
+; LMULMAX1-NEXT:    addi a2, a1, 48
+; LMULMAX1-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vse32.v v28, (a2)
+; LMULMAX1-NEXT:    addi a0, a1, 112
+; LMULMAX1-NEXT:    vse32.v v4, (a0)
+; LMULMAX1-NEXT:    addi a0, a1, 32
+; LMULMAX1-NEXT:    vse32.v v24, (a0)
+; LMULMAX1-NEXT:    addi a0, a1, 16
+; LMULMAX1-NEXT:    vse32.v v20, (a0)
+; LMULMAX1-NEXT:    vse32.v v16, (a1)
+; LMULMAX1-NEXT:    addi a0, a1, 96
+; LMULMAX1-NEXT:    vse32.v v12, (a0)
+; LMULMAX1-NEXT:    addi a0, a1, 80
+; LMULMAX1-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-NEXT:    addi a0, a1, 64
+; LMULMAX1-NEXT:    vl4re8.v v28, (sp) # Unknown-size Folded Reload
+; LMULMAX1-NEXT:    vse32.v v28, (a0)
+; LMULMAX1-NEXT:    csrr a0, vlenb
+; LMULMAX1-NEXT:    slli a0, a0, 2
+; LMULMAX1-NEXT:    add sp, sp, a0
+; LMULMAX1-NEXT:    ret
+  %a = load <32 x i8>, <32 x i8>* %x
+  %b = sext <32 x i8> %a to <32 x i32>
+  store <32 x i32> %b, <32 x i32>* %z
+  ret void
+}