[WIP][RISCV] Exploring directions for vector mem* lowering

[reames]

Summary

Highly experimental, not for review!

This is a follow up to D156249 exploring a few ideas for how we might leverage VL predication in mem* lowering. At the moment, this is purely a discussion piece. I wanted to see what was easily possible, and than bounce around various approaches. It's not clear to me that any of these are clearly profitable (in general).

llvm/include/llvm/CodeGen/TargetLowering.h

bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, LLT Ty,
                           const MachineMemOperand &MMO,
                           unsigned *Fast = nullptr) const;
 
   /// Returns the target specific optimal type for load and store operations as
   /// a result of memset, memcpy, and memmove lowering.
   /// It returns EVT::Other if the type should be determined using generic
   /// target-independent logic.
   virtual EVT
-  getOptimalMemOpType(const MemOp &Op,
+  getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                       const AttributeList & /*FuncAttributes*/) const {
     return MVT::Other;
   }
 
   /// LLT returning variant.
   virtual LLT
-  getOptimalMemOpLLT(const MemOp &Op,
+  getOptimalMemOpLLT(LLVMContext &Context, const MemOp &Op,
                      const AttributeList & /*FuncAttributes*/) const {
     return LLT();
   }
 
   /// Returns true if it's safe to use load / store of the specified type to
   /// expand memcpy / memset inline.
   ///
   /// This is mostly true for all types except for some special cases. For
public:
   };
 
   /// Determines the optimal series of memory ops to replace the memset / memcpy.
   /// Return true if the number of memory ops is below the threshold (Limit).
   /// Note that this is always the case when Limit is ~0.
   /// It returns the types of the sequence of memory ops to perform
   /// memset / memcpy by reference.
   virtual bool
-  findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit,
-                           const MemOp &Op, unsigned DstAS, unsigned SrcAS,
+  findOptimalMemOpLowering(LLVMContext &Context, std::vector<EVT> &MemOps,
+                           unsigned Limit, const MemOp &Op, unsigned DstAS,
+                           unsigned SrcAS,
                            const AttributeList &FuncAttributes) const;
 
   /// Check to see if the specified operand of the specified instruction is a
   /// constant integer.  If so, check to see if there are any bits set in the
   /// constant that are not demanded.  If so, shrink the constant and return
   /// true.
   bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
                               const APInt &DemandedElts,

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
   // only really optimize for size when -Oz (MinSize) is used.
   if (MF.getTarget().getTargetTriple().isOSDarwin())
     return MF.getFunction().hasMinSize();
   return MF.getFunction().hasOptSize();
 }
 
 // Returns a list of types to use for memory op lowering in MemOps. A partial
 // port of findOptimalMemOpLowering in TargetLowering.
-static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
+static bool findGISelOptimalMemOpLowering(LLVMContext &Context, std::vector<LLT> &MemOps,
                                           unsigned Limit, const MemOp &Op,
                                           unsigned DstAS, unsigned SrcAS,
                                           const AttributeList &FuncAttributes,
                                           const TargetLowering &TLI) {
   if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
     return false;
 
-  LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
+  LLT Ty = TLI.getOptimalMemOpLLT(Context, Op, FuncAttributes);
 
   if (Ty == LLT()) {
     // Use the largest scalar type whose alignment constraints are satisfied.
     // We only need to check DstAlign here as SrcAlign is always greater or
     // equal to DstAlign (or zero).
     Ty = LLT::scalar(64);
     if (Op.isFixedDstAlign())
       while (Op.getDstAlign() < Ty.getSizeInBytes() &&
LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
   std::vector<LLT> MemOps;
 
   const auto &DstMMO = **MI.memoperands_begin();
   MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
 
   auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Val, MRI);
   bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
 
-  if (!findGISelOptimalMemOpLowering(MemOps, Limit,
+  if (!findGISelOptimalMemOpLowering(C, MemOps, Limit,
                                      MemOp::Set(KnownLen, DstAlignCanChange,
                                                 Alignment,
                                                 /*IsZeroMemset=*/IsZeroVal,
                                                 /*IsVolatile=*/IsVolatile),
                                      DstPtrInfo.getAddrSpace(), ~0u,
                                      MF.getFunction().getAttributes(), TLI))
     return UnableToLegalize;
 
LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
 
   std::vector<LLT> MemOps;
 
   const auto &DstMMO = **MI.memoperands_begin();
   const auto &SrcMMO = **std::next(MI.memoperands_begin());
   MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
   MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
 
-  if (!findGISelOptimalMemOpLowering(
+  if (!findGISelOptimalMemOpLowering(C,
           MemOps, Limit,
           MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
                       IsVolatile),
           DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
           MF.getFunction().getAttributes(), TLI))
     return UnableToLegalize;
 
   if (DstAlignCanChange) {
LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
   const auto &DstMMO = **MI.memoperands_begin();
   const auto &SrcMMO = **std::next(MI.memoperands_begin());
   MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo();
   MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
 
   // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently due
   // to a bug in it's findOptimalMemOpLowering implementation. For now do the
   // same thing here.
-  if (!findGISelOptimalMemOpLowering(
+  if (!findGISelOptimalMemOpLowering(C,
           MemOps, Limit,
           MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
                       /*IsVolatile*/ true),
           DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
           MF.getFunction().getAttributes(), TLI))
     return UnableToLegalize;
 
   if (DstAlignCanChange) {

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
   bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
   bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
   const MemOp Op = isZeroConstant
                        ? MemOp::Set(Size, DstAlignCanChange, Alignment,
                                     /*IsZeroMemset*/ true, isVol)
                        : MemOp::Copy(Size, DstAlignCanChange, Alignment,
                                      *SrcAlign, isVol, CopyFromConstant);
-  if (!TLI.findOptimalMemOpLowering(
+  if (!TLI.findOptimalMemOpLowering(*DAG.getContext(),
           MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
           SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
     return SDValue();
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(C);
     Align NewAlign = DL.getABITypeAlign(Ty);
 
for (unsigned i = 0; i != NumMemOps; ++i) {
     }
 
     if (!Store.getNode()) {
       // The type might not be legal for the target.  This should only happen
       // if the type is smaller than a legal type, as on PPC, so the right
       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
       // to Load/Store if NVT==VT.
       // FIXME does the case above also need this?
-      EVT NVT = TLI.getTypeToTransformTo(C, VT);
+      EVT NVT = VT.isVector() ? VT : TLI.getTypeToTransformTo(C, VT);
       assert(NVT.bitsGE(VT));
 
       bool isDereferenceable =
         SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
       MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
       if (isDereferenceable)
         SrcMMOFlags |= MachineMemOperand::MODereferenceable;
       if (isConstant)
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
   MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
   if (!SrcAlign || Alignment > *SrcAlign)
     SrcAlign = Alignment;
   assert(SrcAlign && "SrcAlign must be set");
   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
-  if (!TLI.findOptimalMemOpLowering(
+  if (!TLI.findOptimalMemOpLowering(*DAG.getContext(),
           MemOps, Limit,
           MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
                       /*IsVolatile*/ true),
           DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
           MF.getFunction().getAttributes()))
     return SDValue();
 
   if (DstAlignCanChange) {
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
   bool IsZeroVal = isNullConstant(Src);
   unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
 
-  if (!TLI.findOptimalMemOpLowering(
+  if (!TLI.findOptimalMemOpLowering(*DAG.getContext(),
           MemOps, Limit,
           MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
           DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
     return SDValue();
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
     const DataLayout &DL = DAG.getDataLayout();

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

CLI.setDebugLoc(dl)
       .setDiscardResult(!CallOptions.IsReturnValueUsed)
       .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
       .setSExtResult(signExtend)
       .setZExtResult(zeroExtend);
   return LowerCallTo(CLI);
 }
 
 bool TargetLowering::findOptimalMemOpLowering(
-    std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
-    unsigned SrcAS, const AttributeList &FuncAttributes) const {
+    LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op,
+    unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const {
   if (Limit != ~unsigned(0) && Op.isMemcpyWithFixedDstAlign() &&
       Op.getSrcAlign() < Op.getDstAlign())
     return false;
 
-  EVT VT = getOptimalMemOpType(Op, FuncAttributes);
+  EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);
 
   if (VT == MVT::Other) {
     // Use the largest integer type whose alignment constraints are satisfied.
     // We only need to check DstAlign here as SrcAlign is always greater or
     // equal to DstAlign (or zero).
     VT = MVT::i64;
     if (Op.isFixedDstAlign())
       while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&

llvm/lib/Target/AArch64/AArch64ISelLowering.h

public:
   bool isLegalAddImmediate(int64_t) const override;
   bool isLegalICmpImmediate(int64_t) const override;
 
   bool isMulAddWithConstProfitable(SDValue AddNode,
                                    SDValue ConstNode) const override;
 
   bool shouldConsiderGEPOffsetSplit() const override;
 
-  EVT getOptimalMemOpType(const MemOp &Op,
+  EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                           const AttributeList &FuncAttributes) const override;
 
-  LLT getOptimalMemOpLLT(const MemOp &Op,
+  LLT getOptimalMemOpLLT(LLVMContext &Context, const MemOp &Op,
                          const AttributeList &FuncAttributes) const override;
 
   /// Return true if the addressing mode represented by AM is legal for this
   /// target, for a load/store of the specified type.
   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
                              unsigned AS,
                              Instruction *I = nullptr) const override;
 

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

     else
       Builder.CreateCall(StNFunc, {L, R, Address});
   }
 
   return true;
 }
 
 EVT AArch64TargetLowering::getOptimalMemOpType(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+    LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const {
   bool CanImplicitFloat = !FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat);
   bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
   bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
   // Only use AdvSIMD to implement memset of 32-byte and above. It would have
   // taken one instruction to materialize the v2i64 zero and one store (with
   // restrictive addressing mode). Just do i64 stores.
   bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
   auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {

   if (Op.size() >= 8 && AlignmentIsAcceptable(MVT::i64, Align(8)))
     return MVT::i64;
   if (Op.size() >= 4 && AlignmentIsAcceptable(MVT::i32, Align(4)))
     return MVT::i32;
   return MVT::Other;
 }
 
 LLT AArch64TargetLowering::getOptimalMemOpLLT(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+    LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const {
   bool CanImplicitFloat = !FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat);
   bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
   bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
   // Only use AdvSIMD to implement memset of 32-byte and above. It would have
   // taken one instruction to materialize the v2i64 zero and one store (with
   // restrictive addressing mode). Just do i64 stores.
   bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
   auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {

llvm/lib/Target/AMDGPU/SIISelLowering.h

return allowsMisalignedMemoryAccessesImpl(Ty.getSizeInBits(), AddrSpace,
                                               Alignment, Flags, IsFast);
   }
 
   bool allowsMisalignedMemoryAccesses(
       EVT VT, unsigned AS, Align Alignment,
       MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
       unsigned *IsFast = nullptr) const override;
 
-  EVT getOptimalMemOpType(const MemOp &Op,
+  EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                           const AttributeList &FuncAttributes) const override;
 
   bool isMemOpUniform(const SDNode *N) const;
   bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const;
 
   static bool isNonGlobalAddrSpace(unsigned AS);
 
   bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

 bool SITargetLowering::allowsMisalignedMemoryAccesses(
     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
     unsigned *IsFast) const {
   return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace,
                                             Alignment, Flags, IsFast);
 }
 
 EVT SITargetLowering::getOptimalMemOpType(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+LLVMContext &Context,     const MemOp &Op, const AttributeList &FuncAttributes) const {
   // FIXME: Should account for address space here.
 
   // The default fallback uses the private pointer size as a guess for a type to
   // use. Make sure we switch these to 64-bit accesses.
 
   if (Op.size() >= 16 &&
       Op.isDstAligned(Align(4))) // XXX: Should only do for global
     return MVT::v4i32;

llvm/lib/Target/ARM/ARMISelLowering.h

public:
     /// allowsMisalignedMemoryAccesses - Returns true if the target allows
     /// unaligned memory accesses of the specified type. Returns whether it
     /// is "fast" by reference in the second argument.
     bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace,
                                         Align Alignment,
                                         MachineMemOperand::Flags Flags,
                                         unsigned *Fast) const override;
 
-    EVT getOptimalMemOpType(const MemOp &Op,
+    EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                             const AttributeList &FuncAttributes) const override;
 
     bool isTruncateFree(Type *SrcTy, Type *DstTy) const override;
     bool isTruncateFree(EVT SrcVT, EVT DstVT) const override;
     bool isZExtFree(SDValue Val, EVT VT2) const override;
     bool shouldSinkOperands(Instruction *I,
                             SmallVectorImpl<Use *> &Ops) const override;
     Type* shouldConvertSplatType(ShuffleVectorInst* SVI) const override;

llvm/lib/Target/ARM/ARMISelLowering.cpp

     return true;
   }
 
   return false;
 }
 
 
 EVT ARMTargetLowering::getOptimalMemOpType(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+LLVMContext &Context,     const MemOp &Op, const AttributeList &FuncAttributes) const {
   // See if we can use NEON instructions for this...
   if ((Op.isMemcpy() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
       !FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
     unsigned Fast;
     if (Op.size() >= 16 &&
         (Op.isAligned(Align(16)) ||
          (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, Align(1),
                                          MachineMemOperand::MONone, &Fast) &&

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

default:
       llvm_unreachable("Expected a memcpy/move or memset!");
   }
 
   // MemOps will be poplulated with a list of data types that needs to be
   // loaded and stored. That's why we multiply the number of elements by 2 to
   // get the cost for this memcpy.
   std::vector<EVT> MemOps;
   if (getTLI()->findOptimalMemOpLowering(
+          I->getContext(),
           MemOps, Limit, MOp, DstAddrSpace,
           SrcAddrSpace, F->getAttributes()))
     return MemOps.size() * Factor;
 
   // If we can't find an optimal memop lowering, return the default cost
   return -1;
 }
 

llvm/lib/Target/BPF/BPFISelLowering.h

private:
   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
                       SelectionDAG &DAG) const override;
 
   void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
                           SelectionDAG &DAG) const override;
 
-  EVT getOptimalMemOpType(const MemOp &Op,
+  EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                           const AttributeList &FuncAttributes) const override {
     return Op.size() >= 8 ? MVT::i64 : MVT::i32;
   }
 
   bool isIntDivCheap(EVT VT, AttributeList Attr) const override { return true; }
 
   bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                          Type *Ty) const override {

llvm/lib/Target/Hexagon/HexagonISelLowering.h

bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
 
   /// isLegalICmpImmediate - Return true if the specified immediate is legal
   /// icmp immediate, that is the target has icmp instructions which can
   /// compare a register against the immediate without having to materialize
   /// the immediate into a register.
   bool isLegalICmpImmediate(int64_t Imm) const override;
 
-  EVT getOptimalMemOpType(const MemOp &Op,
+  EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                           const AttributeList &FuncAttributes) const override;
 
   bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
                           unsigned AddrSpace, Align Alignment,
                           MachineMemOperand::Flags Flags,
                           unsigned *Fast) const override;
 
   bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace,

llvm/lib/Target/Hexagon/HexagonISelLowering.cpp

 /// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
 /// a need to check it against alignment requirement, probably because the
 /// source does not need to be loaded. If 'IsMemset' is true, that means it's
 /// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
 /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
 /// does not need to be loaded.  It returns EVT::Other if the type should be
 /// determined using generic target-independent logic.
 EVT HexagonTargetLowering::getOptimalMemOpType(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+LLVMContext &Context,     const MemOp &Op, const AttributeList &FuncAttributes) const {
   if (Op.size() >= 8 && Op.isAligned(Align(8)))
     return MVT::i64;
   if (Op.size() >= 4 && Op.isAligned(Align(4)))
     return MVT::i32;
   if (Op.size() >= 2 && Op.isAligned(Align(2)))
     return MVT::i16;
   return MVT::Other;
 }

llvm/lib/Target/Mips/MipsISelLowering.h

private:
     }
 
     bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
                                Type *Ty, unsigned AS,
                                Instruction *I = nullptr) const override;
 
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
 
-    EVT getOptimalMemOpType(const MemOp &Op,
+    EVT getOptimalMemOpType(LLVMContext &Context,     const MemOp &Op,
                             const AttributeList &FuncAttributes) const override;
 
     /// isFPImmLegal - Returns true if the target can instruction select the
     /// specified FP immediate natively. If false, the legalizer will
     /// materialize the FP immediate as a load from a constant pool.
     bool isFPImmLegal(const APFloat &Imm, EVT VT,
                       bool ForCodeSize) const override;
 

llvm/lib/Target/Mips/MipsISelLowering.cpp

 
 bool
 MipsTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
   // The Mips target isn't yet aware of offsets.
   return false;
 }
 
 EVT MipsTargetLowering::getOptimalMemOpType(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+LLVMContext &Context,         const MemOp &Op, const AttributeList &FuncAttributes) const {
   if (Subtarget.hasMips64())
     return MVT::i64;
 
   return MVT::i32;
 }
 
 bool MipsTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                       bool ForCodeSize) const {

llvm/lib/Target/PowerPC/PPCISelLowering.h

public:
 
     bool getTgtMemIntrinsic(IntrinsicInfo &Info,
                             const CallInst &I,
                             MachineFunction &MF,
                             unsigned Intrinsic) const override;
 
     /// It returns EVT::Other if the type should be determined using generic
     /// target-independent logic.
-    EVT getOptimalMemOpType(const MemOp &Op,
+    EVT getOptimalMemOpType(LLVMContext &Context,     const MemOp &Op,
                             const AttributeList &FuncAttributes) const override;
 
     /// Is unaligned memory access allowed for the given type, and is it fast
     /// relative to software emulation.
     bool allowsMisalignedMemoryAccesses(
         EVT VT, unsigned AddrSpace, Align Alignment = Align(1),
         MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
         unsigned *Fast = nullptr) const override;

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

   }
 
   return false;
 }
 
 /// It returns EVT::Other if the type should be determined using generic
 /// target-independent logic.
 EVT PPCTargetLowering::getOptimalMemOpType(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+LLVMContext &Context,         const MemOp &Op, const AttributeList &FuncAttributes) const {
   if (getTargetMachine().getOptLevel() != CodeGenOpt::None) {
     // We should use Altivec/VSX loads and stores when available. For unaligned
     // addresses, unaligned VSX loads are only fast starting with the P8.
     if (Subtarget.hasAltivec() && Op.size() >= 16 &&
         (Op.isAligned(Align(16)) ||
          ((Op.isMemset() && Subtarget.hasVSX()) || Subtarget.hasP8Vector())))
       return MVT::v4i32;
   }

llvm/lib/Target/RISCV/RISCVISelLowering.h

public:
 
   /// Returns true if the target allows unaligned memory accesses of the
   /// specified type.
   bool allowsMisalignedMemoryAccesses(
       EVT VT, unsigned AddrSpace = 0, Align Alignment = Align(1),
       MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
       unsigned *Fast = nullptr) const override;
 
-  EVT getOptimalMemOpType(const MemOp &Op,
+  EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                           const AttributeList &FuncAttributes) const override;
 
+  bool findOptimalMemOpLowering(
+LLVMContext &Context,     std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
+    unsigned SrcAS, const AttributeList &FuncAttributes) const override;
+
   bool splitValueIntoRegisterParts(
       SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
       unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC)
       const override;
 
   SDValue joinRegisterPartsIntoValue(
       SelectionDAG & DAG, const SDLoc &DL, const SDValue *Parts,
       unsigned NumParts, MVT PartVT, EVT ValueVT,

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

   // misaligned accesses.  TODO: Work through the codegen implications of
   // allowing such accesses to be formed, and considered fast.
   if (Fast)
     *Fast = Subtarget.enableUnalignedVectorMem();
   return Subtarget.enableUnalignedVectorMem();
 }
 
 
-EVT RISCVTargetLowering::getOptimalMemOpType(const MemOp &Op,
+EVT RISCVTargetLowering::getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
                                              const AttributeList &FuncAttributes) const {
   if (!Subtarget.hasVInstructions())
     return MVT::Other;
 
-  // Round down to the next smallest LMUL register class.  This is the largest
-  // single load/store we can perform.  TODO: For non-lmul sized operations
-  // smaller than LMUL8, can we do better by using VL?
-  unsigned VLen = Subtarget.getRealMinVLen()/8;
-  unsigned Size = NextPowerOf2(Op.size() & ~(VLen - 1))/2;
-  Size = std::min(Size, 8*VLen);
-  if (Size == 0)
+  if (Op.size() < 16)
     // TODO: Figure out short memops.  For the moment, do the default thing
     // which ends up using scalar sequences.
     return MVT::Other;
 
   // Prefer i8 for non-zero memset as it allows us to avoid materializing
   // a large scalar constant and instead use vmv.v.x/i to do the
   // broadcast.  For everything else, prefer XLenVT to minimize VL and thus
   // maximize the chance we can encode the size in the vsetvli.
   MVT PreferredVT =
     (Op.isMemset() && !Op.isZeroMemset()) ? MVT::i8 : Subtarget.getXLenVT();
 
   // Do we have sufficient alignment for our preferred VT?  If not, revert
   // to byte aligned accesses.
   if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
     if (Op.isFixedDstAlign() &&
         PreferredVT.getStoreSize() > Op.getDstAlign().value())
       PreferredVT = MVT::i8;
     if (Op.isMemcpy() &&
         PreferredVT.getStoreSize() > Op.getSrcAlign().value())
       PreferredVT = MVT::i8;
   }
-  return MVT::getVectorVT(PreferredVT, Size/PreferredVT.getStoreSize());
+
+  // Version 2 - VL toggle for size < LMUL8 + overlap beyond
+  if (Op.size() < Subtarget.getRealMinVLen()) {
+    if (Op.size() % PreferredVT.getStoreSize() == 0)
+      return EVT::getVectorVT(Context, PreferredVT, Op.size()/PreferredVT.getStoreSize());
+    return EVT::getVectorVT(Context, MVT::i8, Op.size());
+  }
+  return MVT::getVectorVT(PreferredVT, Subtarget.getRealMinVLen()/PreferredVT.getStoreSize());
+}
+
+bool RISCVTargetLowering::findOptimalMemOpLowering(
+LLVMContext &Context,     std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
+    unsigned SrcAS, const AttributeList &FuncAttributes) const {
+
+  // Version 0 (Without this change at all) - Round down to LMUL + overlap if needed
+
+#if 1
+  // Version 1  - Full LMUL8 chunks + VL-toggle tail
+  if (Op.size() > 16 && Subtarget.hasVInstructions()) {
+    // Prefer i8 for non-zero memset as it allows us to avoid materializing
+    // a large scalar constant and instead use vmv.v.x/i to do the
+    // broadcast.  For everything else, prefer XLenVT to minimize VL and thus
+    // maximize the chance we can encode the size in the vsetvli.
+    MVT PreferredVT =
+      (Op.isMemset() && !Op.isZeroMemset()) ? MVT::i8 : Subtarget.getXLenVT();
+    
+    // Do we have sufficient alignment for our preferred VT?  If not, revert
+    // to byte aligned accesses.
+    if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
+      if (Op.isFixedDstAlign() &&
+          PreferredVT.getStoreSize() > Op.getDstAlign().value())
+        PreferredVT = MVT::i8;
+      if (Op.isMemcpy() &&
+          PreferredVT.getStoreSize() > Op.getSrcAlign().value())
+        PreferredVT = MVT::i8;
+    }
+
+    unsigned BytesLeft = Op.size();
+    const unsigned FullChunkSize = Subtarget.getRealMinVLen();
+    const unsigned FullChunkElems = FullChunkSize/PreferredVT.getStoreSize();
+    while (BytesLeft >= FullChunkSize) {
+      MemOps.push_back(EVT::getVectorVT(Context, PreferredVT, FullChunkElems));
+      BytesLeft -= FullChunkSize;
+    }
+    if (BytesLeft != 0) {
+      if (BytesLeft % PreferredVT.getStoreSize() == 0)
+        MemOps.push_back(EVT::getVectorVT(Context, PreferredVT, BytesLeft/PreferredVT.getStoreSize()));
+      else
+        MemOps.push_back(EVT::getVectorVT(Context, MVT::i8, BytesLeft));
+    }
+    return true;
+  }
+#endif
+
+  // Other Variants Possible:
+  // - VL control for size < LMUL8 + LMUL8 chunks + overlap
+  // - LMUL8 chunks + overlap via minimum-LMUL
+  // - LMUL8 chunks + some selection of VL and LMUL overlap by size?
+
+  // Note: We may not want to be using LMUL8 as our chunk size here, but that's a
+  // largely separate task.
+                            
+  return TargetLowering::findOptimalMemOpLowering(Context, MemOps, Limit, Op, DstAS,
+                                                  SrcAS, FuncAttributes);
 }
 
 bool RISCVTargetLowering::splitValueIntoRegisterParts(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
     unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
   bool IsABIRegCopy = CC.has_value();
   EVT ValueVT = Val.getValueType();
   if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&

llvm/lib/Target/SystemZ/SystemZISelLowering.h

public:
   bool isLegalAddImmediate(int64_t Imm) const override;
   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
                              unsigned AS,
                              Instruction *I = nullptr) const override;
   bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
                                       MachineMemOperand::Flags Flags,
                                       unsigned *Fast) const override;
   bool
-  findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit,
+  findOptimalMemOpLowering(LLVMContext &Context,     std::vector<EVT> &MemOps, unsigned Limit,
                            const MemOp &Op, unsigned DstAS, unsigned SrcAS,
                            const AttributeList &FuncAttributes) const override;
-  EVT getOptimalMemOpType(const MemOp &Op,
+  EVT getOptimalMemOpType(LLVMContext &Context,     const MemOp &Op,
                           const AttributeList &FuncAttributes) const override;
   bool isTruncateFree(Type *, Type *) const override;
   bool isTruncateFree(EVT, EVT) const override;
 
   bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
                             bool MathUsed) const override {
     // Form add and sub with overflow intrinsics regardless of any extra
     // users of the math result.

llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

if (!SupportedAM.IndexReg)
     // No indexing allowed.
     return AM.Scale == 0;
   else
     // Indexing is OK but no scale factor can be applied.
     return AM.Scale == 0 || AM.Scale == 1;
 }
 
 bool SystemZTargetLowering::findOptimalMemOpLowering(
-    std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
+LLVMContext &Context,         std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
     unsigned SrcAS, const AttributeList &FuncAttributes) const {
   const int MVCFastLen = 16;
 
   if (Limit != ~unsigned(0)) {
     // Don't expand Op into scalar loads/stores in these cases:
     if (Op.isMemcpy() && Op.allowOverlap() && Op.size() <= MVCFastLen)
       return false; // Small memcpy: Use MVC
     if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
       return false; // Small memset (first byte with STC/MVI): Use MVC
     if (Op.isZeroMemset())
       return false; // Memset zero: Use XC
   }
 
-  return TargetLowering::findOptimalMemOpLowering(MemOps, Limit, Op, DstAS,
+  return TargetLowering::findOptimalMemOpLowering(Context, MemOps, Limit, Op, DstAS,
                                                   SrcAS, FuncAttributes);
 }
 
-EVT SystemZTargetLowering::getOptimalMemOpType(const MemOp &Op,
+EVT SystemZTargetLowering::getOptimalMemOpType(LLVMContext &Context,     const MemOp &Op,
                                    const AttributeList &FuncAttributes) const {
   return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
 }
 
 bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
   if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
     return false;
   unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();

llvm/lib/Target/X86/X86ISelLowering.h

public:
 
     /// Return the desired alignment for ByVal aggregate
     /// function arguments in the caller parameter area. For X86, aggregates
     /// that contains are placed at 16-byte boundaries while the rest are at
     /// 4-byte boundaries.
     uint64_t getByValTypeAlignment(Type *Ty,
                                    const DataLayout &DL) const override;
 
-    EVT getOptimalMemOpType(const MemOp &Op,
+    EVT getOptimalMemOpType(LLVMContext &Context,     const MemOp &Op,
                             const AttributeList &FuncAttributes) const override;
 
     /// Returns true if it's safe to use load / store of the
     /// specified type to expand memcpy / memset inline. This is mostly true
     /// for all types except for some special cases. For example, on X86
     /// targets without SSE2 f64 load / store are done with fldl / fstpl which
     /// also does type conversion. Note the specified type doesn't have to be
     /// legal as the hook is used before type legalization.

llvm/lib/Target/X86/X86ISelLowering.cpp

uint64_t X86TargetLowering::getByValTypeAlignment(Type *Ty,
   return Alignment.value();
 }
 
 /// It returns EVT::Other if the type should be determined using generic
 /// target-independent logic.
 /// For vector ops we check that the overall size isn't larger than our
 /// preferred vector width.
 EVT X86TargetLowering::getOptimalMemOpType(
-    const MemOp &Op, const AttributeList &FuncAttributes) const {
+LLVMContext &Context,         const MemOp &Op, const AttributeList &FuncAttributes) const {
   if (!FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
     if (Op.size() >= 16 &&
         (!Subtarget.isUnalignedMem16Slow() || Op.isAligned(Align(16)))) {
       // FIXME: Check if unaligned 64-byte accesses are slow.
       if (Op.size() >= 64 && Subtarget.hasAVX512() &&
           (Subtarget.getPreferVectorWidth() >= 512)) {
         return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;
       }

llvm/test/CodeGen/RISCV/rvv/memcpy-inline.ll

 ; RV64-FAST-NEXT:    vse64.v v8, (a0)
 ; RV64-FAST-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr %dest, ptr %src, i64 16, i1 false)
   ret void
 }
 
 define void @unaligned_memcpy31(ptr nocapture %dest, ptr %src) nounwind {
-; RV32-LABEL: unaligned_memcpy31:
-; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    vse8.v v8, (a0)
-; RV32-NEXT:    addi a1, a1, 15
-; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    addi a0, a0, 15
-; RV32-NEXT:    vse8.v v8, (a0)
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: unaligned_memcpy31:
-; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    vse8.v v8, (a0)
-; RV64-NEXT:    addi a1, a1, 15
-; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    addi a0, a0, 15
-; RV64-NEXT:    vse8.v v8, (a0)
-; RV64-NEXT:    ret
-;
-; RV32-FAST-LABEL: unaligned_memcpy31:
-; RV32-FAST:       # %bb.0: # %entry
-; RV32-FAST-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-FAST-NEXT:    vle32.v v8, (a1)
-; RV32-FAST-NEXT:    vse32.v v8, (a0)
-; RV32-FAST-NEXT:    addi a1, a1, 15
-; RV32-FAST-NEXT:    vle32.v v8, (a1)
-; RV32-FAST-NEXT:    addi a0, a0, 15
-; RV32-FAST-NEXT:    vse32.v v8, (a0)
-; RV32-FAST-NEXT:    ret
+; RV32-BOTH-LABEL: unaligned_memcpy31:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; RV32-BOTH-NEXT:    vle8.v v8, (a1)
+; RV32-BOTH-NEXT:    vse8.v v8, (a0)
+; RV32-BOTH-NEXT:    ret
 ;
-; RV64-FAST-LABEL: unaligned_memcpy31:
-; RV64-FAST:       # %bb.0: # %entry
-; RV64-FAST-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    addi a1, a1, 15
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    addi a0, a0, 15
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    ret
+; RV64-BOTH-LABEL: unaligned_memcpy31:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; RV64-BOTH-NEXT:    vle8.v v8, (a1)
+; RV64-BOTH-NEXT:    vse8.v v8, (a0)
+; RV64-BOTH-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr %dest, ptr %src, i64 31, i1 false)
   ret void
 }
 
 define void @unaligned_memcpy32(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-LABEL: unaligned_memcpy32:
 ; RV32:       # %bb.0: # %entry

 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr %dest, ptr %src, i64 64, i1 false)
   ret void
 }
 
 define void @unaligned_memcpy96(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-LABEL: unaligned_memcpy96:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    li a2, 64
-; RV32-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    vse8.v v8, (a0)
-; RV32-NEXT:    addi a1, a1, 32
+; RV32-NEXT:    li a2, 96
+; RV32-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
 ; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    addi a0, a0, 32
 ; RV32-NEXT:    vse8.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: unaligned_memcpy96:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    li a2, 64
-; RV64-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    vse8.v v8, (a0)
-; RV64-NEXT:    addi a1, a1, 32
+; RV64-NEXT:    li a2, 96
+; RV64-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
 ; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    addi a0, a0, 32
 ; RV64-NEXT:    vse8.v v8, (a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: unaligned_memcpy96:
 ; RV32-FAST:       # %bb.0: # %entry
-; RV32-FAST-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-FAST-NEXT:    vsetivli zero, 24, e32, m8, ta, ma
 ; RV32-FAST-NEXT:    vle32.v v8, (a1)
 ; RV32-FAST-NEXT:    vse32.v v8, (a0)
-; RV32-FAST-NEXT:    addi a1, a1, 32
-; RV32-FAST-NEXT:    vle32.v v8, (a1)
-; RV32-FAST-NEXT:    addi a0, a0, 32
-; RV32-FAST-NEXT:    vse32.v v8, (a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: unaligned_memcpy96:
 ; RV64-FAST:       # %bb.0: # %entry
-; RV64-FAST-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-FAST-NEXT:    vsetivli zero, 12, e64, m8, ta, ma
 ; RV64-FAST-NEXT:    vle64.v v8, (a1)
 ; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    addi a1, a1, 32
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    addi a0, a0, 32
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
 ; RV64-FAST-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr %dest, ptr %src, i64 96, i1 false)
   ret void
 }
 
 define void @unaligned_memcpy128(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-LABEL: unaligned_memcpy128:

 
 define void @unaligned_memcpy196(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-LABEL: unaligned_memcpy196:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    li a2, 128
 ; RV32-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
 ; RV32-NEXT:    vle8.v v8, (a1)
 ; RV32-NEXT:    vse8.v v8, (a0)
-; RV32-NEXT:    addi a1, a1, 68
+; RV32-NEXT:    addi a1, a1, 128
+; RV32-NEXT:    li a2, 68
+; RV32-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
 ; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    addi a0, a0, 68
+; RV32-NEXT:    addi a0, a0, 128
 ; RV32-NEXT:    vse8.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: unaligned_memcpy196:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    li a2, 128
 ; RV64-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
 ; RV64-NEXT:    vle8.v v8, (a1)
 ; RV64-NEXT:    vse8.v v8, (a0)
-; RV64-NEXT:    addi a1, a1, 68
+; RV64-NEXT:    addi a1, a1, 128
+; RV64-NEXT:    li a2, 68
+; RV64-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
 ; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    addi a0, a0, 68
+; RV64-NEXT:    addi a0, a0, 128
 ; RV64-NEXT:    vse8.v v8, (a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: unaligned_memcpy196:
 ; RV32-FAST:       # %bb.0: # %entry
 ; RV32-FAST-NEXT:    li a2, 32
 ; RV32-FAST-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; RV32-FAST-NEXT:    vle32.v v8, (a1)
 ; RV32-FAST-NEXT:    vse32.v v8, (a0)
-; RV32-FAST-NEXT:    addi a1, a1, 68
+; RV32-FAST-NEXT:    addi a1, a1, 128
+; RV32-FAST-NEXT:    vsetivli zero, 17, e32, m8, ta, ma
 ; RV32-FAST-NEXT:    vle32.v v8, (a1)
-; RV32-FAST-NEXT:    addi a0, a0, 68
+; RV32-FAST-NEXT:    addi a0, a0, 128
 ; RV32-FAST-NEXT:    vse32.v v8, (a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: unaligned_memcpy196:
 ; RV64-FAST:       # %bb.0: # %entry
+; RV64-FAST-NEXT:    addi a2, a1, 128
+; RV64-FAST-NEXT:    li a3, 68
+; RV64-FAST-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
+; RV64-FAST-NEXT:    vle8.v v8, (a2)
+; RV64-FAST-NEXT:    addi a2, a0, 128
+; RV64-FAST-NEXT:    vse8.v v8, (a2)
 ; RV64-FAST-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-FAST-NEXT:    vle64.v v8, (a1)
 ; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    addi a1, a1, 68
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    addi a0, a0, 68
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
 ; RV64-FAST-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr %dest, ptr %src, i64 196, i1 false)
   ret void
 }
 
 define void @unaligned_memcpy256(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-LABEL: unaligned_memcpy256:

 ; RV64-BOTH-NEXT:    vse64.v v8, (a0)
 ; RV64-BOTH-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 16, i1 false)
   ret void
 }
 
 define void @aligned_memcpy31(ptr nocapture %dest, ptr %src) nounwind {
-; RV32-LABEL: aligned_memcpy31:
-; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vle32.v v8, (a1)
-; RV32-NEXT:    vse32.v v8, (a0)
-; RV32-NEXT:    addi a1, a1, 15
-; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    addi a0, a0, 15
-; RV32-NEXT:    vse8.v v8, (a0)
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: aligned_memcpy31:
-; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vle64.v v8, (a1)
-; RV64-NEXT:    vse64.v v8, (a0)
-; RV64-NEXT:    addi a1, a1, 15
-; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    addi a0, a0, 15
-; RV64-NEXT:    vse8.v v8, (a0)
-; RV64-NEXT:    ret
-;
-; RV32-FAST-LABEL: aligned_memcpy31:
-; RV32-FAST:       # %bb.0: # %entry
-; RV32-FAST-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-FAST-NEXT:    vle32.v v8, (a1)
-; RV32-FAST-NEXT:    vse32.v v8, (a0)
-; RV32-FAST-NEXT:    addi a1, a1, 15
-; RV32-FAST-NEXT:    vle32.v v8, (a1)
-; RV32-FAST-NEXT:    addi a0, a0, 15
-; RV32-FAST-NEXT:    vse32.v v8, (a0)
-; RV32-FAST-NEXT:    ret
+; RV32-BOTH-LABEL: aligned_memcpy31:
+; RV32-BOTH:       # %bb.0: # %entry
+; RV32-BOTH-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; RV32-BOTH-NEXT:    vle8.v v8, (a1)
+; RV32-BOTH-NEXT:    vse8.v v8, (a0)
+; RV32-BOTH-NEXT:    ret
 ;
-; RV64-FAST-LABEL: aligned_memcpy31:
-; RV64-FAST:       # %bb.0: # %entry
-; RV64-FAST-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    addi a1, a1, 15
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    addi a0, a0, 15
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    ret
+; RV64-BOTH-LABEL: aligned_memcpy31:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; RV64-BOTH-NEXT:    vle8.v v8, (a1)
+; RV64-BOTH-NEXT:    vse8.v v8, (a0)
+; RV64-BOTH-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 31, i1 false)
   ret void
 }
 
 define void @aligned_memcpy32(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-BOTH-LABEL: aligned_memcpy32:
 ; RV32-BOTH:       # %bb.0: # %entry

 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 64, i1 false)
   ret void
 }
 
 define void @aligned_memcpy96(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-BOTH-LABEL: aligned_memcpy96:
 ; RV32-BOTH:       # %bb.0: # %entry
-; RV32-BOTH-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-BOTH-NEXT:    vle32.v v8, (a1)
-; RV32-BOTH-NEXT:    vse32.v v8, (a0)
-; RV32-BOTH-NEXT:    addi a1, a1, 32
+; RV32-BOTH-NEXT:    vsetivli zero, 24, e32, m8, ta, ma
 ; RV32-BOTH-NEXT:    vle32.v v8, (a1)
-; RV32-BOTH-NEXT:    addi a0, a0, 32
 ; RV32-BOTH-NEXT:    vse32.v v8, (a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_memcpy96:
 ; RV64-BOTH:       # %bb.0: # %entry
-; RV64-BOTH-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-BOTH-NEXT:    vsetivli zero, 12, e64, m8, ta, ma
 ; RV64-BOTH-NEXT:    vle64.v v8, (a1)
 ; RV64-BOTH-NEXT:    vse64.v v8, (a0)
-; RV64-BOTH-NEXT:    addi a1, a1, 32
-; RV64-BOTH-NEXT:    vle64.v v8, (a1)
-; RV64-BOTH-NEXT:    addi a0, a0, 32
-; RV64-BOTH-NEXT:    vse64.v v8, (a0)
 ; RV64-BOTH-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 96, i1 false)
   ret void
 }
 
 define void @aligned_memcpy128(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-BOTH-LABEL: aligned_memcpy128:

 
 define void @aligned_memcpy196(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-BOTH-LABEL: aligned_memcpy196:
 ; RV32-BOTH:       # %bb.0: # %entry
 ; RV32-BOTH-NEXT:    li a2, 32
 ; RV32-BOTH-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; RV32-BOTH-NEXT:    vle32.v v8, (a1)
 ; RV32-BOTH-NEXT:    vse32.v v8, (a0)
-; RV32-BOTH-NEXT:    addi a1, a1, 68
+; RV32-BOTH-NEXT:    addi a1, a1, 128
+; RV32-BOTH-NEXT:    vsetivli zero, 17, e32, m8, ta, ma
 ; RV32-BOTH-NEXT:    vle32.v v8, (a1)
-; RV32-BOTH-NEXT:    addi a0, a0, 68
+; RV32-BOTH-NEXT:    addi a0, a0, 128
 ; RV32-BOTH-NEXT:    vse32.v v8, (a0)
 ; RV32-BOTH-NEXT:    ret
 ;
-; RV64-LABEL: aligned_memcpy196:
-; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    addi a2, a1, 68
-; RV64-NEXT:    li a3, 128
-; RV64-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
-; RV64-NEXT:    vle8.v v8, (a2)
-; RV64-NEXT:    addi a2, a0, 68
-; RV64-NEXT:    vse8.v v8, (a2)
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vle64.v v8, (a1)
-; RV64-NEXT:    vse64.v v8, (a0)
-; RV64-NEXT:    ret
-;
-; RV64-FAST-LABEL: aligned_memcpy196:
-; RV64-FAST:       # %bb.0: # %entry
-; RV64-FAST-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    addi a1, a1, 68
-; RV64-FAST-NEXT:    vle64.v v8, (a1)
-; RV64-FAST-NEXT:    addi a0, a0, 68
-; RV64-FAST-NEXT:    vse64.v v8, (a0)
-; RV64-FAST-NEXT:    ret
+; RV64-BOTH-LABEL: aligned_memcpy196:
+; RV64-BOTH:       # %bb.0: # %entry
+; RV64-BOTH-NEXT:    addi a2, a1, 128
+; RV64-BOTH-NEXT:    li a3, 68
+; RV64-BOTH-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
+; RV64-BOTH-NEXT:    vle8.v v8, (a2)
+; RV64-BOTH-NEXT:    addi a2, a0, 128
+; RV64-BOTH-NEXT:    vse8.v v8, (a2)
+; RV64-BOTH-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; RV64-BOTH-NEXT:    vle64.v v8, (a1)
+; RV64-BOTH-NEXT:    vse64.v v8, (a0)
+; RV64-BOTH-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src, i64 196, i1 false)
   ret void
 }
 
 define void @aligned_memcpy256(ptr nocapture %dest, ptr %src) nounwind {
 ; RV32-BOTH-LABEL: aligned_memcpy256:
 ; RV32-BOTH:       # %bb.0: # %entry

llvm/test/CodeGen/RISCV/rvv/memset-inline.ll

 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 64 %a, i8 0, i64 64, i1 0)
   ret void
 }
 
 define void @aligned_bzero_66(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_66:
 ; RV32-BOTH:       # %bb.0:
-; RV32-BOTH-NEXT:    sh zero, 64(a0)
-; RV32-BOTH-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-BOTH-NEXT:    li a1, 128
+; RV32-BOTH-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; RV32-BOTH-NEXT:    vmv.v.i v8, 0
-; RV32-BOTH-NEXT:    vse32.v v8, (a0)
+; RV32-BOTH-NEXT:    li a1, 66
+; RV32-BOTH-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; RV32-BOTH-NEXT:    vse8.v v8, (a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_bzero_66:
 ; RV64-BOTH:       # %bb.0:
-; RV64-BOTH-NEXT:    sh zero, 64(a0)
-; RV64-BOTH-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-BOTH-NEXT:    li a1, 128
+; RV64-BOTH-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; RV64-BOTH-NEXT:    vmv.v.i v8, 0
-; RV64-BOTH-NEXT:    vse64.v v8, (a0)
+; RV64-BOTH-NEXT:    li a1, 66
+; RV64-BOTH-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; RV64-BOTH-NEXT:    vse8.v v8, (a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 64 %a, i8 0, i64 66, i1 0)
   ret void
 }
 
 define void @aligned_bzero_96(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_96:
 ; RV32-BOTH:       # %bb.0:
-; RV32-BOTH-NEXT:    addi a1, a0, 32
-; RV32-BOTH-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-BOTH-NEXT:    li a1, 32
+; RV32-BOTH-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; RV32-BOTH-NEXT:    vmv.v.i v8, 0
-; RV32-BOTH-NEXT:    vse32.v v8, (a1)
+; RV32-BOTH-NEXT:    vsetivli zero, 24, e32, m8, ta, ma
 ; RV32-BOTH-NEXT:    vse32.v v8, (a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_bzero_96:
 ; RV64-BOTH:       # %bb.0:
-; RV64-BOTH-NEXT:    addi a1, a0, 32
-; RV64-BOTH-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-BOTH-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-BOTH-NEXT:    vmv.v.i v8, 0
-; RV64-BOTH-NEXT:    vse64.v v8, (a1)
+; RV64-BOTH-NEXT:    vsetivli zero, 12, e64, m8, ta, ma
 ; RV64-BOTH-NEXT:    vse64.v v8, (a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 64 %a, i8 0, i64 96, i1 0)
   ret void
 }
 
 define void @aligned_bzero_128(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_128: