[X86] Add support for using fast short rep mov for memcpy lowering.

[hjyamauchi]

Summary

Disabled by default behind an option.

[craig.topper]

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

198

Is this needed because we're no longer calling this with just constants we fixed the size of?

[hjyamauchi]

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

198

Do you mean, "is the getCopyToReg call needed"? The size needs to be loaded into the RCX/ECX register for the rep movs instruction and the callers aren't doing that. So, it seems yes. Did you mean in a different way?

[craig.topper]

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

198

I was asking why the getZExtOrTrunc was added. do we create memcpy's where the size type isn't the same size as the pointer?

[hjyamauchi]

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

198

Without the getZExtOrTrunc, I get this fatal error, for example,

Cannot copy EAX to RCX
fatal error: error in backend: Cannot emit physreg copy instruction

which is coming from X86InstrInfo::copyPhysReg.

Re: "do we create memcpy's where the size type isn't the same size as the pointer?" yes, it seems so.

[hjyamauchi]

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

198

The type mismatch is coming from here:

static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
                                         SDValue Chain, ISD::ArgFlagsTy Flags,
                                         SelectionDAG &DAG, const SDLoc &dl) {
  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
  ...

I'm going to fix that instead.

[hjyamauchi]

Update to remove the getZExtOrTrunc.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

3112

Maybe just use getIntPtrConstant instead of getConstant?

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

198

Does that occur with the included test case or some other testing?

[hjyamauchi]

Address comments.

llvm/lib/Target/X86/X86ISelLowering.cpp

3112

My bad. This should actually be an int const rather than an int pointer const. Will change it to check if it's 64 bit instead.

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

198

It occurs in a clang bootstrap build. Added a reduced test below.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

3112

IntPtrConstant is an integer the width of a pointer which I think is what we want.

You can't check for i64 being legal. That doesn't work for GNUX32 or NaCl which use 32-bit pointers on 64 bit.

[hjyamauchi]

Address comment.

llvm/lib/Target/X86/X86ISelLowering.cpp

3112

Done.

[craig.topper]

LGTM

[hjyamauchi]

Rebase.

llvm/lib/Target/X86/X86ISelLowering.cpp

 }
 
 /// Make a copy of an aggregate at address specified by "Src" to address
 /// "Dst" with size and alignment information specified by the specific
 /// parameter attribute. The copy will be passed as a byval function parameter.
 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
                                          SDValue Chain, ISD::ArgFlagsTy Flags,
                                          SelectionDAG &DAG, const SDLoc &dl) {
-  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
+  bool Is64Bit = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
+  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl,
+                                     Is64Bit ? MVT::i64 : MVT::i32);
 
   return DAG.getMemcpy(
       Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
       /*isVolatile*/ false, /*AlwaysInline=*/true,
       /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
 }
 
 /// Return true if the calling convention is one that we can guarantee TCO for.

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/IR/DerivedTypes.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "x86-selectiondag-info"
 
+static cl::opt<bool>
+    UseFSRMForMemcpy("x86-use-fsrm-for-memcpy", cl::Hidden, cl::init(false),
+                     cl::desc("Use fast short rep mov in memcpy lowering"));
+
 bool X86SelectionDAGInfo::isBaseRegConflictPossible(
     SelectionDAG &DAG, ArrayRef<MCPhysReg> ClobberSet) const {
   // We cannot use TRI->hasBasePointer() until *after* we select all basic
   // blocks.  Legalization may introduce new stack temporaries with large
   // alignment requirements.  Fall back to generic code if there are any
   // dynamic stack adjustments (hopefully rare) and the base pointer would
   // conflict if we had to use it.
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
static SDValue emitRepmovs(const X86Subtarget &Subtarget, SelectionDAG &DAG,
                            const SDLoc &dl, SDValue Chain, SDValue Dst,
                            SDValue Src, SDValue Size, MVT AVT) {
   const bool Use64BitRegs = Subtarget.isTarget64BitLP64();
   const unsigned CX = Use64BitRegs ? X86::RCX : X86::ECX;
   const unsigned DI = Use64BitRegs ? X86::RDI : X86::EDI;
   const unsigned SI = Use64BitRegs ? X86::RSI : X86::ESI;
 
   SDValue InFlag;
   Chain = DAG.getCopyToReg(Chain, dl, CX, Size, InFlag);
   InFlag = Chain.getValue(1);
   Chain = DAG.getCopyToReg(Chain, dl, DI, Dst, InFlag);
   InFlag = Chain.getValue(1);
   Chain = DAG.getCopyToReg(Chain, dl, SI, Src, InFlag);
   InFlag = Chain.getValue(1);
 
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
   SDValue Ops[] = {Chain, DAG.getValueType(AVT), InFlag};
SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
   const MCPhysReg ClobberSet[] = {X86::RCX, X86::RSI, X86::RDI,
                                   X86::ECX, X86::ESI, X86::EDI};
   if (isBaseRegConflictPossible(DAG, ClobberSet))
     return SDValue();
 
   const X86Subtarget &Subtarget =
       DAG.getMachineFunction().getSubtarget<X86Subtarget>();
 
+  // If enabled and available, use fast short rep mov.
+  if (UseFSRMForMemcpy && Subtarget.hasFSRM())
+    return emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src, Size, MVT::i8);
+
   /// Handle constant sizes,
   if (ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size))
     return emitConstantSizeRepmov(
         DAG, Subtarget, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
         Size.getValueType(), Alignment.value(), isVolatile, AlwaysInline,
         DstPtrInfo, SrcPtrInfo);
 
   return SDValue();
 }

llvm/test/CodeGen/X86/memcpy-inline-fsrm.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu -x86-use-fsrm-for-memcpy -mattr=-fsrm < %s -o - | FileCheck %s --check-prefix=NOFSRM
+; RUN: llc -mtriple=x86_64-linux-gnu -x86-use-fsrm-for-memcpy -mattr=+fsrm < %s -o - | FileCheck %s --check-prefix=FSRM
+; RUN: llc -mtriple=x86_64-linux-gnu -x86-use-fsrm-for-memcpy -mcpu=haswell < %s | FileCheck %s --check-prefix=NOFSRM
+; RUN: llc -mtriple=x86_64-linux-gnu -x86-use-fsrm-for-memcpy -mcpu=icelake-client < %s | FileCheck %s --check-prefix=FSRM
+; RUN: llc -mtriple=x86_64-linux-gnu -x86-use-fsrm-for-memcpy -mcpu=icelake-server < %s | FileCheck %s --check-prefix=FSRM
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+define void @test1(i8* %a, i8* %b, i64 %s) nounwind {
+; NOFSRM-LABEL: test1
+; NOFSRM:       # %bb.0:
+; NOFSRM:         jmp memcpy
+;
+; FSRM-LABEL: test1
+; FSRM:       # %bb.0:
+; FSRM-NEXT:    movq %rdx, %rcx
+; FSRM-NEXT:    rep;movsb (%rsi), %es:(%rdi)
+; FSRM-NEXT:    retq
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 %s, i1 0)
+  ret void
+}
+
+; Check that we don't crash due to a memcpy size type mismatch error ("Cannot
+; emit physreg copy instruction") in X86InstrInfo::copyPhysReg.
+%struct = type { [4096 x i8] }
+declare void @foo(%struct* byval)
+define void @test2(%struct* %x) {
+  call void @foo(%struct* byval %x)
+  ret void
+}