[SystemZ] Reapply memcmp and memcpy patches

[jonpa]

Summary

The reverted patch applied on top of the refactorization...

[uweigand]

LGTM, thanks!

llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

if (Opcode == SystemZ::CLC) {
       NeedsLoop = true;
   } else {
     NeedsLoop = true;
     LenMinus1Reg = LengthMO.getReg();
   }
 
   // When generating more than one CLC, all but the last will need to
   // branch to the end when a difference is found.
-  MachineBasicBlock *EndMBB = (ImmLength > 256 && Opcode == SystemZ::CLC
-                                   ? SystemZ::splitBlockAfter(MI, MBB)
-                                   : nullptr);
+  MachineBasicBlock *EndMBB =
+      (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
 
   if (NeedsLoop) {
     Register StartCountReg =
       MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
     if (IsImmForm) {
       TII->loadImmediate(*MBB, MI, StartCountReg, ImmLength / 256);
       ImmLength &= 255;
     } else {
if (NeedsLoop) {
     RC = &SystemZ::GR64BitRegClass;
     Register ThisCountReg = MRI.createVirtualRegister(RC);
     Register NextCountReg = MRI.createVirtualRegister(RC);
 
     if (IsRegForm) {
       AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
       StartMBB = SystemZ::emitBlockAfter(MBB);
       LoopMBB = SystemZ::emitBlockAfter(StartMBB);
-      NextMBB = LoopMBB;
-      DoneMBB = SystemZ::emitBlockAfter(LoopMBB);
+      NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
+      DoneMBB = SystemZ::emitBlockAfter(NextMBB);
 
       //  MBB:
       //   # Jump to AllDoneMBB if LenMinus1Reg is -1, or fall thru to StartMBB.
       BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
         .addReg(LenMinus1Reg).addImm(-1);
       BuildMI(MBB, DL, TII->get(SystemZ::BRC))
         .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
         .addMBB(AllDoneMBB);
if (IsRegForm) {
       //   illegal value. The number of bytes processed becomes (%LenMinus1Reg &
       //   0xff) + 1.
       // # Fall through to AllDoneMBB.
       Register RemSrcReg  = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       Register RemDestReg = HaveSingleBase ? RemSrcReg
         : MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg)
         .addReg(StartDestReg).addMBB(StartMBB)
-        .addReg(NextDestReg).addMBB(LoopMBB);
+        .addReg(NextDestReg).addMBB(NextMBB);
       if (!HaveSingleBase)
         BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)
           .addReg(StartSrcReg).addMBB(StartMBB)
-          .addReg(NextSrcReg).addMBB(LoopMBB);
+          .addReg(NextSrcReg).addMBB(NextMBB);
+      MachineInstrBuilder EXRL_MIB =
         BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))
           .addImm(Opcode)
           .addReg(LenMinus1Reg)
           .addReg(RemDestReg).addImm(DestDisp)
           .addReg(RemSrcReg).addImm(SrcDisp);
       MBB->addSuccessor(AllDoneMBB);
       MBB = AllDoneMBB;
+      if (EndMBB) {
+        EXRL_MIB.addReg(SystemZ::CC, RegState::ImplicitDefine);
+        MBB->addLiveIn(SystemZ::CC);
+      }
     }
   }
 
   // Handle any remaining bytes with straight-line code.
   while (ImmLength > 0) {
     uint64_t ThisLength = std::min(ImmLength, uint64_t(256));
     // The previous iteration might have created out-of-range displacements.
     // Apply them using LAY if so.
return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
                                 SystemZ::CCMASK_CMP_GE, 32);
   case SystemZ::ATOMIC_LOAD_UMAX_64:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR,
                                 SystemZ::CCMASK_CMP_GE, 64);
 
   case SystemZ::ATOMIC_CMP_SWAPW:
     return emitAtomicCmpSwapW(MI, MBB);
   case SystemZ::MVCImm:
+  case SystemZ::MVCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::MVC);
   case SystemZ::NCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::NC);
   case SystemZ::OCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::OC);
   case SystemZ::XCImm:
   case SystemZ::XCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::XC);
   case SystemZ::CLCImm:
+  case SystemZ::CLCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::CLC);
   case SystemZ::CLSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::CLST);
   case SystemZ::MVSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::MVST);
   case SystemZ::SRSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::SRST);
   case SystemZ::TBEGIN:

llvm/lib/Target/SystemZ/SystemZInstrFormats.td

 multiclass CompareMemorySS<string mnemonic, bits<8> opcode,
                            SDPatternOperator memop> {
   def "" : SideEffectBinarySSa<mnemonic, opcode>;
   let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
     def Imm : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
                                   imm64:$length),
                           [(set CC, (memop bdaddr12only:$dest, bdaddr12only:$src,
                                            imm64:$length))]>;
+    def Reg : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
+                                  ADDR64:$length),
+                          [(set CC, (memop bdaddr12only:$dest, bdaddr12only:$src,
+                                           ADDR64:$length))]>;
   }
 }
 
 // Define an instruction that operates on two strings, both terminated
 // by the character in R0.  The instruction processes a CPU-determinated
 // number of bytes at a time and sets CC to 3 if the instruction needs
 // to be repeated.  Also define a pseudo instruction that represents
 // the full loop (the main instruction plus the branch on CC==3).

llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp

 
 #include "SystemZTargetMachine.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "systemz-selectiondag-info"
 
+static SDVTList getMemMemVTs(unsigned Op, SelectionDAG &DAG) {
+  return Op == SystemZISD::CLC ? DAG.getVTList(MVT::i32, MVT::Other)
+                               : DAG.getVTList(MVT::Other);
+}
+
 // Emit a mem-mem operation after subtracting one from size, which will be
 // added back during pseudo expansion. As the Reg case emitted here may be
 // converted by DAGCombiner into having an Imm length, they are both emitted
 // the same way.
 static SDValue emitMemMemImm(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
                              SDValue Chain, SDValue Dst, SDValue Src,
                              uint64_t Size) {
-  return DAG.getNode(Op, DL, MVT::Other, Chain, Dst, Src,
+  return DAG.getNode(Op, DL, getMemMemVTs(Op, DAG), Chain, Dst, Src,
                      DAG.getConstant(Size - 1, DL, Src.getValueType()));
 }
 
 static SDValue emitMemMemReg(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
                              SDValue Chain, SDValue Dst, SDValue Src,
                              SDValue Size) {
   SDValue LenMinus1 = DAG.getNode(ISD::ADD, DL, MVT::i64,
                                   DAG.getZExtOrTrunc(Size, DL, MVT::i64),
                                   DAG.getConstant(-1, DL, MVT::i64));
-  return DAG.getNode(Op, DL, MVT::Other, Chain, Dst, Src, LenMinus1);
-}
-
-// Use CLC to compare [Src1, Src1 + Size) with [Src2, Src2 + Size).
-// One is subtracted from size also here, per above.
-static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
-                       SDValue Src1, SDValue Src2, uint64_t Size) {
-  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
-  EVT PtrVT = Src1.getValueType();
-  return DAG.getNode(SystemZISD::CLC, DL, VTs, Chain, Src1, Src2,
-                     DAG.getConstant(Size - 1, DL, PtrVT));
+  return DAG.getNode(Op, DL, getMemMemVTs(Op, DAG), Chain, Dst, Src, LenMinus1);
 }
 
 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
     SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   if (IsVolatile)
     return SDValue();
 
   if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
     return emitMemMemImm(DAG, DL, SystemZISD::MVC, Chain, Dst, Src,
                          CSize->getZExtValue());
-  return SDValue();
+
+  return emitMemMemReg(DAG, DL, SystemZISD::MVC, Chain, Dst, Src, Size);
 }
 
 // Handle a memset of 1, 2, 4 or 8 bytes with the operands given by
 // Chain, Dst, ByteVal and Size.  These cases are expected to use
 // MVI, MVHHI, MVHI and MVGHI respectively.
 static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
                            SDValue Dst, uint64_t ByteVal, uint64_t Size,
                            unsigned Align, MachinePointerInfo DstPtrInfo) {
SDValue SRA = DAG.getNode(ISD::SRA, DL, MVT::i32, SHL,
                             DAG.getConstant(30, DL, MVT::i32));
   return SRA;
 }
 
 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
     SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo,
     MachinePointerInfo Op2PtrInfo) const {
+  SDValue CCReg;
+  // Swap operands to invert CC == 1 vs. CC == 2 cases.
   if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
     uint64_t Bytes = CSize->getZExtValue();
     assert(Bytes > 0 && "Caller should have handled 0-size case");
-    // Swap operands to invert CC == 1 vs. CC == 2 cases.
-    SDValue CCReg = emitCLC(DAG, DL, Chain, Src2, Src1, Bytes);
-    Chain = CCReg.getValue(1);
-    return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain);
-  }
-  return std::make_pair(SDValue(), SDValue());
-}
+    CCReg = emitMemMemImm(DAG, DL, SystemZISD::CLC, Chain, Src2, Src1, Bytes);
+  } else
+    CCReg = emitMemMemReg(DAG, DL, SystemZISD::CLC, Chain, Src2, Src1, Size);
 
 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
     SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
   // Use SRST to find the character.  End is its address on success.
   EVT PtrVT = Src.getValueType();
   SDVTList VTs = DAG.getVTList(PtrVT, MVT::i32, MVT::Other);
   Length = DAG.getZExtOrTrunc(Length, DL, PtrVT);

llvm/test/CodeGen/SystemZ/loop-03.ll

 %6 = type { i64, %4*, i32, i64, i8* }
 %7 = type { i64, i64, %8** }
 %8 = type { i64, i64*, i64*, %4*, i64, i32*, %5, i32, i64, i64 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
 
 define void @fun0(%0*) {
 ; CHECK-LABEL: .LBB0_4
-; CHECK: =>  This Inner Loop Header: Depth=2
+; CHECK: =>  This Inner Loop Header
 ; CHECK-NOT: 16-byte Folded Spill
 ; CHECK-NOT: 16-byte Folded Reload
 
   %2 = load i64, i64* undef, align 8
   %3 = udiv i64 128, %2
   %4 = mul i64 %3, %2
   %5 = load i64, i64* undef, align 8
   switch i32 undef, label %36 [

llvm/test/CodeGen/SystemZ/memcmp-01.ll

 ; CHECK: brctg [[COUNT]], [[LOOP]]
 ; CHECK: clc 0(1,%r3), 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: br %r14
   %res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 769)
   ret i32 %res
 }
+
+define i32 @f14(i8 *%src1, i8 *%src2, i64 %Len) {
+; CHECK-LABEL: f14:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    aghi %r4, -1
+; CHECK-NEXT:    cghi %r4, -1
+; CHECK-NEXT:    je .LBB13_5
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    srlg %r0, %r4, 8
+; CHECK-NEXT:    cgije %r0, 0, .LBB13_4
+; CHECK-NEXT:  .LBB13_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    clc 0(256,%r3), 0(%r2)
+; CHECK-NEXT:    jlh .LBB13_5
+; CHECK-NEXT:  # %bb.3: # in Loop: Header=BB13_2 Depth=1
+; CHECK-NEXT:    la %r3, 256(%r3)
+; CHECK-NEXT:    la %r2, 256(%r2)
+; CHECK-NEXT:    brctg %r0, .LBB13_2
+; CHECK-NEXT:  .LBB13_4:
+; CHECK-NEXT:    exrl %r4, .Ltmp0
+; CHECK-NEXT:  .LBB13_5:
+; CHECK-NEXT:    ipm %r2
+; CHECK-NEXT:    sll %r2, 2
+; CHECK-NEXT:    sra %r2, 30
+; CHECK-NEXT:    br %r14
+  %res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 %Len)
+  ret i32 %res
+}

llvm/test/CodeGen/SystemZ/memcpy-01.ll

; CHECK: br %r14
   %arr = alloca [3200 x i8]
   %dest = getelementptr [3200 x i8], [3200 x i8] *%arr, i64 0, i64 1600
   %src = getelementptr [3200 x i8], [3200 x i8] *%arr, i64 0, i64 0
   call void @foo(i8* %dest, i8* %src)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 1537, i1 false)
   call void @foo(i8* %dest, i8* %src)
   ret void
 }
+
+; Test a variable length loop.
+define void @f17(i8* %dest, i8* %src, i64 %Len) {
+; CHECK-LABEL: f17:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    aghi %r4, -1
+; CHECK-NEXT:    cgibe %r4, -1, 0(%r14)
+; CHECK-NEXT:  .LBB16_1:
+; CHECK-NEXT:    srlg %r0, %r4, 8
+; CHECK-NEXT:    cgije %r0, 0, .LBB16_3
+; CHECK-NEXT:  .LBB16_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    pfd 2, 768(%r2)
+; CHECK-NEXT:    mvc 0(256,%r2), 0(%r3)
+; CHECK-NEXT:    la %r2, 256(%r2)
+; CHECK-NEXT:    la %r3, 256(%r3)
+; CHECK-NEXT:    brctg %r0, .LBB16_2
+; CHECK-NEXT:  .LBB16_3:
+; CHECK-NEXT:    exrl %r4, .Ltmp0
+; CHECK-NEXT:    br %r14
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 %Len, i1 false)
+  ret void
+}
+
+; CHECK:       .Ltmp0:
+; CHECK-NEXT:    mvc 0(1,%r2), 0(%r3)

llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll

 ; RUN: llc -mtriple=s390x-linux-gnu < %s | FileCheck %s
 
-; CHECK-LABEL: tail_memcpy:
-; CHECK: jg memcpy
-define void @tail_memcpy(i8* nocapture %p, i8* nocapture readonly %q, i32 %n) #0 {
-entry:
-  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* %q, i32 %n, i1 false)
-  ret void
-}
-
 ; CHECK-LABEL: tail_memmove:
 ; CHECK: jg memmove
 define void @tail_memmove(i8* nocapture %p, i8* nocapture readonly %q, i32 %n) #0 {
 entry:
   tail call void @llvm.memmove.p0i8.p0i8.i32(i8* %p, i8* %q, i32 %n, i1 false)
   ret void
 }