[SystemZ] Bugfix and refactorization of mem-mem operations

[jonpa]

Summary

This patch fixes the bug that consisted of treating variable / immediate length mem operations (such as memcpy, memset, ...) differently. The variable length case needs to have the length minus 1 passed due to the use of EXRL target instructions. However, the DAGCombiner can convert a register length argument into a constant one, and whenever that happened one byte too little would be performed of the operation.

This is also a refactorization by reducing the number of opcodes and variants involved. For any opcode (variable or constant length), only the length minus one is passed on to the ISD node. The rest of the logic is now instead handled during isel pseudo expansion.

[uweigand]

LGTM, thanks!

llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
   MachineRegisterInfo &MRI = MF.getRegInfo();
   DebugLoc DL = MI.getDebugLoc();
 
   MachineOperand DestBase = earlyUseOperand(MI.getOperand(0));
   uint64_t DestDisp = MI.getOperand(1).getImm();
   MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2));
   uint64_t SrcDisp = MI.getOperand(3).getImm();
   MachineOperand &LengthMO = MI.getOperand(4);
-  uint64_t ImmLength = LengthMO.isImm() ? LengthMO.getImm() : 0;
-  Register LenMinus1Reg =
-      LengthMO.isReg() ? LengthMO.getReg() : SystemZ::NoRegister;
+  bool IsImmForm = LengthMO.isImm();
+  bool IsRegForm = !IsImmForm;
+
+  bool NeedsLoop = false;
+  uint64_t ImmLength = 0;
+  Register LenMinus1Reg = SystemZ::NoRegister;
+  if (IsImmForm) {
+    ImmLength = LengthMO.getImm();
+    ImmLength++; // Add back the '1' subtracted originally.
+    if (ImmLength == 0) {
+      MI.eraseFromParent();
+      return MBB;
+    }
+    if (Opcode == SystemZ::CLC) {
+      if (ImmLength > 3 * 256)
+        // A two-CLC sequence is a clear win over a loop, not least because
+        // it needs only one branch.  A three-CLC sequence needs the same
+        // number of branches as a loop (i.e. 2), but is shorter.  That
+        // brings us to lengths greater than 768 bytes.  It seems relatively
+        // likely that a difference will be found within the first 768 bytes,
+        // so we just optimize for the smallest number of branch
+        // instructions, in order to avoid polluting the prediction buffer
+        // too much.
+        NeedsLoop = true;
+    } else if (ImmLength > 6 * 256)
+      // The heuristic we use is to prefer loops for anything that would
+      // require 7 or more MVCs.  With these kinds of sizes there isn't much
+      // to choose between straight-line code and looping code, since the
+      // time will be dominated by the MVCs themselves.
+      NeedsLoop = true;
+  } else {
+    NeedsLoop = true;
+    LenMinus1Reg = LengthMO.getReg();
+  }
 
   // When generating more than one CLC, all but the last will need to
   // branch to the end when a difference is found.
   MachineBasicBlock *EndMBB = (ImmLength > 256 && Opcode == SystemZ::CLC
                                    ? SystemZ::splitBlockAfter(MI, MBB)
                                    : nullptr);
 
-  // Check for the loop form, in which operand 5 is the trip count.
-  if (MI.getNumExplicitOperands() > 5) {
-    Register StartCountReg = MI.getOperand(5).getReg();
-    bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
+  if (NeedsLoop) {
+    Register StartCountReg =
+      MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
+    if (IsImmForm) {
+      TII->loadImmediate(*MBB, MI, StartCountReg, ImmLength / 256);
+      ImmLength &= 255;
+    } else {
+      BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg)
+        .addReg(LenMinus1Reg)
+        .addReg(0)
+        .addImm(8);
+    }
 
     auto loadZeroAddress = [&]() -> MachineOperand {
       Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0);
       return MachineOperand::CreateReg(Reg, false);
     };
+    bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
     if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
       DestBase = loadZeroAddress();
     if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
       SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
 
     MachineBasicBlock *StartMBB = nullptr;
     MachineBasicBlock *LoopMBB = nullptr;
     MachineBasicBlock *NextMBB = nullptr;
Register ThisDestReg =
         (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RC));
     Register NextSrcReg  = MRI.createVirtualRegister(RC);
     Register NextDestReg =
         (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RC));
     RC = &SystemZ::GR64BitRegClass;
     Register ThisCountReg = MRI.createVirtualRegister(RC);
     Register NextCountReg = MRI.createVirtualRegister(RC);
 
-    if (LengthMO.isReg()) {
+    if (IsRegForm) {
       AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
       StartMBB = SystemZ::emitBlockAfter(MBB);
       LoopMBB = SystemZ::emitBlockAfter(StartMBB);
       NextMBB = LoopMBB;
       DoneMBB = SystemZ::emitBlockAfter(LoopMBB);
 
       //  MBB:
       //   # Jump to AllDoneMBB if LenMinus1Reg is -1, or fall thru to StartMBB.
else {
       NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
 
       //  StartMBB:
       //   # fall through to LoopMBB
       MBB->addSuccessor(LoopMBB);
 
       DestBase = MachineOperand::CreateReg(NextDestReg, false);
       SrcBase = MachineOperand::CreateReg(NextSrcReg, false);
-      ImmLength &= 255;
       if (EndMBB && !ImmLength)
         // If the loop handled the whole CLC range, DoneMBB will be empty with
         // CC live-through into EndMBB, so add it as live-in.
         DoneMBB->addLiveIn(SystemZ::CC);
     }
 
     //  LoopMBB:
     //   %ThisDestReg = phi [ %StartDestReg, StartMBB ],
BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
       .addReg(NextCountReg).addImm(0);
     BuildMI(MBB, DL, TII->get(SystemZ::BRC))
       .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
       .addMBB(LoopMBB);
     MBB->addSuccessor(LoopMBB);
     MBB->addSuccessor(DoneMBB);
 
     MBB = DoneMBB;
-    if (LengthMO.isReg()) {
+    if (IsRegForm) {
       // DoneMBB:
       // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
       // # Use EXecute Relative Long for the remainder of the bytes. The target
       //   instruction of the EXRL will have a length field of 1 since 0 is an
       //   illegal value. The number of bytes processed becomes (%LenMinus1Reg &
       //   0xff) + 1.
       // # Fall through to AllDoneMBB.
       Register RemSrcReg  = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       Register RemDestReg = HaveSingleBase ? RemSrcReg
         : MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg)
         .addReg(StartDestReg).addMBB(StartMBB)
         .addReg(NextDestReg).addMBB(LoopMBB);
       if (!HaveSingleBase)
         BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)
           .addReg(StartSrcReg).addMBB(StartMBB)
           .addReg(NextSrcReg).addMBB(LoopMBB);
-      MRI.constrainRegClass(LenMinus1Reg, &SystemZ::ADDR64BitRegClass);
       BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))
         .addImm(Opcode)
         .addReg(LenMinus1Reg)
         .addReg(RemDestReg).addImm(DestDisp)
         .addReg(RemSrcReg).addImm(SrcDisp);
       MBB->addSuccessor(AllDoneMBB);
       MBB = AllDoneMBB;
     }
case SystemZ::ATOMIC_LOAD_UMAX_32:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
                                 SystemZ::CCMASK_CMP_GE, 32);
   case SystemZ::ATOMIC_LOAD_UMAX_64:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR,
                                 SystemZ::CCMASK_CMP_GE, 64);
 
   case SystemZ::ATOMIC_CMP_SWAPW:
     return emitAtomicCmpSwapW(MI, MBB);
-  case SystemZ::MVCSequence:
-  case SystemZ::MVCLoop:
+  case SystemZ::MVCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::MVC);
-  case SystemZ::NCSequence:
-  case SystemZ::NCLoop:
+  case SystemZ::NCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::NC);
-  case SystemZ::OCSequence:
-  case SystemZ::OCLoop:
+  case SystemZ::OCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::OC);
-  case SystemZ::XCSequence:
-  case SystemZ::XCLoop:
-  case SystemZ::XCLoopVarLen:
+  case SystemZ::XCImm:
+  case SystemZ::XCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::XC);
-  case SystemZ::CLCSequence:
-  case SystemZ::CLCLoop:
+  case SystemZ::CLCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::CLC);
   case SystemZ::CLSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::CLST);
   case SystemZ::MVSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::MVST);
   case SystemZ::SRSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::SRST);
   case SystemZ::TBEGIN:

llvm/lib/Target/SystemZ/SystemZInstrFP.td

let Predicates = [FeatureNoVectorEnhancements1] in {
   def : CopySign128<FP32,  (CPSDRds (EXTRACT_SUBREG FP128:$src1, subreg_h64),
                                     FP32:$src2)>;
   def : CopySign128<FP64,  (CPSDRdd (EXTRACT_SUBREG FP128:$src1, subreg_h64),
                                     FP64:$src2)>;
   def : CopySign128<FP128, (CPSDRdd (EXTRACT_SUBREG FP128:$src1, subreg_h64),
                                     (EXTRACT_SUBREG FP128:$src2, subreg_h64))>;
 }
 
-defm LoadStoreF32  : MVCLoadStore<load, f32,  MVCSequence, 4>;
-defm LoadStoreF64  : MVCLoadStore<load, f64,  MVCSequence, 8>;
-defm LoadStoreF128 : MVCLoadStore<load, f128, MVCSequence, 16>;
+// The length is given as one less for MVCImm.
+defm LoadStoreF32  : MVCLoadStore<load, f32,  MVCImm, 3>;
+defm LoadStoreF64  : MVCLoadStore<load, f64,  MVCImm, 7>;
+defm LoadStoreF128 : MVCLoadStore<load, f128, MVCImm, 15>;
 
 //===----------------------------------------------------------------------===//
 // Load instructions
 //===----------------------------------------------------------------------===//
 
 let canFoldAsLoad = 1, SimpleBDXLoad = 1, mayLoad = 1 in {
   defm LE : UnaryRXPair<"le", 0x78, 0xED64, load, FP32, 4>;
   defm LD : UnaryRXPair<"ld", 0x68, 0xED65, load, FP64, 8>;

llvm/lib/Target/SystemZ/SystemZInstrFormats.td

multiclass CondUnaryRSYPseudoAndMemFold<string mnemonic,
                                         RegisterOperand cls, bits<5> bytes,
                                         AddressingMode mode = bdaddr20only> {
   def "" : CondUnaryRSYPseudo<mnemonic, operator, cls, bytes, mode>;
   def _MemFoldPseudo : MemFoldPseudo_CondMove<mnemonic, cls, bytes, mode>;
 }
 
 // Define an instruction that operates on two fixed-length blocks of memory,
 // and associated pseudo instructions for operating on blocks of any size.
-// The Sequence form uses a straight-line sequence of instructions and
-// the Loop form uses a loop of length-256 instructions followed by
-// another instruction to handle the excess.
-// The LoopVarLen form is for a loop with a non-constant length parameter.
-multiclass MemorySS<string mnemonic, bits<8> opcode,
-                    SDPatternOperator sequence, SDPatternOperator loop> {
+// There are two pseudos for the different cases of when the length is
+// constant or variable. The length operand of a pseudo is actually one less
+// than the intended number of bytes, since the register case needs to use an
+// EXRL with a target instruction that adds one to the length always.
+multiclass MemorySS<string mnemonic, bits<8> opcode, SDPatternOperator memop> {
   def "" : SideEffectBinarySSa<mnemonic, opcode>;
   let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Defs = [CC] in {
-    def Sequence : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
-                                       imm64:$length),
-                           [(sequence bdaddr12only:$dest, bdaddr12only:$src,
-                                      imm64:$length)]>;
-    def Loop : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
-                                   imm64:$length, GR64:$count256),
-                      [(loop bdaddr12only:$dest, bdaddr12only:$src,
-                             imm64:$length, GR64:$count256)]>;
-    def LoopVarLen : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
-                                         GR64:$length, GR64:$count256),
-                            [(loop bdaddr12only:$dest, bdaddr12only:$src,
-                                   GR64:$length, GR64:$count256)]>;
+    def Imm : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
+                             [(memop bdaddr12only:$dest, bdaddr12only:$src,
+    def Reg : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
+                                  ADDR64:$length),
+                             [(memop bdaddr12only:$dest, bdaddr12only:$src,
+                                     ADDR64:$length)]>;
   }
 }
 
 // The same, but setting a CC result as comparison operator.
 multiclass CompareMemorySS<string mnemonic, bits<8> opcode,
-                          SDPatternOperator sequence, SDPatternOperator loop> {
+                           SDPatternOperator memop> {
   def "" : SideEffectBinarySSa<mnemonic, opcode>;
   let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
-    def Sequence : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
-                                       imm64:$length),
-                           [(set CC, (sequence bdaddr12only:$dest, bdaddr12only:$src,
-                                               imm64:$length))]>;
-    def Loop : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
-                                   imm64:$length, GR64:$count256),
-                      [(set CC, (loop bdaddr12only:$dest, bdaddr12only:$src,
-                                      imm64:$length, GR64:$count256))]>;
+    def Imm : Pseudo<(outs), (ins bdaddr12only:$dest, bdaddr12only:$src,
+                          [(set CC, (memop bdaddr12only:$dest, bdaddr12only:$src,
   }
 }
 
 // Define an instruction that operates on two strings, both terminated
 // by the character in R0.  The instruction processes a CPU-determinated
 // number of bytes at a time and sets CC to 3 if the instruction needs
 // to be repeated.  Also define a pseudo instruction that represents
 // the full loop (the main instruction plus the branch on CC==3).

llvm/lib/Target/SystemZ/SystemZInstrInfo.td

 
 // 16-bit immediate stores to 16-, 32- or 64-bit fields.
 def MVHHI : StoreSIL<"mvhhi", 0xE544, truncstorei16, imm32sx16trunc>;
 def MVHI  : StoreSIL<"mvhi",  0xE54C, store,         imm32sx16>;
 def MVGHI : StoreSIL<"mvghi", 0xE548, store,         imm64sx16>;
 
 // Memory-to-memory moves.
 let mayLoad = 1, mayStore = 1 in
-  defm MVC : MemorySS<"mvc", 0xD2, z_mvc, z_mvc_loop>;
+  defm MVC : MemorySS<"mvc", 0xD2, z_mvc>;
 let mayLoad = 1, mayStore = 1, Defs = [CC] in {
   def MVCL  : SideEffectBinaryMemMemRR<"mvcl", 0x0E, GR128, GR128>;
   def MVCLE : SideEffectTernaryMemMemRS<"mvcle", 0xA8, GR128, GR128>;
   def MVCLU : SideEffectTernaryMemMemRSY<"mvclu", 0xEB8E, GR128, GR128>;
 }
 
 // Move right.
 let Predicates = [FeatureMiscellaneousExtensions3],
let CCValues = 0xC, CompareZeroCCMask = 0x8 in {
     defm NG : BinaryRXYAndPseudo<"ng", 0xE380, and, GR64, load, 8>;
   }
 
   // AND to memory
   defm NI : BinarySIPair<"ni", 0x94, 0xEB54, null_frag, imm32zx8>;
 
   // Block AND.
   let mayLoad = 1, mayStore = 1 in
-    defm NC : MemorySS<"nc", 0xD4, z_nc, z_nc_loop>;
+    defm NC : MemorySS<"nc", 0xD4, z_nc>;
 }
 defm : RMWIByte<and, bdaddr12pair, NI>;
 defm : RMWIByte<and, bdaddr20pair, NIY>;
 
 //===----------------------------------------------------------------------===//
 // OR
 //===----------------------------------------------------------------------===//
 
let CCValues = 0xC, CompareZeroCCMask = 0x8 in {
     defm OG : BinaryRXYAndPseudo<"og", 0xE381, or, GR64, load, 8>;
   }
 
   // OR to memory
   defm OI : BinarySIPair<"oi", 0x96, 0xEB56, null_frag, imm32zx8>;
 
   // Block OR.
   let mayLoad = 1, mayStore = 1 in
-    defm OC : MemorySS<"oc", 0xD6, z_oc, z_oc_loop>;
+    defm OC : MemorySS<"oc", 0xD6, z_oc>;
 }
 defm : RMWIByte<or, bdaddr12pair, OI>;
 defm : RMWIByte<or, bdaddr20pair, OIY>;
 
 //===----------------------------------------------------------------------===//
 // XOR
 //===----------------------------------------------------------------------===//
 
let CCValues = 0xC, CompareZeroCCMask = 0x8 in {
     defm XG : BinaryRXYAndPseudo<"xg", 0xE382, xor, GR64, load, 8>;
   }
 
   // XOR to memory
   defm XI : BinarySIPair<"xi", 0x97, 0xEB57, null_frag, imm32zx8>;
 
   // Block XOR.
   let mayLoad = 1, mayStore = 1 in
-    defm XC : MemorySS<"xc", 0xD7, z_xc, z_xc_loop>;
+    defm XC : MemorySS<"xc", 0xD7, z_xc>;
 }
 defm : RMWIByte<xor, bdaddr12pair, XI>;
 defm : RMWIByte<xor, bdaddr20pair, XIY>;
 
 //===----------------------------------------------------------------------===//
 // Combined logical operations
 //===----------------------------------------------------------------------===//
 
let Defs = [CC], CCValues = 0xE, IsLogical = 1 in {
   def CLHHSI : CompareSIL<"clhhsi", 0xE555, z_ucmp, azextloadi16, imm32zx16>;
   def CLFHSI : CompareSIL<"clfhsi", 0xE55D, z_ucmp, load, imm32zx16>;
   def CLGHSI : CompareSIL<"clghsi", 0xE559, z_ucmp, load, imm64zx16>;
 }
 defm : ZXB<z_ucmp, GR64, CLGFR>;
 
 // Memory-to-memory comparison.
 let mayLoad = 1, Defs = [CC] in {
-  defm CLC : CompareMemorySS<"clc", 0xD5, z_clc, z_clc_loop>;
+  defm CLC : CompareMemorySS<"clc", 0xD5, z_clc>;
   def CLCL  : SideEffectBinaryMemMemRR<"clcl", 0x0F, GR128, GR128>;
   def CLCLE : SideEffectTernaryMemMemRS<"clcle", 0xA9, GR128, GR128>;
   def CLCLU : SideEffectTernaryMemMemRSY<"clclu", 0xEB8F, GR128, GR128>;
 }
 
 // String comparison.
 let mayLoad = 1, Defs = [CC] in
   defm CLST : StringRRE<"clst", 0xB25D, z_strcmp>;
let AddedComplexity = 4 in {
 
   def : Pat<(srl GR64:$val, (sub imm32mod64,  GR32:$shift)),
             (SRLG GR64:$val, (LCR GR32:$shift), 0)>;
 
   def : Pat<(rotl GR64:$val, (sub imm32mod64,  GR32:$shift)),
             (RLLG GR64:$val, (LCR GR32:$shift), 0)>;
 }
 
-// Peepholes for turning scalar operations into block operations.
-defm : BlockLoadStore<anyextloadi8, i32, MVCSequence, NCSequence, OCSequence,
-                      XCSequence, 1>;
-defm : BlockLoadStore<anyextloadi16, i32, MVCSequence, NCSequence, OCSequence,
-                      XCSequence, 2>;
-defm : BlockLoadStore<load, i32, MVCSequence, NCSequence, OCSequence,
-                      XCSequence, 4>;
-defm : BlockLoadStore<anyextloadi8, i64, MVCSequence, NCSequence,
-                      OCSequence, XCSequence, 1>;
-defm : BlockLoadStore<anyextloadi16, i64, MVCSequence, NCSequence, OCSequence,
-                      XCSequence, 2>;
-defm : BlockLoadStore<anyextloadi32, i64, MVCSequence, NCSequence, OCSequence,
-                      XCSequence, 4>;
-defm : BlockLoadStore<load, i64, MVCSequence, NCSequence, OCSequence,
-                      XCSequence, 8>;
+// Peepholes for turning scalar operations into block operations.  The length
+// is given as one less for these pseudos.
+defm : BlockLoadStore<anyextloadi8, i32, MVCImm, NCImm, OCImm, XCImm, 0>;
+defm : BlockLoadStore<anyextloadi16, i32, MVCImm, NCImm, OCImm, XCImm, 1>;
+defm : BlockLoadStore<load, i32, MVCImm, NCImm, OCImm, XCImm, 3>;
+defm : BlockLoadStore<anyextloadi8, i64, MVCImm, NCImm, OCImm, XCImm, 0>;
+defm : BlockLoadStore<anyextloadi16, i64, MVCImm, NCImm, OCImm, XCImm, 1>;
+defm : BlockLoadStore<anyextloadi32, i64, MVCImm, NCImm, OCImm, XCImm, 3>;
+defm : BlockLoadStore<load, i64, MVCImm, NCImm, OCImm, XCImm, 7>;
 
 //===----------------------------------------------------------------------===//
 // Mnemonic Aliases
 //===----------------------------------------------------------------------===//
 
 def JCT   : MnemonicAlias<"jct", "brct">;
 def JCTG  : MnemonicAlias<"jctg", "brctg">;
 def JAS   : MnemonicAlias<"jas", "bras">;

llvm/lib/Target/SystemZ/SystemZOperators.td

def SDT_ZMemMemLength       : SDTypeProfile<0, 3,
                                             [SDTCisPtrTy<0>,
                                              SDTCisPtrTy<1>,
                                              SDTCisVT<2, i64>]>;
 def SDT_ZMemMemLengthCC     : SDTypeProfile<1, 3,
                                             [SDTCisVT<0, i32>,
                                              SDTCisPtrTy<1>,
                                              SDTCisPtrTy<2>,
                                              SDTCisVT<3, i64>]>;
-def SDT_ZMemMemLoop         : SDTypeProfile<0, 4,
-                                            [SDTCisPtrTy<0>,
-                                             SDTCisPtrTy<1>,
-                                             SDTCisVT<2, i64>,
-                                             SDTCisVT<3, i64>]>;
-def SDT_ZMemMemLoopCC       : SDTypeProfile<1, 4,
-                                            [SDTCisVT<0, i32>,
-                                             SDTCisPtrTy<1>,
-                                             SDTCisPtrTy<2>,
-                                             SDTCisVT<3, i64>,
-                                             SDTCisVT<4, i64>]>;
 def SDT_ZString             : SDTypeProfile<1, 3,
                                             [SDTCisPtrTy<0>,
                                              SDTCisPtrTy<1>,
                                              SDTCisPtrTy<2>,
                                              SDTCisVT<3, i32>]>;
 def SDT_ZStringCC           : SDTypeProfile<2, 3,
                                             [SDTCisPtrTy<0>,
                                              SDTCisVT<1, i32>,
def z_atomic_store_128  : SDNode<"SystemZISD::ATOMIC_STORE_128",
                                  [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def z_atomic_cmp_swap_128 : SDNode<"SystemZISD::ATOMIC_CMP_SWAP_128",
                                    SDT_ZAtomicCmpSwap128,
                                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad,
                                     SDNPMemOperand]>;
 
 def z_mvc               : SDNode<"SystemZISD::MVC", SDT_ZMemMemLength,
                                  [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
-def z_mvc_loop          : SDNode<"SystemZISD::MVC_LOOP", SDT_ZMemMemLoop,
-                                 [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
 def z_nc                : SDNode<"SystemZISD::NC", SDT_ZMemMemLength,
                                   [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
-def z_nc_loop           : SDNode<"SystemZISD::NC_LOOP", SDT_ZMemMemLoop,
-                                  [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
 def z_oc                : SDNode<"SystemZISD::OC", SDT_ZMemMemLength,
                                   [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
-def z_oc_loop           : SDNode<"SystemZISD::OC_LOOP", SDT_ZMemMemLoop,
-                                  [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
 def z_xc                : SDNode<"SystemZISD::XC", SDT_ZMemMemLength,
                                   [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
-def z_xc_loop           : SDNode<"SystemZISD::XC_LOOP", SDT_ZMemMemLoop,
-                                  [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
 def z_clc               : SDNode<"SystemZISD::CLC", SDT_ZMemMemLengthCC,
                                  [SDNPHasChain, SDNPMayLoad]>;
-def z_clc_loop          : SDNode<"SystemZISD::CLC_LOOP", SDT_ZMemMemLoopCC,
-                                 [SDNPHasChain, SDNPMayLoad]>;
 def z_strcmp            : SDNode<"SystemZISD::STRCMP", SDT_ZStringCC,
                                  [SDNPHasChain, SDNPMayLoad]>;
 def z_stpcpy            : SDNode<"SystemZISD::STPCPY", SDT_ZString,
                                  [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
 def z_search_string     : SDNode<"SystemZISD::SEARCH_STRING", SDT_ZStringCC,
                                  [SDNPHasChain, SDNPMayLoad]>;
 def z_prefetch          : SDNode<"SystemZISD::PREFETCH", SDT_ZPrefetch,
                                  [SDNPHasChain, SDNPMayLoad, SDNPMayStore,

llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp

 
 #include "SystemZTargetMachine.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "systemz-selectiondag-info"
 
-// Decide whether it is best to use a loop or straight-line code for
-// a block operation of Size bytes with source address Src and destination
-// address Dest.  Sequence is the opcode to use for straight-line code
-// (such as MVC) and Loop is the opcode to use for loops (such as MVC_LOOP).
-// Return the chain for the completed operation.
-static SDValue emitMemMem(SelectionDAG &DAG, const SDLoc &DL, unsigned Sequence,
-                          unsigned Loop, SDValue Chain, SDValue Dst,
-                          SDValue Src, uint64_t Size) {
-  EVT PtrVT = Src.getValueType();
-  // The heuristic we use is to prefer loops for anything that would
-  // require 7 or more MVCs.  With these kinds of sizes there isn't
-  // much to choose between straight-line code and looping code,
-  // since the time will be dominated by the MVCs themselves.
-  // However, the loop has 4 or 5 instructions (depending on whether
-  // the base addresses can be proved equal), so there doesn't seem
-  // much point using a loop for 5 * 256 bytes or fewer.  Anything in
-  // the range (5 * 256, 6 * 256) will need another instruction after
-  // the loop, so it doesn't seem worth using a loop then either.
-  // The next value up, 6 * 256, can be implemented in the same
-  // number of straight-line MVCs as 6 * 256 - 1.
-  if (Size > 6 * 256)
-    return DAG.getNode(Loop, DL, MVT::Other, Chain, Dst, Src,
-                       DAG.getConstant(Size, DL, PtrVT),
-                       DAG.getConstant(Size / 256, DL, PtrVT));
-  return DAG.getNode(Sequence, DL, MVT::Other, Chain, Dst, Src,
-                     DAG.getConstant(Size, DL, PtrVT));
+// Emit a mem-mem operation after subtracting one from size, which will be
+// added back during pseudo expansion. As the Reg case emitted here may be
+// converted by DAGCombiner into having an Imm length, they are both emitted
+// the same way.
+static SDValue emitMemMemImm(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
+                             SDValue Chain, SDValue Dst, SDValue Src,
+                             uint64_t Size) {
+  return DAG.getNode(Op, DL, MVT::Other, Chain, Dst, Src,
+                     DAG.getConstant(Size - 1, DL, Src.getValueType()));
+}
+
+static SDValue emitMemMemReg(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
+                             SDValue Chain, SDValue Dst, SDValue Src,
+                             SDValue Size) {
+  SDValue LenMinus1 = DAG.getNode(ISD::ADD, DL, MVT::i64,
+                                  DAG.getZExtOrTrunc(Size, DL, MVT::i64),
+                                  DAG.getConstant(-1, DL, MVT::i64));
+  return DAG.getNode(Op, DL, MVT::Other, Chain, Dst, Src, LenMinus1);
+}
+
+// Use CLC to compare [Src1, Src1 + Size) with [Src2, Src2 + Size).
+// One is subtracted from size also here, per above.
+static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
+                       SDValue Src1, SDValue Src2, uint64_t Size) {
+  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
+  EVT PtrVT = Src1.getValueType();
+  return DAG.getNode(SystemZISD::CLC, DL, VTs, Chain, Src1, Src2,
+                     DAG.getConstant(Size - 1, DL, PtrVT));
 }
 
 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
     SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
   if (IsVolatile)
     return SDValue();
 
   if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
-    return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
-                      Chain, Dst, Src, CSize->getZExtValue());
+    return emitMemMemImm(DAG, DL, SystemZISD::MVC, Chain, Dst, Src,
+                         CSize->getZExtValue());
   return SDValue();
 }
 
 // Handle a memset of 1, 2, 4 or 8 bytes with the operands given by
 // Chain, Dst, ByteVal and Size.  These cases are expected to use
 // MVI, MVHHI, MVHI and MVGHI respectively.
 static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
                            SDValue Dst, uint64_t ByteVal, uint64_t Size,
if (CByte) {
                                       DstPtrInfo.getWithOffset(1), Align(1));
         return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
       }
     }
     assert(Bytes >= 2 && "Should have dealt with 0- and 1-byte cases already");
 
     // Handle the special case of a memset of 0, which can use XC.
     if (CByte && CByte->getZExtValue() == 0)
-      return emitMemMem(DAG, DL, SystemZISD::XC, SystemZISD::XC_LOOP,
-                        Chain, Dst, Dst, Bytes);
+      return emitMemMemImm(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Bytes);
 
     // Copy the byte to the first location and then use MVC to copy
     // it to the rest.
     Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
     SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
                                    DAG.getConstant(1, DL, PtrVT));
-    return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
-                      Chain, DstPlus1, Dst, Bytes - 1);
+    return emitMemMemImm(DAG, DL, SystemZISD::MVC, Chain, DstPlus1, Dst,
+                         Bytes - 1);
   }
 
   // Variable length
-  if (CByte && CByte->getZExtValue() == 0) {
+  if (CByte && CByte->getZExtValue() == 0)
     // Handle the special case of a variable length memset of 0 with XC.
-    SDValue LenMinus1 = DAG.getNode(ISD::ADD, DL, MVT::i64,
-                                    DAG.getZExtOrTrunc(Size, DL, MVT::i64),
-                                    DAG.getConstant(-1, DL, MVT::i64));
-    SDValue TripC = DAG.getNode(ISD::SRL, DL, MVT::i64, LenMinus1,
-                                DAG.getConstant(8, DL, MVT::i64));
-    return DAG.getNode(SystemZISD::XC_LOOP, DL, MVT::Other, Chain, Dst, Dst,
-                       LenMinus1, TripC);
-  }
-  return SDValue();
-}
+    return emitMemMemReg(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Size);
 
-// Use CLC to compare [Src1, Src1 + Size) with [Src2, Src2 + Size),
-// deciding whether to use a loop or straight-line code.
-static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
-                       SDValue Src1, SDValue Src2, uint64_t Size) {
-  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
-  EVT PtrVT = Src1.getValueType();
-  // A two-CLC sequence is a clear win over a loop, not least because it
-  // needs only one branch.  A three-CLC sequence needs the same number
-  // of branches as a loop (i.e. 2), but is shorter.  That brings us to
-  // lengths greater than 768 bytes.  It seems relatively likely that
-  // a difference will be found within the first 768 bytes, so we just
-  // optimize for the smallest number of branch instructions, in order
-  // to avoid polluting the prediction buffer too much.  A loop only ever
-  // needs 2 branches, whereas a straight-line sequence would need 3 or more.
-  if (Size > 3 * 256)
-    return DAG.getNode(SystemZISD::CLC_LOOP, DL, VTs, Chain, Src1, Src2,
-                       DAG.getConstant(Size, DL, PtrVT),
-                       DAG.getConstant(Size / 256, DL, PtrVT));
-  return DAG.getNode(SystemZISD::CLC, DL, VTs, Chain, Src1, Src2,
-                     DAG.getConstant(Size, DL, PtrVT));
+  return SDValue();
 }
 
 // Convert the current CC value into an integer that is 0 if CC == 0,
 // greater than zero if CC == 1 and less than zero if CC >= 2.
 // The sequence starts with IPM, which puts CC into bits 29 and 28
 // of an integer and clears bits 30 and 31.
 static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
                               SelectionDAG &DAG) {

llvm/test/CodeGen/SystemZ/memset-05.ll

 }
 
 ; Test that identical target instructions get reused.
 define void @fun2(i8* %Addr, i32 %Len) {
 ; CHECK-LABEL: fun2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    llgfr %r1, %r3
 ; CHECK-NEXT:    aghi %r1, -1
-; CHECK-NEXT:    srlg %r0, %r1, 8
 ; CHECK-NEXT:    cgije %r1, -1, .LBB2_5
 ; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    srlg %r0, %r1, 8
 ; CHECK-NEXT:    lgr %r3, %r2
 ; CHECK-NEXT:    cgije %r0, 0, .LBB2_4
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    lgr %r3, %r2
-; CHECK-NEXT:    lgr %r4, %r0
 ; CHECK-NEXT:  .LBB2_3: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xc 0(256,%r3), 0(%r3)
 ; CHECK-NEXT:    la %r3, 256(%r3)
-; CHECK-NEXT:    brctg %r4, .LBB2_3
+; CHECK-NEXT:    brctg %r0, .LBB2_3
 ; CHECK-NEXT:  .LBB2_4:
 ; CHECK-NEXT:    exrl %r1, .Ltmp1
 ; CHECK-NEXT:  .LBB2_5:
 ; CHECK-NEXT:    cgije %r1, -1, .LBB2_10
 ; CHECK-NEXT:  # %bb.6:
+; CHECK-NEXT:    srlg %r0, %r1, 8
 ; CHECK-NEXT:    lgr %r3, %r2
 ; CHECK-NEXT:    cgije %r0, 0, .LBB2_9
 ; CHECK-NEXT:  # %bb.7:
 ; CHECK-NEXT:    lgr %r3, %r2
-; CHECK-NEXT:    lgr %r4, %r0
 ; CHECK-NEXT:  .LBB2_8: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xc 0(256,%r3), 0(%r3)
 ; CHECK-NEXT:    la %r3, 256(%r3)
-; CHECK-NEXT:    brctg %r4, .LBB2_8
+; CHECK-NEXT:    brctg %r0, .LBB2_8
 ; CHECK-NEXT:  .LBB2_9:
 ; CHECK-NEXT:    exrl %r1, .Ltmp1
 ; CHECK-NEXT:  .LBB2_10:
 ; CHECK-NEXT:    cgibe %r1, -1, 0(%r14)
 ; CHECK-NEXT:  .LBB2_11:
+; CHECK-NEXT:    srlg %r0, %r1, 8
 ; CHECK-NEXT:    cgije %r0, 0, .LBB2_13
 ; CHECK-NEXT:  .LBB2_12: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xc 0(256,%r2), 0(%r2)
 ; CHECK-NEXT:    la %r2, 256(%r2)
 ; CHECK-NEXT:    brctg %r0, .LBB2_12
 ; CHECK-NEXT:  .LBB2_13:
 ; CHECK-NEXT:    exrl %r1, .Ltmp0
 ; CHECK-NEXT:    br %r14

 ; CHECK-NEXT:    brctg %r0, .LBB3_2
 ; CHECK-NEXT:  .LBB3_3:
 ; CHECK-NEXT:    exrl %r2, .Ltmp2
 ; CHECK-NEXT:    br %r14
   call void @llvm.memset.p0i8.i64(i8* null, i8 0, i64 %Len, i1 false)
   ret void
 }
 
+; Test that a memset with a length argument that DAGCombiner will convert
+; into a constant get the correct number of bytes set.
+@Data = external hidden constant [1024 x i8], align 2
+define void @fun4() {
+; CHECK-LABEL: fun4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, Data
+; CHECK-NEXT:    xc 35(256,%r1), 35(%r1)
+; CHECK-NEXT:    xc 291(256,%r1), 291(%r1)
+; CHECK-NEXT:    xc 547(256,%r1), 547(%r1)
+; CHECK-NEXT:    xc 803(221,%r1), 803(%r1)
+; CHECK-NEXT:    mvghi 0(%r1), 989
+; CHECK-NEXT:    br %r14
+  call void @llvm.memset.p0i8.i64(
+     i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @Data, i64 0, i64 35),
+     i8 0,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 0) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 0, i64 35) to i64), i64 1)),
+     i1 false)
+  %i11 = getelementptr i8, i8* null,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 0) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 0, i64 35) to i64), i64 1))
+  store i8* %i11, i8** undef, align 8
+  ret void
+}
+
+; The same, with a resulting constant length of 0.
+define void @fun5() {
+; CHECK-LABEL: fun5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mvghi 0(%r1), 0
+; CHECK-NEXT:    br %r14
+  call void @llvm.memset.p0i8.i64(
+     i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @Data, i64 0, i64 35),
+     i8 0,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1)),
+     i1 false)
+  %i11 = getelementptr i8, i8* null,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1))
+  store i8* %i11, i8** undef, align 8
+  ret void
+}
+
+; The same, with a resulting constant length of 1.
+define void @fun6() {
+; CHECK-LABEL: fun6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, Data
+; CHECK-NEXT:    xc 35(1,%r1), 35(%r1)
+; CHECK-NEXT:    mvghi 0(%r1), 1
+; CHECK-NEXT:    br %r14
+  call void @llvm.memset.p0i8.i64(
+     i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @Data, i64 0, i64 35),
+     i8 0,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 36) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1)),
+     i1 false)
+  %i11 = getelementptr i8, i8* null,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 36) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1))
+  store i8* %i11, i8** undef, align 8
+  ret void
+}
+
+; The same, with a resulting constant length of 256.
+define void @fun7() {
+; CHECK-LABEL: fun7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, Data
+; CHECK-NEXT:    xc 35(256,%r1), 35(%r1)
+; CHECK-NEXT:    mvghi 0(%r1), 256
+; CHECK-NEXT:    br %r14
+  call void @llvm.memset.p0i8.i64(
+     i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @Data, i64 0, i64 35),
+     i8 0,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 291) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1)),
+     i1 false)
+  %i11 = getelementptr i8, i8* null,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 291) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1))
+  store i8* %i11, i8** undef, align 8
+  ret void
+}
+
+; The same, with a resulting constant length of 257.
+define void @fun8() {
+; CHECK-LABEL: fun8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, Data
+; CHECK-NEXT:    xc 35(256,%r1), 35(%r1)
+; CHECK-NEXT:    xc 291(1,%r1), 291(%r1)
+; CHECK-NEXT:    mvghi 0(%r1), 257
+; CHECK-NEXT:    br %r14
+  call void @llvm.memset.p0i8.i64(
+     i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @Data, i64 0, i64 35),
+     i8 0,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 292) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1)),
+     i1 false)
+  %i11 = getelementptr i8, i8* null,
+     i64 sub (i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 292) to i64), i64 1),
+              i64 add (i64 ptrtoint (i8* getelementptr inbounds ([1024 x i8],
+                                     [1024 x i8]* @Data, i64 1, i64 35) to i64), i64 1))
+  store i8* %i11, i8** undef, align 8
+  ret void
+}
+
 ; CHECK:       .Ltmp2:
 ; CHECK-NEXT:    xc 0(1,%r1), 0(%r1)
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    xc 0(1,%r2), 0(%r2)
 ; CHECK-NEXT:  .Ltmp1:
 ; CHECK-NEXT:    xc 0(1,%r3), 0(%r3)
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
 declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1 immarg)

llvm/test/CodeGen/SystemZ/mverify-optypes.mir

bb.0 (%ir-block.0):
 
     %3:addr64bit = AG %3, %1, 0, %2, implicit-def dead $cc
     %3:addr64bit = AG %3, %1, %0, %2, implicit-def dead $cc
     %3:addr64bit = AG %3, %1, 0, 2, implicit-def dead $cc
 
     %4:addr64bit = LARL @gsrc
     %4:addr64bit = LARL $r2l
 
-    MVCLoop %4, 0, %3, 0, 1680, %0, implicit-def $cc
-    MVCLoop %4, 0, %3, 0, %1, %0, implicit-def $cc
+    MVCImm %4, 0, %3, 0, 1680, implicit-def $cc
+    MVCImm %4, 0, %3, 0, %1, implicit-def $cc
 
     BCR 0, 0, $r2d, implicit $cc
     BCR 0, $r2d, $r2d, implicit $cc
     Return implicit $r2d
 
 ...
 
 # CHECK: *** Bad machine code: Expected a non-register operand. ***

 # CHECK: *** Bad machine code: Addressing mode operands corrupt! ***
 # CHECK: - instruction: %3:addr64bit = AG %3:addr64bit(tied-def 0), %1:addr64bit, 0, 2, implicit-def dead $cc
 
 # CHECK: *** Bad machine code: Expected a non-register operand. ***
 # CHECK: - instruction: %4:addr64bit = LARL $r2l
 # CHECK: - operand 1:   $r2l
 
 # CHECK: *** Bad machine code: Expected a non-register operand. ***
-# CHECK: - instruction: MVCLoop %4:addr64bit, 0, %3:addr64bit, 0, %1:addr64bit, %0:gr64bit, implicit-def $cc
+# CHECK: - instruction: MVCImm %4:addr64bit, 0, %3:addr64bit, 0, %1:addr64bit, implicit-def $cc
 # CHECK: - operand 4:   %1:addr64bit
 
 # CHECK: *** Bad machine code: Expected a non-register operand. ***
 # CHECK: - instruction: BCR 0, $r2d, $r2d, implicit $cc
 # CHECK: - operand 1:   $r2d