Index: llvm/lib/Target/SystemZ/SystemZInstrFP.td
===================================================================
--- llvm/lib/Target/SystemZ/SystemZInstrFP.td
+++ llvm/lib/Target/SystemZ/SystemZInstrFP.td
@@ -438,8 +438,8 @@
     def ADBR : BinaryRRE<"adbr", 0xB31A, any_fadd, FP64,  FP64>;
     def AXBR : BinaryRRE<"axbr", 0xB34A, any_fadd, FP128, FP128>;
   }
-  def AEB : BinaryRXE<"aeb", 0xED0A, any_fadd, FP32, load, 4>;
-  def ADB : BinaryRXE<"adb", 0xED1A, any_fadd, FP64, load, 8>;
+  defm AEB : BinaryRXEAndPseudo<"aeb", 0xED0A, any_fadd, FP32, load, 4>;
+  defm ADB : BinaryRXEAndPseudo<"adb", 0xED1A, any_fadd, FP64, load, 8>;
 }
 
 // Subtraction.
@@ -449,8 +449,8 @@
   def SDBR : BinaryRRE<"sdbr", 0xB31B, any_fsub, FP64,  FP64>;
   def SXBR : BinaryRRE<"sxbr", 0xB34B, any_fsub, FP128, FP128>;
 
-  def SEB : BinaryRXE<"seb",  0xED0B, any_fsub, FP32, load, 4>;
-  def SDB : BinaryRXE<"sdb",  0xED1B, any_fsub, FP64, load, 8>;
+  defm SEB : BinaryRXEAndPseudo<"seb",  0xED0B, any_fsub, FP32, load, 4>;
+  defm SDB : BinaryRXEAndPseudo<"sdb",  0xED1B, any_fsub, FP64, load, 8>;
 }
 
 // Multiplication.
@@ -460,8 +460,8 @@
     def MDBR  : BinaryRRE<"mdbr",  0xB31C, any_fmul, FP64,  FP64>;
     def MXBR  : BinaryRRE<"mxbr",  0xB34C, any_fmul, FP128, FP128>;
   }
-  def MEEB : BinaryRXE<"meeb", 0xED17, any_fmul, FP32, load, 4>;
-  def MDB  : BinaryRXE<"mdb",  0xED1C, any_fmul, FP64, load, 8>;
+  defm MEEB : BinaryRXEAndPseudo<"meeb", 0xED17, any_fmul, FP32, load, 4>;
+  defm MDB  : BinaryRXEAndPseudo<"mdb",  0xED1C, any_fmul, FP64, load, 8>;
 }
 
 // f64 multiplication of two FP32 registers.
@@ -503,8 +503,8 @@
   def MAEBR : TernaryRRD<"maebr", 0xB30E, z_any_fma, FP32, FP32>;
   def MADBR : TernaryRRD<"madbr", 0xB31E, z_any_fma, FP64, FP64>;
 
-  def MAEB : TernaryRXF<"maeb", 0xED0E, z_any_fma, FP32, FP32, load, 4>;
-  def MADB : TernaryRXF<"madb", 0xED1E, z_any_fma, FP64, FP64, load, 8>;
+  defm MAEB : TernaryRXFAndPseudo<"maeb", 0xED0E, z_any_fma, FP32, FP32, load, 4>;
+  defm MADB : TernaryRXFAndPseudo<"madb", 0xED1E, z_any_fma, FP64, FP64, load, 8>;
 }
 
 // Fused multiply-subtract.
@@ -512,8 +512,8 @@
   def MSEBR : TernaryRRD<"msebr", 0xB30F, z_any_fms, FP32, FP32>;
   def MSDBR : TernaryRRD<"msdbr", 0xB31F, z_any_fms, FP64, FP64>;
 
-  def MSEB : TernaryRXF<"mseb", 0xED0F, z_any_fms, FP32, FP32, load, 4>;
-  def MSDB : TernaryRXF<"msdb", 0xED1F, z_any_fms, FP64, FP64, load, 8>;
+  defm MSEB : TernaryRXFAndPseudo<"mseb", 0xED0F, z_any_fms, FP32, FP32, load, 4>;
+  defm MSDB : TernaryRXFAndPseudo<"msdb", 0xED1F, z_any_fms, FP64, FP64, load, 8>;
 }
 
 // Division.
@@ -522,8 +522,8 @@
   def DDBR : BinaryRRE<"ddbr", 0xB31D, any_fdiv, FP64,  FP64>;
   def DXBR : BinaryRRE<"dxbr", 0xB34D, any_fdiv, FP128, FP128>;
 
-  def DEB : BinaryRXE<"deb", 0xED0D, any_fdiv, FP32, load, 4>;
-  def DDB : BinaryRXE<"ddb", 0xED1D, any_fdiv, FP64, load, 8>;
+  defm DEB : BinaryRXEAndPseudo<"deb", 0xED0D, any_fdiv, FP32, load, 4>;
+  defm DDB : BinaryRXEAndPseudo<"ddb", 0xED1D, any_fdiv, FP64, load, 8>;
 }
 
 // Divide to integer.
Index: llvm/lib/Target/SystemZ/SystemZInstrFormats.td
===================================================================
--- llvm/lib/Target/SystemZ/SystemZInstrFormats.td
+++ llvm/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -2911,13 +2911,15 @@
 
 class UnaryVRRa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
                 TypedReg tr1, TypedReg tr2, bits<4> type = 0, bits<4> m4 = 0,
-                bits<4> m5 = 0>
+                bits<4> m5 = 0, string fp_mnemonic = "">
   : InstVRRa<opcode, (outs tr1.op:$V1), (ins tr2.op:$V2),
              mnemonic#"\t$V1, $V2",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V2)))]> {
   let M3 = type;
   let M4 = m4;
   let M5 = m5;
+  let OpKey = fp_mnemonic#!subst("VR", "FP", !cast<string>(tr1.op));
+  let OpType = "reg";
 }
 
 class UnaryVRRaGeneric<string mnemonic, bits<16> opcode, bits<4> m4 = 0,
@@ -3627,7 +3629,7 @@
 
 class BinaryVRRc<string mnemonic, bits<16> opcode, SDPatternOperator operator,
                  TypedReg tr1, TypedReg tr2, bits<4> type = 0, bits<4> m5 = 0,
-                 bits<4> m6 = 0>
+                 bits<4> m6 = 0, string fp_mnemonic = "">
   : InstVRRc<opcode, (outs tr1.op:$V1), (ins tr2.op:$V2, tr2.op:$V3),
              mnemonic#"\t$V1, $V2, $V3",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V2),
@@ -3635,6 +3637,8 @@
   let M4 = type;
   let M5 = m5;
   let M6 = m6;
+  let OpKey = fp_mnemonic#"MemFold"#!subst("VR", "FP", !cast<string>(tr1.op));
+  let OpType = "reg";
 }
 
 class BinaryVRRcGeneric<string mnemonic, bits<16> opcode, bits<4> m5 = 0,
@@ -3986,7 +3990,7 @@
 }
 
 class CompareVRRa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                  TypedReg tr, bits<4> type>
+                  TypedReg tr, bits<4> type, string fp_mnemonic = "">
   : InstVRRa<opcode, (outs), (ins tr.op:$V1, tr.op:$V2),
              mnemonic#"\t$V1, $V2",
              [(set CC, (operator (tr.vt tr.op:$V1), (tr.vt tr.op:$V2)))]> {
@@ -3994,6 +3998,8 @@
   let M3 = type;
   let M4 = 0;
   let M5 = 0;
+  let OpKey = fp_mnemonic#!subst("VR", "FP", !cast<string>(tr.op));
+  let OpType = "reg";
 }
 
 class CompareVRRaGeneric<string mnemonic, bits<16> opcode>
@@ -4407,7 +4413,8 @@
 }
 
 class TernaryVRRe<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                  TypedReg tr1, TypedReg tr2, bits<4> m5 = 0, bits<4> type = 0>
+                  TypedReg tr1, TypedReg tr2, bits<4> m5 = 0, bits<4> type = 0,
+                  string fp_mnemonic = "">
   : InstVRRe<opcode, (outs tr1.op:$V1),
              (ins tr2.op:$V2, tr2.op:$V3, tr1.op:$V4),
              mnemonic#"\t$V1, $V2, $V3, $V4",
@@ -4416,6 +4423,9 @@
                                                   (tr1.vt tr1.op:$V4)))]> {
   let M5 = m5;
   let M6 = type;
+  let OpKey = mnemonic#tr1.op;
+  let OpKey = fp_mnemonic#"MemFold"#!subst("VR", "FP", !cast<string>(tr1.op));
+  let OpType = "reg";
 }
 
 class TernaryVRReFloatGeneric<string mnemonic, bits<16> opcode>
@@ -4785,6 +4795,26 @@
     let hasNoSchedulingInfo = 1;
 }
 
+// Same as MemFoldPseudo but for mapping a W... vector instruction
+class MemFoldPseudo_FP<string mnemonic, RegisterOperand cls, bits<5> bytes,
+                    AddressingMode mode>
+  : MemFoldPseudo<mnemonic, cls, bytes, mode> {
+    let OpKey = mnemonic#"r"#"MemFold"#cls;
+}
+
+class MemFoldPseudo_FPTern<string mnemonic, RegisterOperand cls, bits<5> bytes,
+                           AddressingMode mode>
+  : Pseudo<(outs cls:$R1), (ins cls:$R2, cls:$R3, mode:$XBD2), []> {
+    let OpKey = mnemonic#"r"#"MemFold"#cls;
+    let OpType = "mem";
+    let MemKey = mnemonic#cls;
+    let MemType = "pseudo";
+    let mayLoad = 1;
+    let AccessBytes = bytes;
+    let HasIndex = 1;
+    let hasNoSchedulingInfo = 1;
+}
+
 // Same as MemFoldPseudo but for Load On Condition with CC operands.
 class MemFoldPseudo_CondMove<string mnemonic, RegisterOperand cls, bits<5> bytes,
                              AddressingMode mode>
@@ -5072,7 +5102,6 @@
                               SDPatternOperator operator, RegisterOperand cls,
                               SDPatternOperator load, bits<5> bytes,
                               AddressingMode mode = bdxaddr20only> {
-
   def "" : BinaryRXY<mnemonic, opcode, operator, cls, load, bytes, mode> {
     let MemKey = mnemonic#cls;
     let MemType = "target";
@@ -5099,6 +5128,27 @@
   def _MemFoldPseudo : MemFoldPseudo<mnemonic, cls, bytes, bdxaddr12pair>;
 }
 
+multiclass BinaryRXEAndPseudo<string mnemonic, bits<16> opcode,
+                              SDPatternOperator operator, RegisterOperand cls,
+                              SDPatternOperator load, bits<5> bytes> {
+  def "" : BinaryRXE<mnemonic, opcode, operator, cls, load, bytes> {
+    let MemKey = mnemonic#cls;
+    let MemType = "target";
+  }
+  def _MemFoldPseudo : MemFoldPseudo_FP<mnemonic, cls, bytes, bdxaddr12pair>;
+}
+
+multiclass TernaryRXFAndPseudo<string mnemonic, bits<16> opcode,
+                               SDPatternOperator operator, RegisterOperand cls1,
+                               RegisterOperand cls2, SDPatternOperator load,
+                               bits<5> bytes> {
+  def "" : TernaryRXF<mnemonic, opcode, operator, cls1, cls2, load, bytes> {
+    let MemKey = mnemonic#cls1;
+    let MemType = "target";
+  }
+  def _MemFoldPseudo : MemFoldPseudo_FPTern<mnemonic, cls1, bytes, bdxaddr12pair>;
+}
+
 multiclass CondUnaryRSYPairAndMemFold<string mnemonic, bits<16> opcode,
                                       SDPatternOperator operator,
                                       RegisterOperand cls, bits<5> bytes,
Index: llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
===================================================================
--- llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -993,33 +993,36 @@
     MachineBasicBlock::iterator InsertPt, int FrameIndex,
     LiveIntervals *LIS, VirtRegMap *VRM) const {
   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   unsigned Size = MFI.getObjectSize(FrameIndex);
   unsigned Opcode = MI.getOpcode();
 
+  // Check CC liveness if new instruction introduces a dead def of CC.
+  MCRegUnitIterator CCUnit(SystemZ::CC, TRI);
+  SlotIndex MISlot = SlotIndex();
+  LiveRange *CCLiveRange = nullptr;
+  bool CCLiveAtMI = true;
+  if (LIS) {
+    MISlot = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
+    CCLiveRange = &LIS->getRegUnit(*CCUnit);
+    CCLiveAtMI = CCLiveRange->liveAt(MISlot);
+  }
+  ++CCUnit;
+  assert(!CCUnit.isValid() && "CC only has one reg unit.");
+
   if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
-    if (LIS != nullptr && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) &&
+    if (!CCLiveAtMI && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) &&
         isInt<8>(MI.getOperand(2).getImm()) && !MI.getOperand(3).getReg()) {
-
-      // Check CC liveness, since new instruction introduces a dead
-      // def of CC.
-      MCRegUnitIterator CCUnit(SystemZ::CC, TRI);
-      LiveRange &CCLiveRange = LIS->getRegUnit(*CCUnit);
-      ++CCUnit;
-      assert(!CCUnit.isValid() && "CC only has one reg unit.");
-      SlotIndex MISlot =
-          LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
-      if (!CCLiveRange.liveAt(MISlot)) {
-        // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST
-        MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt,
-                                        MI.getDebugLoc(), get(SystemZ::AGSI))
-                                    .addFrameIndex(FrameIndex)
-                                    .addImm(0)
-                                    .addImm(MI.getOperand(2).getImm());
-        BuiltMI->findRegisterDefOperand(SystemZ::CC)->setIsDead(true);
-        CCLiveRange.createDeadDef(MISlot, LIS->getVNInfoAllocator());
-        return BuiltMI;
-      }
+      // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST
+      MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt,
+                                      MI.getDebugLoc(), get(SystemZ::AGSI))
+        .addFrameIndex(FrameIndex)
+        .addImm(0)
+        .addImm(MI.getOperand(2).getImm());
+      BuiltMI->findRegisterDefOperand(SystemZ::CC)->setIsDead(true);
+      CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator());
+      return BuiltMI;
     }
     return nullptr;
   }
@@ -1173,16 +1176,51 @@
   }
 
   // If the spilled operand is the final one or the instruction is
-  // commutable, try to change <INSN>R into <INSN>.
+  // commutable, try to change <INSN>R into <INSN>.  Don't introduce a def of
+  // CC if it is live and MI does not define it.
   unsigned NumOps = MI.getNumExplicitOperands();
   int MemOpcode = SystemZ::getMemOpcode(Opcode);
-  if (MemOpcode == -1)
+  if (MemOpcode == -1 ||
+      (CCLiveAtMI && !MI.definesRegister(SystemZ::CC) &&
+       get(MemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC)))
     return nullptr;
 
+  // Check if all other vregs have a usable allocation in the case of vector
+  // to FP conversion.
+  const MCInstrDesc &MCID = MI.getDesc();
+  for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) {
+    const MCOperandInfo &MCOI = MCID.OpInfo[I];
+    if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum)
+      continue;
+    const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass);
+    if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) {
+      Register Reg = MI.getOperand(I).getReg();
+      Register PhysReg = Register::isVirtualRegister(Reg)
+                             ? (VRM ? VRM->getPhys(Reg) : Register())
+                             : Reg;
+      if (!PhysReg ||
+          !(SystemZ::FP32BitRegClass.contains(PhysReg) ||
+            SystemZ::FP64BitRegClass.contains(PhysReg) ||
+            SystemZ::VF128BitRegClass.contains(PhysReg)))
+        return nullptr;
+    }
+  }
+  // Fused multiply and add/sub need to have the same dst and accumulator reg.
+  bool FusedFPOp = (Opcode == SystemZ::WFMADB || Opcode == SystemZ::WFMASB ||
+                    Opcode == SystemZ::WFMSDB || Opcode == SystemZ::WFMSSB);
+  if (FusedFPOp) {
+    Register DstReg = VRM->getPhys(MI.getOperand(0).getReg());
+    Register AccReg = VRM->getPhys(MI.getOperand(3).getReg());
+    if (OpNum == 0 || OpNum == 3 || DstReg != AccReg)
+      return nullptr;
+  }
+
   // Try to swap compare operands if possible.
   bool NeedsCommute = false;
   if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR ||
-       MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR) &&
+       MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR ||
+       MI.getOpcode() == SystemZ::WFCDB || MI.getOpcode() == SystemZ::WFCSB ||
+       MI.getOpcode() == SystemZ::WFKDB || MI.getOpcode() == SystemZ::WFKSB) &&
       OpNum == 0 && prepareCompareSwapOperands(MI))
     NeedsCommute = true;
 
@@ -1218,7 +1256,7 @@
     }
   }
 
-  if ((OpNum == NumOps - 1) || NeedsCommute) {
+  if ((OpNum == NumOps - 1) || NeedsCommute || FusedFPOp) {
     const MCInstrDesc &MemDesc = get(MemOpcode);
     uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags);
     assert(AccessBytes != 0 && "Size of access should be known");
@@ -1230,6 +1268,11 @@
       assert(NumOps == 2 && "Expected 2 register operands for a compare.");
       MIB.add(MI.getOperand(NeedsCommute ? 1 : 0));
     }
+    else if (FusedFPOp) {
+      MIB.add(MI.getOperand(0));
+      MIB.add(MI.getOperand(3));
+      MIB.add(MI.getOperand(OpNum == 1 ? 2 : 1));
+    }
     else {
       MIB.add(MI.getOperand(0));
       if (NeedsCommute)
@@ -1247,8 +1290,29 @@
       MIB.addImm(CCValid);
       MIB.addImm(NeedsCommute ? CCMask ^ CCValid : CCMask);
     }
+    if (MIB->definesRegister(SystemZ::CC) &&
+        (!MI.definesRegister(SystemZ::CC) ||
+         MI.registerDefIsDead(SystemZ::CC))) {
+      MIB->addRegisterDead(SystemZ::CC, TRI);
+      if (CCLiveRange)
+        CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator());
+    }
+    // Constrain the register classes if converted from a vector opcode. The
+    // allocated regs are in an FP reg-class per previous check above.
+    for (const MachineOperand &MO : MIB->operands())
+      if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) {
+        unsigned Reg = MO.getReg();
+        if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass)
+          MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass);
+        else if (MRI.getRegClass(Reg) == &SystemZ::VR64BitRegClass)
+          MRI.setRegClass(Reg, &SystemZ::FP64BitRegClass);
+        else if (MRI.getRegClass(Reg) == &SystemZ::VR128BitRegClass)
+          MRI.setRegClass(Reg, &SystemZ::VF128BitRegClass);
+      }
+
     transferDeadCC(&MI, MIB);
     transferMIFlag(&MI, MIB, MachineInstr::NoSWrap);
+    transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept);
     return MIB;
   }
 
Index: llvm/lib/Target/SystemZ/SystemZInstrVector.td
===================================================================
--- llvm/lib/Target/SystemZ/SystemZInstrVector.td
+++ llvm/lib/Target/SystemZ/SystemZInstrVector.td
@@ -1039,10 +1039,12 @@
   let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFA   : BinaryVRRcFloatGeneric<"vfa", 0xE7E3>;
     def VFADB : BinaryVRRc<"vfadb", 0xE7E3, any_fadd, v128db, v128db, 3, 0>;
-    def WFADB : BinaryVRRc<"wfadb", 0xE7E3, any_fadd, v64db, v64db, 3, 8>;
+    def WFADB : BinaryVRRc<"wfadb", 0xE7E3, any_fadd, v64db, v64db, 3, 8, 0,
+                           "adbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
       def VFASB : BinaryVRRc<"vfasb", 0xE7E3, any_fadd, v128sb, v128sb, 2, 0>;
-      def WFASB : BinaryVRRc<"wfasb", 0xE7E3, any_fadd, v32sb, v32sb, 2, 8>;
+      def WFASB : BinaryVRRc<"wfasb", 0xE7E3, any_fadd, v32sb, v32sb, 2, 8, 0,
+                             "aebr">;
       def WFAXB : BinaryVRRc<"wfaxb", 0xE7E3, any_fadd, v128xb, v128xb, 4, 8>;
     }
   }
@@ -1123,10 +1125,12 @@
   let Uses = [FPC], mayRaiseFPException = 1 in {
     def VFD   : BinaryVRRcFloatGeneric<"vfd", 0xE7E5>;
     def VFDDB : BinaryVRRc<"vfddb", 0xE7E5, any_fdiv, v128db, v128db, 3, 0>;
-    def WFDDB : BinaryVRRc<"wfddb", 0xE7E5, any_fdiv, v64db, v64db, 3, 8>;
+    def WFDDB : BinaryVRRc<"wfddb", 0xE7E5, any_fdiv, v64db, v64db, 3, 8, 0,
+                           "ddbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
       def VFDSB : BinaryVRRc<"vfdsb", 0xE7E5, any_fdiv, v128sb, v128sb, 2, 0>;
-      def WFDSB : BinaryVRRc<"wfdsb", 0xE7E5, any_fdiv, v32sb, v32sb, 2, 8>;
+      def WFDSB : BinaryVRRc<"wfdsb", 0xE7E5, any_fdiv, v32sb, v32sb, 2, 8, 0,
+                             "debr">;
       def WFDXB : BinaryVRRc<"wfdxb", 0xE7E5, any_fdiv, v128xb, v128xb, 4, 8>;
     }
   }
@@ -1154,7 +1158,8 @@
   let Uses = [FPC], mayRaiseFPException = 1 in {
     def VLDE  : UnaryVRRaFloatGeneric<"vlde", 0xE7C4>;
     def VLDEB : UnaryVRRa<"vldeb", 0xE7C4, z_any_vextend, v128db, v128sb, 2, 0>;
-    def WLDEB : UnaryVRRa<"wldeb", 0xE7C4, any_fpextend, v64db, v32sb, 2, 8>;
+    def WLDEB : UnaryVRRa<"wldeb", 0xE7C4, any_fpextend, v64db, v32sb, 2, 8, 0,
+                          "ldebr">;
   }
   let Predicates = [FeatureVectorEnhancements1] in {
     let Uses = [FPC], mayRaiseFPException = 1 in {
@@ -1247,10 +1252,12 @@
   let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFM   : BinaryVRRcFloatGeneric<"vfm", 0xE7E7>;
     def VFMDB : BinaryVRRc<"vfmdb", 0xE7E7, any_fmul, v128db, v128db, 3, 0>;
-    def WFMDB : BinaryVRRc<"wfmdb", 0xE7E7, any_fmul, v64db, v64db, 3, 8>;
+    def WFMDB : BinaryVRRc<"wfmdb", 0xE7E7, any_fmul, v64db, v64db, 3, 8, 0,
+                           "mdbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
       def VFMSB : BinaryVRRc<"vfmsb", 0xE7E7, any_fmul, v128sb, v128sb, 2, 0>;
-      def WFMSB : BinaryVRRc<"wfmsb", 0xE7E7, any_fmul, v32sb, v32sb, 2, 8>;
+      def WFMSB : BinaryVRRc<"wfmsb", 0xE7E7, any_fmul, v32sb, v32sb, 2, 8, 0,
+                             "meebr">;
       def WFMXB : BinaryVRRc<"wfmxb", 0xE7E7, any_fmul, v128xb, v128xb, 4, 8>;
     }
   }
@@ -1259,10 +1266,12 @@
   let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFMA   : TernaryVRReFloatGeneric<"vfma", 0xE78F>;
     def VFMADB : TernaryVRRe<"vfmadb", 0xE78F, any_fma, v128db, v128db, 0, 3>;
-    def WFMADB : TernaryVRRe<"wfmadb", 0xE78F, any_fma, v64db, v64db, 8, 3>;
+    def WFMADB : TernaryVRRe<"wfmadb", 0xE78F, any_fma, v64db, v64db, 8, 3,
+                             "madbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
       def VFMASB : TernaryVRRe<"vfmasb", 0xE78F, any_fma, v128sb, v128sb, 0, 2>;
-      def WFMASB : TernaryVRRe<"wfmasb", 0xE78F, any_fma, v32sb, v32sb, 8, 2>;
+      def WFMASB : TernaryVRRe<"wfmasb", 0xE78F, any_fma, v32sb, v32sb, 8, 2,
+                               "maebr">;
       def WFMAXB : TernaryVRRe<"wfmaxb", 0xE78F, any_fma, v128xb, v128xb, 8, 4>;
     }
   }
@@ -1271,10 +1280,12 @@
   let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFMS   : TernaryVRReFloatGeneric<"vfms", 0xE78E>;
     def VFMSDB : TernaryVRRe<"vfmsdb", 0xE78E, any_fms, v128db, v128db, 0, 3>;
-    def WFMSDB : TernaryVRRe<"wfmsdb", 0xE78E, any_fms, v64db, v64db, 8, 3>;
+    def WFMSDB : TernaryVRRe<"wfmsdb", 0xE78E, any_fms, v64db, v64db, 8, 3,
+                             "msdbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
       def VFMSSB : TernaryVRRe<"vfmssb", 0xE78E, any_fms, v128sb, v128sb, 0, 2>;
-      def WFMSSB : TernaryVRRe<"wfmssb", 0xE78E, any_fms, v32sb, v32sb, 8, 2>;
+      def WFMSSB : TernaryVRRe<"wfmssb", 0xE78E, any_fms, v32sb, v32sb, 8, 2,
+                               "msebr">;
       def WFMSXB : TernaryVRRe<"wfmsxb", 0xE78E, any_fms, v128xb, v128xb, 8, 4>;
     }
   }
@@ -1342,10 +1353,12 @@
   let Uses = [FPC], mayRaiseFPException = 1 in {
     def VFSQ   : UnaryVRRaFloatGeneric<"vfsq", 0xE7CE>;
     def VFSQDB : UnaryVRRa<"vfsqdb", 0xE7CE, any_fsqrt, v128db, v128db, 3, 0>;
-    def WFSQDB : UnaryVRRa<"wfsqdb", 0xE7CE, any_fsqrt, v64db, v64db, 3, 8>;
+    def WFSQDB : UnaryVRRa<"wfsqdb", 0xE7CE, any_fsqrt, v64db, v64db, 3, 8, 0,
+                           "sqdbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
       def VFSQSB : UnaryVRRa<"vfsqsb", 0xE7CE, any_fsqrt, v128sb, v128sb, 2, 0>;
-      def WFSQSB : UnaryVRRa<"wfsqsb", 0xE7CE, any_fsqrt, v32sb, v32sb, 2, 8>;
+      def WFSQSB : UnaryVRRa<"wfsqsb", 0xE7CE, any_fsqrt, v32sb, v32sb, 2, 8, 0,
+                             "sqebr">;
       def WFSQXB : UnaryVRRa<"wfsqxb", 0xE7CE, any_fsqrt, v128xb, v128xb, 4, 8>;
     }
   }
@@ -1354,10 +1367,12 @@
   let Uses = [FPC], mayRaiseFPException = 1 in {
     def VFS   : BinaryVRRcFloatGeneric<"vfs", 0xE7E2>;
     def VFSDB : BinaryVRRc<"vfsdb", 0xE7E2, any_fsub, v128db, v128db, 3, 0>;
-    def WFSDB : BinaryVRRc<"wfsdb", 0xE7E2, any_fsub, v64db, v64db, 3, 8>;
+    def WFSDB : BinaryVRRc<"wfsdb", 0xE7E2, any_fsub, v64db, v64db, 3, 8, 0,
+                           "sdbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
       def VFSSB : BinaryVRRc<"vfssb", 0xE7E2, any_fsub, v128sb, v128sb, 2, 0>;
-      def WFSSB : BinaryVRRc<"wfssb", 0xE7E2, any_fsub, v32sb, v32sb, 2, 8>;
+      def WFSSB : BinaryVRRc<"wfssb", 0xE7E2, any_fsub, v32sb, v32sb, 2, 8, 0,
+                             "sebr">;
       def WFSXB : BinaryVRRc<"wfsxb", 0xE7E2, any_fsub, v128xb, v128xb, 4, 8>;
     }
   }
@@ -1383,9 +1398,9 @@
   // Compare scalar.
   let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
     def WFC   : CompareVRRaFloatGeneric<"wfc", 0xE7CB>;
-    def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_any_fcmp, v64db, 3>;
+    def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_any_fcmp, v64db, 3, "cdbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
-      def WFCSB : CompareVRRa<"wfcsb", 0xE7CB, z_any_fcmp, v32sb, 2>;
+      def WFCSB : CompareVRRa<"wfcsb", 0xE7CB, z_any_fcmp, v32sb, 2, "cebr">;
       def WFCXB : CompareVRRa<"wfcxb", 0xE7CB, z_any_fcmp, v128xb, 4>;
     }
   }
@@ -1393,9 +1408,9 @@
   // Compare and signal scalar.
   let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
     def WFK   : CompareVRRaFloatGeneric<"wfk", 0xE7CA>;
-    def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, z_strict_fcmps, v64db, 3>;
+    def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, z_strict_fcmps, v64db, 3, "kdbr">;
     let Predicates = [FeatureVectorEnhancements1] in {
-      def WFKSB : CompareVRRa<"wfksb", 0xE7CA, z_strict_fcmps, v32sb, 2>;
+      def WFKSB : CompareVRRa<"wfksb", 0xE7CA, z_strict_fcmps, v32sb, 2, "kebr">;
       def WFKXB : CompareVRRa<"wfkxb", 0xE7CA, z_strict_fcmps, v128xb, 4>;
     }
   }
Index: llvm/test/CodeGen/SystemZ/foldmemop-vec-binops.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/SystemZ/foldmemop-vec-binops.mir
@@ -0,0 +1,828 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -start-before=greedy %s -o - \
+# RUN:   | FileCheck %s
+#
+# Test folding of a memory operand into an fp memory instruction.
+
+--- |
+  define void @fun0(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun1(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun2(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun3(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun4(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun5(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun6(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun7(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun8(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun9(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun10(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun11(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun12(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun13(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun14(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun15(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun16(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun17(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun18(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun19(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun20(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun21(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun22(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun23(float %arg0, float %arg1, float* %Dst) { ret void }
+
+...
+
+# Test with both orders of operands since some operations are commutative.
+
+# CHECK-LABEL: fun0:
+# CHECK: adb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun0
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFADB %0, %1, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun1:
+# CHECK: adb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun1
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFADB %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+# ADB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun2:
+# CHECK: wfadb   %f0, %v16, %f0
+---
+name:            fun2
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFADB %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun3:
+# CHECK: aeb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun3
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFASB %0, %1, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun4:
+# CHECK: aeb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun4
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFASB %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# AEB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun5:
+# CHECK: wfasb   %f0, %v16, %f0
+---
+name:            fun5
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFASB %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun6:
+# CHECK: sdb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun6
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFSDB %0, %1, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun7:
+# CHECK: wfsdb   %f0, %f1, %f0
+---
+name:            fun7
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFSDB %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+# SDB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun8:
+# CHECK: wfsdb   %f0, %f0, %v16
+---
+name:            fun8
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFSDB %0, %1, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun9:
+# CHECK: seb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun9
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFSSB %0, %1, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun10:
+# CHECK: wfssb   %f0, %f1, %f0
+---
+name:            fun10
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFSSB %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# SEB can't be used if one operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun11:
+# CHECK: wfssb   %f0, %f0, %v16
+---
+name:            fun11
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFSSB %0, %1, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+# CHECK-LABEL: fun12:
+# CHECK: ddb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun12
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFDDB %0, %1, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun13:
+# CHECK: wfddb   %f0, %f1, %f0
+---
+name:            fun13
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFDDB %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+# DDB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun14:
+# CHECK: wfddb   %f0, %f0, %v16
+---
+name:            fun14
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFDDB %0, %1, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun15:
+# CHECK: deb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun15
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFDSB %0, %1, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun16:
+# CHECK: wfdsb   %f0, %f1, %f0
+---
+name:            fun16
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFDSB %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# DEB can't be used if one operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun17:
+# CHECK: wfdsb   %f0, %f0, %v16
+---
+name:            fun17
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFDSB %0, %1, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun18:
+# CHECK: mdb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun18
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMDB %0, %1, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun19:
+# CHECK: mdb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun19
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMDB %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+# MDB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun20:
+# CHECK: wfmdb   %f0, %v16, %f0
+---
+name:            fun20
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMDB %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun21:
+# CHECK: meeb    %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun21
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMSB %0, %1, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun22:
+# CHECK: meeb    %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun22
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMSB %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
+
+
+# MEEB can't be used if one operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun23:
+# CHECK: wfmsb   %f0, %v16, %f0
+---
+name:            fun23
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMSB %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 4 into %ir.Dst)
+    Return
+
+...
Index: llvm/test/CodeGen/SystemZ/foldmemop-vec-cc.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/SystemZ/foldmemop-vec-cc.mir
@@ -0,0 +1,47 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -start-before=greedy %s -o - \
+# RUN:   | FileCheck %s
+#
+# Test that folding does not occur if it would introduce a clobbering of a live CC.
+
+--- |
+  define void @fun0(double %arg0, double %arg1, double* %Dst) { ret void }
+
+...
+
+# CHECK-LABEL: fun0:
+# CHECK: ld      %f1, 160(%r15)          # 8-byte Folded Reload
+# CHECK-NEXT: wfadb   %f0, %f0, %f1
+---
+name:            fun0
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    CDBR %0, %1, implicit-def $cc, implicit $fpc
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFADB %0, %1, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 14, 8, implicit killed $cc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
Index: llvm/test/CodeGen/SystemZ/foldmemop-vec-cmp.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/SystemZ/foldmemop-vec-cmp.mir
@@ -0,0 +1,498 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -start-before=greedy %s -o - \
+# RUN:   | FileCheck %s
+#
+# Test folding of a memory operand into an fp memory instruction.
+
+--- |
+  define void @fun0(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun1(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun2(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun3(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun4(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun5(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun6(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun7(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun8(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun9(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun10(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun11(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun12(float %arg0, float %arg1, float* %Dst) { ret void }
+
+...
+
+# CHECK-LABEL: fun0:
+# CHECK: cdb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun0
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFCDB %0, %1, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun1:
+# CHECK: cdb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun1
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFCDB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CDB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun2:
+# CHECK:      ld      %f0, 160(%r15)                  # 8-byte Folded Reload
+# CHECK-NEXT: wfcdb   %v16, %f0
+---
+name:            fun2
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFCDB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun3:
+# CHECK: ceb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun3
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFCSB %0, %1, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun4:
+# CHECK: ceb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun4
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFCSB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CEB can't be used if one operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun5:
+# CHECK:      lde     %f0, 164(%r15)                  # 4-byte Folded Reload
+# CHECK-NEXT: wfcsb   %v16, %f0
+---
+name:            fun5
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFCSB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun6:
+# CHECK: kdb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun6
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFKDB %0, %1, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun7:
+# CHECK: kdb     %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun7
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFKDB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# KDB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun8:
+# CHECK:      ld      %f0, 160(%r15)                  # 8-byte Folded Reload
+# CHECK-NEXT: wfkdb   %v16, %f0
+---
+name:            fun8
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFKDB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun9:
+# CHECK: keb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun9
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFKSB %0, %1, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun10:
+# CHECK: keb     %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun10
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFKSB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CEB can't be used if one operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun11:
+# CHECK:      lde     %f0, 164(%r15)                  # 4-byte Folded Reload
+# CHECK-NEXT: wfksb   %v16, %f0
+---
+name:            fun11
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    nofpexcept WFKSB %1, %0, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# This test case involves a 128 bit operand (VGMF generating float 1.0).
+# CHECK-LABEL: fun12:
+# CHECK: vgmf    %v0, 2, 8
+# CHECK-NEXT: ceb     %f0, 164(%r15)          # 4-byte Folded Reload
+---
+name:            fun12
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr128bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: gr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s, $f2s, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %1:vr128bit = VGMF 2, 8
+    nofpexcept WFCSB %0, %1.subreg_h32:vr128bit, implicit-def $cc, implicit $fpc
+    %4:gr64bit = LGHI 0
+    %4:gr64bit = LOCGHI %4, 1, 15, 8, implicit killed $cc
+    STG %4, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
Index: llvm/test/CodeGen/SystemZ/foldmemop-vec-fusedfp.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/SystemZ/foldmemop-vec-fusedfp.mir
@@ -0,0 +1,589 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -start-before=greedy %s -o - \
+# RUN:   | FileCheck %s
+#
+# Test folding of a memory operand into an fp memory instruction.
+
+--- |
+  define void @fun0(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun1(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun2(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun3(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun4(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun5(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun6(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun7(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun8(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun9(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun10(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun11(double %arg0, double %arg1, double* %Dst) { ret void }
+  define void @fun12(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun13(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun14(float %arg0, float %arg1, float* %Dst) { ret void }
+  define void @fun15(float %arg0, float %arg1, float* %Dst) { ret void }
+
+...
+
+# CHECK-LABEL: fun0:
+# CHECK: madb    %f0, %f1, 160(%r15)             # 8-byte Folded Reload
+---
+name:            fun0
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMADB %0, %1, %4, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun1:
+# CHECK: madb    %f0, %f1, 160(%r15)             # 8-byte Folded Reload
+---
+name:            fun1
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMADB %1, %0, %4, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+# MADB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun2:
+# CHECK: wfmadb  %f0, %v16, %f1, %f0
+---
+name:            fun2
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMADB %1, %0, %4, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# MADB can't be used unless accumulator and result registers are the same.
+# CHECK-LABEL: fun3:
+# CHECK: wfmadb  %f0, %f0, %f1, %f2
+---
+name:            fun3
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMADB %4, %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun4:
+# CHECK: maeb    %f0, %f1, 160(%r15)             # 4-byte Folded Reload
+---
+name:            fun4
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMASB %0, %1, %4, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun5:
+# CHECK: maeb    %f0, %f1, 160(%r15)             # 4-byte Folded Reload
+---
+name:            fun5
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMASB %1, %0, %4, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# MAEB can't be used if one operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun6:
+# CHECK: wfmasb  %f0, %v16, %f1, %f0
+---
+name:            fun6
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMASB %1, %0, %4, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# MAEB can't be used unless accumulator and result registers are the same.
+# CHECK-LABEL: fun7:
+# CHECK: wfmasb  %f0, %f0, %f1, %f2
+---
+name:            fun7
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMASB %4, %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun8:
+# CHECK: msdb    %f0, %f1, 160(%r15)             # 8-byte Folded Reload
+---
+name:            fun8
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMSDB %0, %1, %4, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun9:
+# CHECK: msdb    %f0, %f1, 160(%r15)             # 8-byte Folded Reload
+---
+name:            fun9
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMSDB %1, %0, %4, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+# MSDB can't be used if one operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun10:
+# CHECK: wfmsdb  %f0, %v16, %f1, %f0
+---
+name:            fun10
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMSDB %1, %0, %4, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# MSDB can't be used unless accumulator and result registers are the same.
+# CHECK-LABEL: fun11:
+# CHECK: wfmsdb  %f0, %f0, %f1, %f2
+---
+name:            fun11
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: fp64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr64bit }
+  - { id: 4, class: fp64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+  - { reg: '$f2d', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp64bit = COPY $f2d
+    %0:fp64bit = COPY $f0d
+    %4:fp64bit = COPY $f2d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr64bit = nofpexcept WFMSDB %4, %1, %0, implicit $fpc
+    VST64 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun12:
+# CHECK: mseb    %f0, %f1, 160(%r15)             # 4-byte Folded Reload
+---
+name:            fun12
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMSSB %0, %1, %4, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# CHECK-LABEL: fun13:
+# CHECK: mseb    %f0, %f1, 160(%r15)             # 4-byte Folded Reload
+---
+name:            fun13
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMSSB %1, %0, %4, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# MSEB can't be used if one operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun14:
+# CHECK: wfmssb  %f0, %v16, %f1, %f0
+---
+name:            fun14
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:vr32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMSSB %1, %0, %4, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
+
+
+# MSEB can't be used unless accumulator and result registers are the same.
+# CHECK-LABEL: fun15:
+# CHECK: wfmssb  %f0, %f0, %f1, %f2
+---
+name:            fun15
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: vr32bit }
+  - { id: 4, class: fp32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+  - { reg: '$f2s', virtual-reg: '%1' }
+  - { reg: '$r2d', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d, $f2d, $r2d
+  
+    %2:addr64bit = COPY $r2d
+    %1:fp32bit = COPY $f2s
+    %0:fp32bit = COPY $f0s
+    %4:fp32bit = COPY $f2s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %3:vr32bit = nofpexcept WFMSSB %4, %1, %0, implicit $fpc
+    VST32 %3, %2, 0, $noreg :: (store 8 into %ir.Dst)
+    Return
+
+...
Index: llvm/test/CodeGen/SystemZ/foldmemop-vec-unary.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/SystemZ/foldmemop-vec-unary.mir
@@ -0,0 +1,215 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -start-before=greedy %s -o - \
+# RUN:   | FileCheck %s
+#
+# Test folding of a memory operand into an fp memory instruction.
+
+--- |
+  define double @fun0(float %arg0) { ret double 0.0 }
+  define double @fun1(float %arg0) { ret double 0.0 }
+  define double @fun2(float %arg0) { ret double 0.0 }
+  define double @fun3(double %arg0) { ret double 0.0 }
+  define double @fun4(double %arg0) { ret double 0.0 }
+  define float @fun5(float %arg0) { ret float 0.0 }
+  define float @fun6(float %arg0) { ret float 0.0 }
+
+...
+
+
+# CHECK-LABEL: fun0:
+# CHECK: ldeb    %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun0
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s
+  
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %1:vr64bit = nofpexcept WLDEB %0, implicit $fpc
+    $f0d = COPY %1
+    Return implicit $f0d
+
+...
+
+
+# LDEB can't be used if dst operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun1:
+# CHECK:      lde     %f0, 164(%r15)                  # 4-byte Folded Reload
+# CHECK-NEXT: wldeb   %v16, %f0
+---
+name:            fun1
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s
+  
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %1:vr64bit = nofpexcept WLDEB %0, implicit $fpc
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    $f0d = COPY %1
+    Return implicit $f0d
+
+...
+
+
+# Spilling the destination of an fp extension needs an extra store instruction.
+# CHECK-LABEL: fun2:
+# CHECK:      ldebr   %f0, %f0
+# CHECK-NEXT: std     %f0, 160(%r15)                  # 8-byte Folded Spill
+---
+name:            fun2
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: fp64bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s
+  
+    %0:fp32bit = COPY $f0s
+    %1:fp64bit = nofpexcept WLDEB %0, implicit $fpc
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    $f0d = COPY %1
+    Return implicit $f0d
+
+...
+
+
+# CHECK-LABEL: fun3:
+# CHECK: sqdb    %f0, 160(%r15)                  # 8-byte Folded Reload
+---
+name:            fun3
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d
+  
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %1:vr64bit = nofpexcept WFSQDB %0, implicit $fpc
+    $f0d = COPY %1
+    Return implicit $f0d
+
+...
+
+
+# SQDB can't be used if dst operand is a VR64 (and not FP64).
+# CHECK-LABEL: fun4:
+# CHECK:      ld      %f0, 160(%r15)                  # 8-byte Folded Reload
+# CHECK-NEXT: wfsqdb  %v16, %f0
+---
+name:            fun4
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp64bit }
+  - { id: 1, class: vr64bit }
+liveins:
+  - { reg: '$f0d', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0d
+  
+    %0:fp64bit = COPY $f0d
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %1:vr64bit = nofpexcept WFSQDB %0, implicit $fpc
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    $f0d = COPY %1
+    Return implicit $f0d
+
+...
+
+
+# CHECK-LABEL: fun5:
+# CHECK: sqeb    %f0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun5
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s
+  
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %1:vr32bit = nofpexcept WFSQSB %0, implicit $fpc
+    $f0s = COPY %1
+    Return implicit $f0d
+
+...
+
+
+# SQEB can't be used if dst operand is a VR32 (and not FP32).
+# CHECK-LABEL: fun6:
+# CHECK:      lde     %f0, 164(%r15)                  # 4-byte Folded Reload
+# CHECK-NEXT: wfsqsb  %v16, %f0
+---
+name:            fun6
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fp32bit }
+  - { id: 1, class: vr32bit }
+liveins:
+  - { reg: '$f0s', virtual-reg: '%0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $f0s
+  
+    %0:fp32bit = COPY $f0s
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    %1:vr32bit = nofpexcept WFSQSB %0, implicit $fpc
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    $f0s = COPY %1
+    Return implicit $f0s
+
+...
Index: llvm/test/CodeGen/SystemZ/int-cmp-56.mir
===================================================================
--- llvm/test/CodeGen/SystemZ/int-cmp-56.mir
+++ llvm/test/CodeGen/SystemZ/int-cmp-56.mir
@@ -18,7 +18,7 @@
 # Test CGR -> CG
 # CHECK: name:            fun1
 # CHECK: CG %10, %stack.0, 0, $noreg, implicit-def $cc :: (load 8 from %stack.0)
-# CHECK-NEXT: %12:gr64bit = LOCGHI %12, 8, 14, 12, implicit killed $cc
+# CHECK-NEXT: %12:gr64bit = LOCGHI %12, 8, 14, 12, implicit $cc
 ---
 name:            fun1
 alignment:       16
@@ -95,7 +95,7 @@
 # Test CLGR -> CLG
 # CHECK: name:            fun2
 # CHECK: CLG %10, %stack.0, 0, $noreg, implicit-def $cc :: (load 8 from %stack.0)
-# CHECK-NEXT: %12:gr64bit = LOCGHI %12, 8, 14, 12, implicit killed $cc
+# CHECK-NEXT: %12:gr64bit = LOCGHI %12, 8, 14, 12, implicit $cc
 ---
 name:            fun2
 alignment:       16
@@ -172,7 +172,7 @@
 # Test CR -> C
 # CHECK: name:            fun3
 # CHECK: C %10, %stack.0, 0, $noreg, implicit-def $cc :: (load 4 from %stack.0)
-# CHECK: %12:gr32bit = LOCHIMux %12, 8, 14, 12, implicit killed $cc
+# CHECK: %12:gr32bit = LOCHIMux %12, 8, 14, 12, implicit $cc
 ---
 name:            fun3
 alignment:       16
@@ -249,7 +249,7 @@
 # Test CLR -> CL
 # CHECK: name:            fun4
 # CHECK: CL %10, %stack.0, 0, $noreg, implicit-def $cc :: (load 4 from %stack.0)
-# CHECK: %12:gr32bit = LOCHIMux %12, 8, 14, 12, implicit killed $cc
+# CHECK: %12:gr32bit = LOCHIMux %12, 8, 14, 12, implicit $cc
 ---
 name:            fun4
 alignment:       16