AMDGPU: Change m0 initialization handling to help LDS

[arsenm]

Summary

Initialize m0 to the default value for LDS in the entry block,
and remove the initialization around DS instruction uses.

Treat the LDS value as the default, and insert writes of the default around other uses.
Spills need to still do save restore, since we don't know the point where it is being spilled (and could be spilled in a sequence involving inlineasm).

This isn't an ideal solution. Unfortunately this needs to add m0 as a physreg live in to every block for now right after instruction selection which is discouraged. Inserting a copy from the initial value to m0 in each block works, but misses many of the cases where we want to eliminate m0 usage. The live ins are added too aggressively, making more defs appear alive than they really are. Better would be to always use save/restore, but there are missing optimizations to eliminate redundant ones. Also missing are optimizations to generally hoist the same m0 def into predecessor blocks. MachineLICM handles some, but it doesn't handle all loops, or diamonds and other simple control flow. The worst code quality regressions are around SGPR spills at -O0 when using scalar stores, but I'm not sure how much of a concern that is.

[tony-tye]

lib/Target/AMDGPU/SIInstrInfo.cpp

3791–3792

I would say neither. We want to totally disable the bounds checking for normal code, hence the MAXINT value seems the best choice.

An address sanitizer may want to insert range check code and pay the performance cost. But this would be on a per variable basis, not on the entire LDS. The hardware ensures that one wave cannot corrupt the LDS of another work-group regardless of M0 value.

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

private:
   bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs,
                    const R600InstrInfo *TII);
   bool FoldOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
   bool FoldDotOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
 
   bool isConstantLoad(const MemSDNode *N, int cbID) const;
   bool isUniformBr(const SDNode *N) const;
 
-  SDNode *glueCopyToM0(SDNode *N) const;
-
   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
   bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
   bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
                                        SDValue& Offset);
   bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
   bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
   bool isDSOffsetLegal(const SDValue &Base, unsigned Offset,
                        unsigned OffsetBits) const;
case AMDGPU::REG_SEQUENCE: {
     SDValue SubRegOp = N->getOperand(OpNo + 1);
     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
                                                               SubRegIdx);
   }
   }
 }
 
-SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
-  if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
-      cast<MemSDNode>(N)->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
-    return N;
-
-  const SITargetLowering& Lowering =
-      *static_cast<const SITargetLowering*>(getTargetLowering());
-
-  // Write max value to m0 before each load operation
-
-  SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
-                                 CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
-
-  SDValue Glue = M0.getValue(1);
-
-  SmallVector <SDValue, 8> Ops;
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-     Ops.push_back(N->getOperand(i));
-  }
-  Ops.push_back(Glue);
-  CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
-
-  return N;
-}
-
 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) {
   switch (NumVectorElts) {
   case 1:
     return AMDGPU::SReg_32_XM0RegClassID;
   case 2:
     return AMDGPU::SReg_64RegClassID;
   case 4:
     return AMDGPU::SReg_128RegClassID;
   case 8:
     return AMDGPU::SReg_256RegClassID;
   case 16:
     return AMDGPU::SReg_512RegClassID;
   }
 
   llvm_unreachable("invalid vector size");
 }
 
 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
   unsigned int Opc = N->getOpcode();
   if (N->isMachineOpcode()) {
     N->setNodeId(-1);
     return;   // Already selected.
   }
 
-  if (isa<AtomicSDNode>(N) ||
-      (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC))
-    N = glueCopyToM0(N);
-
   switch (Opc) {
   default: break;
   // We are selecting i64 ADD here instead of custom lower it during
   // DAG legalization, so we can fold some i64 ADDs used for address
   // calculation into the LOAD and STORE instructions.
   case ISD::ADD:
   case ISD::ADDC:
   case ISD::ADDE:
const SDValue Ops[] = {
       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
     };
 
     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
                                           N->getValueType(0), Ops));
     return;
   }
-  case ISD::LOAD:
-  case ISD::STORE: {
-    N = glueCopyToM0(N);
-    break;
-  }
-
   case AMDGPUISD::BFE_I32:
   case AMDGPUISD::BFE_U32: {
     if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
       break;
 
     // There is a scalar version available, but unlike the vector version which
     // has a separate operand for the offset and width, the scalar version packs
     // the width and offset into a single operand. Try to move to the scalar

lib/Target/AMDGPU/AMDGPUISelLowering.h

enum NodeType : unsigned {
   SAMPLED,
   SAMPLEL,
 
   // These cvt_f32_ubyte* nodes need to remain consecutive and in order.
   CVT_F32_UBYTE0,
   CVT_F32_UBYTE1,
   CVT_F32_UBYTE2,
   CVT_F32_UBYTE3,
+
+  // These are the same as the standard nodes except they have glue input/output
+  // for the implicit m0 use.
+  INSERT_VECTOR_ELT_INDIRECT,
+  EXTRACT_VECTOR_ELT_INDIRECT,
+
   /// This node is for VLIW targets and it is used to represent a vector
   /// that is stored in consecutive registers with the same channel.
   /// For example:
   ///   |X  |Y|Z|W|
   /// T0|v.x| | | |
   /// T1|v.y| | | |
   /// T2|v.z| | | |
   /// T3|v.w| | | |

lib/Target/AMDGPU/AMDGPUISelLowering.cpp

const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(SAMPLE)
   NODE_NAME_CASE(SAMPLEB)
   NODE_NAME_CASE(SAMPLED)
   NODE_NAME_CASE(SAMPLEL)
   NODE_NAME_CASE(CVT_F32_UBYTE0)
   NODE_NAME_CASE(CVT_F32_UBYTE1)
   NODE_NAME_CASE(CVT_F32_UBYTE2)
   NODE_NAME_CASE(CVT_F32_UBYTE3)
+  NODE_NAME_CASE(INSERT_VECTOR_ELT_INDIRECT)
+  NODE_NAME_CASE(EXTRACT_VECTOR_ELT_INDIRECT)
   NODE_NAME_CASE(BUILD_VERTICAL_VECTOR)
   NODE_NAME_CASE(CONST_DATA_PTR)
   NODE_NAME_CASE(PC_ADD_REL_OFFSET)
   NODE_NAME_CASE(KILL)
   NODE_NAME_CASE(DUMMY_CHAIN)
   case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
   NODE_NAME_CASE(SENDMSG)
   NODE_NAME_CASE(SENDMSGHALT)

lib/Target/AMDGPU/AMDGPUInstrInfo.td

 def AMDGPUFmasOp : SDTypeProfile<1, 4,
   [SDTCisFP<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisInt<4>]
 >;
 
 def AMDGPUKillSDT : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 
 //===----------------------------------------------------------------------===//
 // AMDGPU DAG Nodes
+
+def extractelt_indirect : SDNode<"AMDGPUISD::EXTRACT_VECTOR_ELT_INDIRECT", SDTVecExtract,
+  [SDNPOutGlue]
+>;
+def insertelt_indirect : SDNode<"AMDGPUISD::INSERT_VECTOR_ELT_INDIRECT", SDTVecInsert,
+  [SDNPOutGlue]
+>;
+
 //
 
 def AMDGPUconstdata_ptr : SDNode<
   "AMDGPUISD::CONST_DATA_PTR", SDTypeProfile <1, 1, [SDTCisVT<0, iPTR>,
                                                      SDTCisVT<0, iPTR>]>
 >;
 
 // This argument to this node is a dword address.

 def AMDGPUumed3 : SDNode<"AMDGPUISD::UMED3", AMDGPUDTIntTernaryOp,
   []
 >;
 
 def AMDGPUfmed3 : SDNode<"AMDGPUISD::FMED3", SDTFPTernaryOp, []>;
 
 def AMDGPUsendmsg : SDNode<"AMDGPUISD::SENDMSG",
                     SDTypeProfile<0, 1, [SDTCisInt<0>]>,
-                    [SDNPHasChain, SDNPInGlue]>;
+                    [SDNPHasChain, SDNPInGlue, SDNPOutGlue]>;
 
 def AMDGPUsendmsghalt : SDNode<"AMDGPUISD::SENDMSGHALT",
                     SDTypeProfile<0, 1, [SDTCisInt<0>]>,
-                    [SDNPHasChain, SDNPInGlue]>;
+                    [SDNPHasChain, SDNPInGlue, SDNPOutGlue]>;
 
 def AMDGPUinterp_mov : SDNode<"AMDGPUISD::INTERP_MOV",
                         SDTypeProfile<1, 3, [SDTCisFP<0>]>,
-                        [SDNPInGlue]>;
+                        [SDNPInGlue, SDNPOutGlue]>;
 
 def AMDGPUinterp_p1 : SDNode<"AMDGPUISD::INTERP_P1",
                       SDTypeProfile<1, 3, [SDTCisFP<0>]>,
                       [SDNPInGlue, SDNPOutGlue]>;
 
 def AMDGPUinterp_p2 : SDNode<"AMDGPUISD::INTERP_P2",
                       SDTypeProfile<1, 4, [SDTCisFP<0>]>,
-                      [SDNPInGlue]>;
+                      [SDNPInGlue, SDNPOutGlue]>;
 
 
 def AMDGPUkill : SDNode<"AMDGPUISD::KILL", AMDGPUKillSDT,
   [SDNPHasChain, SDNPSideEffect]>;
 
 // SI+ export
 def AMDGPUExportOp : SDTypeProfile<0, 8, [
   SDTCisInt<0>,       // i8 tgt

lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
   }
 }
 
 bool AMDGPUInstructionSelector::selectG_ADD(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
   MachineFunction *MF = BB->getParent();
   MachineRegisterInfo &MRI = MF->getRegInfo();
   unsigned Size = RBI.getSizeInBits(I.getOperand(0).getReg(), MRI, TRI);
-  unsigned DstLo = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-  unsigned DstHi = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+  unsigned DstLo = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
+  unsigned DstHi = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
 
   if (Size != 64)
     return false;
 
   DebugLoc DL = I.getDebugLoc();
 
   MachineOperand Lo1(getSubOperand64(I.getOperand(1), AMDGPU::sub0));
   MachineOperand Lo2(getSubOperand64(I.getOperand(2), AMDGPU::sub0));
bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
   if (Size == 32) {
     I.setDesc(TII.get(AMDGPU::S_MOV_B32));
     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
   }
 
   assert(Size == 64);
 
   DebugLoc DL = I.getDebugLoc();
-  unsigned LoReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-  unsigned HiReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+  unsigned LoReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
+  unsigned HiReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
   const APInt &Imm = I.getOperand(1).getCImm()->getValue();
 
   BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), LoReg)
           .addImm(Imm.trunc(32).getZExtValue());
 
   BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg)
           .addImm(Imm.ashr(32).getZExtValue());
 
if (Subtarget.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS &&
                                    .addReg(PtrReg)
                                    .addImm(EncodedImm)
                                    .addImm(0); // glc
       return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);
     }
 
     if (isUInt<32>(GEPInfo.Imm)) {
       Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_SGPR, LoadSize);
-      unsigned OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+      unsigned OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
       BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), OffsetReg)
               .addImm(GEPInfo.Imm);
 
       MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg)
                                    .addReg(PtrReg)
                                    .addReg(OffsetReg)
                                    .addImm(0); // glc
       return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

 /// If the PHI's definition class is constrained to <vgpr> then the coalescer
 /// will be unable to perform the COPY removal from the above example  which
 /// ultimately led to the creation of an illegal COPY.
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
 #include "AMDGPUSubtarget.h"
 #include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 
static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
   case AMDGPU::V_MOV_B64_PSEUDO:
     SMovOp = AMDGPU::S_MOV_B64;
     break;
   }
   Imm = ImmOp->getImm();
   return true;
 }
 
+static bool usesDefaultM0Value(const SIInstrInfo &TII, const MachineInstr &MI) {
+  return TII.isDS(MI) || TII.mayAccessFlatAddressSpace(MI);
+}
+
 bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
   const SIInstrInfo *TII = ST.getInstrInfo();
+  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+
   MDT = &getAnalysis<MachineDominatorTree>();
 
-  SmallVector<MachineInstr *, 16> Worklist;
 
+  MachineBasicBlock &Entry = *MF.begin();
+  for (const MachineInstr &MI : MRI.use_instructions(AMDGPU::M0)) {
+    if (usesDefaultM0Value(*TII, MI)) {
+      MFI->setNeedsM0Intialization();
+      break;
+    }
+  }
+
+  bool NeedM0 = false;
+  if (MFI->needsM0Initialization()) {
+    NeedM0 = true;
+    TII->emitSetM0ToDefaultValue(Entry, Entry.begin(), DebugLoc());
+  } else {
+    NeedM0 = MRI.isPhysRegUsed(AMDGPU::M0);
+    if (NeedM0) {
+      BuildMI(Entry, Entry.begin(), DebugLoc(),
+              TII->get(AMDGPU::IMPLICIT_DEF), AMDGPU::M0);
+    }
+  }
+
+  SmallVector<MachineInstr *, 16> Worklist;
   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
                                                   BI != BE; ++BI) {
 
     MachineBasicBlock &MBB = *BI;
+    if (NeedM0 && (&MBB != &Entry)) {
+      assert(!MBB.isLiveIn(AMDGPU::M0));
+      MBB.addLiveIn(AMDGPU::M0);
+    }
+
     for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
          I != E; ++I) {
       MachineInstr &MI = *I;
 
       switch (MI.getOpcode()) {
       default:
         continue;
       case AMDGPU::COPY: {

lib/Target/AMDGPU/SIISelLowering.h

public:
   MachineSDNode *wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL,
                                 SDValue Ptr) const;
   MachineSDNode *buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr,
                            uint32_t RsrcDword1, uint64_t RsrcDword2And3) const;
   std::pair<unsigned, const TargetRegisterClass *>
   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
                                StringRef Constraint, MVT VT) const override;
   ConstraintType getConstraintType(StringRef Constraint) const override;
-  SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL,
-                   SDValue V) const;
+
+  SDValue getNodeWithM0Use(SelectionDAG &DAG, unsigned Opc,
+                           const SDLoc &SL, EVT VT,
+                           SDValue Chain,  ArrayRef<SDValue> Ops,
+                           SDValue M0Val) const;
+
+  SDValue getNodeWithM0SaveRestore(SelectionDAG &DAG, unsigned Opc,
+                                   const SDLoc &SL, EVT VT,
+                                   ArrayRef<SDValue> Ops) const;
+
+  SDValue getDefaultM0Value(SelectionDAG &DAG, const SDLoc &SL) const;
+
+  SDValue getNodeWithM0UseRestoreDefault(SelectionDAG &DAG, unsigned Opc,
+                                         const SDLoc &SL, EVT VT,
+                                         SDValue Chain,  ArrayRef<SDValue> Ops,
+                                         SDValue M0Val = SDValue()) const;
+
 };
 
 } // End namespace llvm
 
 #endif

lib/Target/AMDGPU/SIISelLowering.cpp

SITargetLowering::SITargetLowering(const TargetMachine &TM,
   // with > 4 elements.
   for (MVT VT : {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64}) {
     for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
       switch (Op) {
       case ISD::LOAD:
       case ISD::STORE:
       case ISD::BUILD_VECTOR:
       case ISD::BITCAST:
-      case ISD::EXTRACT_VECTOR_ELT:
-      case ISD::INSERT_VECTOR_ELT:
+        //case ISD::EXTRACT_VECTOR_ELT:
+        //case ISD::INSERT_VECTOR_ELT:
       case ISD::INSERT_SUBVECTOR:
       case ISD::EXTRACT_SUBVECTOR:
       case ISD::SCALAR_TO_VECTOR:
         break;
       case ISD::CONCAT_VECTORS:
         setOperationAction(Op, VT, Custom);
         break;
       default:
         setOperationAction(Op, VT, Expand);
         break;
       }
     }
   }
 
+#if 1
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom);
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i32, Custom);
+
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
+
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8f32, Custom);
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i32, Custom);
+
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16f32, Custom);
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f32, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i32, Custom);
+#endif
+
   // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
   // is expanded to avoid having two separate loops in case the index is a VGPR.
 
   // Most operations are naturally 32-bit vector operations. We only support
   // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
   for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) {
     setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
     AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32);
static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
   const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
 
   unsigned SubReg;
   std::tie(SubReg, Offset)
     = computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset);
 
   bool UseGPRIdxMode = ST.hasVGPRIndexMode() && EnableVGPRIndexMode;
 
+  const DebugLoc &DL = MI.getDebugLoc();
+  MachineBasicBlock::iterator I(&MI);
+  TII->emitSetM0ToDefaultValue(MBB, std::next(I), DL);
+
   if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, true)) {
     MachineBasicBlock::iterator I(&MI);
     const DebugLoc &DL = MI.getDebugLoc();
 
     if (UseGPRIdxMode) {
       // TODO: Look at the uses to avoid the copy. This may require rescheduling
       // to avoid interfering with other uses, so probably requires a new
       // optimization pass.
       BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
         .addReg(SrcReg, RegState::Undef, SubReg)
         .addReg(SrcReg, RegState::Implicit)
         .addReg(AMDGPU::M0, RegState::Implicit);
       BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
     } else {
       BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
         .addReg(SrcReg, RegState::Undef, SubReg)
         .addReg(SrcReg, RegState::Implicit);
     }
 
     MI.eraseFromParent();
 
     return &MBB;
   }
 
-  const DebugLoc &DL = MI.getDebugLoc();
-  MachineBasicBlock::iterator I(&MI);
-
   unsigned PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
   unsigned InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
   BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
 
   if (UseGPRIdxMode) {
     MachineInstr *SetOn = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
       .addImm(0) // Reset inside loop.
BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst)
         .add(*SrcVec)
         .add(*Val)
         .addImm(SubReg);
 
     MI.eraseFromParent();
     return &MBB;
   }
 
+  const DebugLoc &DL = MI.getDebugLoc();
+  MachineBasicBlock::iterator I(&MI);
+
+  TII->emitSetM0ToDefaultValue(MBB, std::next(I), DL);
+
   if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, false)) {
     MachineBasicBlock::iterator I(&MI);
     const DebugLoc &DL = MI.getDebugLoc();
 
     if (UseGPRIdxMode) {
       BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_indirect))
           .addReg(SrcVec->getReg(), RegState::Undef, SubReg) // vdst
           .add(*Val)
if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, false)) {
 
     MI.eraseFromParent();
     return &MBB;
   }
 
   if (Val->isReg())
     MRI.clearKillFlags(Val->getReg());
 
-  const DebugLoc &DL = MI.getDebugLoc();
-
   if (UseGPRIdxMode) {
     MachineBasicBlock::iterator I(&MI);
 
     MachineInstr *SetOn = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
       .addImm(0) // Reset inside loop.
       .addImm(VGPRIndexMode::DST_ENABLE);
     SetOn->getOperand(3).setIsUndef();
 
SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   return SDValue();
 }
 
 void SITargetLowering::ReplaceNodeResults(SDNode *N,
                                           SmallVectorImpl<SDValue> &Results,
                                           SelectionDAG &DAG) const {
   switch (N->getOpcode()) {
   case ISD::INSERT_VECTOR_ELT: {
-    if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
+    if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG)) {
+      if (Res.getNode() != N)
         Results.push_back(Res);
+    }
     return;
   }
   case ISD::EXTRACT_VECTOR_ELT: {
-    if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
+    if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG)) {
+      if (Res.getNode() != N)
         Results.push_back(Res);
+    }
     return;
   }
   default:
     break;
   }
 }
 
 /// \brief Helper function for LowerBRCOND
SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
   return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc);;
 }
 
 SDValue SITargetLowering::getSegmentAperture(unsigned AS,
                                              SelectionDAG &DAG) const {
 
   if (Subtarget->hasApertureRegs()) { // Read from Aperture Registers directly.
-    unsigned RegNo = (AS == AMDGPUAS::LOCAL_ADDRESS) ? AMDGPU::SRC_SHARED_BASE :
-                                                       AMDGPU::SRC_PRIVATE_BASE;
-    return CreateLiveInRegister(DAG, &AMDGPU::SReg_32RegClass, RegNo, MVT::i32);
+    unsigned Reg = (AS == AMDGPUAS::LOCAL_ADDRESS) ? AMDGPU::SRC_SHARED_BASE :
+    return CreateLiveInRegister(DAG, &AMDGPU::SReg_32_XM0RegClass,
+                                Reg, MVT::i32);
   }
 
   SDLoc SL;
   MachineFunction &MF = DAG.getMachineFunction();
   SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
   unsigned UserSGPR = Info->getQueuePtrUserSGPR();
   assert(UserSGPR != AMDGPU::NoRegister);
 
SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
 
   return DAG.getUNDEF(ASC->getValueType(0));
 }
 
 SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDValue Idx = Op.getOperand(2);
   if (isa<ConstantSDNode>(Idx))
-    return SDValue();
+    return Op;
 
-  // Avoid stack access for dynamic indexing.
   SDLoc SL(Op);
+  EVT ResultVT = Op.getValueType();
   SDValue Vec = Op.getOperand(0);
+
+  if (Vec.getValueType().getSizeInBits() != 32) {
+    SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1), Op.getOperand(2) };
+    return getNodeWithM0UseRestoreDefault(DAG, AMDGPUISD::INSERT_VECTOR_ELT_INDIRECT,
+                                          SL, ResultVT, SDValue(), Ops);
+  }
+
+  // Avoid stack access for dynamic indexing.
   SDValue Val = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Op.getOperand(1));
 
   // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
   SDValue ExtVal = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Val);
 
   // Convert vector index to bit-index.
   SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx,
                                   DAG.getConstant(16, SL, MVT::i32));
SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
                                                   SelectionDAG &DAG) const {
   SDLoc SL(Op);
 
   EVT ResultVT = Op.getValueType();
   SDValue Vec = Op.getOperand(0);
   SDValue Idx = Op.getOperand(1);
 
   if (const ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
-    SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
+    if (Vec.getValueType().getSizeInBits() != 32) {
+      return Op;
+    }
 
+    SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
     if (CIdx->getZExtValue() == 1) {
       Result = DAG.getNode(ISD::SRL, SL, MVT::i32, Result,
                            DAG.getConstant(16, SL, MVT::i32));
     } else {
       assert(CIdx->getZExtValue() == 0);
     }
 
     if (ResultVT.bitsLT(MVT::i32))
       Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Result);
     return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result);
   }
 
+  if (Vec.getValueType().getSizeInBits() != 32) {
+    SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) };
+    return getNodeWithM0UseRestoreDefault(DAG, AMDGPUISD::EXTRACT_VECTOR_ELT_INDIRECT,
+                                          SL, ResultVT, SDValue(), Ops);
+  }
+
   SDValue Sixteen = DAG.getConstant(16, SL, MVT::i32);
 
   // Convert vector index to bit-index.
   SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, Sixteen);
 
   SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
   SDValue Elt = DAG.getNode(ISD::SRL, SL, MVT::i32, BC, ScaledIdx);
 
SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
   // FIXME: Use a PseudoSourceValue once those can be assigned an address space.
   MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
 
   return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Align,
                      MachineMemOperand::MODereferenceable |
                          MachineMemOperand::MOInvariant);
 }
 
-SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
-                                   const SDLoc &DL, SDValue V) const {
+SDValue SITargetLowering::getNodeWithM0Use(SelectionDAG &DAG, unsigned Opc,
+                                           const SDLoc &SL, EVT VT,
+                                           SDValue InputChain,
+                                           ArrayRef<SDValue> Ops,
+                                           SDValue M0Val) const {
+  SDValue Chain = InputChain ? InputChain : DAG.getEntryNode();
+  SDValue OrigM0 = DAG.getCopyFromReg(Chain, SL, AMDGPU::M0, MVT::i32, SDValue());
+
+#if 1
+  SDValue NewM0 = DAG.getCopyToReg(OrigM0.getValue(1), SL, AMDGPU::M0,
+                                   M0Val, OrigM0.getValue(2));
+#else
   // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
   // the destination register.
   //
   // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
   // so we will end up with redundant moves to m0.
   //
   // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
 
   // A Null SDValue creates a glue result.
-  SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue,
-                                  V, Chain);
-  return SDValue(M0, 0);
+  SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, SL, MVT::Other, MVT::Glue,
+                                  M0Val, Chain, OrigM0.getValue(2));
+  SDValue NewM0(M0, 0);
+
+#endif
+
+  SDVTList VTList = (InputChain && VT != MVT::Other) ?
+    DAG.getVTList(VT, MVT::Other, MVT::Glue) : DAG.getVTList(VT, MVT::Glue);
+
+  SmallVector<SDValue, 6> NodeOps;
+
+  if (InputChain)
+    NodeOps.push_back(NewM0); // Chain
+
+  NodeOps.append(Ops.begin(), Ops.end()); // Operands
+  NodeOps.push_back(NewM0.getValue(1)); // Glue
+
+  SDValue Node = DAG.getNode(Opc, SL, VTList, NodeOps);
+
+  SDValue OutChain = InputChain ? Node : NewM0;
+  SDValue OutGlue = Node.getValue(Node->getNumValues() - 1);
+#if 1
+  SDValue RestoreM0 = DAG.getCopyToReg(OutChain, SL, AMDGPU::M0,
+                                       OrigM0, OutGlue);
+#else
+  SDNode *InitM0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, SL, MVT::Other, MVT::Glue,
+                                  OrigM0, OutChain, OutGlue);
+  SDValue RestoreM0(InitM0, 0);
+#endif
+  if (InputChain)
+    return RestoreM0;
+
+  SDValue NewRoot = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
+                                RestoreM0, DAG.getRoot());
+  DAG.setRoot(NewRoot);
+  return Node;
+}
+
+SDValue SITargetLowering::getNodeWithM0SaveRestore(SelectionDAG &DAG, unsigned Opc,
+                                                   const SDLoc &SL, EVT VT,
+                                                   ArrayRef<SDValue> Ops) const {
+  SDValue SaveM0 = DAG.getCopyFromReg(DAG.getEntryNode(), SL, AMDGPU::M0, MVT::i32,
+                                      SDValue());
+
+  SmallVector<SDValue, 4> OpsWithGlue(Ops.begin(), Ops.end());
+  OpsWithGlue.push_back(SaveM0.getValue(2));
+
+  SDVTList VTs = DAG.getVTList(VT, MVT::Glue);
+  SDValue Op = DAG.getNode(Opc, SL, VTs, OpsWithGlue);
+
+#if 1
+  SDValue RestoreM0 = DAG.getCopyToReg(SaveM0.getValue(1), SL, AMDGPU::M0,
+                                       SaveM0.getValue(0), Op.getValue(1));
+#else
+  SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, SL, MVT::Other, MVT::Glue,
+                                  SaveM0, SaveM0.getValue(1), Op.getValue(1));
+  SDValue RestoreM0(M0, 0);
+
+#endif
+  SDValue NewRoot = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
+                                RestoreM0, DAG.getRoot());
+  DAG.setRoot(NewRoot);
+  return Op;
+}
+
+SDValue SITargetLowering::getDefaultM0Value(SelectionDAG &DAG,
+                                            const SDLoc &SL) const {
+  return DAG.getConstant(-1, SL, MVT::i32);
+  //return DAG.getTargetConstant(-1, SL, MVT::i32);
+}
+
+SDValue SITargetLowering::getNodeWithM0UseRestoreDefault(SelectionDAG &DAG,
+                                                         unsigned Opc,
+                                                         const SDLoc &SL, EVT VT,
+                                                         SDValue InputChain,
+                                                         ArrayRef<SDValue> Ops,
+                                                         SDValue M0Val) const {
+  SDValue Chain = InputChain ? InputChain : DAG.getEntryNode();
+  SDVTList VTList = InputChain ?
+    DAG.getVTList(VT, MVT::Other, MVT::Glue) : DAG.getVTList(VT, MVT::Glue);
+
+
+  //SDValue M0Reg = DAG.getRegister(AMDGPU::M0, MVT::i32);
+  //SDValue NewM0 = copyToM0(DAG, OrigM0.getValue(1), SL, M0Val,
+
+  SmallVector<SDValue, 6> NodeOps;
+
+  SDValue Node;
+  SDValue OutChain = Chain;
+  if (M0Val) {
+    // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
+    // the destination register.
+    //
+    //
+    // We don't use CopyToReg, because MachineCSE won't combine COPY
+    // instructions, so we will end up with redundant moves to m0.
+    //
+    // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
+    //
+    // A Null SDValue creates a glue result.
+    SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, SL, MVT::Other,
+                                    MVT::Glue, M0Val, Chain);
+    SDValue NewM0(M0, 0);
+
+    if (InputChain)
+      NodeOps.push_back(NewM0); // Chain
+    NodeOps.append(Ops.begin(), Ops.end()); // Operands
+    NodeOps.push_back(NewM0.getValue(1)); // Glue
+
+    Node = DAG.getNode(Opc, SL, VTList, NodeOps);
+    OutChain = InputChain ? Node : NewM0;
+  } else {
+    NodeOps.append(Ops.begin(), Ops.end()); // Operands
+    Node = DAG.getNode(Opc, SL, VTList, NodeOps);
+  }
+
+  SDValue OutGlue = Node.getValue(Node->getNumValues() - 1);
+  SDValue RestoreM0 = DAG.getCopyToReg(OutChain, SL, AMDGPU::M0,
+                                       getDefaultM0Value(DAG, SL), OutGlue);
+  if (InputChain)
+    return RestoreM0;
+
+  SDValue NewRoot = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
+                                RestoreM0, DAG.getRoot());
+  DAG.setRoot(NewRoot);
+  return Node;
 }
 
 SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG,
                                                  SDValue Op,
                                                  MVT VT,
                                                  unsigned Offset) const {
   SDLoc SL(Op);
   SDValue Param = LowerParameter(DAG, MVT::i32, MVT::i32, SL,
case AMDGPUIntrinsic::SI_vs_load_input:
     return DAG.getNode(AMDGPUISD::LOAD_INPUT, DL, VT,
                        Op.getOperand(1),
                        Op.getOperand(2),
                        Op.getOperand(3));
   case AMDGPUIntrinsic::SI_packf16:
     if (Op.getOperand(1).isUndef() && Op.getOperand(2).isUndef())
       return DAG.getUNDEF(MVT::i32);
     return Op;
-  case Intrinsic::amdgcn_interp_mov: {
-    SDValue M0 = copyToM0(DAG, DAG.getEntryNode(), DL, Op.getOperand(4));
-    SDValue Glue = M0.getValue(1);
-    return DAG.getNode(AMDGPUISD::INTERP_MOV, DL, MVT::f32, Op.getOperand(1),
-                       Op.getOperand(2), Op.getOperand(3), Glue);
-  }
-  case Intrinsic::amdgcn_interp_p1: {
-    SDValue M0 = copyToM0(DAG, DAG.getEntryNode(), DL, Op.getOperand(4));
-    SDValue Glue = M0.getValue(1);
-    return DAG.getNode(AMDGPUISD::INTERP_P1, DL, MVT::f32, Op.getOperand(1),
-                       Op.getOperand(2), Op.getOperand(3), Glue);
+  case Intrinsic::amdgcn_interp_mov:
+  case Intrinsic::amdgcn_interp_p1:  {
+    unsigned Opc = IntrinsicID == Intrinsic::amdgcn_interp_mov ?
+      AMDGPUISD::INTERP_MOV : AMDGPUISD::INTERP_P1;
+
+    SDValue M0 = Op.getOperand(4);
+    SDValue Ops[] = { Op.getOperand(1), Op.getOperand(2), Op.getOperand(3) };
+    return getNodeWithM0UseRestoreDefault(DAG, Opc, DL,
+                                          MVT::f32, SDValue(), Ops, M0);
   }
   case Intrinsic::amdgcn_interp_p2: {
-    SDValue M0 = copyToM0(DAG, DAG.getEntryNode(), DL, Op.getOperand(5));
-    SDValue Glue = SDValue(M0.getNode(), 1);
-    return DAG.getNode(AMDGPUISD::INTERP_P2, DL, MVT::f32, Op.getOperand(1),
-                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(4),
-                       Glue);
+    SDValue M0 = Op.getOperand(5);
+    SDValue Ops[] = {
+      Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), Op.getOperand(4)
+    };
+
+    return getNodeWithM0UseRestoreDefault(DAG, AMDGPUISD::INTERP_P2, DL,
+                                          MVT::f32, SDValue(), Ops, M0);
   }
   case Intrinsic::amdgcn_sin:
     return DAG.getNode(AMDGPUISD::SIN_HW, DL, VT, Op.getOperand(1));
 
   case Intrinsic::amdgcn_cos:
     return DAG.getNode(AMDGPUISD::COS_HW, DL, VT, Op.getOperand(1));
 
   case Intrinsic::amdgcn_log_clamp: {

 SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
                                               SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   SDLoc DL(Op);
   SDValue Chain = Op.getOperand(0);
   unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
 
   switch (IntrinsicID) {
-      case Intrinsic::amdgcn_exp: {
     const ConstantSDNode *Tgt = cast<ConstantSDNode>(Op.getOperand(2));
     const ConstantSDNode *En = cast<ConstantSDNode>(Op.getOperand(3));
     const ConstantSDNode *Done = cast<ConstantSDNode>(Op.getOperand(8));
     const ConstantSDNode *VM = cast<ConstantSDNode>(Op.getOperand(9));
 
     const SDValue Ops[] = {
       Chain,
       DAG.getTargetConstant(Tgt->getZExtValue(), DL, MVT::i8), // tgt
case Intrinsic::amdgcn_exp_compr: {
     unsigned Opc = Done->isNullValue() ?
       AMDGPUISD::EXPORT : AMDGPUISD::EXPORT_DONE;
     return DAG.getNode(Opc, DL, Op->getVTList(), Ops);
   }
   case Intrinsic::amdgcn_s_sendmsg:
   case Intrinsic::amdgcn_s_sendmsghalt: {
     unsigned NodeOp = (IntrinsicID == Intrinsic::amdgcn_s_sendmsg) ?
       AMDGPUISD::SENDMSG : AMDGPUISD::SENDMSGHALT;
-    Chain = copyToM0(DAG, Chain, DL, Op.getOperand(3));
-    SDValue Glue = Chain.getValue(1);
-    return DAG.getNode(NodeOp, DL, MVT::Other, Chain,
-                       Op.getOperand(2), Glue);
+    SDValue M0 = Op.getOperand(3);
+    return getNodeWithM0Use(DAG, NodeOp, DL, MVT::Other,
+                            Chain, { Op.getOperand(2) }, M0);
   }
   case AMDGPUIntrinsic::SI_tbuffer_store: {
     SDValue Ops[] = {
       Chain,
       Op.getOperand(2),
       Op.getOperand(3),
       Op.getOperand(4),
       Op.getOperand(5),

lib/Target/AMDGPU/SIInstrInfo.h

public:
   ScheduleHazardRecognizer *
   CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
                                  const ScheduleDAG *DAG) const override;
 
   ScheduleHazardRecognizer *
   CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const override;
 
   bool isBasicBlockPrologue(const MachineInstr &MI) const override;
+
+  void emitSetM0ToDefaultValue(MachineBasicBlock &MBB,
+                               MachineBasicBlock::iterator I,
+                               const DebugLoc &DL) const;
 };
 
 namespace AMDGPU {
   LLVM_READONLY
   int getVOPe64(uint16_t Opcode);
 
   LLVM_READONLY
   int getVOPe32(uint16_t Opcode);

lib/Target/AMDGPU/SIInstrInfo.cpp

 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
   return new GCNHazardRecognizer(MF);
 }
 
 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const {
   return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
          MI.modifiesRegister(AMDGPU::EXEC, &RI);
 }
+
+void SIInstrInfo::emitSetM0ToDefaultValue(MachineBasicBlock &MBB,
+                                          MachineBasicBlock::iterator I,
+                                          const DebugLoc &DL) const {
+  // TODO: Restrict to compile time LDS size or initialize with dynamic size
+  // from register?
+  BuildMI(MBB, I, DL, get(AMDGPU::S_MOV_B32), AMDGPU::M0)
+    .addImm(-1);
+}

lib/Target/AMDGPU/SIInstructions.td

def SI_RETURN : SPseudoInstSI <
   let isTerminator = 1;
   let isBarrier = 1;
   let isReturn = 1;
   let hasSideEffects = 1;
   let hasNoSchedulingInfo = 1;
   let DisableWQM = 1;
 }
 
-let Defs = [M0, EXEC],
-  UseNamedOperandTable = 1 in {
+// These use and define m0, but restore it to its default value.
+let Defs = [M0, EXEC], UseNamedOperandTable = 1 in {
 
 class SI_INDIRECT_SRC<RegisterClass rc> : VPseudoInstSI <
   (outs VGPR_32:$vdst),
   (ins rc:$src, VS_32:$idx, i32imm:$offset)> {
   let usesCustomInserter = 1;
 }
 
 class SI_INDIRECT_DST<RegisterClass rc> : VPseudoInstSI <

 
 /********** ====================== **********/
 /**********   Indirect addressing  **********/
 /********** ====================== **********/
 
 multiclass SI_INDIRECT_Pattern <ValueType vt, ValueType eltvt, string VecSize> {
   // Extract with offset
   def : Pat<
-    (eltvt (extractelt vt:$src, (MOVRELOffset i32:$idx, (i32 imm:$offset)))),
+    (eltvt (extractelt_indirect vt:$src, (MOVRELOffset i32:$idx, (i32 imm:$offset)))),
     (!cast<Instruction>("SI_INDIRECT_SRC_"#VecSize) $src, $idx, imm:$offset)
   >;
 
   // Insert with offset
   def : Pat<
-    (insertelt vt:$src, eltvt:$val, (MOVRELOffset i32:$idx, (i32 imm:$offset))),
+    (insertelt_indirect vt:$src, eltvt:$val, (MOVRELOffset i32:$idx, (i32 imm:$offset))),
     (!cast<Instruction>("SI_INDIRECT_DST_"#VecSize) $src, $idx, imm:$offset, $val)
   >;
 }
 
 defm : SI_INDIRECT_Pattern <v2f32, f32, "V2">;
 defm : SI_INDIRECT_Pattern <v4f32, f32, "V4">;
 defm : SI_INDIRECT_Pattern <v8f32, f32, "V8">;
 defm : SI_INDIRECT_Pattern <v16f32, f32, "V16">;

lib/Target/AMDGPU/SIMachineFunctionInfo.h

public:
   unsigned PSInputEna;
 
 
   unsigned ScratchOffsetReg;
   unsigned NumUserSGPRs;
   unsigned NumSystemSGPRs;
 
 private:
+  bool NeedsM0Initialization;
   bool HasSpilledSGPRs;
   bool HasSpilledVGPRs;
   bool HasNonSpillStackObjects;
 
   unsigned NumSpilledSGPRs;
   unsigned NumSpilledVGPRs;
 
   // Feature bits required for inputs passed in user SGPRs.
public:
   unsigned getQueuePtrUserSGPR() const {
     return QueuePtrUserSGPR;
   }
 
   unsigned getPrivateMemoryPtrUserSGPR() const {
     return PrivateMemoryPtrUserSGPR;
   }
 
+  bool needsM0Initialization() const {
+    return NeedsM0Initialization;
+  }
+
+  /// The default m0 value is to support LDS accesses. If this is true, other
+  /// uses need to restore this value after mutating m0.
+  bool shouldPreserveDefaultM0Value() const {
+    return NeedsM0Initialization;
+  }
+
+  void setNeedsM0Intialization() {
+    NeedsM0Initialization = true;
+  }
+
   bool hasSpilledSGPRs() const {
     return HasSpilledSGPRs;
   }
 
   void setHasSpilledSGPRs(bool Spill = true) {
     HasSpilledSGPRs = Spill;
   }
 

lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

: AMDGPUMachineFunction(MF),
     FlatWorkGroupSizes(0, 0),
     WavesPerEU(0, 0),
     DebuggerWorkGroupIDStackObjectIndices({{0, 0, 0}}),
     DebuggerWorkItemIDStackObjectIndices({{0, 0, 0}}),
     LDSWaveSpillSize(0),
     PSInputEna(0),
     NumUserSGPRs(0),
     NumSystemSGPRs(0),
+    NeedsM0Initialization(false),
     HasSpilledSGPRs(false),
     HasSpilledVGPRs(false),
     HasNonSpillStackObjects(false),
     NumSpilledSGPRs(0),
     NumSpilledVGPRs(0),
     PrivateSegmentBuffer(false),
     DispatchPtr(false),
     QueuePtr(false),

lib/Target/AMDGPU/SIRegisterInfo.td

def SReg_32_XM0 : RegisterClass<"AMDGPU", [i32, f32, i16, f16], 32,
   (add SReg_32_XM0_XEXEC, EXEC_LO, EXEC_HI)> {
   let AllocationPriority = 7;
 }
 
 // Register class for all scalar registers (SGPRs + Special Registers)
 def SReg_32 : RegisterClass<"AMDGPU", [i32, f32, i16, f16], 32,
   (add SReg_32_XM0, M0_CLASS, EXEC_LO, EXEC_HI)> {
   let AllocationPriority = 7;
+  let isAllocatable = 0;
 }
 
 def SGPR_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64], 32, (add SGPR_64Regs)> {
   let CopyCost = 1;
   let AllocationPriority = 8;
 }
 
 def TTMP_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64], 32, (add TTMP_64Regs)> {

test/CodeGen/AMDGPU/control-flow-fastregalloc.ll

 ; VGPR: buffer_store_dword [[LOAD0]], off, s[0:3], s7 ; 4-byte Folded Spill
 
 ; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}
 
 ; GCN: s_waitcnt vmcnt(0) expcnt(0)
 ; GCN: mask branch [[ENDIF:BB[0-9]+_[0-9]+]]
 
 ; GCN: {{^}}BB{{[0-9]+}}_1: ; %if
-; GCN: s_mov_b32 m0, -1
+; GCN-NOT: m0
 ; GCN: ds_read_b32 [[LOAD1:v[0-9]+]]
 ; VMEM: buffer_load_dword [[RELOAD_LOAD0:v[0-9]+]], off, s[0:3], s7 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload
 ; VGPR: buffer_load_dword [[RELOAD_LOAD0:v[0-9]+]], off, s[0:3], s7 ; 4-byte Folded Reload
 ; GCN: s_waitcnt vmcnt(0)
 
 ; Spill val register
 ; GCN: v_add_i32_e32 [[VAL:v[0-9]+]], vcc, [[LOAD1]], [[RELOAD_LOAD0]]
 ; GCN: buffer_store_dword [[VAL]], off, s[0:3], s7 offset:[[VAL_OFFSET:[0-9]+]] ; 4-byte Folded Spill

test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll

-; RUN: llc -O0 -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=MOVREL  %s
+; RUN: llc -O0 -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=MOVREL %s
+; RUN: llc -O0 -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-vgpr-index-mode -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=IDXMODE %s
 
 ; FIXME: Merge into indirect-addressing-si.ll
 
 ; Make sure that TwoAddressInstructions keeps src0 as subregister sub0
 ; of the tied implicit use and def of the super register.
 
-; CHECK-LABEL: {{^}}insert_wo_offset:
-; CHECK: s_load_dword [[IN:s[0-9]+]]
-; CHECK: s_mov_b32 m0, [[IN]]
-; CHECK: v_movreld_b32_e32 v[[ELT0:[0-9]+]]
-; CHECK-NEXT: buffer_store_dwordx4 v{{\[}}[[ELT0]]:
-define void @insert_wo_offset(<4 x float> addrspace(1)* %out, i32 %in) {
+; GCN-LABEL: {{^}}insert_wo_offset:
+; GCN: ; implicit-def: %M0
+; GCN-DAG: s_load_dword [[IN:s[0-9]+]]
+; GCN: s_mov_b32 [[NEG1:s[0-9]+]], -1{{$}}
+
+; MOVREL: s_mov_b32 m0, [[IN]]
+; MOVREL-NEXT: v_movreld_b32_e32 v[[ELT0:[0-9]+]]
+
+; IDXMODE: s_set_gpr_idx_on [[IN]], dst
+; IDXMODE-NEXT: v_mov_b32_e32 v[[ELT0:[0-9]+]], v
+; IDXMODE-NEXT: s_set_gpr_idx_off
+
+; GCN-NEXT: s_mov_b32 m0, -1
+; GCN-NEXT: s_mov_b32 m0, [[NEG1]]
+; GCN-NEXT: buffer_store_dwordx4 v{{\[}}[[ELT0]]:
+define void @insert_wo_offset(<4 x float> addrspace(1)* %out, i32 %in) #0 {
 entry:
   %ins = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %in
   store <4 x float> %ins, <4 x float> addrspace(1)* %out
   ret void
 }
 
+attributes #0 = { nounwind }

test/CodeGen/AMDGPU/lds-m0-init-in-loop.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; Make sure that m0 is not reinitialized in the loop.
 
 ; GCN-LABEL: {{^}}copy_local_to_global_loop_m0_init:
+; Initialize in entry block only
+; GCN: s_mov_b32 m0, -1
+; GCN-NOT: m0
 ; GCN: s_cbranch_scc1 BB0_3
+; GCN-NOT: m0
 
-; Initialize in preheader
-; GCN: s_mov_b32 m0, -1
 
 ; GCN: BB0_2:
 ; GCN-NOT: m0
 ; GCN: ds_read_b32
 ; GCN-NOT: m0
 ; GCN: buffer_store_dword
+; GCN-NOT: m0
 
 ; GCN: s_cbranch_scc0 BB0_2
 
 ; GCN: BB0_3:
 ; GCN-NEXT: s_endpgm
 define void @copy_local_to_global_loop_m0_init(i32 addrspace(1)* noalias nocapture %out, i32 addrspace(3)* noalias nocapture readonly %in, i32 %n) #0 {
 bb:
   %tmp = icmp sgt i32 %n, 0
.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
   store i32 %tmp2, i32 addrspace(1)* %tmp3, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %tmp4 = add nuw nsw i32 %i.01, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, %n
   br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
 }
 
+; GCN-LABEL: {{^}}diamond_lds_m0_init:
+; GCN: s_mov_b32 m0, -1
+; GCN-NOT: m0
+; GCN: ds_read_b32
+; GCN-NOT: m0
+; GCN: ds_read_b32
+; GCN-NOT: s_mov_b32 m0
+define void @diamond_lds_m0_init(i32 addrspace(1)* noalias nocapture %out0, i32 addrspace(1)* noalias nocapture %out1, i32 addrspace(3)* noalias nocapture readonly %in0, i32 addrspace(3)* noalias nocapture readonly %in1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %then, label %else
+
+then:
+  %val0 = load volatile i32, i32 addrspace(3)* %in0
+  store volatile i32 %val0, i32 addrspace(1)* %out0
+  call void asm "", ""()
+  br label %endif
+
+else:
+  %val1 = load volatile i32, i32 addrspace(3)* %in1
+  store volatile i32 %val1, i32 addrspace(1)* %out1
+  call void asm "", ""()
+  br label %endif
+
+endif:
+  ret void
+}
+
 attributes #0 = { nounwind }

test/CodeGen/AMDGPU/llvm.amdgcn.interp.ll

 ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
 ; RUN: llc -march=amdgcn -mcpu=kabini -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,16BANK %s
 ; RUN: llc -march=amdgcn -mcpu=stoney -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,16BANK %s
 
 
 ; GCN-LABEL: {{^}}v_interp:
 ; GCN-NOT: s_wqm
 ; GCN: s_mov_b32 m0, s{{[0-9]+}}
 ; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}}
 ; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}}
 ; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}}
 ; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p0, attr0.x{{$}}
-define amdgpu_ps void @v_interp(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg, <2 x float>) {
+define amdgpu_ps void @v_interp(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg %m0, <2 x float> %arg3) {
 main_body:
-  %i = extractelement <2 x float> %4, i32 0
-  %j = extractelement <2 x float> %4, i32 1
-  %p0_0 = call float @llvm.amdgcn.interp.p1(float %i, i32 0, i32 0, i32 %3)
-  %p1_0 = call float @llvm.amdgcn.interp.p2(float %p0_0, float %j, i32 0, i32 0, i32 %3)
-  %p0_1 = call float @llvm.amdgcn.interp.p1(float %i, i32 1, i32 0, i32 %3)
-  %p1_1 = call float @llvm.amdgcn.interp.p2(float %p0_1, float %j, i32 1, i32 0, i32 %3)
-  %const = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %3)
+  %i = extractelement <2 x float> %arg3, i32 0
+  %j = extractelement <2 x float> %arg3, i32 1
+  %p0_0 = call float @llvm.amdgcn.interp.p1(float %i, i32 0, i32 0, i32 %m0)
+  %p1_0 = call float @llvm.amdgcn.interp.p2(float %p0_0, float %j, i32 0, i32 0, i32 %m0)
+  %p0_1 = call float @llvm.amdgcn.interp.p1(float %i, i32 1, i32 0, i32 %m0)
+  %p1_1 = call float @llvm.amdgcn.interp.p2(float %p0_1, float %j, i32 1, i32 0, i32 %m0)
+  %const = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0)
   %w = fadd float %p1_1, %const
   call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %p0_0, float %p0_0, float %p1_1, float %w)
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_interp_p1:
 ; GCN: s_movk_i32 m0, 0x100
 ; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}}
define amdgpu_ps void @v_interp_mov(float %x, float %j) {
 
   store volatile float %mov_9, float addrspace(1)* undef
   store volatile float %mov_10, float addrspace(1)* undef
   store volatile float %mov_11, float addrspace(1)* undef
   store volatile float %mov_12, float addrspace(1)* undef
   ret void
 }
 
+; FIXME: Dead def of m0 emitted
 ; SI won't merge ds memory operations, because of the signed offset bug, so
 ; we only have check lines for VI.
 ; VI-LABEL: v_interp_readnone:
 ; VI: s_mov_b32 m0, 0
 ; VI-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
 ; VI-DAG: v_interp_mov_f32 v{{[0-9]+}}, p0, attr0.x{{$}}
 ; VI: s_mov_b32 m0, -1{{$}}
 ; VI: ds_write2_b32 v{{[0-9]+}}, [[ZERO]], [[ZERO]] offset1:4
 define amdgpu_ps void @v_interp_readnone(float addrspace(3)* %lds) {
   store float 0.0, float addrspace(3)* %lds
   %tmp1 = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 0)
   %tmp2 = getelementptr float, float addrspace(3)* %lds, i32 4
   store float 0.0, float addrspace(3)* %tmp2
   call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %tmp1, float %tmp1, float %tmp1, float %tmp1)
   ret void
 }
 
+; GCN-LABEL: {{^}}v_interp_lds_restore:
+; GCN: s_wqm
+; GCN-NEXT: s_mov_b32 m0, s6
+; GCN-NEXT: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}}
+; GCN: s_mov_b32 m0, -1{{$}}
+; GCN-NOT: m0
+; GCN: ds_write_b32
+; GCN-NEXT: s_mov_b32 m0, s6
+
+; GCN: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}}
+; GCN-NEXT: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}}
+; GCN-NEXT: v_interp_mov_f32 v{{[0-9]+}}, p0, attr0.x{{$}}
+; GCN-NOT: m0
+define amdgpu_ps void @v_interp_lds_restore(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg %m0, <2 x float> %arg3) {
+main_body:
+  %i = extractelement <2 x float> %arg3, i32 0
+  %j = extractelement <2 x float> %arg3, i32 1
+  %p0_0 = call float @llvm.amdgcn.interp.p1(float %i, i32 0, i32 0, i32 %m0)
+  %p1_0 = call float @llvm.amdgcn.interp.p2(float %p0_0, float %j, i32 0, i32 0, i32 %m0)
+  store volatile float 4.0, float addrspace(3)* undef
+  %p0_1 = call float @llvm.amdgcn.interp.p1(float %i, i32 1, i32 0, i32 %m0)
+  %p1_1 = call float @llvm.amdgcn.interp.p2(float %p0_1, float %j, i32 1, i32 0, i32 %m0)
+  %const = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0)
+  %w = fadd float %p1_1, %const
+  call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %p0_0, float %p0_0, float %p1_1, float %w)
+  ret void
+}
+
 ; Thest that v_interp_p1 uses different source and destination registers
 ; on 16 bank LDS chips.
 
 ; GCN-LABEL: {{^}}v_interp_p1_bank16_bug:
 ; 16BANK-NOT: v_interp_p1_f32 [[DST:v[0-9]+]], [[DST]]
 define amdgpu_ps void @v_interp_p1_bank16_bug([6 x <16 x i8>] addrspace(2)* byval %arg, [17 x <16 x i8>] addrspace(2)* byval %arg13, [17 x <4 x i32>] addrspace(2)* byval %arg14, [34 x <8 x i32>] addrspace(2)* byval %arg15, float inreg %arg16, i32 inreg %arg17, <2 x i32> %arg18, <2 x i32> %arg19, <2 x i32> %arg20, <3 x i32> %arg21, <2 x i32> %arg22, <2 x i32> %arg23, <2 x i32> %arg24, float %arg25, float %arg26, float %arg27, float %arg28, float %arg29, float %arg30, i32 %arg31, float %arg32, float %arg33) {
 main_body:
   %i.i = extractelement <2 x i32> %arg19, i32 0
main_body:
   %tmp36 = call i32 @llvm.SI.packf16(float %tmp, float %tmp34)
   %tmp37 = bitcast i32 %tmp36 to float
   %tmp38 = call i32 @llvm.SI.packf16(float %tmp35, float 1.000000e+00)
   %tmp39 = bitcast i32 %tmp38 to float
   call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %tmp37, float %tmp39, float %tmp37, float %tmp39)
   ret void
 }
 
+; Put the interps out of the entry block, obscuring the dead def of m0
+; in the entry.
+; FIXME: getting dead def
+; GCN-LABEL: {{^}}v_interp_no_dead_setup:
+; GCN: ; implicit-def: %M0
+; GCN-NOT: s_mov_b32 m0
+; GCN: s_cbranch_scc1
+
+; GCN: s_mov_b32 m0, s6
+; GCN: v_interp_p1_f32
+
+; FIXME: Dead
+; GCN: s_mov_b32 m0, -1
+define amdgpu_ps void @v_interp_no_dead_setup(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg %m0, <2 x float> %arg3) {
+entry:
+  %i = extractelement <2 x float> %arg3, i32 0
+  %j = extractelement <2 x float> %arg3, i32 1
+  call void asm sideeffect "", ""()
+  br i1 undef, label %if, label %endif
+
+if:
+  %p0_0 = call float @llvm.amdgcn.interp.p1(float %i, i32 0, i32 0, i32 %m0)
+  %p1_0 = call float @llvm.amdgcn.interp.p2(float %p0_0, float %j, i32 0, i32 0, i32 %m0)
+  %p0_1 = call float @llvm.amdgcn.interp.p1(float %i, i32 1, i32 0, i32 %m0)
+  %p1_1 = call float @llvm.amdgcn.interp.p2(float %p0_1, float %j, i32 1, i32 0, i32 %m0)
+  %const = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0)
+  %w = fadd float %p1_1, %const
+  store volatile float %p0_0, float addrspace(1)* undef
+  store volatile float %p0_1, float addrspace(1)* undef
+  store volatile float %w, float addrspace(1)* undef
+  br label %endif
+
+endif:
+  ret void
+}
+
 declare float @llvm.fabs.f32(float) #0
 declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #0
 declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #0
 declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) #0
 declare i32 @llvm.SI.packf16(float, float) #0
 declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float)
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }

test/CodeGen/AMDGPU/llvm.amdgcn.sendmsg.ll

 ; VI-NEXT: s_nop 0
 ; GCN-NEXT: sendmsg(MSG_GS_DONE, GS_OP_NOP)
 ; GCN-NEXT: s_endpgm
 define amdgpu_gs void @sendmsg(i32 inreg %a) #0 {
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 %a)
   ret void
 }
 
+; GCN-LABEL: {{^}}test_interrupt_undef_m0:
+; GCN-NOT: s_mov_b32 m0
+; GCN: s_sendmsg sendmsg(MSG_INTERRUPT)
+; GCN-NOT: s_mov_b32 m0
+define void @test_interrupt_undef_m0() {
+body:
+  call void @llvm.amdgcn.s.sendmsg(i32 1, i32 undef);
+  ret void
+}
+
+; FIXME: Should not get any m0 def
+; GCN-LABEL: {{^}}test_interrupt_undef_m0_nonentry:
+; GCN: ; implicit-def: %M0
+; GCN-NOT: s_mov_b32 m0
+; GCN: s_cbranch_scc1
+; GCN: s_mov_b32 s0, m0
+; GCN: s_sendmsg sendmsg(MSG_INTERRUPT)
+; GCN-NOT: s_mov_b32 m0
+define void @test_interrupt_undef_m0_nonentry() {
+entry:
+  br i1 undef, label %if, label %endif
+
+if:
+  call void @llvm.amdgcn.s.sendmsg(i32 1, i32 undef)
+  br label %endif
+
+endif:
+  ret void
+}
+
 ; GCN-LABEL: {{^}}sendmsghalt:
 ; GCN: s_mov_b32 m0, s0
 ; VI-NEXT: s_nop 0
 ; GCN-NEXT: s_sendmsghalt sendmsg(MSG_INTERRUPT)
 ; GCN-NEXT: s_endpgm
 define void @sendmsghalt(i32 inreg %a) #0 {
   call void @llvm.amdgcn.s.sendmsghalt(i32 1, i32 %a)
   ret void

test/CodeGen/AMDGPU/regcoalesce-dbg.mir

 # RUN: llc -march=amdgcn -run-pass simple-register-coalescing -o - %s | FileCheck %s
 
 # Test that register coalescing does not allow a call to
 # LIS->getInstructionIndex with a DBG_VALUE instruction, which does not have
 # a slot index.
 
 # CHECK: %13.sub2 = S_MOV_B32 0
 # CHECK: DBG_VALUE{{.*}}debug-use %13.sub2
 
 --- |
   define void @test(i32 addrspace(1)* %out) { ret void }
-  
   !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !4, producer: "llvm", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4)
   !1 = !DILocalVariable(name: "a", scope: !2, file: !4, line: 126, type: !6)
   !2 = distinct !DISubprogram(name: "test", scope: !4, file: !4, line: 1, type: !3, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, variables: !5)
   !3 = !DISubroutineType(types: !4)
   !4 = !{null}
   !5 = !{!1}
   !6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64, align: 32)
   !7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
   !8 = !DIExpression()
   !9 = !DILocation(line: 126, column: 9, scope: !2)
 
 ...
 ---
 name:            test
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: sreg_32_xm0_xexec }
-  - { id: 6, class: sreg_32 }
-  - { id: 7, class: sreg_32 }
+  - { id: 6, class: sreg_32_xm0 }
+  - { id: 7, class: sreg_32_xm0 }
   - { id: 8, class: sreg_32_xm0 }
   - { id: 9, class: sreg_64 }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
   - { id: 13, class: sgpr_128 }
   - { id: 14, class: sreg_32_xm0 }
   - { id: 15, class: sreg_64 }
   - { id: 16, class: vgpr_32 }
   - { id: 17, class: vreg_64 }
   - { id: 18, class: vgpr_32 }
   - { id: 19, class: vreg_64 }
   - { id: 20, class: vreg_64 }
-liveins:         
+liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }
   - { reg: '%vgpr0', virtual-reg: '%3' }
 body:             |
   bb.0:
     liveins: %sgpr0_sgpr1, %vgpr0
-  
     %3 = COPY killed %vgpr0
     %0 = COPY killed %sgpr0_sgpr1
     %4 = S_LOAD_DWORDX2_IMM %0, 9, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
     %5 = S_LOAD_DWORD_IMM killed %0, 13, 0 :: (non-temporal dereferenceable invariant load 4 from `i32 addrspace(2)* undef`)
     %18 = V_ASHRREV_I32_e32 31, %3, implicit %exec
     undef %19.sub0 = COPY killed %3
     %19.sub1 = COPY killed %18
     %10 = S_MOV_B32 61440

test/CodeGen/AMDGPU/shl_add_ptr.ll

define void @load_shl_base_lds_max_offset(i8 addrspace(1)* %out, i8 addrspace(3)* %lds, i32 addrspace(1)* %add_use) #0 {
   store i8 %val0, i8 addrspace(1)* %out
   ret void
 }
 
 ; The two globals are placed adjacent in memory, so the same base
 ; pointer can be used with an offset into the second one.
 
 ; SI-LABEL: {{^}}load_shl_base_lds_2:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
 ; SI: s_mov_b32 m0, -1
+; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+
 ; SI-NEXT: ds_read2st64_b32 {{v\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:1 offset1:9
 ; SI: s_endpgm
 define void @load_shl_base_lds_2(float addrspace(1)* %out) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 64
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds0, i32 0, i32 %idx.0
   %val0 = load float, float addrspace(3)* %arrayidx0, align 4
   %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds1, i32 0, i32 %idx.0

test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir

 tracksRegLiveness: true
 registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: sreg_64_xexec }
-  - { id: 6, class: sreg_32 }
-  - { id: 7, class: sreg_32 }
+  - { id: 6, class: sreg_32_xm0 }
+  - { id: 7, class: sreg_32_xm0 }
   - { id: 8, class: sreg_32_xm0 }
   - { id: 9, class: sreg_64 }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
   - { id: 13, class: sgpr_128 }
   - { id: 14, class: sreg_32_xm0 }
   - { id: 15, class: sreg_64 }
   - { id: 16, class: sgpr_128 }
   - { id: 17, class: vgpr_32 }
   - { id: 18, class: vreg_64 }
   - { id: 19, class: vgpr_32 }
   - { id: 20, class: vreg_64 }
   - { id: 21, class: sreg_32_xm0 }
-  - { id: 22, class: sreg_32 }
-  - { id: 23, class: sreg_32 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_32_xm0 }
   - { id: 24, class: vgpr_32 }
   - { id: 25, class: vreg_64 }
   - { id: 26, class: vgpr_32 }
   - { id: 27, class: vreg_64 }
   - { id: 28, class: vreg_64 }
   - { id: 29, class: vgpr_32 }
 liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }

 tracksRegLiveness: true
 registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: sreg_64_xexec }
-  - { id: 6, class: sreg_32 }
-  - { id: 7, class: sreg_32 }
+  - { id: 6, class: sreg_32_xm0 }
+  - { id: 7, class: sreg_32_xm0 }
   - { id: 8, class: sreg_32_xm0 }
   - { id: 9, class: sreg_64 }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
   - { id: 13, class: sgpr_128 }
   - { id: 14, class: sreg_32_xm0 }
   - { id: 15, class: sreg_64 }
   - { id: 16, class: sgpr_128 }
   - { id: 17, class: vgpr_32 }
   - { id: 18, class: vreg_64 }
   - { id: 19, class: vgpr_32 }
   - { id: 20, class: vreg_64 }
   - { id: 21, class: sreg_32_xm0 }
-  - { id: 22, class: sreg_32 }
-  - { id: 23, class: sreg_32 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_32_xm0 }
   - { id: 24, class: vgpr_32 }
   - { id: 25, class: vreg_64 }
   - { id: 26, class: vgpr_32 }
   - { id: 27, class: vreg_64 }
   - { id: 28, class: vreg_64 }
   - { id: 29, class: vgpr_32 }
 liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }

 tracksRegLiveness: true
 registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: sreg_64_xexec }
-  - { id: 6, class: sreg_32 }
-  - { id: 7, class: sreg_32 }
+  - { id: 6, class: sreg_32_xm0 }
+  - { id: 7, class: sreg_32_xm0 }
   - { id: 8, class: sreg_32_xm0 }
   - { id: 9, class: sreg_64 }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
   - { id: 13, class: sgpr_128 }
   - { id: 14, class: sreg_32_xm0 }
   - { id: 15, class: sreg_64 }
   - { id: 16, class: sgpr_128 }
   - { id: 17, class: vgpr_32 }
   - { id: 18, class: vreg_64 }
   - { id: 19, class: vgpr_32 }
   - { id: 20, class: vreg_64 }
   - { id: 21, class: sreg_32_xm0 }
-  - { id: 22, class: sreg_32 }
-  - { id: 23, class: sreg_32 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_32_xm0 }
   - { id: 24, class: vgpr_32 }
   - { id: 25, class: vreg_64 }
   - { id: 26, class: vgpr_32 }
   - { id: 27, class: vreg_64 }
   - { id: 28, class: vreg_64 }
   - { id: 29, class: vgpr_32 }
 liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }

 tracksRegLiveness: true
 registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: sreg_64_xexec }
-  - { id: 6, class: sreg_32 }
-  - { id: 7, class: sreg_32 }
+  - { id: 6, class: sreg_32_xm0 }
+  - { id: 7, class: sreg_32_xm0 }
   - { id: 8, class: sreg_32_xm0 }
   - { id: 9, class: sreg_64 }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
   - { id: 13, class: sgpr_128 }
   - { id: 14, class: sreg_32_xm0 }
   - { id: 15, class: sreg_64 }
   - { id: 16, class: sgpr_128 }
   - { id: 17, class: vgpr_32 }
   - { id: 18, class: vreg_64 }
   - { id: 19, class: vgpr_32 }
   - { id: 20, class: vreg_64 }
   - { id: 21, class: sreg_32_xm0 }
-  - { id: 22, class: sreg_32 }
-  - { id: 23, class: sreg_32 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_32_xm0 }
   - { id: 24, class: vgpr_32 }
   - { id: 25, class: vreg_64 }
   - { id: 26, class: vgpr_32 }
   - { id: 27, class: vreg_64 }
   - { id: 28, class: vreg_64 }
   - { id: 29, class: vgpr_32 }
 liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }

 tracksRegLiveness: true
 registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: sreg_64_xexec }
-  - { id: 6, class: sreg_32 }
-  - { id: 7, class: sreg_32 }
+  - { id: 6, class: sreg_32_xm0 }
+  - { id: 7, class: sreg_32_xm0 }
   - { id: 8, class: sreg_32_xm0 }
   - { id: 9, class: sreg_64 }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
   - { id: 13, class: sgpr_128 }
   - { id: 14, class: sreg_32_xm0 }
   - { id: 15, class: sreg_64 }
   - { id: 16, class: sgpr_128 }
   - { id: 17, class: vgpr_32 }
   - { id: 18, class: vreg_64 }
   - { id: 19, class: vgpr_32 }
   - { id: 20, class: vreg_64 }
   - { id: 21, class: sreg_32_xm0 }
-  - { id: 22, class: sreg_32 }
-  - { id: 23, class: sreg_32 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_32_xm0 }
   - { id: 24, class: vgpr_32 }
   - { id: 25, class: vreg_64 }
   - { id: 26, class: vgpr_32 }
   - { id: 27, class: vreg_64 }
   - { id: 28, class: vreg_64 }
   - { id: 29, class: vgpr_32 }
 liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }

 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
-  - { id: 2, class: sgpr_32 }
+  - { id: 2, class: sreg_32_xm0 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: sreg_64_xexec }
-  - { id: 6, class: sreg_32 }
-  - { id: 7, class: sreg_32 }
+  - { id: 6, class: sreg_32_xm0 }
+  - { id: 7, class: sreg_32_xm0 }
   - { id: 8, class: sreg_32_xm0 }
   - { id: 9, class: sreg_64 }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
   - { id: 13, class: sgpr_128 }
   - { id: 14, class: sreg_32_xm0 }
   - { id: 15, class: sreg_64 }
   - { id: 16, class: sgpr_128 }
   - { id: 17, class: vgpr_32 }
   - { id: 18, class: vreg_64 }
   - { id: 19, class: vgpr_32 }
   - { id: 20, class: vreg_64 }
   - { id: 21, class: sreg_32_xm0 }
-  - { id: 22, class: sreg_32 }
-  - { id: 23, class: sreg_32 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_32_xm0 }
   - { id: 24, class: vgpr_32 }
   - { id: 25, class: vreg_64 }
   - { id: 26, class: vgpr_32 }
   - { id: 27, class: vreg_64 }
   - { id: 28, class: vreg_64 }
   - { id: 29, class: vgpr_32 }
 liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }

test/CodeGen/AMDGPU/spill-m0.ll

endif:
   store i32 %foo, i32 addrspace(1)* %out
   ret void
 }
 
 @lds = internal addrspace(3) global [64 x float] undef
 
 ; m0 is killed, so it isn't necessary during the entry block spill to preserve it
 ; GCN-LABEL: {{^}}spill_kill_m0_lds:
+; GCN: s_mov_b32 m0, -1
+; GCN: s_mov_b32 s1, -1
 ; GCN: s_mov_b32 m0, s6
 ; GCN: v_interp_mov_f32
 
-; TOSMEM-NOT: s_m0
 ; TOSMEM: s_mov_b32 m0, s7
 ; TOSMEM-NEXT: s_buffer_store_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 4-byte Folded Spill
-; TOSMEM-NOT: m0
+; TOSMEM-NEXT: m0, vcc_hi
+; TOSMEM-NEXT: vcc_hi, m0
 
-; TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s7, 0x100
-; TOSMEM: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill
-; TOSMEM-NOT: m0
+; TOSMEM-NEXT: s_add_u32 m0, s7, 0x100
+; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill
+; TOSMEM-NEXT: s_mov_b32 m0, vcc_hi
 
 ; TOSMEM: s_mov_b64 exec,
 ; TOSMEM: s_cbranch_execz
 ; TOSMEM: s_branch
 
 ; TOSMEM: BB{{[0-9]+_[0-9]+}}:
+; TOSMEM-NEXT: s_mov_b32 vcc_hi, m0
 ; TOSMEM-NEXT: s_add_u32 m0, s7, 0x100
 ; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload
-
+; TOSMEM-NEXT: m0, vcc_hi
 
 ; GCN-NOT: v_readlane_b32 m0
 ; GCN-NOT: s_buffer_store_dword m0
 ; GCN-NOT: s_buffer_load_dword m0
 define amdgpu_ps void @spill_kill_m0_lds(<16 x i8> addrspace(2)* inreg %arg, <16 x i8> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, i32 inreg %m0) #0 {
 main_body:
   %tmp = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0)
   %cmp = fcmp ueq float 0.000000e+00, %tmp

 ; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill
 ; TOSMEM: s_mov_b32 m0, vcc_hi
 
 ; TOSMEM: s_mov_b64 exec,
 ; TOSMEM: s_cbranch_execz
 ; TOSMEM: s_branch
 
 ; TOSMEM: BB{{[0-9]+_[0-9]+}}:
+; TOSMEM-NEXT: s_mov_b32 vcc_hi, m0
 ; TOSMEM-NEXT: s_mov_b32 m0, s3
 ; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload
+; TOSMEM-NEXT: s_mov_b32 m0, vcc_hi
 
 ; GCN-NOT: v_readlane_b32 m0
 ; GCN-NOT: s_buffer_store_dword m0
 ; GCN-NOT: s_buffer_load_dword m0
 define void @m0_unavailable_spill(i32 %m0.arg) #0 {
 main_body:
   %m0 = call i32 asm sideeffect "; def $0, 1", "={M0}"() #0
   %tmp = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0.arg)

 
 endif:
   ret void
 }
 
 ; GCN-LABEL: {{^}}restore_m0_lds:
 ; TOSMEM: s_load_dwordx2 [[REG:s\[[0-9]+:[0-9]+\]]]
 ; TOSMEM: s_cmp_eq_u32
-; TOSMEM-NOT: m0
-; TOSMEM: s_mov_b32 m0, s3
+; TOSMEM-NEXT: s_mov_b32 vcc_hi, m0
+; TOSMEM-NEXT: s_mov_b32 m0, s3
 ; TOSMEM: s_buffer_store_dwordx2 [[REG]], s[88:91], m0 ; 8-byte Folded Spill
-; TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s3, 0x200
-; TOSMEM: s_buffer_store_dword s{{[0-9]+}}, s[88:91], m0 ; 4-byte Folded Spill
-; TOSMEM-NOT: m0
+; TOSMEM-NEXT: s_mov_b32 m0, vcc_hi
+; TOSMEM-NEXT: s_mov_b32 vcc_hi, m0
+; TOSMEM-NEXT: s_add_u32 m0, s3, 0x200
+; TOSMEM-NEXT: s_buffer_store_dword s{{[0-9]+}}, s[88:91], m0 ; 4-byte Folded Spill
+; TOSMEM-NEXT: s_mov_b32 m0, vcc_hi
 ; TOSMEM: s_cbranch_scc1
 
-; TOSMEM: s_mov_b32 m0, -1
-
 ; TOSMEM: s_mov_b32 vcc_hi, m0
-; TOSMEM: s_mov_b32 m0, s3
-; TOSMEM: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[88:91], m0 ; 8-byte Folded Reload
-; TOSMEM: s_mov_b32 m0, vcc_hi
+; TOSMEM-NEXT: s_mov_b32 m0, s3
+; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[88:91], m0 ; 8-byte Folded Reload
+; TOSMEM-NEXT: s_mov_b32 m0, vcc_hi
 ; TOSMEM: s_waitcnt lgkmcnt(0)
 
 ; TOSMEM: ds_write_b64
-
-; TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s3, 0x200
+; TOSMEM-NEXT: s_mov_b32 vcc_hi, m0
+; TOSMEM-NEXT: s_add_u32 m0, s3, 0x200
 ; TOSMEM: s_buffer_load_dword s0, s[88:91], m0 ; 4-byte Folded Reload
-; TOSMEM-NOT: m0
+; TOSMEM-NEXT: s_mov_b32 m0, vcc_hi
 ; TOSMEM: s_waitcnt lgkmcnt(0)
 ; TOSMEM-NOT: m0
 ; TOSMEM: s_mov_b32 m0, s0
 ; TOSMEM: ; use m0
 
 ; TOSMEM: s_dcache_wb
 ; TOSMEM: s_endpgm
 define void @restore_m0_lds(i32 %arg) {

test/CodeGen/MIR/AMDGPU/fold-imm-f16-f32.mir

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
 frameInfo:
   isFrameAddressTaken: false
   isReturnAddressTaken: false
   hasStackMap:     false

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
 frameInfo:
   isFrameAddressTaken: false

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
   - { id: 16, class: vgpr_32 }
 frameInfo:

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
   - { id: 16, class: vgpr_32 }
   - { id: 17, class: vgpr_32 }

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
 frameInfo:
   isFrameAddressTaken: false

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
   - { id: 16, class: vgpr_32 }
   - { id: 17, class: vgpr_32 }

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
 frameInfo:
   isFrameAddressTaken: false

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
 frameInfo:
   isFrameAddressTaken: false

 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sreg_64 }
-  - { id: 1, class: sreg_32 }
+  - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64 }
-  - { id: 5, class: sreg_32 }
+  - { id: 5, class: sreg_32_xm0 }
   - { id: 6, class: sreg_64 }
-  - { id: 7, class: sreg_32 }
-  - { id: 8, class: sreg_32 }
-  - { id: 9, class: sreg_32 }
+  - { id: 7, class: sreg_32_xm0 }
+  - { id: 8, class: sreg_32_xm0 }
+  - { id: 9, class: sreg_32_xm0 }
   - { id: 10, class: sreg_128 }
   - { id: 11, class: vgpr_32 }
   - { id: 12, class: vgpr_32 }
   - { id: 13, class: vgpr_32 }
   - { id: 14, class: vgpr_32 }
   - { id: 15, class: vgpr_32 }
 frameInfo:
   isFrameAddressTaken: false