[AMDGPU] Use null for dead sdst operand

[foad]

Nice!

[rampitec]

Rebased tests.

llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp

public:
   bool isKUImmOperand(const MachineOperand &Src) const;
   bool isKImmOrKUImmOperand(const MachineOperand &Src, bool &IsUnsigned) const;
   bool isReverseInlineImm(const MachineOperand &Src, int32_t &ReverseImm) const;
   void copyExtraImplicitOps(MachineInstr &NewMI, MachineInstr &MI) const;
   void shrinkScalarCompare(MachineInstr &MI) const;
   void shrinkMIMG(MachineInstr &MI) const;
   void shrinkMadFma(MachineInstr &MI) const;
   bool shrinkScalarLogicOp(MachineInstr &MI) const;
+  bool tryReplaceDeadSDST(MachineInstr &MI) const;
   bool instAccessReg(iterator_range<MachineInstr::const_mop_iterator> &&R,
                      Register Reg, unsigned SubReg) const;
   bool instReadsReg(const MachineInstr *MI, unsigned Reg,
                     unsigned SubReg) const;
   bool instModifiesReg(const MachineInstr *MI, unsigned Reg,
                        unsigned SubReg) const;
   TargetInstrInfo::RegSubRegPair getSubRegForIndex(Register Reg, unsigned Sub,
                                                    unsigned I) const;
for (auto Iter = std::next(MovT.getIterator()),
     }
 
     return Next;
   }
 
   return nullptr;
 }
 
+// If an instruction has dead sdst replace it with NULL register on gfx10+
+bool SIShrinkInstructions::tryReplaceDeadSDST(MachineInstr &MI) const {
+  if (ST->getGeneration() < AMDGPUSubtarget::GFX10)
+    return false;
+
+  MachineOperand *Op = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);
+  if (!Op)
+    return false;
+  Register SDstReg = Op->getReg();
+  if (SDstReg.isPhysical() || !MRI->use_nodbg_empty(SDstReg))
+    return false;
+
+  Op->setReg(ST->isWave32() ? AMDGPU::SGPR_NULL : AMDGPU::SGPR_NULL64);
+  return true;
+}
+
 bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
 
   MRI = &MF.getRegInfo();
   ST = &MF.getSubtarget<GCNSubtarget>();
   TII = ST->getInstrInfo();
   TRI = &TII->getRegisterInfo();
for (I = MBB.begin(); I != MBB.end(); I = Next) {
       if (MI.getOpcode() == AMDGPU::V_MAD_F32_e64 ||
           MI.getOpcode() == AMDGPU::V_FMA_F32_e64 ||
           MI.getOpcode() == AMDGPU::V_MAD_F16_e64 ||
           MI.getOpcode() == AMDGPU::V_FMA_F16_e64) {
         shrinkMadFma(MI);
         continue;
       }
 
-      if (!TII->hasVALU32BitEncoding(MI.getOpcode()))
+      if (!TII->hasVALU32BitEncoding(MI.getOpcode())) {
+        // If there is no chance we will shrink it and use VCC as sdst to get
+        // a 32 bit form try to replace dead sdst with NULL.
+        tryReplaceDeadSDST(MI);
         continue;
+      }
 
       if (!TII->canShrink(MI, *MRI)) {
         // Try commuting the instruction and see if that enables us to shrink
         // it.
         if (!MI.isCommutable() || !TII->commuteInstruction(MI) ||
-            !TII->canShrink(MI, *MRI))
+            !TII->canShrink(MI, *MRI)) {
+          tryReplaceDeadSDST(MI);
           continue;
         }
+      }
 
       int Op32 = AMDGPU::getVOPe32(MI.getOpcode());
 
       if (TII->isVOPC(Op32)) {
         MachineOperand &Op0 = MI.getOperand(0);
         if (Op0.isReg()) {
           // Exclude VOPCX instructions as these don't explicitly write a
           // dst.

llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll

 ; GFX9-LABEL: div_scale_s_s_true:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
 ; GFX9-NEXT:    v_div_scale_f32 v0, s[0:1], s2, v0, s2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: div_scale_s_s_true:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, s2, s3, s2
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s2, s3, s2
 ; GFX10-NEXT:    ; return to shader part epilog
   %div.scale = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %src0, float %src1, i1 true)
   %result = extractvalue { float, i1 } %div.scale, 0
   ret float %result
 }
 
 define amdgpu_ps float @div_scale_s_s_false(float inreg %src0, float inreg %src1) {
 ; GFX9-LABEL: div_scale_s_s_false:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
 ; GFX9-NEXT:    v_div_scale_f32 v0, s[0:1], v0, v0, s2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: div_scale_s_s_false:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, s3, s3, s2
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s3, s3, s2
 ; GFX10-NEXT:    ; return to shader part epilog
   %div.scale = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %src0, float %src1, i1 false)
   %result = extractvalue { float, i1 } %div.scale, 0
   ret float %result
 }
 
 declare float @llvm.fma.f32(float, float, float) #0
 declare float @llvm.amdgcn.div.fmas.f32(float, float, float, i1) #1
 declare { float, i1 } @llvm.amdgcn.div.scale.f32(float, float, i1 immarg) #1
 declare i1 @llvm.amdgcn.class.f32(float, i32) #1
 
 attributes #0 = { nounwind readnone speculatable willreturn }
 attributes #1 = { nounwind readnone speculatable }

llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f32.ll

 ; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
 ; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
 ; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_fdiv_f32:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, s4, v1, v1, v0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, null, v1, v1, v0
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, vcc_lo, v0, v1, v0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX10-IEEE-NEXT:    v_fma_f32 v4, -v2, v3, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v3, v4, v3
 ; GFX10-IEEE-NEXT:    v_mul_f32_e32 v4, v5, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, v5
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v4, v6, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v2, -v2, v4, v5
 ; GFX10-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v4
 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
 ; GFX10-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLUSH-LABEL: v_fdiv_f32:
 ; GFX10-FLUSH:       ; %bb.0:
 ; GFX10-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-FLUSH-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, s4, v1, v1, v0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, null, v1, v1, v0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, vcc_lo, v0, v1, v0
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v3, v5, v3
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v5, v6, v3

 ; GFX89-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
 ; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
 ; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_fdiv_f32_ulp25:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, s4, v1, v1, v0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, null, v1, v1, v0
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, vcc_lo, v0, v1, v0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX10-IEEE-NEXT:    v_fma_f32 v4, -v2, v3, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v3, v4, v3
 ; GFX10-IEEE-NEXT:    v_mul_f32_e32 v4, v5, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, v5
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v4, v6, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v2, -v2, v4, v5

 ; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v1, v1, v3, v4
 ; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v0, v1, v0, 1.0
 ; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_rcp_f32:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v1, s4, v0, v0, 1.0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v1, null, v0, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, vcc_lo, 1.0, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX10-IEEE-NEXT:    v_fma_f32 v3, -v1, v2, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v2, v3, v2
 ; GFX10-IEEE-NEXT:    v_mul_f32_e32 v3, v4, v2
 ; GFX10-IEEE-NEXT:    v_fma_f32 v5, -v1, v3, v4
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v3, v5, v2
 ; GFX10-IEEE-NEXT:    v_fma_f32 v1, -v1, v3, v4
 ; GFX10-IEEE-NEXT:    v_div_fmas_f32 v1, v1, v2, v3
 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v0, v1, v0, 1.0
 ; GFX10-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLUSH-LABEL: v_rcp_f32:
 ; GFX10-FLUSH:       ; %bb.0:
 ; GFX10-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-FLUSH-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v1, s4, v0, v0, 1.0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v1, null, v0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v3, vcc_lo, 1.0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v4, -v1, v2, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v2, v4, v2
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v4, v3, v2
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v5, -v1, v4, v3
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v4, v5, v2

 ; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v1, v1, v3, v4
 ; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v0, v1, v0, 1.0
 ; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_rcp_f32_arcp:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v1, s4, v0, v0, 1.0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v1, null, v0, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, vcc_lo, 1.0, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX10-IEEE-NEXT:    v_fma_f32 v3, -v1, v2, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v2, v3, v2
 ; GFX10-IEEE-NEXT:    v_mul_f32_e32 v3, v4, v2
 ; GFX10-IEEE-NEXT:    v_fma_f32 v5, -v1, v3, v4
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v3, v5, v2
 ; GFX10-IEEE-NEXT:    v_fma_f32 v1, -v1, v3, v4
 ; GFX10-IEEE-NEXT:    v_div_fmas_f32 v1, v1, v2, v3
 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v0, v1, v0, 1.0
 ; GFX10-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLUSH-LABEL: v_rcp_f32_arcp:
 ; GFX10-FLUSH:       ; %bb.0:
 ; GFX10-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-FLUSH-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v1, s4, v0, v0, 1.0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v1, null, v0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v3, vcc_lo, 1.0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v4, -v1, v2, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v2, v4, v2
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v4, v3, v2
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v5, -v1, v4, v3
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v4, v5, v2

 ; GFX89-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
 ; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
 ; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_fdiv_f32_arcp_ulp25:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, s4, v1, v1, v0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, null, v1, v1, v0
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, vcc_lo, v0, v1, v0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX10-IEEE-NEXT:    v_fma_f32 v4, -v2, v3, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v3, v4, v3
 ; GFX10-IEEE-NEXT:    v_mul_f32_e32 v4, v5, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, v5
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v4, v6, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v2, -v2, v4, v5

 ; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v2, v5, v2, v4
 ; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
 ; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_fdiv_v2f32:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, s4, v2, v2, v0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, s4, v3, v3, v1
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, null, v2, v2, v0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, null, v3, v3, v1
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v10, vcc_lo, v0, v2, v0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v6, v4
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v7, v5
 ; GFX10-IEEE-NEXT:    v_fma_f32 v8, -v4, v6, 1.0
 ; GFX10-IEEE-NEXT:    v_fma_f32 v9, -v5, v7, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v6, v8, v6
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v8, s4, v1, v3, v1
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v7, v9, v7

 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v1, v5, v3, v1
 ; GFX10-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLUSH-LABEL: v_fdiv_v2f32:
 ; GFX10-FLUSH:       ; %bb.0:
 ; GFX10-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-FLUSH-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, s4, v2, v2, v0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, null, v2, v2, v0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v6, vcc_lo, v0, v2, v0
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v5, v4
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v5, v7, v5
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v7, v6, v5
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v8, -v4, v7, v6
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v7, v8, v5
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v4, -v4, v7, v6
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v6, s4, v3, v3, v1
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v6, null, v3, v3, v1
 ; GFX10-FLUSH-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v5, v6
 ; GFX10-FLUSH-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, vcc_lo, v1, v3, v1
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v4, -v6, v5, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v5, v4, v5
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v4, v2, v5

 ; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
 ; GFX89-IEEE-NEXT:    v_div_fixup_f32 v1, v5, v3, v1
 ; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_fdiv_v2f32_ulp25:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, s4, v2, v2, v0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, s4, v3, v3, v1
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, null, v2, v2, v0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, null, v3, v3, v1
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v10, vcc_lo, v0, v2, v0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v6, v4
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v7, v5
 ; GFX10-IEEE-NEXT:    v_fma_f32 v8, -v4, v6, 1.0
 ; GFX10-IEEE-NEXT:    v_fma_f32 v9, -v5, v7, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v6, v8, v6
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v8, s4, v1, v3, v1
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v7, v9, v7

 ; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v2, v3, v2, v5
 ; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v1, 1.0
 ; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_rcp_v2f32:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, s4, v0, v0, 1.0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v3, s4, v1, v1, 1.0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, null, v0, v0, 1.0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v3, null, v1, v1, 1.0
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v8, vcc_lo, 1.0, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v5, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
 ; GFX10-IEEE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v4, v6, v4
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v6, s4, 1.0, v1, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v5, v7, v5

 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v1, v3, v1, 1.0
 ; GFX10-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLUSH-LABEL: v_rcp_v2f32:
 ; GFX10-FLUSH:       ; %bb.0:
 ; GFX10-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-FLUSH-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, s4, v0, v0, 1.0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, null, v0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, vcc_lo, 1.0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v3, v5, v3
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v5, v6, v3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, s4, v1, v1, 1.0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, null, v1, v1, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v3, v4
 ; GFX10-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, vcc_lo, 1.0, v1, 1.0
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v5, -v4, v3, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v3, v5, v3
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v5, v2, v3

 ; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v2, v3, v2, v5
 ; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v1, 1.0
 ; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_rcp_v2f32_arcp:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, s4, v0, v0, 1.0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v3, s4, v1, v1, 1.0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v2, null, v0, v0, 1.0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v3, null, v1, v1, 1.0
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v8, vcc_lo, 1.0, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v5, v3
 ; GFX10-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
 ; GFX10-IEEE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v4, v6, v4
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v6, s4, 1.0, v1, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v5, v7, v5

 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v0, 1.0
 ; GFX10-IEEE-NEXT:    v_div_fixup_f32 v1, v3, v1, 1.0
 ; GFX10-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLUSH-LABEL: v_rcp_v2f32_arcp:
 ; GFX10-FLUSH:       ; %bb.0:
 ; GFX10-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-FLUSH-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, s4, v0, v0, 1.0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, null, v0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, vcc_lo, 1.0, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v3, v5, v3
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v5, v6, v3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 0
-; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, s4, v1, v1, 1.0
+; GFX10-FLUSH-NEXT:    v_div_scale_f32 v4, null, v1, v1, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
 ; GFX10-FLUSH-NEXT:    v_rcp_f32_e32 v3, v4
 ; GFX10-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v0, 1.0
 ; GFX10-FLUSH-NEXT:    v_div_scale_f32 v2, vcc_lo, 1.0, v1, 1.0
 ; GFX10-FLUSH-NEXT:    s_denorm_mode 3
 ; GFX10-FLUSH-NEXT:    v_fma_f32 v5, -v4, v3, 1.0
 ; GFX10-FLUSH-NEXT:    v_fmac_f32_e32 v3, v5, v3
 ; GFX10-FLUSH-NEXT:    v_mul_f32_e32 v5, v2, v3

 ; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
 ; GFX89-IEEE-NEXT:    v_div_fixup_f32 v1, v5, v3, v1
 ; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-IEEE-LABEL: v_fdiv_v2f32_arcp_ulp25:
 ; GFX10-IEEE:       ; %bb.0:
 ; GFX10-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-IEEE-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, s4, v2, v2, v0
-; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, s4, v3, v3, v1
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v4, null, v2, v2, v0
+; GFX10-IEEE-NEXT:    v_div_scale_f32 v5, null, v3, v3, v1
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v10, vcc_lo, v0, v2, v0
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v6, v4
 ; GFX10-IEEE-NEXT:    v_rcp_f32_e32 v7, v5
 ; GFX10-IEEE-NEXT:    v_fma_f32 v8, -v4, v6, 1.0
 ; GFX10-IEEE-NEXT:    v_fma_f32 v9, -v5, v7, 1.0
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v6, v8, v6
 ; GFX10-IEEE-NEXT:    v_div_scale_f32 v8, s4, v1, v3, v1
 ; GFX10-IEEE-NEXT:    v_fmac_f32_e32 v7, v9, v7

llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f64.ll

 ; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fdiv_f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[4:5], s4, v[2:3], v[2:3], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[4:5], null, v[2:3], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_mul_f64 v[8:9], v[10:11], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[8:9], v[10:11]

 ; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fdiv_f64_ulp25:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[4:5], s4, v[2:3], v[2:3], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[4:5], null, v[2:3], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_mul_f64 v[8:9], v[10:11], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[8:9], v[10:11]

 ; GFX9-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_rcp_f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[2:3], s4, v[0:1], v[0:1], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[2:3], null, v[0:1], v[0:1], 1.0
 ; GFX10-NEXT:    v_div_scale_f64 v[8:9], vcc_lo, 1.0, v[0:1], 1.0
 ; GFX10-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
 ; GFX10-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
 ; GFX10-NEXT:    v_mul_f64 v[6:7], v[8:9], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[6:7], v[8:9]

 ; GFX9-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_rcp_f64_arcp:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[2:3], s4, v[0:1], v[0:1], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[2:3], null, v[0:1], v[0:1], 1.0
 ; GFX10-NEXT:    v_div_scale_f64 v[8:9], vcc_lo, 1.0, v[0:1], 1.0
 ; GFX10-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
 ; GFX10-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
 ; GFX10-NEXT:    v_mul_f64 v[6:7], v[8:9], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[6:7], v[8:9]

 ; GFX9-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_rcp_f64_ulp25:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[2:3], s4, v[0:1], v[0:1], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[2:3], null, v[0:1], v[0:1], 1.0
 ; GFX10-NEXT:    v_div_scale_f64 v[8:9], vcc_lo, 1.0, v[0:1], 1.0
 ; GFX10-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
 ; GFX10-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
 ; GFX10-NEXT:    v_mul_f64 v[6:7], v[8:9], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[6:7], v[8:9]

 ; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fdiv_f64_arcp_ulp25:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[4:5], s4, v[2:3], v[2:3], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[4:5], null, v[2:3], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_mul_f64 v[8:9], v[10:11], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[8:9], v[10:11]

 ; GFX9-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
 ; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fdiv_v2f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[8:9], s4, v[4:5], v[4:5], v[0:1]
-; GFX10-NEXT:    v_div_scale_f64 v[10:11], s4, v[6:7], v[6:7], v[2:3]
+; GFX10-NEXT:    v_div_scale_f64 v[8:9], null, v[4:5], v[4:5], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[10:11], null, v[6:7], v[6:7], v[2:3]
 ; GFX10-NEXT:    v_div_scale_f64 v[20:21], vcc_lo, v[0:1], v[4:5], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
 ; GFX10-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
 ; GFX10-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0

 ; GFX9-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
 ; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fdiv_v2f64_ulp25:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[8:9], s4, v[4:5], v[4:5], v[0:1]
-; GFX10-NEXT:    v_div_scale_f64 v[10:11], s4, v[6:7], v[6:7], v[2:3]
+; GFX10-NEXT:    v_div_scale_f64 v[8:9], null, v[4:5], v[4:5], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[10:11], null, v[6:7], v[6:7], v[2:3]
 ; GFX10-NEXT:    v_div_scale_f64 v[20:21], vcc_lo, v[0:1], v[4:5], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
 ; GFX10-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
 ; GFX10-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0

 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
 ; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_rcp_v2f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[4:5], s4, v[0:1], v[0:1], 1.0
-; GFX10-NEXT:    v_div_scale_f64 v[6:7], s4, v[2:3], v[2:3], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[4:5], null, v[0:1], v[0:1], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[6:7], null, v[2:3], v[2:3], 1.0
 ; GFX10-NEXT:    v_div_scale_f64 v[16:17], vcc_lo, 1.0, v[0:1], 1.0
 ; GFX10-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
 ; GFX10-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0

 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
 ; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_rcp_v2f64_arcp:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[4:5], s4, v[0:1], v[0:1], 1.0
-; GFX10-NEXT:    v_div_scale_f64 v[6:7], s4, v[2:3], v[2:3], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[4:5], null, v[0:1], v[0:1], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[6:7], null, v[2:3], v[2:3], 1.0
 ; GFX10-NEXT:    v_div_scale_f64 v[16:17], vcc_lo, 1.0, v[0:1], 1.0
 ; GFX10-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
 ; GFX10-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0

 ; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
 ; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_rcp_v2f64_ulp25:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[4:5], s4, v[0:1], v[0:1], 1.0
-; GFX10-NEXT:    v_div_scale_f64 v[6:7], s4, v[2:3], v[2:3], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[4:5], null, v[0:1], v[0:1], 1.0
+; GFX10-NEXT:    v_div_scale_f64 v[6:7], null, v[2:3], v[2:3], 1.0
 ; GFX10-NEXT:    v_div_scale_f64 v[16:17], vcc_lo, 1.0, v[0:1], 1.0
 ; GFX10-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
 ; GFX10-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0

 ; GFX9-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
 ; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fdiv_v2f64_arcp_ulp25:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_div_scale_f64 v[8:9], s4, v[4:5], v[4:5], v[0:1]
-; GFX10-NEXT:    v_div_scale_f64 v[10:11], s4, v[6:7], v[6:7], v[2:3]
+; GFX10-NEXT:    v_div_scale_f64 v[8:9], null, v[4:5], v[4:5], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[10:11], null, v[6:7], v[6:7], v[2:3]
 ; GFX10-NEXT:    v_div_scale_f64 v[20:21], vcc_lo, v[0:1], v[4:5], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
 ; GFX10-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
 ; GFX10-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3] glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_load_dword v2, v0, s[2:3] offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, v2, v2, v1
+; GFX10-NEXT:    v_div_scale_f32 v0, null, v2, v2, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
 
   %a = load volatile float, float addrspace(1)* %gep.0, align 4

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3] glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_load_dword v2, v0, s[2:3] offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, v1, v2, v1
+; GFX10-NEXT:    v_div_scale_f32 v0, null, v1, v2, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
 
   %a = load volatile float, float addrspace(1)* %gep.0, align 4

 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[2:3] glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[2:3], v4, s[2:3] offset:8 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, v[2:3], v[2:3], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, v[2:3], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr double, double addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1
 
   %a = load volatile double, double addrspace(1)* %gep.0, align 8

 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[2:3] glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[2:3], v4, s[2:3] offset:8 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, v[0:1], v[2:3], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, v[0:1], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr double, double addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1
 
   %a = load volatile double, double addrspace(1)* %gep.0, align 8

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, v0, v0, s0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, v0, v0, s0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr float, float addrspace(1)* %in, i32 %tid
 
   %b = load float, float addrspace(1)* %gep, align 4
 
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 false)

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, s0, v0, s0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s0, v0, s0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr float, float addrspace(1)* %in, i32 %tid
 
   %b = load float, float addrspace(1)* %gep, align 4
 
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, s0, s0, v0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s0, s0, v0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr float, float addrspace(1)* %in, i32 %tid
 
   %a = load float, float addrspace(1)* %gep, align 4
 
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 false)

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, v0, s0, v0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, v0, s0, v0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr float, float addrspace(1)* %in, i32 %tid
 
   %a = load float, float addrspace(1)* %gep, align 4
 
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s0, v[0:1], v[0:1], s[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, v[0:1], v[0:1], s[0:1]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr double, double addrspace(1)* %in, i32 %tid
 
   %b = load double, double addrspace(1)* %gep, align 8
 
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 false)

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s0, s[0:1], v[0:1], s[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, s[0:1], v[0:1], s[0:1]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr double, double addrspace(1)* %in, i32 %tid
 
   %b = load double, double addrspace(1)* %gep, align 8
 
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true)

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s0, s[0:1], s[0:1], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, s[0:1], s[0:1], v[0:1]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr double, double addrspace(1)* %in, i32 %tid
 
   %a = load double, double addrspace(1)* %gep, align 8
 
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 false)

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s0, v[0:1], s[0:1], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, v[0:1], s[0:1], v[0:1]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr double, double addrspace(1)* %in, i32 %tid
 
   %a = load double, double addrspace(1)* %gep, align 8
 
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true)

 ; GFX10-LABEL: test_div_scale_f32_all_scalar_1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x2
 ; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x4c
 ; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x70
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, s5, s5, s4
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s5, s5, s4
 ; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_endpgm
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 false)
   %result0 = extractvalue { float, i1 } %result, 0
   store float %result0, float addrspace(1)* %out, align 4
   ret void
 }
 

 ; GFX10-LABEL: test_div_scale_f32_all_scalar_2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x2
 ; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x4c
 ; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x70
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s0, s4, s5, s4
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s4, s5, s4
 ; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_endpgm
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 true)
   %result0 = extractvalue { float, i1 } %result, 0
   store float %result0, float addrspace(1)* %out, align 4
   ret void
 }
 

 ; GFX10-LABEL: test_div_scale_f64_all_scalar_1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x74
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, s[4:5], s[4:5], s[2:3]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, s[4:5], s[4:5], s[2:3]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 false)
   %result0 = extractvalue { double, i1 } %result, 0
   store double %result0, double addrspace(1)* %out, align 8
   ret void
 }
 

 ; GFX10-LABEL: test_div_scale_f64_all_scalar_2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x74
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, s[2:3], s[4:5], s[2:3]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, s[2:3], s[4:5], s[2:3]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true)
   %result0 = extractvalue { double, i1 } %result, 0
   store double %result0, double addrspace(1)* %out, align 8
   ret void
 }
 

 ; GFX10-LABEL: test_div_scale_f32_inline_imm_num:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, v0, v0, 1.0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, v0, v0, 1.0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %a = load float, float addrspace(1)* %gep.0, align 4
 
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 1.0, float %a, i1 false)
   %result0 = extractvalue { float, i1 } %result, 0

 ; GFX10-LABEL: test_div_scale_f32_inline_imm_den:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, 2.0, 2.0, v0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, 2.0, 2.0, v0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %a = load float, float addrspace(1)* %gep.0, align 4
 
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float 2.0, i1 false)
   %result0 = extractvalue { float, i1 } %result, 0

 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3] glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_load_dword v2, v0, s[2:3] offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, v2, v2, v0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, v2, v2, v0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
 
   %a = load volatile float, float addrspace(1)* %gep.0, align 4
   %b = load volatile float, float addrspace(1)* %gep.1, align 4

 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3] glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_load_dword v2, v0, s[2:3] offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v2
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, v0, v0, v1
+; GFX10-NEXT:    v_div_scale_f32 v0, null, v0, v0, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
 
   %a = load volatile float, float addrspace(1)* %gep.0, align 4

 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: test_div_scale_f32_val_undef_val:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, s0, s0, 0x41000000
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s0, s0, 0x41000000
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 8.0, float undef, i1 false)
   %result0 = extractvalue { float, i1 } %result, 0
   store float %result0, float addrspace(1)* %out, align 4
   ret void
 }
 

 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: test_div_scale_f32_undef_val_val:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, 0x41000000, 0x41000000, s0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, 0x41000000, 0x41000000, s0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float undef, float 8.0, i1 false)
   %result0 = extractvalue { float, i1 } %result, 0
   store float %result0, float addrspace(1)* %out, align 4
   ret void
 }
 

 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: test_div_scale_f32_undef_undef_val:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s2, s0, s0, s0
+; GFX10-NEXT:    v_div_scale_f32 v0, null, s0, s0, s0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float undef, float undef, i1 false)
   %result0 = extractvalue { float, i1 } %result, 0
   store float %result0, float addrspace(1)* %out, align 4
   ret void
 }
 

 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: test_div_scale_f64_val_undef_val:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_mov_b32 s2, 0
 ; GFX10-NEXT:    s_mov_b32 s3, 0x40200000
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    v_div_scale_f64 v[0:1], null, s[0:1], s[0:1], s[2:3]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX10-NEXT:    s_endpgm
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double 8.0, double undef, i1 false)
   %result0 = extractvalue { double, i1 } %result, 0
   store double %result0, double addrspace(1)* %out, align 8
   ret void

llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll

 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_i64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v3, v4, v3
 ; GFX10-NEXT:    v_mul_lo_u32 v2, v5, v2
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v3, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = mul i64 %num, %den
   ret i64 %result
 }
 

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v2, v2, v3
 ; GFX10-NEXT:    v_mul_lo_u32 v5, v6, v5
 ; GFX10-NEXT:    v_mul_lo_u32 v8, v7, v4
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v6, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v6, v3, 0
 ; GFX10-NEXT:    v_add3_u32 v2, v5, v8, v2
-; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s4, v6, v4, v[1:2]
-; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s4, v7, v3, v[1:2]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v6, v4, v[1:2]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v7, v3, v[1:2]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = mul i96 %num, %den
   ret i96 %result
 }
 
 define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
 ; GFX7-LABEL: s_mul_i128:
 ; GFX7:       ; %bb.0:

 ; GFX10-LABEL: v_mul_i128:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v10, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v4
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v8, v6, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v8, v6, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v8, v7
 ; GFX10-NEXT:    v_mul_lo_u32 v6, v9, v6
-; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s4, v9, v5, v[0:1]
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v8, v4, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s4, v10, v4, v[11:12]
+; GFX10-NEXT:    v_mad_u64_u32 v[11:12], null, v9, v5, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v8, v4, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[11:12], null, v10, v4, v[11:12]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v11
 ; GFX10-NEXT:    v_mad_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2]
 ; GFX10-NEXT:    v_mul_lo_u32 v5, v10, v5
 ; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s4, v9, v4, v[1:2]
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v7, s4, v12, v7, s4
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v7, v6, vcc_lo
 ; GFX10-NEXT:    v_add3_u32 v3, v4, v5, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]

 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v16, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v17, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v27, v6, v9
 ; GFX10-NEXT:    v_mul_lo_u32 v28, v5, v10
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v7, v8
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v16, v14, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s4, v16, v12, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v16, v14, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[18:19], null, v16, v12, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v30, v17, v14
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v17, v13, v[0:1]
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v2, v12, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v17, v13, v[0:1]
 ; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s4, v17, v11, v[18:19]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s4
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s5, v3, v11, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v2, v12, v[0:1]
 ; GFX10-NEXT:    v_mad_u64_u32 v[18:19], vcc_lo, v2, v10, v[18:19]
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v22, vcc_lo, 0, v20, vcc_lo
-; GFX10-NEXT:    v_mad_u64_u32 v[20:21], s4, v16, v10, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v10, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[20:21], null, v16, v10, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v3, v11, v[0:1]
 ; GFX10-NEXT:    v_mad_u64_u32 v[18:19], vcc_lo, v3, v9, v[18:19]
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v24, vcc_lo, 0, v22, vcc_lo
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v5, v9, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v10, v[0:1]
 ; GFX10-NEXT:    v_mad_u64_u32 v[18:19], vcc_lo, v4, v8, v[18:19]
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v26, vcc_lo, 0, v24, vcc_lo
-; GFX10-NEXT:    v_mad_u64_u32 v[22:23], s4, v6, v8, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v5, v9, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[22:23], null, v6, v8, v[0:1]
 ; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v17, v9, v[20:21]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v20, v22
 ; GFX10-NEXT:    v_mad_u64_u32 v[21:22], vcc_lo, v2, v8, v[0:1]
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v29, vcc_lo, 0, v25, vcc_lo
 ; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v16, v13, v[19:20]
 ; GFX10-NEXT:    v_mov_b32_e32 v20, v18
 ; GFX10-NEXT:    v_mov_b32_e32 v19, v22
 ; GFX10-NEXT:    v_mul_lo_u32 v22, v16, v15
 ; GFX10-NEXT:    v_mad_u64_u32 v[24:25], vcc_lo, v17, v12, v[0:1]
 ; GFX10-NEXT:    v_mad_u64_u32 v[14:15], s6, v16, v11, v[19:20]
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s5, v16, v8, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v16, v8, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v20, v4, v11
 ; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s6
 ; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s5, v2, v11, v[24:25]
 ; GFX10-NEXT:    v_mul_lo_u32 v25, v3, v12
 ; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s6, v17, v10, v[14:15]
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s6, 0, v6, s6
 ; GFX10-NEXT:    v_mul_lo_u32 v24, v2, v13
 ; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s7, v3, v10, v[18:19]

llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll

 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GFX10-NEXT:    v_trunc_f32_e32 v1, v1
 ; GFX10-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v1
 ; GFX10-NEXT:    v_add_f32_e32 v0, v2, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v2, v1
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v4, s10, v2
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s14, s10, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s10, v3, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, s11, v3
 ; GFX10-NEXT:    v_mul_hi_u32 v6, v2, v0
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v4, v5
 ; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v3, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, v3, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v2, v1
 ; GFX10-NEXT:    v_mul_hi_u32 v8, v3, v1

 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v5, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v5, v7, v6
 ; GFX10-NEXT:    v_add_co_u32 v0, s14, v4, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s14
 ; GFX10-NEXT:    v_add_co_u32 v3, vcc_lo, v3, v0
 ; GFX10-NEXT:    v_add3_u32 v1, v5, v4, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v4, s11, v3
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v2, v1, vcc_lo
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s14, s10, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s10, v3, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, s10, v2
 ; GFX10-NEXT:    v_mul_hi_u32 v6, v2, v0
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v4
 ; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v3, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, v3, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v2, v1
 ; GFX10-NEXT:    v_mul_hi_u32 v8, v3, v1

 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s10
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v3, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v3, v5, v4
 ; GFX10-NEXT:    v_add_co_u32 v2, s10, v2, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s10
 ; GFX10-NEXT:    v_mul_lo_u32 v4, s9, v2
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v2, 1
 ; GFX10-NEXT:    v_add3_u32 v3, v3, v0, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s10, s8, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s8, v2, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, s8, v3
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v3, vcc_lo
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v4
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v6, 1
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v7, vcc_lo
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v8, s1, v1
 ; GFX10-NEXT:    v_sub_co_u32 v0, vcc_lo, s0, v0
 ; GFX10-NEXT:    v_sub_co_ci_u32_e64 v1, s0, s1, v1, vcc_lo

 ; GFX10-NEXT:    s_addc_u32 s11, s15, s16
 ; GFX10-NEXT:    v_add_f32_e32 v0, v1, v0
 ; GFX10-NEXT:    s_xor_b64 s[10:11], s[10:11], s[16:17]
 ; GFX10-NEXT:    s_mov_b32 s13, s12
 ; GFX10-NEXT:    v_cvt_f32_u32_e32 v1, s11
 ; GFX10-NEXT:    v_cvt_f32_u32_e32 v2, s10
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX10-NEXT:    s_xor_b64 s[14:15], s[6:7], s[12:13]
+; GFX10-NEXT:    s_sub_u32 s3, 0, s10
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; GFX10-NEXT:    s_subb_u32 s6, 0, s11
 ; GFX10-NEXT:    v_add_f32_e32 v1, v1, v2
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v1, v1
 ; GFX10-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v0
 ; GFX10-NEXT:    v_trunc_f32_e32 v2, v2
 ; GFX10-NEXT:    v_mul_f32_e32 v3, 0x5f7ffffc, v1
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v2
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v5, v2
 ; GFX10-NEXT:    v_mul_f32_e32 v4, 0x2f800000, v3
 ; GFX10-NEXT:    v_add_f32_e32 v0, v1, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v7, s20, v5
 ; GFX10-NEXT:    v_trunc_f32_e32 v4, v4
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v6, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v4
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s3, s20, v6, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s20, v6, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v8, s21, v6
 ; GFX10-NEXT:    v_add_f32_e32 v2, v2, v3
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v4
-; GFX10-NEXT:    s_sub_u32 s3, 0, s10
-; GFX10-NEXT:    s_subb_u32 s6, 0, s11
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v4, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v9, s3, v3
 ; GFX10-NEXT:    v_add3_u32 v7, v1, v7, v8
 ; GFX10-NEXT:    v_mul_lo_u32 v10, v5, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v11, v6, v0
-; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s7, s3, v4, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, s3, v4, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v8, s6, v4
 ; GFX10-NEXT:    v_mul_lo_u32 v12, v6, v7
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v5, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v13, v5, v7
 ; GFX10-NEXT:    v_mul_hi_u32 v14, v6, v7
 ; GFX10-NEXT:    v_mul_hi_u32 v7, v5, v7
 ; GFX10-NEXT:    v_add3_u32 v2, v2, v9, v8
 ; GFX10-NEXT:    v_add_co_u32 v10, s7, v10, v12
 ; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_u32 v0, s7, v13, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v8, v3, v1
 ; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s7
 ; GFX10-NEXT:    v_mul_lo_u32 v15, v4, v2
 ; GFX10-NEXT:    v_add_co_u32 v10, s7, v10, v11
 ; GFX10-NEXT:    v_mul_hi_u32 v9, v4, v1
 ; GFX10-NEXT:    v_mul_hi_u32 v1, v3, v1
 ; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_u32 v0, s7, v0, v14
 ; GFX10-NEXT:    v_mul_lo_u32 v14, v3, v2
 ; GFX10-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s7
 ; GFX10-NEXT:    v_add_nc_u32_e32 v10, v12, v10
 ; GFX10-NEXT:    v_add_co_u32 v8, s7, v8, v15
 ; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s7
-; GFX10-NEXT:    v_mul_hi_u32 v16, v4, v2
 ; GFX10-NEXT:    v_add_nc_u32_e32 v11, v13, v11
+; GFX10-NEXT:    v_mul_hi_u32 v16, v4, v2
 ; GFX10-NEXT:    v_add_co_u32 v1, s7, v14, v1
 ; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_u32 v0, s7, v0, v10
 ; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_u32 v8, s7, v8, v9
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_u32 v9, s7, v1, v16
 ; GFX10-NEXT:    v_add3_u32 v7, v11, v10, v7
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v8, v12, v8
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v5, v7, vcc_lo
 ; GFX10-NEXT:    v_mul_hi_u32 v2, v3, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v10, v13, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s7, s20, v6, 0
 ; GFX10-NEXT:    v_add_co_u32 v7, s7, v9, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, v13, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s20, v6, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v9, s21, v6
 ; GFX10-NEXT:    v_mul_lo_u32 v11, s20, v5
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s7
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v7
 ; GFX10-NEXT:    v_add3_u32 v2, v10, v8, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v8, v5, v0
 ; GFX10-NEXT:    v_add3_u32 v7, v1, v11, v9
 ; GFX10-NEXT:    v_mul_hi_u32 v10, v6, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v5, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v3, v2, vcc_lo
 ; GFX10-NEXT:    v_mul_lo_u32 v12, v6, v7
-; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s7, s3, v4, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, s3, v4, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v9, s6, v4
 ; GFX10-NEXT:    v_mul_lo_u32 v11, s3, v3
 ; GFX10-NEXT:    v_mul_lo_u32 v13, v5, v7
 ; GFX10-NEXT:    v_mul_hi_u32 v14, v6, v7
 ; GFX10-NEXT:    v_mul_hi_u32 v7, v5, v7
 ; GFX10-NEXT:    v_add_co_u32 v8, s3, v8, v12
 ; GFX10-NEXT:    v_mul_lo_u32 v15, v3, v1
 ; GFX10-NEXT:    v_mul_hi_u32 v16, v4, v1

 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s3
 ; GFX10-NEXT:    v_mul_hi_u32 v2, v3, v2
 ; GFX10-NEXT:    v_add_nc_u32_e32 v6, v13, v6
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v8
 ; GFX10-NEXT:    v_add3_u32 v5, v7, v0, v5
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
 ; GFX10-NEXT:    v_mul_hi_u32 v8, s14, v4
 ; GFX10-NEXT:    v_add3_u32 v2, v6, v1, v2
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s3, s8, v9, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s8, v9, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v6, s9, v9
 ; GFX10-NEXT:    v_mul_lo_u32 v7, s8, v5
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v3, v2, vcc_lo
 ; GFX10-NEXT:    v_mul_lo_u32 v3, s15, v4
 ; GFX10-NEXT:    v_mul_hi_u32 v4, s15, v4
 ; GFX10-NEXT:    v_mul_lo_u32 v10, s14, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v11, s15, v2
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v7, v6

 ; GFX10-NEXT:    v_add_nc_u32_e32 v3, v8, v10
 ; GFX10-NEXT:    v_add_co_u32 v4, s0, v4, v1
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
 ; GFX10-NEXT:    v_sub_co_u32 v8, s0, v12, s8
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e64 v10, s0, 0, v0, s0
 ; GFX10-NEXT:    v_add3_u32 v2, v3, v1, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v6, v19, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v20, vcc_lo
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, s10, v4, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s10, v4, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v7, s10, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v11, s11, v4
 ; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v17
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v16
 ; GFX10-NEXT:    v_mov_b32_e32 v16, 0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, v12, v8, s0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc_lo
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v7, v11

llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll

 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; GFX10-NEXT:    v_trunc_f32_e32 v1, v1
 ; GFX10-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v1
 ; GFX10-NEXT:    v_add_f32_e32 v0, v2, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v2, v1
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v4, s0, v2
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s2, s0, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s0, v3, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, s1, v3
 ; GFX10-NEXT:    v_mul_hi_u32 v6, v2, v0
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v4, v5
 ; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v3, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, v3, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v2, v1
 ; GFX10-NEXT:    v_mul_hi_u32 v8, v3, v1

 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v5, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v5, v7, v6
 ; GFX10-NEXT:    v_add_co_u32 v0, s2, v4, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s2
 ; GFX10-NEXT:    v_add_co_u32 v3, vcc_lo, v3, v0
 ; GFX10-NEXT:    v_add3_u32 v1, v5, v4, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v4, s1, v3
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v2, v1, vcc_lo
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s2, s0, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s0, v3, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, s0, v2
 ; GFX10-NEXT:    v_mul_hi_u32 v6, v2, v0
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v4
 ; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v3, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, v3, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v2, v1
 ; GFX10-NEXT:    v_mul_hi_u32 v8, v3, v1

 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v3, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v3, v5, v4
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, v2, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
 ; GFX10-NEXT:    v_mul_lo_u32 v4, s11, v2
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v2, 1
 ; GFX10-NEXT:    v_add3_u32 v3, v3, v0, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, s10, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s10, v2, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v5, s10, v3
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v3, vcc_lo
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v4
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v6, 1
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v7, vcc_lo
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v8, s9, v1
 ; GFX10-NEXT:    v_sub_co_u32 v9, vcc_lo, s8, v0
 ; GFX10-NEXT:    v_sub_co_ci_u32_e64 v10, s0, s9, v1, vcc_lo

 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v6, v3
 ; GFX10-NEXT:    v_add_f32_e32 v0, v4, v0
 ; GFX10-NEXT:    v_add_f32_e32 v1, v5, v1
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v4, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v10, s2, v6
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v5, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v8, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v7, s0, v4
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s6, s0, v5, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s0, v5, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v9, s1, v5
-; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s6, s2, v8, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[2:3], null, s2, v8, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v11, s3, v8
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v7, v9
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v4, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v9, v5, v0
 ; GFX10-NEXT:    v_add3_u32 v3, v3, v10, v11
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v4, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v12, v5, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v13, v4, v1

 ; GFX10-NEXT:    v_add_nc_u32_e32 v9, v13, v9
 ; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s6
 ; GFX10-NEXT:    v_mul_hi_u32 v3, v6, v3
 ; GFX10-NEXT:    v_add_co_u32 v2, s6, v2, v10
 ; GFX10-NEXT:    v_add_nc_u32_e32 v11, v16, v11
 ; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s6
 ; GFX10-NEXT:    v_add3_u32 v1, v9, v7, v1
 ; GFX10-NEXT:    v_add_co_u32 v5, vcc_lo, v5, v0
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
 ; GFX10-NEXT:    v_add3_u32 v3, v11, v10, v3
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v4, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v2
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v6, v3, vcc_lo
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s6, s0, v5, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s0, v5, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v7, s1, v5
 ; GFX10-NEXT:    v_mul_lo_u32 v9, s0, v4
-; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s0, s2, v8, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[2:3], null, s2, v8, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v10, s3, v8
 ; GFX10-NEXT:    v_mul_lo_u32 v11, s2, v6
 ; GFX10-NEXT:    v_mul_lo_u32 v12, v4, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v13, v5, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v0, v4, v0
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v9, v7
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v6, v2
 ; GFX10-NEXT:    v_mul_hi_u32 v9, v8, v2

 ; GFX10-NEXT:    v_add_nc_u32_e32 v5, v8, v5
 ; GFX10-NEXT:    v_mul_hi_u32 v8, s10, v2
 ; GFX10-NEXT:    v_add_co_u32 v4, s0, v0, v4
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
 ; GFX10-NEXT:    v_mul_hi_u32 v2, s11, v2
 ; GFX10-NEXT:    v_mul_hi_u32 v11, s10, v3
 ; GFX10-NEXT:    v_add_co_u32 v6, s0, v6, v7
 ; GFX10-NEXT:    v_add3_u32 v5, v5, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, s12, v4, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s12, v4, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v12, s13, v4
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s0
 ; GFX10-NEXT:    v_mul_lo_u32 v13, s12, v5
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, v10, v2
 ; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s0
 ; GFX10-NEXT:    v_add_co_u32 v6, s0, v6, v8
 ; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s0
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, v2, v11
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s0
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v13, v12
 ; GFX10-NEXT:    v_add_nc_u32_e32 v6, v7, v6
 ; GFX10-NEXT:    v_mul_hi_u32 v3, s11, v3
-; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v7, v10, v8
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v8, s9, v1
 ; GFX10-NEXT:    v_sub_co_u32 v10, vcc_lo, s8, v0
 ; GFX10-NEXT:    v_sub_co_ci_u32_e64 v11, s0, s9, v1, vcc_lo
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v0, vcc_lo, s13, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s12, v10
-; GFX10-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
 ; GFX10-NEXT:    v_sub_co_u32 v8, vcc_lo, v10, s12
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e64 v12, s0, 0, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_le_u32_e64 s0, s13, v11
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v0, vcc_lo, s13, v0, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s0
 ; GFX10-NEXT:    v_cmp_le_u32_e64 s0, s12, v8
 ; GFX10-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s0
 ; GFX10-NEXT:    v_cmp_le_u32_e64 s0, s13, v12
 ; GFX10-NEXT:    v_cndmask_b32_e64 v15, 0, -1, s0
 ; GFX10-NEXT:    v_add_co_u32 v16, s0, v4, 1
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v17, s0, 0, v5, s0
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, s13, v11
 ; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v1, s0
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, s13, v12
 ; GFX10-NEXT:    v_cndmask_b32_e64 v14, v15, v14, s0
 ; GFX10-NEXT:    v_add_co_u32 v6, s0, v2, v6
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
 ; GFX10-NEXT:    v_add_co_u32 v15, s0, v16, 1
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v18, s0, 0, v17, s0
-; GFX10-NEXT:    v_add3_u32 v3, v7, v1, v3
-; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s0, s14, v6, 0
 ; GFX10-NEXT:    v_mul_lo_u32 v19, s15, v6
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_add3_u32 v3, v7, v1, v3
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, s14, v6, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v18, s0, 0, v17, s0
 ; GFX10-NEXT:    v_mul_lo_u32 v7, s14, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc_lo
 ; GFX10-NEXT:    v_sub_co_u32 v16, s0, v8, s12
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e64 v20, s0, 0, v0, s0
-; GFX10-NEXT:    v_add3_u32 v2, v2, v7, v19
-; GFX10-NEXT:    v_sub_co_u32 v7, s0, s10, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v18, vcc_lo
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_add3_u32 v2, v2, v7, v19
+; GFX10-NEXT:    v_sub_co_u32 v7, s0, s10, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v15, vcc_lo
 ; GFX10-NEXT:    v_sub_co_ci_u32_e64 v13, s1, s11, v2, s0
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v2, s11, v2
 ; GFX10-NEXT:    v_cmp_ne_u32_e64 s1, 0, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_le_u32_e64 s2, s15, v13
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e64 s2, s15, v13
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e64 v2, s0, s15, v2, s0
 ; GFX10-NEXT:    v_cmp_le_u32_e64 s0, s14, v7
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, v8, v16, s1
 ; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s2
 ; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v20, s1
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, s15, v13
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s0
 ; GFX10-NEXT:    v_sub_co_u32 v14, s0, v7, s14

llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll

 ; GFX10W64-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10W64-NEXT:    buffer_atomic_add v1, off, s[12:15], 0 glc
 ; GFX10W64-NEXT:  .LBB1_2:
 ; GFX10W64-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10W64-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX10W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10W64-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s8, v0, s[0:1]
+; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], null, s8, v0, s[0:1]
 ; GFX10W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10W64-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10W64-NEXT:    s_endpgm
 ;
 ; GFX10W32-LABEL: add_i32_uniform:
 ; GFX10W32:       ; %bb.0: ; %entry
 ; GFX10W32-NEXT:    s_clause 0x1
 ; GFX10W32-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24

 ; GFX10W32-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10W32-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
 ; GFX10W32-NEXT:  .LBB1_2:
 ; GFX10W32-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX10W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], s0, s4, v0, s[0:1]
+; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], null, s4, v0, s[0:1]
 ; GFX10W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10W32-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10W32-NEXT:    s_endpgm
 entry:
   %old = call i32 @llvm.amdgcn.raw.buffer.atomic.add(i32 %additive, <4 x i32> %inout, i32 0, i32 0, i32 0)
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll

 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  .LBB1_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1064-NEXT:    s_mov_b32 s6, -1
-; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s8, v0, s[0:1]
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], null, s8, v0, s[0:1]
 ; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i32_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_clause 0x1
 ; GFX1032-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX1032-NEXT:    s_load_dword s2, s[0:1], 0x34

 ; GFX1032-NEXT:    buffer_gl1_inv
 ; GFX1032-NEXT:  .LBB1_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s6, -1
-; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s0, s2, v0, s[0:1]
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v0, s[0:1]
 ; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i32 addrspace(1)* %inout, i32 %additive acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 

 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  .LBB3_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s3, v1
-; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v2, 5, s[2:3]
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24

 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:    buffer_gl1_inv
 ; GFX1032-NEXT:  .LBB3_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s3, v1
-; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s2, v2, 5, s[2:3]
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(1)* %inout, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void

 ; GFX1064-NEXT:  .LBB4_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1064-NEXT:    s_mov_b32 s6, -1
-; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s2, v2, s[0:1]
-; GFX1064-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s3, v2, v[1:2]
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[0:1]
+; GFX1064-NEXT:    v_mad_u64_u32 v[1:2], null, s3, v2, v[1:2]
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_clause 0x1
 ; GFX1032-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX1032-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34

 ; GFX1032-NEXT:  .LBB4_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s6, -1
-; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s0, s2, v2, s[0:1]
-; GFX1032-NEXT:    v_mad_u64_u32 v[1:2], s0, s3, v2, v[1:2]
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[0:1]
+; GFX1032-NEXT:    v_mad_u64_u32 v[1:2], null, s3, v2, v[1:2]
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(1)* %inout, i64 %additive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 

 ; GFX1064-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  .LBB10_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s2, v2, 0
-; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
-; GFX1064-NEXT:    s_mov_b32 s6, -1
-; GFX1064-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], s3, v2, v[4:5]
+; GFX1064-NEXT:    v_mad_u64_u32 v[3:4], null, s2, v2, 0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    v_mad_u64_u32 v[4:5], null, s3, v2, v[4:5]
 ; GFX1064-NEXT:    v_sub_co_u32 v0, vcc, s0, v3
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s1, v1, vcc
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: sub_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry

 ; GFX1032-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:    buffer_gl1_inv
 ; GFX1032-NEXT:  .LBB10_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mad_u64_u32 v[3:4], s0, s2, v2, 0
+; GFX1032-NEXT:    v_mad_u64_u32 v[3:4], null, s2, v2, 0
+; GFX1032-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s6, -1
-; GFX1032-NEXT:    v_mad_u64_u32 v[4:5], s0, s3, v2, v[4:5]
-; GFX1032-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT:    v_mad_u64_u32 v[4:5], null, s3, v2, v[4:5]
 ; GFX1032-NEXT:    v_sub_co_u32 v0, vcc_lo, s0, v3
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(1)* %inout, i64 %subitive acq_rel
   store i64 %old, i64 addrspace(1)* %out

llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll

 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  .LBB1_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s6, v0, s[0:1]
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], null, s6, v0, s[0:1]
 ; GFX1064-NEXT:    s_mov_b32 s6, -1
 ; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i32_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_clause 0x1
 ; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24

 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  .LBB1_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s6, -1
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s0, s2, v0, s[0:1]
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v0, s[0:1]
 ; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: add_i32_uniform:
 ; GFX1164:       ; %bb.0: ; %entry
 ; GFX1164-NEXT:    s_clause 0x1
 ; GFX1164-NEXT:    s_load_b64 s[4:5], s[0:1], 0x24
 ; GFX1164-NEXT:    s_load_b32 s6, s[0:1], 0x2c

 ; GFX1164-NEXT:    ds_add_rtn_u32 v1, v1, v2
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    buffer_gl0_inv
 ; GFX1164-NEXT:  .LBB1_2:
 ; GFX1164-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1164-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX1164-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s6, v0, s[0:1]
+; GFX1164-NEXT:    v_mad_u64_u32 v[1:2], null, s6, v0, s[0:1]
 ; GFX1164-NEXT:    s_mov_b32 s6, -1
 ; GFX1164-NEXT:    buffer_store_b32 v1, off, s[4:7], 0
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: add_i32_uniform:
 ; GFX1132:       ; %bb.0: ; %entry
 ; GFX1132-NEXT:    s_clause 0x1
 ; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x24

 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    buffer_gl0_inv
 ; GFX1132-NEXT:  .LBB1_2:
 ; GFX1132-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX1132-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX1132-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1132-NEXT:    s_mov_b32 s6, -1
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_mad_u64_u32 v[1:2], s0, s0, v0, s[2:3]
+; GFX1132-NEXT:    v_mad_u64_u32 v[1:2], null, s0, v0, s[2:3]
 ; GFX1132-NEXT:    buffer_store_b32 v1, off, s[4:7], 0
 ; GFX1132-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %additive acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 

 ; GFX1064-NEXT:    ds_add_rtn_u64 v[0:1], v1, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  .LBB4_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s3, v1
-; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v2, 5, s[2:3]
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry

 ; GFX1032-NEXT:    ds_add_rtn_u64 v[0:1], v1, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  .LBB4_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s3, v1
-; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s2, v2, 5, s[2:3]
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: add_i64_constant:
 ; GFX1164:       ; %bb.0: ; %entry

 ; GFX1164-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1164-NEXT:    ds_add_rtn_u64 v[0:1], v1, v[0:1]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    buffer_gl0_inv
 ; GFX1164-NEXT:  .LBB4_2:
 ; GFX1164-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1164-NEXT:    v_readfirstlane_b32 s3, v1
-; GFX1164-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v2, 5, s[2:3]
+; GFX1164-NEXT:    v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
 ; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1164-NEXT:    s_mov_b32 s2, -1
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: add_i64_constant:
 ; GFX1132:       ; %bb.0: ; %entry

 ; GFX1132-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1132-NEXT:    ds_add_rtn_u64 v[0:1], v1, v[0:1]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    buffer_gl0_inv
 ; GFX1132-NEXT:  .LBB4_2:
 ; GFX1132-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1132-NEXT:    v_readfirstlane_b32 s3, v1
-; GFX1132-NEXT:    v_mad_u64_u32 v[0:1], s2, v2, 5, s[2:3]
+; GFX1132-NEXT:    v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1132-NEXT:    s_mov_b32 s2, -1
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1132-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out

 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  .LBB5_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s2, v2, s[4:5]
-; GFX1064-NEXT:    v_mad_u64_u32 v[1:2], s[2:3], s3, v2, v[1:2]
-; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5]
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_mad_u64_u32 v[1:2], null, s3, v2, v[1:2]
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1

 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  .LBB5_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1032-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s2, s2, v2, s[4:5]
-; GFX1032-NEXT:    v_mad_u64_u32 v[1:2], s2, s3, v2, v[1:2]
-; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5]
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_mad_u64_u32 v[1:2], null, s3, v2, v[1:2]
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: add_i64_uniform:
 ; GFX1164:       ; %bb.0: ; %entry
 ; GFX1164-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX1164-NEXT:    s_mov_b64 s[4:5], exec

 ; GFX1164-NEXT:    ds_add_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    buffer_gl0_inv
 ; GFX1164-NEXT:  .LBB5_2:
 ; GFX1164-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1164-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1164-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s2, v2, s[4:5]
-; GFX1164-NEXT:    v_mad_u64_u32 v[3:4], s[2:3], s3, v2, v[1:2]
-; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5]
 ; GFX1164-NEXT:    s_mov_b32 s2, -1
+; GFX1164-NEXT:    v_mad_u64_u32 v[3:4], null, s3, v2, v[1:2]
+; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX1164-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: add_i64_uniform:
 ; GFX1132:       ; %bb.0: ; %entry
 ; GFX1132-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s5, exec_lo

 ; GFX1132-NEXT:    ds_add_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    buffer_gl0_inv
 ; GFX1132-NEXT:  .LBB5_2:
 ; GFX1132-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1132-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1132-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_mad_u64_u32 v[0:1], s2, s2, v2, s[4:5]
-; GFX1132-NEXT:    v_mad_u64_u32 v[3:4], s2, s3, v2, v[1:2]
-; GFX1132-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5]
 ; GFX1132-NEXT:    s_mov_b32 s2, -1
+; GFX1132-NEXT:    v_mad_u64_u32 v[3:4], null, s3, v2, v[1:2]
+; GFX1132-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1132-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX1132-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1132-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 %additive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }

 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  .LBB12_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], s2, v2, 0
-; GFX1064-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX1064-NEXT:    v_mad_u64_u32 v[4:5], s[2:3], s3, v2, v[4:5]
+; GFX1064-NEXT:    v_mad_u64_u32 v[3:4], null, s2, v2, 0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1064-NEXT:    v_mad_u64_u32 v[4:5], null, s3, v2, v[4:5]
 ; GFX1064-NEXT:    v_sub_co_u32 v0, vcc, s2, v3
-; GFX1064-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: sub_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s5, exec_lo

 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  .LBB12_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mad_u64_u32 v[3:4], s2, s2, v2, 0
-; GFX1032-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX1032-NEXT:    v_mad_u64_u32 v[4:5], s2, s3, v2, v[4:5]
+; GFX1032-NEXT:    v_mad_u64_u32 v[3:4], null, s2, v2, 0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1032-NEXT:    v_mad_u64_u32 v[4:5], null, s3, v2, v[4:5]
 ; GFX1032-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v3
-; GFX1032-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: sub_i64_uniform:
 ; GFX1164:       ; %bb.0: ; %entry
 ; GFX1164-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], exec

 ; GFX1164-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1164-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1164-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    buffer_gl0_inv
 ; GFX1164-NEXT:  .LBB12_2:
 ; GFX1164-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], s2, v2, 0
-; GFX1164-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX1164-NEXT:    v_mad_u64_u32 v[5:6], s[2:3], s3, v2, v[4:5]
+; GFX1164-NEXT:    v_mad_u64_u32 v[3:4], null, s2, v2, 0
 ; GFX1164-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1164-NEXT:    v_mad_u64_u32 v[5:6], null, s3, v2, v[4:5]
 ; GFX1164-NEXT:    v_sub_co_u32 v0, vcc, s2, v3
-; GFX1164-NEXT:    v_mov_b32_e32 v1, v5
+; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1164-NEXT:    s_mov_b32 s2, -1
+; GFX1164-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX1164-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc
 ; GFX1164-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: sub_i64_uniform:
 ; GFX1132:       ; %bb.0: ; %entry
 ; GFX1132-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s5, exec_lo

 ; GFX1132-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1132-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1132-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    buffer_gl0_inv
 ; GFX1132-NEXT:  .LBB12_2:
 ; GFX1132-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_mad_u64_u32 v[3:4], s2, s2, v2, 0
-; GFX1132-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX1132-NEXT:    v_mad_u64_u32 v[5:6], s2, s3, v2, v[4:5]
+; GFX1132-NEXT:    v_mad_u64_u32 v[3:4], null, s2, v2, 0
 ; GFX1132-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX1132-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1132-NEXT:    v_mad_u64_u32 v[5:6], null, s3, v2, v[4:5]
 ; GFX1132-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v3
-; GFX1132-NEXT:    v_mov_b32_e32 v1, v5
+; GFX1132-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1132-NEXT:    s_mov_b32 s2, -1
+; GFX1132-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX1132-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
 ; GFX1132-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
 ; GFX1132-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 %subitive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll

 ; GFX10W64-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10W64-NEXT:    buffer_atomic_add v1, off, s[12:15], 0 glc
 ; GFX10W64-NEXT:  .LBB1_2:
 ; GFX10W64-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10W64-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX10W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10W64-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s8, v0, s[0:1]
+; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], null, s8, v0, s[0:1]
 ; GFX10W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10W64-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10W64-NEXT:    s_endpgm
 ;
 ; GFX10W32-LABEL: add_i32_uniform:
 ; GFX10W32:       ; %bb.0: ; %entry
 ; GFX10W32-NEXT:    s_clause 0x1
 ; GFX10W32-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24

 ; GFX10W32-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10W32-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
 ; GFX10W32-NEXT:  .LBB1_2:
 ; GFX10W32-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX10W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], s0, s4, v0, s[0:1]
+; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], null, s4, v0, s[0:1]
 ; GFX10W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10W32-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10W32-NEXT:    s_endpgm
 entry:
   %old = call i32 @llvm.amdgcn.raw.buffer.atomic.add(i32 %additive, <4 x i32> %inout, i32 0, i32 0, i32 0)
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll

 ; GFX10W64-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10W64-NEXT:    buffer_atomic_add v1, v2, s[12:15], 0 idxen glc
 ; GFX10W64-NEXT:  .LBB1_2:
 ; GFX10W64-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10W64-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX10W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10W64-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s8, v0, s[0:1]
+; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], null, s8, v0, s[0:1]
 ; GFX10W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10W64-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10W64-NEXT:    s_endpgm
 ;
 ; GFX10W32-LABEL: add_i32_uniform:
 ; GFX10W32:       ; %bb.0: ; %entry
 ; GFX10W32-NEXT:    s_clause 0x1
 ; GFX10W32-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24

 ; GFX10W32-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10W32-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
 ; GFX10W32-NEXT:  .LBB1_2:
 ; GFX10W32-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX10W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], s0, s4, v0, s[0:1]
+; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], null, s4, v0, s[0:1]
 ; GFX10W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10W32-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10W32-NEXT:    s_endpgm
 entry:
   %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 %additive, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 0)
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/carryout-selection.ll

 ;
 ; VI: v_add_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; VI: v_addc_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX1010: v_add_co_u32 v{{[0-9]+}}, [[CARRY:s[0-9]+]], s{{[0-9]+}}, v{{[0-9]+}}
-; GFX1010: v_add_co_ci_u32_e64 v{{[0-9]+}}, [[CARRY]], s{{[0-9]+}}, 0, [[CARRY]]
+; GFX1010: v_add_co_ci_u32_e64 v{{[0-9]+}}, null, s{{[0-9]+}}, 0, [[CARRY]]
 define amdgpu_kernel void @vadd64rr(i64 addrspace(1)* %out, i64 %a) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %add = add i64 %a, %tid.ext
   store i64 %add, i64 addrspace(1)* %out
   ret void
 }

 ; VI: v_mov_b32_e32 v1, 0x1234
 ; VI: v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ;
 ; GFX9: v_add_co_u32_e32 v0, vcc, 0x56789876, v0
 ; GFX9: v_mov_b32_e32 v1, 0x1234
 ; GFX9: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ;
 ; GFX1010: v_add_co_u32 v{{[0-9]+}}, [[CARRY:s[0-9]+]], 0x56789876, v{{[0-9]+}}
-; GFX1010: v_add_co_ci_u32_e64 v{{[0-9]+}}, [[CARRY]], 0, 0x1234, [[CARRY]]
+; GFX1010: v_add_co_ci_u32_e64 v{{[0-9]+}}, null, 0, 0x1234, [[CARRY]]
 define amdgpu_kernel void @vadd64ri(i64 addrspace(1)* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %add = add i64 20015998343286, %tid.ext
   store i64 %add, i64 addrspace(1)* %out
   ret void
 }

 ;
 ; VI: v_add_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v0
 ; VI: v_addc_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v0
 ; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX1010: v_add_co_u32 v{{[0-9]+}}, [[CARRY:s[0-9]+]], s{{[0-9]+}}, v0
-; GFX1010: v_add_co_ci_u32_e64 v{{[0-9]+}}, [[CARRY]], s{{[0-9]+}}, 0, [[CARRY]]
+; GFX1010: v_add_co_ci_u32_e64 v{{[0-9]+}}, null, s{{[0-9]+}}, 0, [[CARRY]]
 define amdgpu_kernel void @vuaddo64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %uadd = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %tid.ext)
   %val = extractvalue { i64, i1 } %uadd, 0
   %carry = extractvalue { i64, i1 } %uadd, 1
   store i64 %val, i64 addrspace(1)* %out, align 8
   store i1 %carry, i1 addrspace(1)* %carryout

 ;
 ; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; VI: v_subbrev_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9: v_subbrev_co_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX1010: v_sub_co_u32 v{{[0-9]+}}, [[CARRY:s[0-9]+]], s{{[0-9]+}}, v{{[0-9]+}}
-; GFX1010: v_sub_co_ci_u32_e64 v{{[0-9]+}}, [[CARRY]], s{{[0-9]+}}, 0, [[CARRY]]
+; GFX1010: v_sub_co_ci_u32_e64 v{{[0-9]+}}, null, s{{[0-9]+}}, 0, [[CARRY]]
 define amdgpu_kernel void @vsub64rr(i64 addrspace(1)* %out, i64 %a) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %sub = sub i64 %a, %tid.ext
   store i64 %sub, i64 addrspace(1)* %out
   ret void
 }

 ; VI: v_mov_b32_e32 v1, 0x1234
 ; VI: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
 ;
 ; GFX9: v_sub_co_u32_e32 v0, vcc, 0x56789876, v0
 ; GFX9: v_mov_b32_e32 v1, 0x1234
 ; GFX9: v_subbrev_co_u32_e32 v1, vcc, 0, v1, vcc
 ;
 ; GFX1010: v_sub_co_u32 v{{[0-9]+}}, [[CARRY:s[0-9]+]], 0x56789876, v{{[0-9]+}}
-; GFX1010: v_sub_co_ci_u32_e64 v{{[0-9]+}}, [[CARRY]], 0x1234, 0, [[CARRY]]
+; GFX1010: v_sub_co_ci_u32_e64 v{{[0-9]+}}, null, 0x1234, 0, [[CARRY]]
 define amdgpu_kernel void @vsub64ri(i64 addrspace(1)* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %sub = sub i64 20015998343286, %tid.ext
   store i64 %sub, i64 addrspace(1)* %out
   ret void
 }

 ;
 ; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v0
 ; VI: v_subbrev_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v0
 ; GFX9: v_subbrev_co_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 ;
 ; GFX1010: v_sub_co_u32 v{{[0-9]+}}, [[CARRY:s[0-9]+]], s{{[0-9]+}}, v0
-; GFX1010: v_sub_co_ci_u32_e64 v{{[0-9]+}}, [[CARRY]], s{{[0-9]+}}, 0, [[CARRY]]
+; GFX1010: v_sub_co_ci_u32_e64 v{{[0-9]+}}, null, s{{[0-9]+}}, 0, [[CARRY]]
 define amdgpu_kernel void @vusubo64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %usub = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %a, i64 %tid.ext)
   %val = extractvalue { i64, i1 } %usub, 0
   %carry = extractvalue { i64, i1 } %usub, 1
   store i64 %val, i64 addrspace(1)* %out, align 8
   store i1 %carry, i1 addrspace(1)* %carryout

llvm/test/CodeGen/AMDGPU/frem.ll

 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dword v1, v0, s[6:7]
 ; GFX10-NEXT:    global_load_dword v2, v0, s[2:3] offset:16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v4, s0, v2, v2, v1
+; GFX10-NEXT:    v_div_scale_f32 v4, null, v2, v2, v1
 ; GFX10-NEXT:    v_div_scale_f32 v3, vcc_lo, v1, v2, v1
 ; GFX10-NEXT:    v_rcp_f32_e32 v5, v4
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v6, -v4, v5, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v5, v6, v5
 ; GFX10-NEXT:    v_mul_f32_e32 v6, v3, v5
 ; GFX10-NEXT:    v_fma_f32 v7, -v4, v6, v3
 ; GFX10-NEXT:    v_fmac_f32_e32 v6, v7, v5

 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v12, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v12, s[6:7]
 ; GFX10-NEXT:    global_load_dwordx2 v[2:3], v12, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[4:5], s0, v[2:3], v[2:3], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[4:5], null, v[2:3], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_div_scale_f64 v[8:9], vcc_lo, v[0:1], v[2:3], v[0:1]
 ; GFX10-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]

 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[6:7]
 ; GFX10-NEXT:    global_load_dwordx2 v[2:3], v4, s[2:3] offset:32
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v6, s0, v3, v3, v1
+; GFX10-NEXT:    v_div_scale_f32 v6, null, v3, v3, v1
 ; GFX10-NEXT:    v_div_scale_f32 v5, vcc_lo, v1, v3, v1
 ; GFX10-NEXT:    v_rcp_f32_e32 v7, v6
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v8, -v6, v7, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v7, v8, v7
 ; GFX10-NEXT:    v_mul_f32_e32 v8, v5, v7
 ; GFX10-NEXT:    v_fma_f32 v9, -v6, v8, v5
 ; GFX10-NEXT:    v_fmac_f32_e32 v8, v9, v7
 ; GFX10-NEXT:    v_fma_f32 v5, -v6, v8, v5
 ; GFX10-NEXT:    s_denorm_mode 12
 ; GFX10-NEXT:    v_div_fmas_f32 v5, v5, v7, v8
 ; GFX10-NEXT:    v_div_fixup_f32 v5, v5, v3, v1
 ; GFX10-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX10-NEXT:    v_fma_f32 v1, -v5, v3, v1
-; GFX10-NEXT:    v_div_scale_f32 v5, s0, v2, v2, v0
+; GFX10-NEXT:    v_div_scale_f32 v5, null, v2, v2, v0
 ; GFX10-NEXT:    v_div_scale_f32 v3, vcc_lo, v0, v2, v0
 ; GFX10-NEXT:    v_rcp_f32_e32 v6, v5
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v7, -v5, v6, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v6, v7, v6
 ; GFX10-NEXT:    v_mul_f32_e32 v7, v3, v6
 ; GFX10-NEXT:    v_fma_f32 v8, -v5, v7, v3
 ; GFX10-NEXT:    v_fmac_f32_e32 v7, v8, v6

 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx4 v[0:3], v8, s[6:7]
 ; GFX10-NEXT:    global_load_dwordx4 v[4:7], v8, s[2:3] offset:64
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v10, s0, v7, v7, v3
+; GFX10-NEXT:    v_div_scale_f32 v10, null, v7, v7, v3
 ; GFX10-NEXT:    v_div_scale_f32 v9, vcc_lo, v3, v7, v3
 ; GFX10-NEXT:    v_rcp_f32_e32 v11, v10
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v12, -v10, v11, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v11, v12, v11
 ; GFX10-NEXT:    v_mul_f32_e32 v12, v9, v11
 ; GFX10-NEXT:    v_fma_f32 v13, -v10, v12, v9
 ; GFX10-NEXT:    v_fmac_f32_e32 v12, v13, v11
 ; GFX10-NEXT:    v_fma_f32 v9, -v10, v12, v9
 ; GFX10-NEXT:    s_denorm_mode 12
 ; GFX10-NEXT:    v_div_fmas_f32 v9, v9, v11, v12
 ; GFX10-NEXT:    v_div_fixup_f32 v9, v9, v7, v3
 ; GFX10-NEXT:    v_trunc_f32_e32 v9, v9
 ; GFX10-NEXT:    v_fma_f32 v3, -v9, v7, v3
-; GFX10-NEXT:    v_div_scale_f32 v9, s0, v6, v6, v2
+; GFX10-NEXT:    v_div_scale_f32 v9, null, v6, v6, v2
 ; GFX10-NEXT:    v_div_scale_f32 v7, vcc_lo, v2, v6, v2
 ; GFX10-NEXT:    v_rcp_f32_e32 v10, v9
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v11, -v9, v10, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v10, v11, v10
 ; GFX10-NEXT:    v_mul_f32_e32 v11, v7, v10
 ; GFX10-NEXT:    v_fma_f32 v12, -v9, v11, v7
 ; GFX10-NEXT:    v_fmac_f32_e32 v11, v12, v10
 ; GFX10-NEXT:    v_fma_f32 v7, -v9, v11, v7
 ; GFX10-NEXT:    s_denorm_mode 12
 ; GFX10-NEXT:    v_div_fmas_f32 v7, v7, v10, v11
 ; GFX10-NEXT:    v_div_fixup_f32 v7, v7, v6, v2
 ; GFX10-NEXT:    v_trunc_f32_e32 v7, v7
 ; GFX10-NEXT:    v_fma_f32 v2, -v7, v6, v2
-; GFX10-NEXT:    v_div_scale_f32 v7, s0, v5, v5, v1
+; GFX10-NEXT:    v_div_scale_f32 v7, null, v5, v5, v1
 ; GFX10-NEXT:    v_div_scale_f32 v6, vcc_lo, v1, v5, v1
 ; GFX10-NEXT:    v_rcp_f32_e32 v9, v7
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v10, -v7, v9, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v9, v10, v9
 ; GFX10-NEXT:    v_mul_f32_e32 v10, v6, v9
 ; GFX10-NEXT:    v_fma_f32 v11, -v7, v10, v6
 ; GFX10-NEXT:    v_fmac_f32_e32 v10, v11, v9
 ; GFX10-NEXT:    v_fma_f32 v6, -v7, v10, v6
 ; GFX10-NEXT:    s_denorm_mode 12
 ; GFX10-NEXT:    v_div_fmas_f32 v6, v6, v9, v10
 ; GFX10-NEXT:    v_div_fixup_f32 v6, v6, v5, v1
 ; GFX10-NEXT:    v_trunc_f32_e32 v6, v6
 ; GFX10-NEXT:    v_fma_f32 v1, -v6, v5, v1
-; GFX10-NEXT:    v_div_scale_f32 v6, s0, v4, v4, v0
+; GFX10-NEXT:    v_div_scale_f32 v6, null, v4, v4, v0
 ; GFX10-NEXT:    v_div_scale_f32 v5, vcc_lo, v0, v4, v0
 ; GFX10-NEXT:    v_rcp_f32_e32 v7, v6
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v9, -v6, v7, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v7, v9, v7
 ; GFX10-NEXT:    v_mul_f32_e32 v9, v5, v7
 ; GFX10-NEXT:    v_fma_f32 v10, -v6, v9, v5
 ; GFX10-NEXT:    v_fmac_f32_e32 v9, v10, v7

 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v16, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx4 v[0:3], v16, s[6:7]
 ; GFX10-NEXT:    global_load_dwordx4 v[4:7], v16, s[2:3] offset:64
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_div_scale_f64 v[8:9], s0, v[6:7], v[6:7], v[2:3]
+; GFX10-NEXT:    v_div_scale_f64 v[8:9], null, v[6:7], v[6:7], v[2:3]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[10:11], v[8:9]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
 ; GFX10-NEXT:    v_div_scale_f64 v[12:13], vcc_lo, v[2:3], v[6:7], v[2:3]
 ; GFX10-NEXT:    v_mul_f64 v[14:15], v[12:13], v[10:11]
 ; GFX10-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[14:15], v[12:13]
 ; GFX10-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15]
 ; GFX10-NEXT:    v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
 ; GFX10-NEXT:    v_trunc_f64_e32 v[8:9], v[8:9]
 ; GFX10-NEXT:    v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
-; GFX10-NEXT:    v_div_scale_f64 v[6:7], s0, v[4:5], v[4:5], v[0:1]
+; GFX10-NEXT:    v_div_scale_f64 v[6:7], null, v[4:5], v[4:5], v[0:1]
 ; GFX10-NEXT:    v_rcp_f64_e32 v[8:9], v[6:7]
 ; GFX10-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
 ; GFX10-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
 ; GFX10-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
 ; GFX10-NEXT:    v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1]
 ; GFX10-NEXT:    v_mul_f64 v[12:13], v[10:11], v[8:9]
 ; GFX10-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]

llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll

 ; GFX9-NEXT:    global_atomic_swap v0, v0, v1, s[2:3] offset:2048 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_xchg_saddr_i32_rtn_2048:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc, 0x800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_atomic_swap v0, v[2:3], v1, off glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    buffer_gl0_inv
 ; GFX10-NEXT:    buffer_gl1_inv

llvm/test/CodeGen/AMDGPU/global-saddr-load.ll

 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:-4096
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg4096:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0xfffff000, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -4096
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 ; SGPR base with maximum negative gfx9 immediate offset -1
 define amdgpu_ps float @global_load_saddr_i8_offset_neg4097(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_neg4097:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_add_u32 s0, s2, 0xffffefff
 ; GFX9-NEXT:    s_addc_u32 s1, s3, -1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[0:1]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg4097:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0xfffff000, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -4097
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 ; SGPR base with maximum negative gfx9 immediate offset -2
 define amdgpu_ps float @global_load_saddr_i8_offset_neg4098(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_neg4098:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_add_u32 s0, s2, 0xffffeffe
 ; GFX9-NEXT:    s_addc_u32 s1, s3, -1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[0:1]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg4098:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0xfffff000, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -4098
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr

 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:-2049
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg2049:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0xfffff800, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -2049
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 ; SGPR base with maximum negative gfx10 immediate offset - 1
 define amdgpu_ps float @global_load_saddr_i8_offset_neg2050(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_neg2050:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:-2050
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg2050:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0xfffff800, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -2050
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr

 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 1, v1, vcc
 ; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_4294967296:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], 1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 4294967296
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 define amdgpu_ps float @global_load_saddr_i8_offset_4294967297(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_4294967297:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 1, v1, vcc
 ; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_4294967297:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], 1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 4294967297
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 define amdgpu_ps float @global_load_saddr_i8_offset_4294971391(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_4294971391:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_add_u32 s0, s2, 0xfff
 ; GFX9-NEXT:    s_addc_u32 s1, s3, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[0:1]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_4294971391:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0x800, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], 1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 4294971391
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 define amdgpu_ps float @global_load_saddr_i8_offset_4294971392(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_4294971392:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_add_u32 s0, s2, 0x1000
 ; GFX9-NEXT:    s_addc_u32 s1, s3, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[0:1]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_4294971392:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0x1000, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], 1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 4294971392
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 define amdgpu_ps float @global_load_saddr_i8_offset_neg4294967295(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_neg4294967295:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-4095
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg4294967295:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0x800, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -4294967295
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 define amdgpu_ps float @global_load_saddr_i8_offset_neg4294967296(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_neg4294967296:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg4294967296:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -4294967296
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr
 }
 
 define amdgpu_ps float @global_load_saddr_i8_offset_neg4294967297(i8 addrspace(1)* inreg %sbase) {
 ; GFX9-LABEL: global_load_saddr_i8_offset_neg4294967297:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    v_add_co_u32_e64 v0, vcc, 0, s2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_offset_neg4294967297:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], 0, s2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], -1, s3, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s3, s[0:1]
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 -4294967297
   %load = load i8, i8 addrspace(1)* %gep0
   %zext = zext i8 %load to i32
   %to.vgpr = bitcast i32 %zext to float
   ret float %to.vgpr

 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:4095
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_zext_vgpr_offset_4095:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc, 0x800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %gep0, i64 4095

 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_zext_vgpr_offset_4096:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc, 0x1000, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %gep0, i64 4096

 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:-4096
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg4096:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc, 0xfffff000, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %gep0, i64 -4096

 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX9-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg4097:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc, 0xfffff000, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %gep0, i64 -4097

 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:2048
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_zext_vgpr_offset_2048:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc, 0x800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %gep0, i64 2048

 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:-2049
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg2049:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc, 0xfffff800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %gep0, i64 -2049

 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] offset:4095
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: global_load_saddr_i8_zext_vgpr_offset_4095_gep_order:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32 v0, s[0:1], s2, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s[0:1], s3, 0, s[0:1]
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc, 0x800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, i8 addrspace(1)* %sbase, i64 4095
   %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %gep0, i64 %zext.offset

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll

 ; GFX1013-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x34
 ; GFX1013-NEXT:    v_mov_b32_e32 v6, 4.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v7, 0x40a00000
 ; GFX1013-NEXT:    v_mov_b32_e32 v8, 0x40c00000
 ; GFX1013-NEXT:    v_mov_b32_e32 v9, 0x40e00000
 ; GFX1013-NEXT:    v_mov_b32_e32 v10, 0x41000000
 ; GFX1013-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1013-NEXT:    v_add_co_u32 v2, s4, s4, v0
-; GFX1013-NEXT:    v_add_co_ci_u32_e64 v3, s4, s5, 0, s4
+; GFX1013-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, 0, s4
 ; GFX1013-NEXT:    v_add_co_u32 v4, s4, s6, v0
-; GFX1013-NEXT:    v_add_co_ci_u32_e64 v5, s4, s7, 0, s4
+; GFX1013-NEXT:    v_add_co_ci_u32_e64 v5, null, s7, 0, s4
 ; GFX1013-NEXT:    flat_load_dword v0, v[2:3]
 ; GFX1013-NEXT:    flat_load_dword v1, v[4:5]
 ; GFX1013-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1013-NEXT:    v_mov_b32_e32 v3, 1.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v4, 2.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v5, 0x40400000
 ; GFX1013-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1013-NEXT:    image_bvh_intersect_ray v[0:3], v[0:15], s[0:3]

 ; GFX1030-NEXT:    v_mov_b32_e32 v9, 0x40e00000
 ; GFX1030-NEXT:    v_mov_b32_e32 v8, 0x40c00000
 ; GFX1030-NEXT:    v_mov_b32_e32 v7, 0x40a00000
 ; GFX1030-NEXT:    v_mov_b32_e32 v6, 4.0
 ; GFX1030-NEXT:    v_mov_b32_e32 v5, 0x40400000
 ; GFX1030-NEXT:    v_mov_b32_e32 v4, 2.0
 ; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1030-NEXT:    v_add_co_u32 v0, s4, s4, v2
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, s4, s5, 0, s4
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s4
 ; GFX1030-NEXT:    v_add_co_u32 v2, s4, s6, v2
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, s4, s7, 0, s4
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, s7, 0, s4
 ; GFX1030-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX1030-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX1030-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 1.0
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    image_bvh_intersect_ray v[0:3], v[0:15], s[0:3]
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1030-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]

 ; GFX1013:       ; %bb.0: ; %main_body
 ; GFX1013-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX1013-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX1013-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x34
 ; GFX1013-NEXT:    v_mov_b32_e32 v6, 0x46004500
 ; GFX1013-NEXT:    v_mov_b32_e32 v7, 0x48004700
 ; GFX1013-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1013-NEXT:    v_add_co_u32 v2, s4, s4, v0
-; GFX1013-NEXT:    v_add_co_ci_u32_e64 v3, s4, s5, 0, s4
+; GFX1013-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, 0, s4
 ; GFX1013-NEXT:    v_add_co_u32 v4, s4, s6, v0
-; GFX1013-NEXT:    v_add_co_ci_u32_e64 v5, s4, s7, 0, s4
+; GFX1013-NEXT:    v_add_co_ci_u32_e64 v5, null, s7, 0, s4
 ; GFX1013-NEXT:    flat_load_dword v0, v[2:3]
 ; GFX1013-NEXT:    flat_load_dword v1, v[4:5]
 ; GFX1013-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1013-NEXT:    v_mov_b32_e32 v3, 1.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v4, 2.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v5, 0x44004200
 ; GFX1013-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1013-NEXT:    image_bvh_intersect_ray v[0:3], v[0:7], s[0:3] a16
 ; GFX1013-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1013-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; GFX1013-NEXT:    s_endpgm
 ;
 ; GFX1030-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
 ; GFX1030:       ; %bb.0: ; %main_body
 ; GFX1030-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX1030-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; GFX1030-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x34
 ; GFX1030-NEXT:    v_mov_b32_e32 v4, 2.0
 ; GFX1030-NEXT:    v_mov_b32_e32 v5, 0x44004200
 ; GFX1030-NEXT:    v_mov_b32_e32 v6, 0x46004500
 ; GFX1030-NEXT:    v_mov_b32_e32 v7, 0x48004700
 ; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1030-NEXT:    v_add_co_u32 v0, s4, s4, v2
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, s4, s5, 0, s4
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s4
 ; GFX1030-NEXT:    v_add_co_u32 v2, s4, s6, v2
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, s4, s7, 0, s4
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v3, null, s7, 0, s4
 ; GFX1030-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX1030-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX1030-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 1.0
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    image_bvh_intersect_ray v[0:3], v[0:7], s[0:3] a16
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1030-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]

 ; GFX1013-NEXT:    v_mov_b32_e32 v6, 0x40400000
 ; GFX1013-NEXT:    v_mov_b32_e32 v7, 4.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v8, 0x40a00000
 ; GFX1013-NEXT:    v_mov_b32_e32 v9, 0x40c00000
 ; GFX1013-NEXT:    v_mov_b32_e32 v10, 0x40e00000
 ; GFX1013-NEXT:    v_mov_b32_e32 v11, 0x41000000
 ; GFX1013-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1013-NEXT:    v_add_co_u32 v0, s4, s4, v0
-; GFX1013-NEXT:    v_add_co_ci_u32_e64 v1, s4, s5, 0, s4
+; GFX1013-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s4
 ; GFX1013-NEXT:    flat_load_dword v2, v[0:1]
 ; GFX1013-NEXT:    v_mov_b32_e32 v0, 0xb36211c7
 ; GFX1013-NEXT:    v_mov_b32_e32 v1, 0x102
 ; GFX1013-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1013-NEXT:    image_bvh64_intersect_ray v[0:3], v[0:15], s[0:3]
 ; GFX1013-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1013-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; GFX1013-NEXT:    s_endpgm

 ; GFX1030-NEXT:    v_mov_b32_e32 v9, 0x40c00000
 ; GFX1030-NEXT:    v_mov_b32_e32 v8, 0x40a00000
 ; GFX1030-NEXT:    v_mov_b32_e32 v7, 4.0
 ; GFX1030-NEXT:    v_mov_b32_e32 v6, 0x40400000
 ; GFX1030-NEXT:    v_mov_b32_e32 v5, 2.0
 ; GFX1030-NEXT:    v_mov_b32_e32 v4, 1.0
 ; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1030-NEXT:    v_add_co_u32 v0, s4, s4, v0
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, s4, s5, 0, s4
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s4
 ; GFX1030-NEXT:    flat_load_dword v2, v[0:1]
 ; GFX1030-NEXT:    v_mov_b32_e32 v1, 0x102
 ; GFX1030-NEXT:    v_mov_b32_e32 v0, 0xb36211c7
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    image_bvh64_intersect_ray v[0:3], v[0:15], s[0:3]
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1030-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; GFX1030-NEXT:    s_endpgm

 ; GFX1013-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1013-NEXT:    v_mov_b32_e32 v4, 1.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v5, 2.0
 ; GFX1013-NEXT:    v_mov_b32_e32 v6, 0x44004200
 ; GFX1013-NEXT:    v_mov_b32_e32 v7, 0x46004500
 ; GFX1013-NEXT:    v_mov_b32_e32 v8, 0x48004700
 ; GFX1013-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1013-NEXT:    v_add_co_u32 v0, s4, s4, v0
-; GFX1013-NEXT:    v_add_co_ci_u32_e64 v1, s4, s5, 0, s4
+; GFX1013-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s4
 ; GFX1013-NEXT:    flat_load_dword v2, v[0:1]
 ; GFX1013-NEXT:    v_mov_b32_e32 v0, 0xb36211c6
 ; GFX1013-NEXT:    v_mov_b32_e32 v1, 0x102
 ; GFX1013-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1013-NEXT:    image_bvh64_intersect_ray v[0:3], v[0:15], s[0:3] a16
 ; GFX1013-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1013-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; GFX1013-NEXT:    s_endpgm
 ;
 ; GFX1030-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
 ; GFX1030:       ; %bb.0: ; %main_body
 ; GFX1030-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX1030-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x34
 ; GFX1030-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX1030-NEXT:    v_mov_b32_e32 v5, 2.0
 ; GFX1030-NEXT:    v_mov_b32_e32 v4, 1.0
 ; GFX1030-NEXT:    v_mov_b32_e32 v6, 0x44004200
 ; GFX1030-NEXT:    v_mov_b32_e32 v7, 0x46004500
 ; GFX1030-NEXT:    v_mov_b32_e32 v8, 0x48004700
 ; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1030-NEXT:    v_add_co_u32 v0, s4, s4, v0
-; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, s4, s5, 0, s4
+; GFX1030-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s4
 ; GFX1030-NEXT:    flat_load_dword v2, v[0:1]
 ; GFX1030-NEXT:    v_mov_b32_e32 v1, 0x102
 ; GFX1030-NEXT:    v_mov_b32_e32 v0, 0xb36211c6
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    image_bvh64_intersect_ray v[0:3], v[0:15], s[0:3] a16
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1030-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; GFX1030-NEXT:    s_endpgm

llvm/test/CodeGen/AMDGPU/llvm.mulo.ll

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: umulo_i64_v_v:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s4, v5, v2, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s4, v5, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[9:10], null, v5, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[11:12], null, v5, v3, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v5, v5, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v4, v4, v3
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
 ; GFX10-NEXT:    v_add3_u32 v1, v1, v4, v5
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v6, v9
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v7, v10, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v12, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v11
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: umulo_i64_v_v:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v1
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v4, v2, 0
-; GFX11-NEXT:    v_mad_u64_u32 v[6:7], s0, v4, v3, 0
-; GFX11-NEXT:    v_mad_u64_u32 v[9:10], s0, v5, v2, 0
-; GFX11-NEXT:    v_mad_u64_u32 v[11:12], s0, v5, v3, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v5, v2, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v5, v3, 0
 ; GFX11-NEXT:    v_mov_b32_e32 v8, v1
 ; GFX11-NEXT:    v_mul_lo_u32 v5, v5, v2
 ; GFX11-NEXT:    v_mul_lo_u32 v4, v4, v3
 ; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
 ; GFX11-NEXT:    v_add3_u32 v1, v1, v4, v5
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v6, v9
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v7, v10, vcc_lo

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: smulo_i64_v_v:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s4, v5, v2, 0
-; GFX10-NEXT:    v_mad_i64_i32 v[11:12], s4, v5, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[9:10], null, v5, v2, 0
+; GFX10-NEXT:    v_mad_i64_i32 v[11:12], null, v5, v3, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v1
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
 ; GFX10-NEXT:    v_mul_lo_u32 v8, v5, v2
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v9
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v7, v10, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v12, vcc_lo
 ; GFX10-NEXT:    v_mul_lo_u32 v9, v4, v3

 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: smulo_i64_v_v:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v1
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v4, v2, 0
-; GFX11-NEXT:    v_mad_u64_u32 v[6:7], s0, v4, v3, 0
-; GFX11-NEXT:    v_mad_u64_u32 v[9:10], s0, v5, v2, 0
-; GFX11-NEXT:    v_mad_i64_i32 v[11:12], s0, v5, v3, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v5, v2, 0
+; GFX11-NEXT:    v_mad_i64_i32 v[11:12], null, v5, v3, 0
 ; GFX11-NEXT:    v_mov_b32_e32 v8, v1
 ; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
 ; GFX11-NEXT:    v_mul_lo_u32 v8, v5, v2
 ; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v9
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v7, v10, vcc_lo
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v12, vcc_lo
 ; GFX11-NEXT:    v_mul_lo_u32 v9, v4, v3

llvm/test/CodeGen/AMDGPU/mad_64_32.ll

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_sextops:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v0
-; GFX11-NEXT:    v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3]
+; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %mul, %arg2
   ret i64 %mad
 }
 

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_sextops_commute:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v0
-; GFX11-NEXT:    v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3]
+; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %arg2, %mul
   ret i64 %mad
 }
 

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_u64_u32_zextops:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v5, v4, v[2:3]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = zext i32 %arg0 to i64
   %sext1 = zext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %mul, %arg2
   ret i64 %mad
 }
 

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_u64_u32_zextops_commute:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v5, v4, v[2:3]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = zext i32 %arg0 to i64
   %sext1 = zext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %arg2, %mul
   ret i64 %mad
 }
 

 ; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v7, v4, vcc
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v5, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_sextops_i32_i128:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_mad_u64_u32 v[6:7], s0, v0, v1, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v0, v1, 0
 ; GFX11-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v14, 31, v0
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v15, 31, v1
-; GFX11-NEXT:    v_mad_u64_u32 v[9:10], s0, v14, v1, v[7:8]
+; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v14, v1, v[7:8]
 ; GFX11-NEXT:    v_mov_b32_e32 v7, v10
 ; GFX11-NEXT:    v_mov_b32_e32 v10, v8
-; GFX11-NEXT:    v_mad_u64_u32 v[11:12], s0, v0, v15, v[9:10]
-; GFX11-NEXT:    v_mad_i64_i32 v[9:10], s0, v1, v14, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v0, v15, v[9:10]
+; GFX11-NEXT:    v_mad_i64_i32 v[9:10], null, v1, v14, 0
 ; GFX11-NEXT:    v_mov_b32_e32 v8, v12
+; GFX11-NEXT:    v_mad_i64_i32 v[12:13], null, v15, v0, v[9:10]
 ; GFX11-NEXT:    v_add_co_u32 v7, s0, v7, v8
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v8, s0, 0, 0, s0
-; GFX11-NEXT:    v_mad_i64_i32 v[12:13], s0, v15, v0, v[9:10]
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v14, v15, v[7:8]
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v8, null, 0, 0, s0
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v14, v15, v[7:8]
 ; GFX11-NEXT:    v_mov_b32_e32 v7, v11
 ; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v0, v12
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v1, v13, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v6, v2
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v7, v3, vcc_lo
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_sextops_i32_i63:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v0
-; GFX11-NEXT:    v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3]
+; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i63
   %sext1 = sext i32 %arg1 to i63
   %mul = mul i63 %sext0, %sext1
   %mad = add i63 %mul, %arg2
   ret i63 %mad
 }
 

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_sextops_i31_i63:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_bfe_i32 v4, v1, 0, 31
 ; GFX11-NEXT:    v_bfe_i32 v5, v0, 0, 31
-; GFX11-NEXT:    v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3]
+; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i31 %arg0 to i63
   %sext1 = sext i31 %arg1 to i63
   %mul = mul i63 %sext0, %sext1
   %mad = add i63 %mul, %arg2
   ret i63 %mad
 }
 

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_extops_i32_i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v5, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v5, v4, v[2:3]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3]
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v5, 31, v5
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v1
-; GFX11-NEXT:    v_mad_u64_u32 v[1:2], s0, v5, v4, v[3:4]
+; GFX11-NEXT:    v_mad_u64_u32 v[1:2], null, v5, v4, v[3:4]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %ext0 = sext i32 %arg0 to i64
   %ext1 = zext i32 %arg1 to i64
   %mul = mul i64 %ext0, %ext1
   %mad = add i64 %mul, %arg2
   ret i64 %mad
 }
 

 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_u64_u32_bitops:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v3, v2, v[4:5]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v3, v2, v[4:5]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %trunc.lhs = and i64 %arg0, 4294967295
   %trunc.rhs = and i64 %arg1, 4294967295
   %mul = mul i64 %trunc.lhs, %trunc.rhs
   %add = add i64 %mul, %arg2
   ret i64 %add
 }
 

 ;
 ; GFX11-LABEL: mad_u64_u32_bitops_lhs_mask_small:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX11-NEXT:    v_mov_b32_e32 v6, v1
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v2, v3, v[4:5]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v2, v3, v[4:5]
 ; GFX11-NEXT:    v_and_b32_e32 v5, 1, v6
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v1
-; GFX11-NEXT:    v_mad_u64_u32 v[1:2], s0, v5, v3, v[4:5]
+; GFX11-NEXT:    v_mad_u64_u32 v[1:2], null, v5, v3, v[4:5]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %trunc.lhs = and i64 %arg0, 8589934591
   %trunc.rhs = and i64 %arg1, 4294967295
   %mul = mul i64 %trunc.lhs, %trunc.rhs
   %add = add i64 %mul, %arg2
   ret i64 %add
 }
 

 ; GFX9-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_u64_u32_bitops_rhs_mask_small:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v6, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v6, v2, v[4:5]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v6, v2, v[4:5]
 ; GFX11-NEXT:    v_and_b32_e32 v4, 1, v3
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v1
-; GFX11-NEXT:    v_mad_u64_u32 v[1:2], s0, v6, v4, v[3:4]
+; GFX11-NEXT:    v_mad_u64_u32 v[1:2], null, v6, v4, v[3:4]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %trunc.lhs = and i64 %arg0, 4294967295
   %trunc.rhs = and i64 %arg1, 8589934591
   %mul = mul i64 %trunc.lhs, %trunc.rhs
   %add = add i64 %mul, %arg2
   ret i64 %add
 }
 

 ; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_bitops:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    v_mad_i64_i32 v[0:1], s0, v3, v2, v[4:5]
+; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v3, v2, v[4:5]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %shl.lhs = shl i64 %arg0, 32
   %trunc.lhs = ashr i64 %shl.lhs, 32
   %shl.rhs = shl i64 %arg1, 32
   %trunc.rhs = ashr i64 %shl.rhs, 32
   %mul = mul i64 %trunc.lhs, %trunc.rhs
   %add = add i64 %mul, %arg2
   ret i64 %add

 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_unpack_i64ops:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_mad_u64_u32 v[2:3], s0, v1, v0, v[0:1]
+; GFX11-NEXT:    v_mad_u64_u32 v[2:3], null, v1, v0, v[0:1]
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX11-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %tmp4 = lshr i64 %arg0, 32
   %tmp5 = and i64 %arg0, 4294967295
   %mul = mul nuw i64 %tmp4, %tmp5
   %mad = add i64 %mul, %arg0
   ret i64 %mad

 ; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_twice:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_mad_i64_i32 v[6:7], s0, v0, v1, v[2:3]
-; GFX11-NEXT:    v_mad_i64_i32 v[2:3], s0, v0, v1, v[4:5]
+; GFX11-NEXT:    v_mad_i64_i32 v[6:7], null, v0, v1, v[2:3]
+; GFX11-NEXT:    v_mad_i64_i32 v[2:3], null, v0, v1, v[4:5]
 ; GFX11-NEXT:    v_xor_b32_e32 v0, v6, v2
 ; GFX11-NEXT:    v_xor_b32_e32 v1, v7, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad1 = add i64 %mul, %arg2
   %mad2 = add i64 %mul, %arg3

 ; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_thrice:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_mad_i64_i32 v[8:9], s0, v0, v1, 0
+; GFX11-NEXT:    v_mad_i64_i32 v[8:9], null, v0, v1, 0
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v8, v2
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v9, v3, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v8, v4
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v8, v6
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v9, v7, vcc_lo
 ; GFX11-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v3

 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i64_i32_secondary_use:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_mad_i64_i32 v[4:5], s0, v0, v1, 0
+; GFX11-NEXT:    v_mad_i64_i32 v[4:5], null, v0, v1, 0
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v4, v2
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v5, v3, vcc_lo
 ; GFX11-NEXT:    v_xor_b32_e32 v0, v0, v4
 ; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v5
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1

 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: mad_i48_i48:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v6, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v7, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v7, v2, v[4:5]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v7, v2, v[4:5]
 ; GFX11-NEXT:    v_mul_lo_u32 v3, v7, v3
 ; GFX11-NEXT:    v_mul_lo_u32 v2, v6, v2
 ; GFX11-NEXT:    v_add3_u32 v1, v2, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %m = mul i48 %arg0, %arg1
   %a = add i48 %m, %arg2
   ret i48 %a
 }
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone speculatable }

llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx900 --verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1010 --verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1100 --verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX11 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 --verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 --verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX11 %s
 
 define amdgpu_ps float @mad_i32_vvv(i32 %a, i32 %b, i32 %c) {
 ; GFX9-LABEL: mad_i32_vvv:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, v1, v[2:3]
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vvv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, v1, v[2:3]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[2:3]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vvv:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v4, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v4, v3, v[2:3]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v3, v[2:3]
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c) {

 define amdgpu_ps float @mad_i32_vvc(i32 %a, i32 %b) {
 ; GFX9-LABEL: mad_i32_vvc:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, v1, 42
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vvc:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, v1, 42
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, 42
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vvc:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v3, v2, 42
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v3, v2, 42
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, 42
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_vvi(i32 %a, i32 %b) {
 ; GFX9-LABEL: mad_i32_vvi:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x12d687
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, v1, v[2:3]
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vvi:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, v1, 0x12d687
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, 0x12d687
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vvi:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v3, v2, 0x12d687
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v3, v2, 0x12d687
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, 1234567
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_vcv(i32 %a, i32 %c) {
 ; GFX9-LABEL: mad_i32_vcv:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 42, v[1:2]
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vcv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, 42, v[1:2]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, 42, v[1:2]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vcv:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_mad_u64_u32 v[2:3], s0, v0, 42, v[1:2]
+; GFX11-NEXT:    v_mad_u64_u32 v[2:3], null, v0, 42, v[1:2]
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, 42
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_vcc(i32 %a) {
 ; GFX9-LABEL: mad_i32_vcc:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 42, 43
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vcc:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, 42, 43
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, 42, 43
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vcc:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v2, 42, 43
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v2, 42, 43
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, 42
   %add = add i32 %mul, 43
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_vvs(i32 %a, i32 %b, i32 inreg %c) {
 ; GFX9-LABEL: mad_i32_vvs:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, v1, s[0:1]
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vvs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, v1, s[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, s[0:1]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vvs:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v3, v2, s[0:1]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v3, v2, s[0:1]
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_vsv(i32 %a, i32 inreg %b, i32 %c) {
 ; GFX9-LABEL: mad_i32_vsv:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, s0, v[1:2]
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vsv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, s0, v[1:2]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, s0, v[1:2]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vsv:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_mad_u64_u32 v[2:3], s0, v0, s0, v[1:2]
+; GFX11-NEXT:    v_mad_u64_u32 v[2:3], null, v0, s0, v[1:2]
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_svv(i32 inreg %a, i32 %b, i32 %c) {
 ; GFX9-LABEL: mad_i32_svv:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s0, v0, v[1:2]
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_svv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, s0, v0, v[1:2]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s0, v0, v[1:2]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_svv:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_mad_u64_u32 v[2:3], s0, s0, v0, v[1:2]
+; GFX11-NEXT:    v_mad_u64_u32 v[2:3], null, s0, v0, v[1:2]
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_vss(i32 %a, i32 inreg %b, i32 inreg %c) {
 ; GFX9-LABEL: mad_i32_vss:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GFX9-NEXT:    v_add_u32_e32 v0, s1, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_vss:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_mov_b32 s2, s1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, s0, s[2:3]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, s0, s[2:3]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_vss:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX11-NEXT:    s_mov_b32 s2, s1
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, v2, s0, s[2:3]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v2, s0, s[2:3]
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_svs(i32 inreg %a, i32 %b, i32 inreg %c) {
 ; GFX9-LABEL: mad_i32_svs:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GFX9-NEXT:    v_add_u32_e32 v0, s1, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_svs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_mov_b32 s2, s1
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, s0, v0, s[2:3]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s0, v0, s[2:3]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_svs:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX11-NEXT:    s_mov_b32 s2, s1
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], s0, s0, v2, s[2:3]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, s0, v2, s[2:3]
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 
 define amdgpu_ps float @mad_i32_ssv(i32 inreg %a, i32 inreg %b, i32 %c) {
 ; GFX9-LABEL: mad_i32_ssv:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_mul_i32 s0, s0, s1
 ; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: mad_i32_ssv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, s0, s1, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s0, s1, v[0:1]
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: mad_i32_ssv:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_mad_u64_u32 v[1:2], s0, s0, s1, v[0:1]
+; GFX11-NEXT:    v_mad_u64_u32 v[1:2], null, s0, s1, v[0:1]
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX11-NEXT:    ; return to shader part epilog
   %mul = mul i32 %a, %b
   %add = add i32 %mul, %c
   %cast = bitcast i32 %add to float
   ret float %cast
 }
 

llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll

 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-WGP-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] slc
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
 ; GFX10-WGP-NEXT:    s_endpgm
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_load_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-CU-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] slc
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
 ; GFX10-CU-NEXT:    s_endpgm
 ;
 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_1:

 ; GFX940-TGSPLIT-NEXT:    s_endpgm
 ;
 ; GFX11-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX11-WGP:       ; %bb.0: ; %entry
 ; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
-; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] slc dlc
+; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] slc
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_load_1:
 ; GFX11-CU:       ; %bb.0: ; %entry
 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
-; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] slc dlc
+; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] slc
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-CU-NEXT:    s_endpgm
     i32* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()

 ; GFX10-WGP:       ; %bb.0: ; %entry
 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX10-WGP-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX10-WGP-NEXT:    flat_load_dword v2, v[1:2]
-; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2 glc slc
 ; GFX10-WGP-NEXT:    s_endpgm
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_store_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX10-CU-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX10-CU-NEXT:    flat_load_dword v2, v[1:2]
-; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2 glc slc
 ; GFX10-CU-NEXT:    s_endpgm
 ;
 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_1:
 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
 ; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0

 ; GFX11-WGP:       ; %bb.0: ; %entry
 ; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX11-WGP-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX11-WGP-NEXT:    flat_load_b32 v2, v[1:2]
-; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 glc slc dlc
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_store_1:
 ; GFX11-CU:       ; %bb.0: ; %entry
 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX11-CU-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[1:2]
-; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 glc slc dlc
 ; GFX11-CU-NEXT:    s_endpgm
     i32* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val = load i32, i32* %in, align 4
   %out.gep = getelementptr inbounds i32, i32* %out, i32 %tid
   store i32 %val, i32* %out.gep, !nontemporal !0
   ret void
 }
 
 !0 = !{i32 1}
 declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll

 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-WGP-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
 ; GFX10-WGP-NEXT:    s_endpgm
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_load_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-CU-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
 ; GFX10-CU-NEXT:    s_endpgm
 ;

 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
 ; GFX11-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX11-WGP:       ; %bb.0: ; %entry
 ; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc dlc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_load_1:
 ; GFX11-CU:       ; %bb.0: ; %entry
 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc dlc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-CU-NEXT:    s_endpgm
     i32* %in, i32* %out) {

 ; GFX10-WGP:       ; %bb.0: ; %entry
 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX10-WGP-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX10-WGP-NEXT:    flat_load_dword v2, v[1:2]
-; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-WGP-NEXT:    s_endpgm
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_store_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX10-CU-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX10-CU-NEXT:    flat_load_dword v2, v[1:2]
-; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-CU-NEXT:    s_endpgm
 ;
 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_1:
 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0

 ; GFX11-WGP:       ; %bb.0: ; %entry
 ; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX11-WGP-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX11-WGP-NEXT:    flat_load_b32 v2, v[1:2]
-; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 dlc
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_store_1:
 ; GFX11-CU:       ; %bb.0: ; %entry
 ; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
 ; GFX11-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX11-CU-NEXT:    v_add_co_u32 v0, s0, s2, v0
 ; GFX11-CU-NEXT:    flat_load_b32 v2, v[1:2]
-; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 dlc
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    s_endpgm
     i32* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val = load i32, i32* %in, align 4

llvm/test/CodeGen/AMDGPU/offset-split-global.ll

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_neg_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0xfffff000, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -4096
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_neg_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0xffffe000, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -8192
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_2x_neg_11bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0xfffff000, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -4096
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_2x_neg_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0xffffe000, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -8192
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_2x_neg_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0xffffc000, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -16384
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589936639
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0x800, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589936640
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0x800, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589938687
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0x1000, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589938688
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0x1800, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589942783
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

 ; GFX9-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, 0x2000, s0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 2, s1, s0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_byte v[0:1], v0, off
 ; GFX10-NEXT:    s_endpgm
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 8589942784
   %load = load volatile i8, i8 addrspace(1)* %gep, align 1
   store i8 %load, i8 addrspace(1)* undef
   ret void

llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff8000, v2
 ; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, s34, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, s35, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v2, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 0x1000
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v10, vcc_lo, v0, 0x2000
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
 ; GFX10-NEXT:    global_load_dwordx2 v[8:9], v[4:5], off offset:-2048

 ; GFX10-NEXT:    s_movk_i32 s1, 0x7f
 ; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 3, v[1:2]
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xfe000000, v3
 ; GFX10-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX10-NEXT:    v_add_co_u32 v3, vcc_lo, v0, s34
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, s35, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, s34, v2
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, 0x5000, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s35, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v6, vcc_lo
 ; GFX10-NEXT:  .LBB1_1: ; %for.cond.preheader
 ; GFX10-NEXT:    ; =>This Loop Header: Depth=1
 ; GFX10-NEXT:    ; Child Loop BB1_2 Depth 2
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v3
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX10-NEXT:    s_mov_b32 s2, 0
 ; GFX10-NEXT:  .LBB1_2: ; %for.body

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff8000, v2
 ; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, s34, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, s35, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v2, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, 0x800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v0, 0x1000
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, 0x1000, v0
 ; GFX10-NEXT:    s_clause 0x4

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff8000, v2
 ; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, s34, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, s35, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v2, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 0xfffff800
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
 ; GFX10-NEXT:    global_load_dwordx2 v[8:9], v[4:5], off offset:-2048
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, 0, v0

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff8000, v2
 ; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, s34, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, s35, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v2, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 0x80000000
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    global_load_dword v6, v[0:1], off
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7ffff800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    s_clause 0x2

 ; GFX10-NEXT:    s_mov_b64 s[0:1], s[40:41]
 ; GFX10-NEXT:    s_mov_b64 s[2:3], s[42:43]
 ; GFX10-NEXT:    s_mov_b32 s32, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff8000, v0
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, s36, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s37, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s37, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v14, s0, s38, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v15, s0, s39, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v15, null, s39, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 0x1800
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v14, 0x3000
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v15, vcc_lo
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[6:7], v[2:3], off offset:-2048
 ; GFX10-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, 0x2000, v0

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff8000, v2
 ; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, s34, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, s35, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v2, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, 0x3800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, 0x3000, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[8:9], v[0:1], off

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff8000, v2
 ; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v2, s0, s34, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, s35, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v2, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800, v4
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v5, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, 0, v4
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, -1, v5, vcc_lo
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off

llvm/test/CodeGen/AMDGPU/saddo.ll

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s6, s2
 ; GFX10-NEXT:    s_addc_u32 s1, s7, s3
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s2, s[2:3], 0
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s3, s[0:1], s[6:7]
 ; GFX10-NEXT:    s_xor_b32 s2, s2, s3
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s2
 ; GFX10-NEXT:    v_add_co_u32 v0, s0, s0, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
   %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %sadd, 0
   %carry = extractvalue { i64, i1 } %sadd, 1
   %ext = zext i1 %carry to i64
   %add2 = add i64 %val, %ext
   store i64 %add2, i64 addrspace(1)* %out, align 8

llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll

 ; MUBUF-NEXT:  ; %bb.1: ; %if.then4.i
 ; MUBUF-NEXT:    v_add_nc_u32_e64 v0, 4, 0x4000
 ; MUBUF-NEXT:    s_mov_b32 s0, 0x41c64e6d
 ; MUBUF-NEXT:    s_clause 0x1
 ; MUBUF-NEXT:    buffer_load_dword v1, v0, s[36:39], 0 offen
 ; MUBUF-NEXT:    buffer_load_dword v2, v0, s[36:39], 0 offen offset:4
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    v_add_nc_u32_e32 v0, v2, v1
-; MUBUF-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, s0, 0x3039
+; MUBUF-NEXT:    v_mad_u64_u32 v[0:1], null, v0, s0, 0x3039
 ; MUBUF-NEXT:    buffer_store_dword v0, v0, s[36:39], 0 offen
 ; MUBUF-NEXT:  .LBB0_2: ; %shader_eval_surface.exit
 ; MUBUF-NEXT:    s_endpgm
 ;
 ; FLATSCR-LABEL: kernel_background_evaluate:
 ; FLATSCR:       ; %bb.0: ; %entry
 ; FLATSCR-NEXT:    s_add_u32 s2, s2, s5
 ; FLATSCR-NEXT:    s_movk_i32 s32, 0x6000

 ; FLATSCR-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; FLATSCR-NEXT:    s_cbranch_execz .LBB0_2
 ; FLATSCR-NEXT:  ; %bb.1: ; %if.then4.i
 ; FLATSCR-NEXT:    s_movk_i32 vcc_lo, 0x4000
 ; FLATSCR-NEXT:    s_mov_b32 s0, 0x41c64e6d
 ; FLATSCR-NEXT:    scratch_load_dwordx2 v[0:1], off, vcc_lo offset:4
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_nc_u32_e32 v0, v1, v0
-; FLATSCR-NEXT:    v_mad_u64_u32 v[0:1], s0, v0, s0, 0x3039
+; FLATSCR-NEXT:    v_mad_u64_u32 v[0:1], null, v0, s0, 0x3039
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s0
 ; FLATSCR-NEXT:  .LBB0_2: ; %shader_eval_surface.exit
 ; FLATSCR-NEXT:    s_endpgm
 entry:
   %sd = alloca < 1339 x i32>, align 8192, addrspace(5)
   %state = alloca <4 x i32>, align 16, addrspace(5)
   %rslt = call i32 @svm_eval_nodes(float addrspace(5)* %kg, <1339 x i32> addrspace(5)* %sd, <4 x i32> addrspace(5)* %state, i32 0, i32 4194304)
   %cmp = icmp eq i32 %rslt, 0

llvm/test/CodeGen/AMDGPU/udiv.ll

 ; GFX1030-NEXT:    v_add_co_u32 v4, vcc_lo, v5, v4
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v6, v8, vcc_lo
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v9, vcc_lo
 ; GFX1030-NEXT:    v_add_co_u32 v3, vcc_lo, v4, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v5, vcc_lo
 ; GFX1030-NEXT:    v_add_co_u32 v5, vcc_lo, v2, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, s4, v4, vcc_lo
 ; GFX1030-NEXT:    v_mul_hi_u32 v8, v0, v5
-; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], s4, v1, v5, 0
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], s4, v0, v6, 0
-; GFX1030-NEXT:    v_mad_u64_u32 v[6:7], s4, v1, v6, 0
+; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], null, v1, v5, 0
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, v0, v6, 0
+; GFX1030-NEXT:    v_mad_u64_u32 v[6:7], null, v1, v6, 0
 ; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v8, v2
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v3, v5, vcc_lo
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v7, vcc_lo
 ; GFX1030-NEXT:    v_add_co_u32 v5, vcc_lo, v2, v6
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v3, vcc_lo
-; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], s4, 0x186a0, v5, 0
-; GFX1030-NEXT:    v_mad_u64_u32 v[3:4], s4, 0x186a0, v6, v[3:4]
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], null, 0x186a0, v5, 0
+; GFX1030-NEXT:    v_mad_u64_u32 v[3:4], null, 0x186a0, v6, v[3:4]
 ; GFX1030-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v2
 ; GFX1030-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
 ; GFX1030-NEXT:    v_subrev_co_u32 v2, vcc_lo, 0x186a0, v0
 ; GFX1030-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX1030-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x1869f, v2
 ; GFX1030-NEXT:    v_cmp_eq_u32_e64 s4, 0, v1
 ; GFX1030-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc_lo
 ; GFX1030-NEXT:    v_add_co_u32 v4, vcc_lo, v5, 2

llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll

define amdgpu_kernel void @livevariables_update_missed_block(i8 addrspace(1)* %src1) {
   ; SI-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_NE_U32_e64_]], %bb.5, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
   ; SI-NEXT:   S_BRANCH %bb.2
   ; SI-NEXT: {{  $}}
   ; SI-NEXT: bb.1.if.then:
   ; SI-NEXT:   successors: %bb.7(0x80000000)
   ; SI-NEXT: {{  $}}
   ; SI-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.src1.kernarg.offset.cast, align 4, addrspace 4)
   ; SI-NEXT:   [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, killed %50, 0, implicit $exec
-  ; SI-NEXT:   %43:vgpr_32, dead %45:sreg_32_xm0_xexec = V_ADDC_U32_e64 0, killed [[S_LOAD_DWORDX2_IMM]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+  ; SI-NEXT:   %43:vgpr_32, dead $sgpr_null = V_ADDC_U32_e64 0, killed [[S_LOAD_DWORDX2_IMM]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
   ; SI-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[V_ADD_CO_U32_e64_]], %subreg.sub0, killed %43, %subreg.sub1
   ; SI-NEXT:   [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8) from %ir.i10, addrspace 1)
   ; SI-NEXT:   [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
   ; SI-NEXT:   GLOBAL_STORE_BYTE killed [[V_MOV_B]], killed [[GLOBAL_LOAD_UBYTE]], 0, 0, implicit $exec :: (store (s8) into `i8 addrspace(1)* null`, addrspace 1)
   ; SI-NEXT:   S_BRANCH %bb.7
   ; SI-NEXT: {{  $}}
   ; SI-NEXT: bb.2.if.then9:
   ; SI-NEXT:   successors: %bb.4(0x40000000), %bb.3(0x40000000)

llvm/test/CodeGen/AMDGPU/wave32.ll

bb:
   %tmp2 = load i64, i64 addrspace(1)* %arg, align 8
   %tmp3 = udiv i64 %tmp1, %tmp2
   %tmp4 = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 2
   store i64 %tmp3, i64 addrspace(1)* %tmp4, align 8
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_div_scale_f32:
-; GFX1032: v_div_scale_f32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX1064: v_div_scale_f32 v{{[0-9]+}}, s[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX1032: v_div_scale_f32 v{{[0-9]+}}, null, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX1064: v_div_scale_f32 v{{[0-9]+}}, null, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @test_div_scale_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
 
   %a = load volatile float, float addrspace(1)* %gep.0, align 4
   %b = load volatile float, float addrspace(1)* %gep.1, align 4
 
   %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 false) nounwind readnone
   %result0 = extractvalue { float, i1 } %result, 0
   store float %result0, float addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_div_scale_f64:
-; GFX1032: v_div_scale_f64 v[{{[0-9:]+}}], s{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], v[{{[0-9:]+}}]
-; GFX1064: v_div_scale_f64 v[{{[0-9:]+}}], s[{{[0-9:]+}}], v[{{[0-9:]+}}], v[{{[0-9:]+}}], v[{{[0-9:]+}}]
+; GFX1032: v_div_scale_f64 v[{{[0-9:]+}}], null, v[{{[0-9:]+}}], v[{{[0-9:]+}}], v[{{[0-9:]+}}]
+; GFX1064: v_div_scale_f64 v[{{[0-9:]+}}], null, v[{{[0-9:]+}}], v[{{[0-9:]+}}], v[{{[0-9:]+}}]
 define amdgpu_kernel void @test_div_scale_f64(double addrspace(1)* %out, double addrspace(1)* %aptr, double addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr double, double addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1
 
   %a = load volatile double, double addrspace(1)* %gep.0, align 8
   %b = load volatile double, double addrspace(1)* %gep.1, align 8
 
   %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double %a, double %b, i1 true) nounwind readnone
   %result0 = extractvalue { double, i1 } %result, 0
   store double %result0, double addrspace(1)* %out, align 8
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_mad_i64_i32:
-; GFX1032: v_mad_i64_i32 v[{{[0-9:]+}}], s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
-; GFX1064: v_mad_i64_i32 v[{{[0-9:]+}}], s[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
+; GFX1032: v_mad_i64_i32 v[{{[0-9:]+}}], null, v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
+; GFX1064: v_mad_i64_i32 v[{{[0-9:]+}}], null, v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
 define i64 @test_mad_i64_i32(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %mul, %arg2
   ret i64 %mad
 }
 
 ; GCN-LABEL: {{^}}test_mad_u64_u32:
-; GFX1032: v_mad_u64_u32 v[{{[0-9:]+}}], s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
-; GFX1064: v_mad_u64_u32 v[{{[0-9:]+}}], s[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
+; GFX1032: v_mad_u64_u32 v[{{[0-9:]+}}], null, v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
+; GFX1064: v_mad_u64_u32 v[{{[0-9:]+}}], null, v{{[0-9]+}}, v{{[0-9]+}}, v[{{[0-9:]+}}]
 define i64 @test_mad_u64_u32(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
   %sext0 = zext i32 %arg0 to i64
   %sext1 = zext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %mul, %arg2
   ret i64 %mad
 }
 

 exit:
   %cond = phi i1 [false, %entry], [%cmp1, %bb]
   %result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float %c, i1 %cond) nounwind readnone
   store float %result, float addrspace(1)* %gep.out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}fdiv_f32:
-; GFX1032: v_div_scale_f32 v{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GFX1064: v_div_scale_f32 v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; GFX1032: v_div_scale_f32 v{{[0-9]+}}, null, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; GFX1064: v_div_scale_f32 v{{[0-9]+}}, null, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 ; GCN: v_rcp_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX1032: v_div_scale_f32 v{{[0-9]+}}, vcc_lo, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 ; GFX1064: v_div_scale_f32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
 
 ; GCN-NOT: vcc
 ; GCN: v_div_fmas_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @fdiv_f32(float addrspace(1)* %out, float %a, float %b) #0 {
 entry: