[AMDGPU] SIFoldOperands: eagerly delete dead copies

[foad]

Summary

This is cheap to implement, means less work for future passes like
MachineDCE, and slightly improves the folding in some cases.

[arsenm]

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

1789

Slightly worried about the case where the only remaining copy user is a debug instruction

[foad]

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

1789

Good point. What can you do in that case?

[foad]

Use eraseFromParentAndMarkDBGValuesForRemoval.

[arsenm]

Can you add a test for the DBG_VALUE case?

[rampitec]

LGTM with the DBG_VALUE test requested by Matt.

[foad]

Add a DBG_VALUE test case.

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

for (auto &MI : make_early_inc_range(*MBB)) {
       //
       //    %3 = COPY %vgpr0; VGPR_32:%3
       //    ...
       //    %vgpr0 = V_MOV_B32_e32 1, implicit %exec
       if (!MI.getOperand(0).getReg().isVirtual())
         continue;
 
       foldInstOperand(MI, OpToFold);
+
+      // If we managed to fold all uses of this copy then we might as well
+      // delete it now.
+      if (MRI->use_nodbg_empty(MI.getOperand(0).getReg()))
+        MI.eraseFromParentAndMarkDBGValuesForRemoval();
     }
   }
   return true;
 }

llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll

 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_movk_i32 s4, 0xff
 ; SI-NEXT:    v_and_b32_e32 v0, s4, v0
 ; SI-NEXT:    v_ffbh_u32_e32 v2, v0
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
 ; SI-NEXT:    v_ffbh_u32_e32 v3, 0
 ; SI-NEXT:    v_cmp_eq_u32_e64 vcc, 0, 0
 ; SI-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; SI-NEXT:    v_mov_b32_e32 v3, 0xbe
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
-; SI-NEXT:    v_sub_i32_e32 v4, vcc, v3, v2
+; SI-NEXT:    v_sub_i32_e32 v4, vcc, 0xbe, v2
 ; SI-NEXT:    v_lshl_b64 v[2:3], v[0:1], v2
 ; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; SI-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v3
 ; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v4, vcc
 ; SI-NEXT:    v_and_b32_e32 v3, s4, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 23, v0
 ; SI-NEXT:    s_mov_b32 s4, 0

 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_movk_i32 s4, 0xff
 ; VI-NEXT:    v_and_b32_e32 v0, s4, v0
 ; VI-NEXT:    v_ffbh_u32_e32 v2, v0
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 32, v2
 ; VI-NEXT:    v_ffbh_u32_e32 v3, 0
 ; VI-NEXT:    v_cmp_eq_u32_e64 vcc, 0, 0
 ; VI-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; VI-NEXT:    v_mov_b32_e32 v3, 0xbe
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
-; VI-NEXT:    v_sub_u32_e32 v4, vcc, v3, v2
+; VI-NEXT:    v_sub_u32_e32 v4, vcc, 0xbe, v2
 ; VI-NEXT:    v_lshlrev_b64 v[2:3], v2, v[0:1]
 ; VI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; VI-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v3
 ; VI-NEXT:    v_cndmask_b32_e32 v0, 0, v4, vcc
 ; VI-NEXT:    v_and_b32_e32 v3, s4, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 23, v0
 ; VI-NEXT:    s_mov_b32 s4, 0

llvm/test/CodeGen/AMDGPU/flat-scratch-fold-fi.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-enable-flat-scratch -run-pass=si-fold-operands -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
 
 ---
 name:            test_fold_fi_scratch_load_vgpr
 stack:
   - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
 body:             |
   bb.0.entry:
     ; GCN-LABEL: name: test_fold_fi_scratch_load_vgpr
-    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     ; GCN: [[SCRATCH_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = SCRATCH_LOAD_DWORD_SADDR %stack.0, 4, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
     ; GCN: S_ENDPGM 0
     %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     %1:vgpr_32 = SCRATCH_LOAD_DWORD %0:vgpr_32, 4, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
     S_ENDPGM 0
 
 ...
 
 ---
 name:            test_fold_fi_scratch_load_sgpr
 stack:
   - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
 body:             |
   bb.0.entry:
     ; GCN-LABEL: name: test_fold_fi_scratch_load_sgpr
-    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 %stack.0
     ; GCN: [[SCRATCH_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = SCRATCH_LOAD_DWORD_SADDR %stack.0, 4, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
     ; GCN: S_ENDPGM 0
     %0:sgpr_32 = S_MOV_B32 %stack.0
     %1:vgpr_32 = SCRATCH_LOAD_DWORD_SADDR %0:sgpr_32, 4, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
     S_ENDPGM 0
 
 ...
 
 ---
 name:            test_fold_fi_scratch_store_vgpr
 stack:
   - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
 body:             |
   bb.0.entry:
     ; GCN-LABEL: name: test_fold_fi_scratch_store_vgpr
-    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN: SCRATCH_STORE_DWORD_SADDR [[DEF]], %stack.0, 4, 0, implicit $exec, implicit $flat_scr :: (store 4 into %stack.0, addrspace 5)
     ; GCN: S_ENDPGM 0
     %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     %1:vgpr_32 = IMPLICIT_DEF
     SCRATCH_STORE_DWORD %1:vgpr_32, %0:vgpr_32, 4, 0, implicit $exec, implicit $flat_scr :: (store 4 into %stack.0, addrspace 5)
     S_ENDPGM 0
 

 
 ---
 name:            test_fold_fi_scratch_store_sgpr
 stack:
   - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
 body:             |
   bb.0.entry:
     ; GCN-LABEL: name: test_fold_fi_scratch_store_sgpr
-    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 %stack.0
     ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN: SCRATCH_STORE_DWORD_SADDR [[DEF]], %stack.0, 4, 0, implicit $exec, implicit $flat_scr :: (store 4 into %stack.0, addrspace 5)
     ; GCN: S_ENDPGM 0
     %0:sgpr_32 = S_MOV_B32 %stack.0
     %1:vgpr_32 = IMPLICIT_DEF
     SCRATCH_STORE_DWORD_SADDR %1:vgpr_32, %0:sgpr_32, 4, 0, implicit $exec, implicit $flat_scr :: (store 4 into %stack.0, addrspace 5)
     S_ENDPGM 0
 
 ...

llvm/test/CodeGen/AMDGPU/fold-cndmask-wave32.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=gfx1030 -run-pass si-fold-operands -verify-machineinstrs -o - %s | FileCheck %s
 
 ---
 name:            fold_cndmask
 tracksRegLiveness: true
 registers:
 body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: fold_cndmask
     ; CHECK: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
     ; CHECK: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    ; CHECK: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     %0:sreg_32_xm0_xexec = IMPLICIT_DEF
     %1:sreg_32 = S_MOV_B32 0
     %2:vgpr_32 = COPY %1:sreg_32
     %3:vgpr_32 = V_CNDMASK_B32_e64 0, %1:sreg_32, 0, %2:vgpr_32, %0:sreg_32_xm0_xexec, implicit $exec
     S_ENDPGM 0, implicit %3
 ...

llvm/test/CodeGen/AMDGPU/fold-cndmask.mir

 # RUN: llc -march=amdgcn -run-pass si-fold-operands -verify-machineinstrs -o - %s | FileCheck %s
 
 # CHECK: %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
 # CHECK: %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
 # CHECK: %4:vgpr_32 = COPY %3
-# CHECK: %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
 # CHECK: %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
 # CHECK: %7:vgpr_32 = COPY %3
 
 ---
 name:            fold_cndmask
 tracksRegLiveness: true
 registers:
   - { id: 0, class: sgpr_64 }

llvm/test/CodeGen/AMDGPU/fold-imm-f16-f32.mir

bb.0 (%ir-block.0):
     BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     BUFFER_STORE_SHORT_OFFSET killed %16, %10, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     BUFFER_STORE_DWORD_OFFSET killed %17, %10, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
 ---
 # CHECK-LABEL: name: add_i32_1_multi_f16_use
-# CHECK: %13:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
 # CHECK: %14:vgpr_32 = V_ADD_F16_e32 1, killed %11, implicit $mode, implicit $exec
 # CHECK: %15:vgpr_32 = V_ADD_F16_e32 1, killed %12, implicit $mode, implicit $exec
 
 
 name:            add_i32_1_multi_f16_use
 alignment:       1
 exposesReturnsTwice: false
 legalized:       false
bb.0 (%ir-block.0):
     BUFFER_STORE_SHORT_OFFSET killed %14, %10, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
 ---
 
 # CHECK-LABEL: name: add_i32_m2_one_f32_use_multi_f16_use
-# CHECK: %14:vgpr_32 = V_MOV_B32_e32 -2, implicit $exec
 # CHECK: %15:vgpr_32 = V_ADD_F16_e32 -2, %11, implicit $mode, implicit $exec
 # CHECK: %16:vgpr_32 = V_ADD_F16_e32 -2, %12, implicit $mode, implicit $exec
 # CHECK: %17:vgpr_32 = V_ADD_F32_e32 -2, killed %13, implicit $mode, implicit $exec
 
 name:            add_i32_m2_one_f32_use_multi_f16_use
 alignment:       1
 exposesReturnsTwice: false
 legalized:       false

llvm/test/CodeGen/AMDGPU/fold-operands-order.mir

 # RUN: llc -mtriple=amdgcn--amdhsa -mcpu=hawaii -verify-machineinstrs -run-pass si-fold-operands -o - %s | FileCheck -check-prefix=GCN %s
 ...
 ---
 
 # Blocks should be processed in program order to make sure folds
 # aren't made in users before the def is seen.
 
 # GCN-LABEL: name: mov_in_use_list_2x{{$}}
-# GCN: %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-# GCN-NEXT: %3:vgpr_32 = COPY undef %0
-
-# GCN: %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+# GCN: %3:vgpr_32 = COPY undef %0
 
 
 name: mov_in_use_list_2x
 tracksRegLiveness: true
 registers:
   - { id: 0, class: vgpr_32, preferred-register: '' }
   - { id: 1, class: vgpr_32, preferred-register: '' }
   - { id: 2, class: vgpr_32, preferred-register: '' }

llvm/test/CodeGen/AMDGPU/fold-readlane.mir

 # RUN: llc -march=amdgcn -run-pass si-fold-operands -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
 
 # GCN-LABEL: name: fold-imm-readfirstlane{{$}}
 # GCN: %1:sreg_32_xm0 = S_MOV_B32 123
 ---
 name: fold-imm-readfirstlane
 tracksRegLiveness: true
 body:             |
   bb.0:
     %0:vgpr_32 = V_MOV_B32_e32 123, implicit $exec
     %1:sreg_32_xm0 = V_READFIRSTLANE_B32 %0, implicit $exec
     S_NOP 0, implicit %1
 ...
 
+# GCN-LABEL: name: fold-imm-readfirstlane-dbgvalue{{$}}
+# GCN: %1:sreg_32_xm0 = S_MOV_B32 123
+# GCN: DBG_VALUE $noreg, 0, 0
+---
+name: fold-imm-readfirstlane-dbgvalue
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:vgpr_32 = V_MOV_B32_e32 123, implicit $exec
+    %1:sreg_32_xm0 = V_READFIRSTLANE_B32 %0, implicit $exec
+    DBG_VALUE %0, 0, 0
+    S_NOP 0, implicit %1
+...
+
 # GCN-LABEL: name: fold-imm-readfirstlane-readfirstlane{{$}}
-# GCN: %0:vgpr_32 = V_MOV_B32_e32 123, implicit $exec
 # GCN: %1:sreg_32_xm0 = S_MOV_B32 123
-# GCN: %2:vgpr_32 = V_MOV_B32_e32 123, implicit $exec
 # GCN: %3:sreg_32_xm0 = COPY %1
 
 ---
 name: fold-imm-readfirstlane-readfirstlane
 tracksRegLiveness: true
 body:             |
   bb.0:
     %0:vgpr_32 = V_MOV_B32_e32 123, implicit $exec
     %1:sreg_32_xm0 = V_READFIRSTLANE_B32 %0, implicit $exec
     %2:vgpr_32 = COPY %1
     %3:sreg_32_xm0 = V_READFIRSTLANE_B32 %2, implicit $exec
     S_NOP 0, implicit %3
 ...
 
 
 # GCN-LABEL: name: fold-copy-readfirstlane{{$}}
 # GCN: %0:sreg_32_xm0 = COPY $sgpr10
-# GCN: %1:vgpr_32 = COPY %0
 # GCN: %2:sreg_32_xm0 = COPY %0
 ---
 name: fold-copy-readfirstlane
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr10
     %0:sreg_32_xm0 = COPY $sgpr10

llvm/test/CodeGen/AMDGPU/huge-number-operand-folds.mir

 # RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -run-pass=si-fold-operands  %s -o - | FileCheck -check-prefix=GCN %s
 
 # We were storing fold candidate uses in an unsigned char, which this exceeds.
 # The use operand overflows and the expected register operand hits the immediate 0.
 # We never have more than a handful of non-implicit operands, so don't try to fold into
 # implicit operands to avoid this problem.
 
 ---
 name: op_idx_overflows_uchar
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr12_sgpr13_sgpr14_sgpr15
 
     ; GCN-LABEL: name: op_idx_overflows_uchar
-    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; GCN: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:sreg_32 = S_MOV_B32 0
     %1:vgpr_32 = COPY %0
     S_ENDPGM 0, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1
 ...

llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll

 ; GCN-LABEL: DiffBase:
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ;
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048

llvm/test/CodeGen/AMDGPU/udiv64.ll

 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v12, v6, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v5, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
 ; GCN-NEXT:    v_lshrrev_b32_e32 v2, 17, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GCN-NEXT:    s_mov_b32 s4, 0x8000
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, v2
 ; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s4, v4
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0x8000, v4
 ; GCN-NEXT:    v_subb_u32_e64 v5, s[4:5], v5, v1, vcc
 ; GCN-NEXT:    v_sub_i32_e64 v6, s[4:5], v4, v0
 ; GCN-NEXT:    v_subbrev_u32_e64 v5, s[4:5], 0, v5, s[4:5]
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
 ; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v1

 ; GCN-IR-LABEL: v_test_udiv_pow2_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
 ; GCN-IR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GCN-IR-NEXT:    v_cndmask_b32_e32 v8, v3, v2, vcc
-; GCN-IR-NEXT:    s_movk_i32 s6, 0xffd0
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, s6, v8
+; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffd0, v8
 ; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[6:7], 0, -1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
 ; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[4:5]
 ; GCN-IR-NEXT:    s_mov_b32 s8, 0x8000
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], vcc
 ; GCN-IR-NEXT:    v_mov_b32_e32 v2, s8
 ; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[4:5]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0

llvm/test/CodeGen/AMDGPU/urem64.ll

 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v5, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
 ; GCN-NEXT:    v_lshrrev_b32_e32 v2, 17, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v0, v2
-; GCN-NEXT:    s_mov_b32 s4, 0x8000
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s4, v2
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0x8000, v2
 ; GCN-NEXT:    v_subb_u32_e64 v4, s[4:5], v4, v1, vcc
 ; GCN-NEXT:    v_sub_i32_e64 v5, s[4:5], v2, v0
 ; GCN-NEXT:    v_subbrev_u32_e64 v6, s[6:7], 0, v4, s[4:5]
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[6:7], v6, v1
 ; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[6:7]
 ; GCN-NEXT:    v_cmp_ge_u32_e64 s[6:7], v5, v0
 ; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[6:7]
 ; GCN-NEXT:    v_cmp_eq_u32_e64 s[6:7], v6, v1