[ANDGPU] getMemOperandsWithOffset: support BUF non-stack-access instructions with resource but no vaddr

[foad]

Summary

This enables clustering for many more BUF instructions.

Depends on D73867.

[merge_guards_bot]

Unit tests: fail. 62405 tests passed, 1 failed and 839 were skipped.

failed: libc++.std/thread/thread_mutex/thread_mutex_requirements/thread_sharedtimedmutex_requirements/thread_sharedtimedmutex_class/try_lock.pass.cpp

clang-tidy: pass.

clang-format: pass.

Build artifacts: diff.json, clang-tidy.txt, clang-format.patch, CMakeCache.txt, console-log.txt, test-results.xml

Pre-merge checks is in beta. Report issue. Please join beta or enable it for your project.

[rampitec]

LGTM

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

if (SOffset && SOffset->isReg()) {
       const MachineOperand *OffsetImm =
         getNamedOperand(LdSt, AMDGPU::OpName::offset);
       BaseOps.push_back(RSrc);
       BaseOps.push_back(SOffset);
       Offset = OffsetImm->getImm();
       return true;
     }
 
-    BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
-    if (!BaseOp)
+    BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
+    if (!BaseOp) // e.g. BUFFER_WBINVL1_VOL
       return false;
-    const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
-    BaseOps.push_back(RSrc);
+    BaseOps.push_back(BaseOp);
+
+    BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
+    if (BaseOp)
       BaseOps.push_back(BaseOp);
 
     const MachineOperand *OffsetImm =
         getNamedOperand(LdSt, AMDGPU::OpName::offset);
     Offset = OffsetImm->getImm();
     if (SOffset) // soffset can be an inline immediate.
       Offset += SOffset->getImm();
     return true;
   }

llvm/test/CodeGen/AMDGPU/bswap.ll

 ; VI-NEXT:    v_alignbit_b32 v1, s11, v1, 8
 ; VI-NEXT:    s_bfe_u32 s6, s11, 0x80010
 ; VI-NEXT:    v_and_b32_e32 v3, s16, v3
 ; VI-NEXT:    v_and_b32_e32 v1, s17, v1
 ; VI-NEXT:    s_lshr_b32 s4, s11, 24
 ; VI-NEXT:    s_lshl_b32 s6, s6, 8
 ; VI-NEXT:    s_or_b32 s4, s6, s4
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v3
-; VI-NEXT:    v_or_b32_e32 v5, s4, v1
+; VI-NEXT:    v_or_b32_e32 v6, s4, v1
 ; VI-NEXT:    v_mov_b32_e32 v1, s8
 ; VI-NEXT:    v_alignbit_b32 v3, s9, v1, 24
 ; VI-NEXT:    v_alignbit_b32 v1, s9, v1, 8
 ; VI-NEXT:    s_bfe_u32 s6, s9, 0x80010
 ; VI-NEXT:    s_lshl_b64 s[14:15], s[10:11], 8
 ; VI-NEXT:    v_and_b32_e32 v3, s16, v3
 ; VI-NEXT:    v_and_b32_e32 v1, s17, v1
 ; VI-NEXT:    s_lshr_b32 s4, s9, 24
 ; VI-NEXT:    s_lshl_b32 s6, s6, 8
-; VI-NEXT:    s_and_b32 s13, s15, s18
-; VI-NEXT:    s_lshl_b64 s[14:15], s[10:11], 24
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v3
 ; VI-NEXT:    s_or_b32 s4, s6, s4
-; VI-NEXT:    v_or_b32_e32 v3, s4, v1
+; VI-NEXT:    s_and_b32 s13, s15, s18
+; VI-NEXT:    s_lshl_b64 s[14:15], s[10:11], 24
+; VI-NEXT:    v_or_b32_e32 v4, s4, v1
 ; VI-NEXT:    s_lshl_b32 s4, s10, 8
 ; VI-NEXT:    s_and_b32 s15, s15, s19
 ; VI-NEXT:    s_mov_b32 s14, s12
 ; VI-NEXT:    s_or_b64 s[14:15], s[14:15], s[12:13]
 ; VI-NEXT:    s_lshl_b32 s13, s10, 24
 ; VI-NEXT:    s_and_b32 s11, s4, s16
 ; VI-NEXT:    s_mov_b32 s10, s12
 ; VI-NEXT:    s_or_b64 s[10:11], s[12:13], s[10:11]
 ; VI-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
 ; VI-NEXT:    s_lshl_b64 s[14:15], s[8:9], 8
 ; VI-NEXT:    s_and_b32 s13, s15, s18
 ; VI-NEXT:    s_lshl_b64 s[14:15], s[8:9], 24
 ; VI-NEXT:    s_lshl_b32 s4, s8, 8
 ; VI-NEXT:    s_and_b32 s15, s15, s19
 ; VI-NEXT:    s_mov_b32 s14, s12
 ; VI-NEXT:    s_or_b64 s[14:15], s[14:15], s[12:13]
 ; VI-NEXT:    s_lshl_b32 s13, s8, 24
 ; VI-NEXT:    s_and_b32 s9, s4, s16
 ; VI-NEXT:    s_mov_b32 s8, s12
 ; VI-NEXT:    s_or_b64 s[8:9], s[12:13], s[8:9]
 ; VI-NEXT:    s_or_b64 s[8:9], s[8:9], s[14:15]
-; VI-NEXT:    v_mov_b32_e32 v4, s9
-; VI-NEXT:    v_mov_b32_e32 v6, s11
-; VI-NEXT:    buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:16
+; VI-NEXT:    v_mov_b32_e32 v5, s9
+; VI-NEXT:    v_mov_b32_e32 v7, s11
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
   %val = load <4 x i64>, <4 x i64> addrspace(1)* %in, align 32
   %bswap = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %val) nounwind readnone
   store <4 x i64> %bswap, <4 x i64> addrspace(1)* %out, align 32
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll

 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[0:1]
 ; VI-NEXT:    s_mov_b32 s4, s0
 ; VI-NEXT:    s_mov_b32 s5, s1
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
-; VI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0 offset:2
+; VI-NEXT:    buffer_store_byte v1, off, s[4:7], 0 offset:2
 ; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <3 x i8>, <3 x i8> addrspace(1)* %in, i32 %tid.x
   %val = load <3 x i8>, <3 x i8> addrspace(1)* %gep, align 4
   store <3 x i8> %val, <3 x i8> addrspace(1)* %out, align 4
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll

define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind {
   store i32 %ret, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_cttz_zero_undef_i64_with_select:
 ; SI-NOSDWA: v_or_b32_e32
 ; SI-NOSDWA: v_or_b32_e32
 ; SI-NOSDWA: v_or_b32_e32
-; SI-SDWA: v_or_b32_sdwa
 ; SI-NOSDWA: v_or_b32_e32
+; SI-NOSDWA: v_or_b32_e32 [[VAL1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; SI-NOSDWA: v_or_b32_e32 [[VAL2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]]
+; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL2]]
+; SI-SDWA: v_or_b32_e32
+; SI-SDWA: v_or_b32_sdwa
+; SI-SDWA: v_or_b32_e32 [[VAL1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; SI-SDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]]
+; SI-SDWA: v_or_b32_e32
 ; SI-SDWA: v_or_b32_sdwa
-; SI: v_or_b32_e32 [[VAL1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_or_b32_e32 [[VAL2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]]
-; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL2]]
+; SI-SDWA: v_or_b32_e32 [[VAL2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; SI-SDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL2]]
 ; SI: v_cmp_eq_u32_e32 vcc, 0
 ; SI: v_cmp_ne_u64_e32 vcc, 0
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]]
 define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(i64 addrspace(1)* noalias %out, i64 addrspace(1)* nocapture readonly %arrayidx) nounwind {
   %val = load i64, i64 addrspace(1)* %arrayidx, align 1
   %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
   %cttz_ret = icmp ne i64 %val, 0
   %ret = select i1 %cttz_ret, i64 %cttz, i64 32

llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll

 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, 0
 ; SI-NEXT:    s_mov_b32 s3, s7
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[0:3], 0 addr64 offset:1
-; SI-NEXT:    buffer_load_ubyte v4, v[0:1], s[0:3], 0 addr64 offset:2
-; SI-NEXT:    buffer_load_ubyte v5, v[0:1], s[0:3], 0 addr64 offset:3
-; SI-NEXT:    buffer_load_ubyte v6, v[0:1], s[0:3], 0 addr64 offset:4
-; SI-NEXT:    buffer_load_ubyte v7, v[0:1], s[0:3], 0 addr64 offset:5
-; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64 offset:6
+; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[0:3], 0 addr64 offset:5
+; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[0:3], 0 addr64 offset:6
+; SI-NEXT:    buffer_load_ubyte v4, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    buffer_load_ubyte v5, v[0:1], s[0:3], 0 addr64 offset:1
+; SI-NEXT:    buffer_load_ubyte v6, v[0:1], s[0:3], 0 addr64 offset:2
+; SI-NEXT:    buffer_load_ubyte v7, v[0:1], s[0:3], 0 addr64 offset:3
+; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64 offset:4
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
-; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v5
-; SI-NEXT:    v_or_b32_e32 v2, v2, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v5
+; SI-NEXT:    v_or_b32_e32 v1, v1, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT:    v_or_b32_e32 v3, v3, v6
-; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0 offset:24
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
-; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    v_cvt_f32_ubyte1_e32 v5, v3
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v4, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v7
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v7, v3
+; SI-NEXT:    v_or_b32_e32 v3, v5, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v2, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT:    v_cvt_f32_ubyte1_e32 v5, v0
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v4, v0
+; SI-NEXT:    v_or_b32_e32 v0, v2, v1
 ; SI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v0
 ; SI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v0
 ; SI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v0
 ; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; SI-NEXT:    buffer_store_dword v7, off, s[4:7], 0 offset:24
 ; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:16
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: load_v7i8_to_v7f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c

llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll

 ; SI-NEXT:    s_mov_b32 s4, 0xffff
 ; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], s8
 ; SI-NEXT:    s_mov_b32 s8, 0x50005
 ; SI-NEXT:    s_and_b32 s9, s5, s8
 ; SI-NEXT:    s_and_b32 s8, s4, s8
 ; SI-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
 ; SI-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
 ; SI-NEXT:    v_mov_b32_e32 v0, s5
+; SI-NEXT:    v_mov_b32_e32 v1, s4
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0 offset:4
-; SI-NEXT:    v_mov_b32_e32 v0, s4
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v3i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x8
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
 ; VI-NEXT:    s_mov_b32 s5, 0

llvm/test/CodeGen/AMDGPU/llvm.memcpy.ll

 define amdgpu_kernel void @test_memcpy_const_string_align4(i8 addrspace(1)* noalias %out) nounwind {
   %str = bitcast [16 x i8] addrspace(4)* @hello.align4 to i8 addrspace(4)*
   call void @llvm.memcpy.p1i8.p2i8.i64(i8 addrspace(1)* align 4 %out, i8 addrspace(4)* align 4 %str, i64 32, i1 false)
   ret void
 }
 
 ; FUNC-LABEL: {{^}}test_memcpy_const_string_align1:
 ; SI-NOT: buffer_load
-; SI: v_mov_b32_e32 v{{[0-9]+}}, 0x69
+; SI: v_mov_b32_e32 v{{[0-9]+}}, 0x
 ; SI: buffer_store_byte
 ; SI: buffer_store_byte
 ; SI: buffer_store_byte
 ; SI: buffer_store_byte
 ; SI: buffer_store_byte
 ; SI: buffer_store_byte
 ; SI: buffer_store_byte
 ; SI: buffer_store_byte

llvm/test/CodeGen/AMDGPU/merge-stores.ll

define amdgpu_kernel void @merge_local_store_4_constants_i32(i32 addrspace(3)* %out) #0 {
   store i32 333, i32 addrspace(3)* %out.gep.3
   store i32 1234, i32 addrspace(3)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}merge_global_store_5_constants_i32:
 ; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 9{{$}}
 ; GCN-DAG: v_mov_b32_e32 v[[HI4:[0-9]+]], -12{{$}}
-; GCN: buffer_store_dwordx4 v{{\[}}[[LO]]:[[HI4]]{{\]}}
 ; GCN: v_mov_b32_e32 v[[HI:[0-9]+]], 11{{$}}
+; GCN: buffer_store_dwordx4 v{{\[}}[[LO]]:[[HI4]]{{\]}}
 ; GCN: buffer_store_dword v[[HI]]
 define amdgpu_kernel void @merge_global_store_5_constants_i32(i32 addrspace(1)* %out) {
   store i32 9, i32 addrspace(1)* %out, align 4
   %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
   store i32 12, i32 addrspace(1)* %idx1, align 4
   %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
   store i32 16, i32 addrspace(1)* %idx2, align 4
   %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3

llvm/test/CodeGen/AMDGPU/sdiv.ll

 ; TONGA-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x24
 ; TONGA-NEXT:    s_mov_b32 s11, 0xf000
 ; TONGA-NEXT:    s_mov_b32 s10, -1
 ; TONGA-NEXT:    s_mov_b32 s2, s10
 ; TONGA-NEXT:    s_mov_b32 s3, s11
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
 ; TONGA-NEXT:    s_mov_b32 s0, s14
 ; TONGA-NEXT:    s_mov_b32 s1, s15
-; TONGA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; TONGA-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; TONGA-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; TONGA-NEXT:    s_mov_b32 s14, 0x4f800000
 ; TONGA-NEXT:    s_mov_b32 s8, s12
 ; TONGA-NEXT:    s_mov_b32 s9, s13
 ; TONGA-NEXT:    s_waitcnt vmcnt(1)
+; TONGA-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
+; TONGA-NEXT:    s_waitcnt vmcnt(0)
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
 ; TONGA-NEXT:    v_add_u32_e32 v4, vcc, v9, v4
-; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
 ; TONGA-NEXT:    v_xor_b32_e32 v4, v4, v9
 ; TONGA-NEXT:    v_xor_b32_e32 v15, v8, v9
 ; TONGA-NEXT:    v_cvt_f32_u32_e32 v9, v4
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
 ; TONGA-NEXT:    v_add_u32_e32 v5, vcc, v11, v5
 ; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v8, v0
 ; TONGA-NEXT:    v_rcp_iflag_f32_e32 v9, v9
 ; TONGA-NEXT:    v_xor_b32_e32 v5, v5, v11

 ; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s15, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s14, -1
 ; GFX9-NEXT:    s_mov_b32 s2, s14
 ; GFX9-NEXT:    s_mov_b32 s3, s15
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s0, s10
 ; GFX9-NEXT:    s_mov_b32 s1, s11
-; GFX9-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; GFX9-NEXT:    buffer_load_dwordx4 v[4:7], off, s[0:3], 0 offset:16
 ; GFX9-NEXT:    s_mov_b32 s4, 0x4f800000
 ; GFX9-NEXT:    s_mov_b32 s12, s8
 ; GFX9-NEXT:    s_mov_b32 s13, s9
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v8, 31, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_ashrrev_i32_e32 v9, 31, v4
 ; GFX9-NEXT:    v_add_u32_e32 v4, v4, v9
 ; GFX9-NEXT:    v_add_u32_e32 v0, v0, v8
 ; GFX9-NEXT:    v_xor_b32_e32 v4, v4, v9
 ; GFX9-NEXT:    v_xor_b32_e32 v16, v8, v9
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v8
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v4
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v5
 ; GFX9-NEXT:    v_add_u32_e32 v5, v5, v11

 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; TONGA-LABEL: v_sdiv_i8:
 ; TONGA:       ; %bb.0:
 ; TONGA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; TONGA-NEXT:    s_mov_b32 s3, 0xf000
 ; TONGA-NEXT:    s_mov_b32 s2, -1
+; TONGA-NEXT:    s_mov_b32 s10, s2
+; TONGA-NEXT:    s_mov_b32 s11, s3
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
+; TONGA-NEXT:    s_mov_b32 s8, s6
+; TONGA-NEXT:    s_mov_b32 s9, s7
+; TONGA-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; TONGA-NEXT:    buffer_load_sbyte v1, off, s[8:11], 0 offset:1
 ; TONGA-NEXT:    s_mov_b32 s0, s4
 ; TONGA-NEXT:    s_mov_b32 s1, s5
-; TONGA-NEXT:    s_mov_b32 s4, s6
-; TONGA-NEXT:    s_mov_b32 s5, s7
-; TONGA-NEXT:    s_mov_b32 s6, s2
-; TONGA-NEXT:    s_mov_b32 s7, s3
-; TONGA-NEXT:    buffer_load_sbyte v0, off, s[4:7], 0 offset:1
-; TONGA-NEXT:    buffer_load_sbyte v2, off, s[4:7], 0
 ; TONGA-NEXT:    s_waitcnt vmcnt(1)
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v1, v0
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v3, v0
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v3, v2
-; TONGA-NEXT:    v_xor_b32_e32 v0, v2, v0
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v2, v1
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v1
 ; TONGA-NEXT:    v_or_b32_e32 v0, 1, v0
-; TONGA-NEXT:    v_mul_f32_e32 v2, v3, v4
-; TONGA-NEXT:    v_trunc_f32_e32 v2, v2
-; TONGA-NEXT:    v_mad_f32 v3, -v2, v1, v3
-; TONGA-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v1|
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; TONGA-NEXT:    v_mul_f32_e32 v1, v3, v4
+; TONGA-NEXT:    v_trunc_f32_e32 v1, v1
+; TONGA-NEXT:    v_mad_f32 v3, -v1, v2, v3
+; TONGA-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_sdiv_i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    s_mov_b32 s10, s2
+; GFX9-NEXT:    s_mov_b32 s11, s3
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s8, s6
+; GFX9-NEXT:    s_mov_b32 s9, s7
+; GFX9-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GFX9-NEXT:    buffer_load_sbyte v1, off, s[8:11], 0 offset:1
 ; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    s_mov_b32 s4, s6
-; GFX9-NEXT:    s_mov_b32 s5, s7
-; GFX9-NEXT:    s_mov_b32 s6, s2
-; GFX9-NEXT:    s_mov_b32 s7, s3
-; GFX9-NEXT:    buffer_load_sbyte v0, off, s[4:7], 0 offset:1
-; GFX9-NEXT:    buffer_load_sbyte v2, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, v0
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, v2
-; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, v1
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v1
 ; GFX9-NEXT:    v_or_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_mul_f32_e32 v2, v3, v4
-; GFX9-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v2
-; GFX9-NEXT:    v_mad_f32 v2, -v2, v1, v3
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GFX9-NEXT:    v_mul_f32_e32 v1, v3, v4
+; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v1
+; GFX9-NEXT:    v_mad_f32 v1, -v1, v2, v3
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v0, v4, v0
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; EG-LABEL: v_sdiv_i8:
 ; EG:       ; %bb.0:

 ; GCN-LABEL: v_sdiv_i23:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_mov_b32 s10, s6
 ; GCN-NEXT:    s_mov_b32 s11, s7
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s4, s0
-; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_mov_b32 s8, s2
 ; GCN-NEXT:    s_mov_b32 s9, s3
 ; GCN-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:2
 ; GCN-NEXT:    buffer_load_ubyte v3, off, s[8:11], 0 offset:6
 ; GCN-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
 ; GCN-NEXT:    buffer_load_ushort v2, off, s[8:11], 0 offset:4
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_waitcnt vmcnt(3)
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(2)
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_or_b32_e32 v1, v2, v3

 ; TONGA-NEXT:    s_mov_b32 s2, -1
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
 ; TONGA-NEXT:    s_mov_b32 s0, s4
 ; TONGA-NEXT:    s_mov_b32 s1, s5
 ; TONGA-NEXT:    s_mov_b32 s4, s6
 ; TONGA-NEXT:    s_mov_b32 s5, s7
 ; TONGA-NEXT:    s_mov_b32 s6, s2
 ; TONGA-NEXT:    s_mov_b32 s7, s3
-; TONGA-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0 offset:2
-; TONGA-NEXT:    buffer_load_ushort v1, off, s[4:7], 0 offset:4
-; TONGA-NEXT:    buffer_load_ubyte v2, off, s[4:7], 0 offset:6
-; TONGA-NEXT:    buffer_load_ushort v3, off, s[4:7], 0
-; TONGA-NEXT:    s_waitcnt vmcnt(3)
-; TONGA-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; TONGA-NEXT:    s_waitcnt vmcnt(1)
-; TONGA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; TONGA-NEXT:    v_or_b32_e32 v1, v1, v2
-; TONGA-NEXT:    v_bfe_i32 v1, v1, 0, 23
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v2, v1
+; TONGA-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; TONGA-NEXT:    buffer_load_ubyte v1, off, s[4:7], 0 offset:2
+; TONGA-NEXT:    buffer_load_ushort v2, off, s[4:7], 0 offset:4
+; TONGA-NEXT:    buffer_load_ubyte v3, off, s[4:7], 0 offset:6
+; TONGA-NEXT:    s_waitcnt vmcnt(2)
+; TONGA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; TONGA-NEXT:    v_or_b32_e32 v0, v0, v1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_or_b32_e32 v0, v3, v0
+; TONGA-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; TONGA-NEXT:    v_or_b32_e32 v2, v2, v3
+; TONGA-NEXT:    v_bfe_i32 v2, v2, 0, 23
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v3, v2
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 23
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v3, v0
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v1
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v1, v0
+; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v2
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v3
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
 ; TONGA-NEXT:    v_or_b32_e32 v0, 1, v0
-; TONGA-NEXT:    v_mul_f32_e32 v1, v3, v4
-; TONGA-NEXT:    v_trunc_f32_e32 v1, v1
-; TONGA-NEXT:    v_mad_f32 v3, -v1, v2, v3
-; TONGA-NEXT:    v_cvt_i32_f32_e32 v1, v1
-; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; TONGA-NEXT:    v_mul_f32_e32 v2, v1, v4
+; TONGA-NEXT:    v_trunc_f32_e32 v2, v2
+; TONGA-NEXT:    v_mad_f32 v1, -v2, v3, v1
+; TONGA-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v3|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_sdiv_i23:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_mov_b32 s1, s5
 ; GFX9-NEXT:    s_mov_b32 s4, s6
 ; GFX9-NEXT:    s_mov_b32 s5, s7
 ; GFX9-NEXT:    s_mov_b32 s6, s2
 ; GFX9-NEXT:    s_mov_b32 s7, s3
-; GFX9-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0 offset:2
-; GFX9-NEXT:    buffer_load_ushort v1, off, s[4:7], 0 offset:4
-; GFX9-NEXT:    buffer_load_ubyte v2, off, s[4:7], 0 offset:6
-; GFX9-NEXT:    buffer_load_ushort v3, off, s[4:7], 0
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT:    v_bfe_i32 v1, v1, 0, 23
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, v1
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; GFX9-NEXT:    buffer_load_ubyte v1, off, s[4:7], 0 offset:2
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[4:7], 0 offset:4
+; GFX9-NEXT:    buffer_load_ubyte v3, off, s[4:7], 0 offset:6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX9-NEXT:    v_bfe_i32 v2, v2, 0, 23
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, v2
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 23
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, v0
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v3
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
 ; GFX9-NEXT:    v_or_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_mul_f32_e32 v1, v3, v4
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v1
-; GFX9-NEXT:    v_mad_f32 v1, -v1, v2, v3
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
+; GFX9-NEXT:    v_mul_f32_e32 v2, v1, v4
+; GFX9-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v2
+; GFX9-NEXT:    v_mad_f32 v1, -v2, v3, v1
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v3|
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v0, v4, v0
 ; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 23
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; EG-LABEL: v_sdiv_i23:
 ; EG:       ; %bb.0:

 ; GCN-LABEL: v_sdiv_i24:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; GCN-NEXT:    s_mov_b32 s7, 0xf000
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_mov_b32 s10, s6
 ; GCN-NEXT:    s_mov_b32 s11, s7
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s4, s0
-; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_mov_b32 s8, s2
 ; GCN-NEXT:    s_mov_b32 s9, s3
 ; GCN-NEXT:    buffer_load_sbyte v1, off, s[8:11], 0 offset:2
 ; GCN-NEXT:    buffer_load_sbyte v3, off, s[8:11], 0 offset:6
 ; GCN-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
 ; GCN-NEXT:    buffer_load_ushort v2, off, s[8:11], 0 offset:4
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_waitcnt vmcnt(3)
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(2)
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_or_b32_e32 v2, v2, v3

 ; TONGA-NEXT:    s_mov_b32 s2, -1
 ; TONGA-NEXT:    s_waitcnt lgkmcnt(0)
 ; TONGA-NEXT:    s_mov_b32 s0, s4
 ; TONGA-NEXT:    s_mov_b32 s1, s5
 ; TONGA-NEXT:    s_mov_b32 s4, s6
 ; TONGA-NEXT:    s_mov_b32 s5, s7
 ; TONGA-NEXT:    s_mov_b32 s6, s2
 ; TONGA-NEXT:    s_mov_b32 s7, s3
-; TONGA-NEXT:    buffer_load_sbyte v0, off, s[4:7], 0 offset:2
-; TONGA-NEXT:    buffer_load_ushort v1, off, s[4:7], 0 offset:4
-; TONGA-NEXT:    buffer_load_sbyte v2, off, s[4:7], 0 offset:6
-; TONGA-NEXT:    buffer_load_ushort v3, off, s[4:7], 0
-; TONGA-NEXT:    s_waitcnt vmcnt(3)
-; TONGA-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; TONGA-NEXT:    s_waitcnt vmcnt(1)
-; TONGA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; TONGA-NEXT:    v_or_b32_e32 v1, v1, v2
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v1, v1
+; TONGA-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; TONGA-NEXT:    buffer_load_sbyte v1, off, s[4:7], 0 offset:2
+; TONGA-NEXT:    buffer_load_ushort v2, off, s[4:7], 0 offset:4
+; TONGA-NEXT:    buffer_load_sbyte v3, off, s[4:7], 0 offset:6
+; TONGA-NEXT:    s_waitcnt vmcnt(2)
+; TONGA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; TONGA-NEXT:    v_or_b32_e32 v0, v0, v1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_or_b32_e32 v3, v3, v0
-; TONGA-NEXT:    v_cvt_f32_i32_e32 v3, v3
-; TONGA-NEXT:    v_xor_b32_e32 v0, v0, v2
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v1
-; TONGA-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
-; TONGA-NEXT:    v_or_b32_e32 v0, 1, v0
-; TONGA-NEXT:    v_mul_f32_e32 v2, v3, v4
-; TONGA-NEXT:    v_trunc_f32_e32 v2, v2
-; TONGA-NEXT:    v_mad_f32 v3, -v2, v1, v3
-; TONGA-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v1|
-; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; TONGA-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; TONGA-NEXT:    v_or_b32_e32 v2, v2, v3
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; TONGA-NEXT:    v_cvt_f32_i32_e32 v0, v0
+; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v3
+; TONGA-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; TONGA-NEXT:    v_or_b32_e32 v1, 1, v1
+; TONGA-NEXT:    v_mul_f32_e32 v3, v0, v4
+; TONGA-NEXT:    v_trunc_f32_e32 v3, v3
+; TONGA-NEXT:    v_mad_f32 v0, -v3, v2, v0
+; TONGA-NEXT:    v_cvt_i32_f32_e32 v3, v3
+; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
+; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v3
 ; TONGA-NEXT:    v_bfe_i32 v0, v0, 0, 24
 ; TONGA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; TONGA-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v_sdiv_i24:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000

llvm/test/CodeGen/AMDGPU/shl.ll

 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_mov_b32 s8, s2
 ; GCN-NEXT:    s_mov_b32 s9, s3
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_mov_b32 s14, 0
 ; GCN-NEXT:    s_mov_b32 s15, s7
 ; GCN-NEXT:    s_mov_b64 s[12:13], s[2:3]
+; GCN-NEXT:    buffer_load_dword v2, off, s[8:11], 0
 ; GCN-NEXT:    buffer_load_dword v0, v[0:1], s[12:15], 0 addr64 offset:4
-; GCN-NEXT:    buffer_load_dword v1, off, s[8:11], 0
 ; GCN-NEXT:    s_mov_b32 s4, s0
 ; GCN-NEXT:    s_mov_b32 s0, 0xffff
 ; GCN-NEXT:    s_mov_b32 s5, s1
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GCN-NEXT:    v_lshl_b32_e32 v0, v1, v0
-; GCN-NEXT:    v_lshl_b32_e32 v1, v2, v3
+; GCN-NEXT:    v_lshl_b32_e32 v0, v2, v0
+; GCN-NEXT:    v_lshl_b32_e32 v1, v1, v3
 ; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; EG-LABEL: shl_v2i16:
 ; EG:       ; %bb.0: