[GlobalISel] Add new combine to convert G_MUL to G_SHL.

[aemerson]

Summary

For pow2 constants we should use G_SHL for pattern matching (and perf) purposes later.

[aemerson]

@arsenm Matt when I run the update_llc_test_checks script on the llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll test, it crashes with a backtrace like this:

Assertion failed: (Opcode < NumOpcodes && "Invalid opcode!"), function get, file /Users/aemerson/work/oss/co1/src/llvm/include/llvm/MC/MCInstrInfo.h, line 4
frame #4: 0x00000001000bdef3 llc`llvm::MCInstrInfo::get(this=0x0000000110010248, Opcode=4294967295) const at MCInstrInfo.h:45:5
frame #5: 0x000000010083b8d7 llc`llvm::SIInstrInfo::getMCOpcodeFromPseudo(this=0x0000000110010240, Opcode=3758) const at SIInstrInfo.h:939:12
frame #6: 0x00000001008277e3 llc`llvm::SIInstrInfo::getInstSizeInBytes(this=0x0000000110010240, MI=0x000000010f879b28) const at SIInstrInfo.cpp:6067:29
frame #7: 0x0000000101f9bf44 llc`(anonymous namespace)::BranchRelaxation::computeBlockSize(this=0x000000010f03fb60, MBB=0x000000010f8bbf28) const at BranchRelaxation.cpp:168:18
frame #8: 0x0000000101f9b7be llc`(anonymous namespace)::BranchRelaxation::scanFunction(this=0x000000010f03fb60) at BranchRelaxation.cpp:158:39
frame #9: 0x0000000101f9b264 llc`(anonymous namespace)::BranchRelaxation::runOnMachineFunction(this=0x000000010f03fb60, mf=0x000000010f0566f0) at BranchRelaxation.cpp:555:3

I'm replacing a G_MUL with an equivalent G_SHL here, is there an issue with doing this for AMDGPU?

[arsenm]

In D73659#1847892, @aemerson wrote:

@arsenm Matt when I run the update_llc_test_checks script on the llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll test, it crashes with a backtrace like this:

Assertion failed: (Opcode < NumOpcodes && "Invalid opcode!"), function get, file /Users/aemerson/work/oss/co1/src/llvm/include/llvm/MC/MCInstrInfo.h, line 4
frame #4: 0x00000001000bdef3 llc`llvm::MCInstrInfo::get(this=0x0000000110010248, Opcode=4294967295) const at MCInstrInfo.h:45:5
frame #5: 0x000000010083b8d7 llc`llvm::SIInstrInfo::getMCOpcodeFromPseudo(this=0x0000000110010240, Opcode=3758) const at SIInstrInfo.h:939:12
frame #6: 0x00000001008277e3 llc`llvm::SIInstrInfo::getInstSizeInBytes(this=0x0000000110010240, MI=0x000000010f879b28) const at SIInstrInfo.cpp:6067:29
frame #7: 0x0000000101f9bf44 llc`(anonymous namespace)::BranchRelaxation::computeBlockSize(this=0x000000010f03fb60, MBB=0x000000010f8bbf28) const at BranchRelaxation.cpp:168:18
frame #8: 0x0000000101f9b7be llc`(anonymous namespace)::BranchRelaxation::scanFunction(this=0x000000010f03fb60) at BranchRelaxation.cpp:158:39
frame #9: 0x0000000101f9b264 llc`(anonymous namespace)::BranchRelaxation::runOnMachineFunction(this=0x000000010f03fb60, mf=0x000000010f0566f0) at BranchRelaxation.cpp:555:3

I'm replacing a G_MUL with an equivalent G_SHL here, is there an issue with doing this for AMDGPU?

No, this is probably one of the more important optimizations to do. I think the problem you're hitting here is there's some confusion in the selection predicates for sub targets which removed the non-reversed operand forms. I think D73483 should fix that when committed

llvm/include/llvm/Target/GlobalISel/Combine.td

138

lsl->shl to match the instruction name?

[arsenm]

llvm/test/CodeGen/AArch64/GlobalISel/combine-mul-to-lsl.mir

72

Add some vector cases to make sure they at least don't crash?

[aemerson]

I've made the test XFAIL for now. Address other comments.

[aemerson]

Forgot to rename test file.

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

public:
   ///     $addr = G_INDEXED_STORE $val, $base, $offset
   ///     [...]
   ///     $whatever = COPY $addr
   bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0);
 
   bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo);
   bool applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo);
 
+  bool matchCombineMulToLsl(MachineInstr &MI, unsigned &ShiftVal);
+  bool applyCombineMulToLsl(MachineInstr &MI, unsigned &ShiftVal);
+
   /// Try to transform \p MI by using all of the above
   /// combine functions. Returns true if changed.
   bool tryCombine(MachineInstr &MI);
 
 private:
   // Memcpy family optimization helpers.
   bool optimizeMemcpy(MachineInstr &MI, Register Dst, Register Src,
                       unsigned KnownLen, unsigned DstAlign, unsigned SrcAlign,

llvm/include/llvm/Target/GlobalISel/Combine.td

 class GIApplyKind;
 class GIApplyKindWithArgs;
 
 def copy_prop : GICombineRule<
   (defs root:$d),
   (match (COPY $d, $s):$mi,
          [{ return Helper.matchCombineCopy(*${mi}); }]),
   (apply [{ Helper.applyCombineCopy(*${mi}); }])>;
-def trivial_combines : GICombineGroup<[copy_prop]>;
 
 def extending_loads : GICombineRule<
   (defs root:$root, extending_load_matchdata:$matchinfo),
   (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD):$root,
          [{ return Helper.matchCombineExtendingLoads(*${root}, ${matchinfo}); }]),
   (apply [{ Helper.applyCombineExtendingLoads(*${root}, ${matchinfo}); }])>;
 def combines_for_extload: GICombineGroup<[extending_loads]>;
 

 
 def ptr_add_immed_matchdata : GIDefMatchData<"PtrAddChain">;
 def ptr_add_immed_chain : GICombineRule<
   (defs root:$d, ptr_add_immed_matchdata:$matchinfo),
   (match (wip_match_opcode G_PTR_ADD):$d,
          [{ return Helper.matchPtrAddImmedChain(*${d}, ${matchinfo}); }]),
   (apply [{ Helper.applyPtrAddImmedChain(*${d}, ${matchinfo}); }])>;
 
+def mul_to_lsl_matchdata : GIDefMatchData<"unsigned">;
+def mul_to_lsl : GICombineRule<
+  (defs root:$d, mul_to_lsl_matchdata:$matchinfo),
+  (match (G_MUL $d, $op1, $op2):$mi,
+         [{ return Helper.matchCombineMulToLsl(*${mi}, ${matchinfo}); }]),
+  (apply [{ Helper.applyCombineMulToLsl(*${mi}, ${matchinfo}); }])>;
+
+def trivial_combines : GICombineGroup<[copy_prop, mul_to_lsl]>;
 def all_combines : GICombineGroup<[trivial_combines, ptr_add_immed_chain,
     combines_for_extload, combine_indexed_load_store]>;
+

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetMachine.h"
 
 #define DEBUG_TYPE "gi-combiner"
 
 using namespace llvm;
 
 // Option to allow testing of the combiner while no targets know about indexed
 // addressing.
bool CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
   auto NewOffset = MIB.buildConstant(OffsetTy, MatchInfo.Imm);
   Observer.changingInstr(MI);
   MI.getOperand(1).setReg(MatchInfo.Base);
   MI.getOperand(2).setReg(NewOffset.getReg(0));
   Observer.changedInstr(MI);
   return true;
 }
 
+bool CombinerHelper::matchCombineMulToLsl(MachineInstr &MI,
+                                          unsigned &ShiftVal) {
+  assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
+  auto MaybeImmVal =
+      getConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
+  if (!MaybeImmVal || !isPowerOf2_64(MaybeImmVal->Value))
+    return false;
+  ShiftVal = Log2_64(MaybeImmVal->Value);
+  return true;
+}
+
+bool CombinerHelper::applyCombineMulToLsl(MachineInstr &MI,
+                                          unsigned &ShiftVal) {
+  assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
+  MachineIRBuilder MIB(MI);
+  LLT ShiftTy = MRI.getType(MI.getOperand(0).getReg());
+  auto ShiftCst = MIB.buildConstant(ShiftTy, ShiftVal);
+  Observer.changingInstr(MI);
+  MI.setDesc(MIB.getTII().get(TargetOpcode::G_SHL));
+  MI.getOperand(2).setReg(ShiftCst.getReg(0));
+  Observer.changedInstr(MI);
+  return true;
+}
+
 bool CombinerHelper::tryCombine(MachineInstr &MI) {
   if (tryCombineCopy(MI))
     return true;
   if (tryCombineExtendingLoads(MI))
     return true;
   if (tryCombineIndexedLoadStore(MI))
     return true;
   return false;
 }

llvm/test/CodeGen/AArch64/GlobalISel/combine-mul-to-lsl.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s
+...
+---
+name:            mul_to_lsl
+alignment:       4
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: mul_to_lsl
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; CHECK: $x0 = COPY [[SHL]](s64)
+    ; CHECK: RET_ReallyLR implicit-def $x0
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = G_CONSTANT i64 4
+    %2:_(s64) = G_MUL %0, %1(s64)
+    $x0 = COPY %2(s64)
+    RET_ReallyLR implicit-def $x0
+...
+---
+name:            mul_to_lsl_16
+alignment:       4
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: mul_to_lsl_16
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; CHECK: $x0 = COPY [[SHL]](s64)
+    ; CHECK: RET_ReallyLR implicit-def $x0
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = G_CONSTANT i64 16
+    %2:_(s64) = G_MUL %0, %1(s64)
+    $x0 = COPY %2(s64)
+    RET_ReallyLR implicit-def $x0
+...
+---
+name:            mul_to_lsl_non_pow2
+alignment:       4
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: mul_to_lsl_non_pow2
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 10
+    ; CHECK: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY]], [[C]]
+    ; CHECK: $x0 = COPY [[MUL]](s64)
+    ; CHECK: RET_ReallyLR implicit-def $x0
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = G_CONSTANT i64 10
+    %2:_(s64) = G_MUL %0, %1(s64)
+    $x0 = COPY %2(s64)
+    RET_ReallyLR implicit-def $x0
+...

llvm/test/CodeGen/AMDGPU/GlobalISel/lds-relocs.ll

 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefixes=GCN %s
 ; FIXME: Merge with DAG test
 
 @lds.external = external unnamed_addr addrspace(3) global [0 x i32]
 @lds.defined = unnamed_addr addrspace(3) global [8 x i32] undef, align 8
 
 ; GCN-LABEL: {{^}}test_basic:
-; GCN: s_add_u32 s0, lds.defined@abs32@lo, s2 ; encoding: [0xff,0x02,0x00,0x80,A,A,A,A]
-; GCN: v_add_u32_e32 v0, lds.external@abs32@lo, v0 ; encoding: [0xff,0x00,0x00,0x68,A,A,A,A]
+; GCN: s_add_u32 s0, lds.defined@abs32@lo, s0 ; encoding: [0xff,0x00,0x00,0x80,A,A,A,A]
+; GCN: v_mov_b32_e32 v2, s0 ; encoding: [0x00,0x02,0x04,0x7e]
 
 ; GCN: .globl lds.external
 ; GCN: .amdgpu_lds lds.external, 0, 4
 ; GCN: .globl lds.defined
 ; GCN: .amdgpu_lds lds.defined, 32, 8
 define amdgpu_gs float @test_basic(i32 inreg %wave, i32 %arg1) #0 {
 main_body:
   %gep0 = getelementptr [0 x i32], [0 x i32] addrspace(3)* @lds.external, i32 0, i32 %arg1

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll

; GFX9-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_dec_ret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    flat_atomic_dec v0, v[0:1], v4 glc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v2, s3
+; CI-NEXT:    v_add_i32_e32 v3, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 20, v3
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; CI-NEXT:    flat_atomic_dec v2, v[2:3], v4 glc
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dword v[2:3], v0
+; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_dec_ret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    flat_atomic_dec v0, v[0:1], v4 glc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 20, v3
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; VI-NEXT:    flat_atomic_dec v2, v[2:3], v4 glc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dword v[2:3], v0
+; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: global_atomic_dec_ret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
 ; GFX9-NEXT:    v_mul_lo_u32 v1, 4, v1
 ; GFX9-NEXT:    v_mul_hi_u32 v3, 4, v0
; GFX9-NEXT:    s_endpgm
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
   store i32 %result, i32 addrspace(1)* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(i32 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_dec_noret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_dec_noret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: global_atomic_dec_noret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX9-NEXT:    s_endpgm
   %gep = getelementptr i32, i32* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(i32* %out, i32* %ptr) #0 {
 ; CI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    flat_atomic_dec v0, v[0:1], v4 glc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v2, s3
+; CI-NEXT:    v_add_i32_e32 v3, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 20, v3
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; CI-NEXT:    flat_atomic_dec v2, v[2:3], v4 glc
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dword v[2:3], v0
+; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    flat_atomic_dec v0, v[0:1], v4 glc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 20, v3
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; VI-NEXT:    flat_atomic_dec v2, v[2:3], v4 glc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dword v[2:3], v0
+; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
 ; GFX9-NEXT:    v_mul_lo_u32 v1, 4, v1
 ; GFX9-NEXT:    v_mul_hi_u32 v3, 4, v0
; GFX9-NEXT:    s_endpgm
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false)
   store i32 %result, i32* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(i32* %ptr) #0 {
 ; CI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX9-NEXT:    s_endpgm
   %gep = getelementptr i64, i64* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(i64* %out, i64* %ptr) #0 {
 ; CI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    v_mov_b32_e32 v5, 0
-; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[4:5] glc
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v4, s3
+; CI-NEXT:    v_add_i32_e32 v5, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v4, vcc, 40, v5
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[4:5], v[2:3] glc
+; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    v_mov_b32_e32 v5, 0
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[4:5] glc
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s3
+; VI-NEXT:    v_add_u32_e32 v5, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 40, v5
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[4:5], v[2:3] glc
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
 ; GFX9-NEXT:    v_mul_lo_u32 v1, 8, v1
 ; GFX9-NEXT:    v_mul_hi_u32 v3, 8, v0
; GFX9-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false)
   store i64 %result, i64* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(i64* %ptr) #0 {
 ; CI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v4, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v0
-; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v0
-; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
 ; GFX9-NEXT:    v_mul_lo_u32 v1, 8, v1
; GFX9-NEXT:    s_endpgm
   ret void
 }
 
 @lds0 = internal addrspace(3) global [512 x i32] undef
 
 define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_dec_shl_base_lds_0:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_mul_lo_u32 v5, v0, 4
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mov_b32_e32 v6, 9
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; CI-NEXT:    v_add_i32_e32 v0, vcc, 8, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 9
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    v_add_i32_e32 v5, vcc, 0, v5
-; CI-NEXT:    v_add_i32_e32 v5, vcc, 8, v5
-; CI-NEXT:    ds_dec_rtn_u32 v5, v5, v6
-; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
+; CI-NEXT:    ds_dec_rtn_u32 v3, v0, v1
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
-; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
-; CI-NEXT:    v_mov_b32_e32 v2, s0
-; CI-NEXT:    flat_store_dword v[0:1], v4
-; CI-NEXT:    flat_store_dword v[2:3], v5
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v3
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_dec_shl_base_lds_0:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_mul_lo_u32 v5, v0, 4
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mov_b32_e32 v6, 9
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 2, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 8, v0
+; VI-NEXT:    v_mov_b32_e32 v1, 9
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    v_add_u32_e32 v5, vcc, 0, v5
-; VI-NEXT:    v_add_u32_e32 v5, vcc, 8, v5
-; VI-NEXT:    ds_dec_rtn_u32 v5, v5, v6
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
+; VI-NEXT:    ds_dec_rtn_u32 v3, v0, v1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
-; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
-; VI-NEXT:    v_mov_b32_e32 v2, s0
-; VI-NEXT:    flat_store_dword v[0:1], v4
-; VI-NEXT:    flat_store_dword v[2:3], v5
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v3
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: atomic_dec_shl_base_lds_0:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mul_lo_u32 v1, 4, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_add_u32_e32 v3, 2, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 9
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0, v1
; GFX9-NEXT:    s_endpgm
   %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_dec_ret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    v_mov_b32_e32 v5, 0
-; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[4:5] glc
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v4, s3
+; CI-NEXT:    v_add_i32_e32 v5, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v4, vcc, 40, v5
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[4:5], v[2:3] glc
+; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_dec_ret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    v_mov_b32_e32 v5, 0
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[4:5] glc
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s3
+; VI-NEXT:    v_add_u32_e32 v5, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 40, v5
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[4:5], v[2:3] glc
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: global_atomic_dec_ret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
 ; GFX9-NEXT:    v_mul_lo_u32 v1, 8, v1
 ; GFX9-NEXT:    v_mul_hi_u32 v3, 8, v0
; GFX9-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
   store i64 %result, i64 addrspace(1)* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(i64 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_dec_noret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v4, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v0
-; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_dec_noret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v0
-; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: global_atomic_dec_noret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    v_mul_lo_u32 v2, 0, v0
 ; GFX9-NEXT:    v_mul_lo_u32 v1, 8, v1
; GFX9-NEXT:    s_endpgm
   ret void
 }
 
 @lds1 = internal addrspace(3) global [512 x i64] undef, align 8
 
 define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_dec_shl_base_lds_0_i64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_mul_lo_u32 v7, v0, 8
+; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_add_i32_e32 v6, vcc, 2, v0
+; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 16, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, 9
-; CI-NEXT:    v_add_i32_e32 v7, vcc, 0, v7
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
-; CI-NEXT:    v_add_i32_e32 v7, vcc, 16, v7
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    ds_dec_rtn_u64 v[0:1], v7, v[0:1]
+; CI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1]
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
-; CI-NEXT:    v_mov_b32_e32 v5, s1
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
-; CI-NEXT:    v_mov_b32_e32 v4, s0
-; CI-NEXT:    flat_store_dword v[2:3], v6
-; CI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; CI-NEXT:    flat_store_dword v[2:3], v4
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_dec_shl_base_lds_0_i64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_mul_lo_u32 v7, v0, 8
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 2, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 16, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, 9
-; VI-NEXT:    v_add_u32_e32 v7, vcc, 0, v7
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
-; VI-NEXT:    v_add_u32_e32 v7, vcc, 16, v7
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    ds_dec_rtn_u64 v[0:1], v7, v[0:1]
+; VI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
-; VI-NEXT:    v_mov_b32_e32 v4, s0
-; VI-NEXT:    flat_store_dword v[2:3], v6
-; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    flat_store_dword v[2:3], v4
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ; GFX9-LABEL: atomic_dec_shl_base_lds_0_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_mul_lo_u32 v3, 8, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 9
 ; GFX9-NEXT:    v_add_u32_e32 v4, 2, v0
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0, v3

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll

; GFX9-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_inc_ret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    flat_atomic_inc v0, v[0:1], v4 glc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v2, s3
+; CI-NEXT:    v_add_i32_e32 v3, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 20, v3
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; CI-NEXT:    flat_atomic_inc v2, v[2:3], v4 glc
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dword v[2:3], v0
+; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_inc_ret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    flat_atomic_inc v0, v[0:1], v4 glc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 20, v3
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; VI-NEXT:    flat_atomic_inc v2, v[2:3], v4 glc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dword v[2:3], v0
+; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_inc_ret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 4
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 4
-; GFX9-NEXT:    v_mul_lo_u32 v4, v0, 4
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:    v_mov_b32_e32 v5, s1
-; GFX9-NEXT:    v_add3_u32 v3, v1, v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 20, v1
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 42
-; GFX9-NEXT:    global_atomic_inc v0, v[0:1], v4, off glc
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v3, vcc
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 20, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v5, vcc
+; GFX9-NEXT:    global_atomic_inc v2, v[2:3], v4, off glc
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32 addrspace(1)* %ptr, i32 %id
   %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id
   %gep = getelementptr i32, i32 addrspace(1)* %gep.tid, i32 5
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
   store i32 %result, i32 addrspace(1)* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(i32 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_inc_noret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_inc_noret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_inc_noret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 4
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 4
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 4
-; GFX9-NEXT:    v_add3_u32 v1, v1, v2, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 20, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    global_atomic_inc v0, v[0:1], v2, off glc
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32 addrspace(1)* %ptr, i32 %id
   %gep = getelementptr i32, i32 addrspace(1)* %gep.tid, i32 5
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 @lds0 = internal addrspace(3) global [512 x i32] undef, align 4
 
 define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_inc_shl_base_lds_0_i32:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_mul_lo_u32 v5, v0, 4
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mov_b32_e32 v6, 9
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; CI-NEXT:    v_add_i32_e32 v0, vcc, 8, v0
+; CI-NEXT:    v_mov_b32_e32 v1, 9
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    v_add_i32_e32 v5, vcc, 0, v5
-; CI-NEXT:    v_add_i32_e32 v5, vcc, 8, v5
-; CI-NEXT:    ds_inc_rtn_u32 v5, v5, v6
-; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
+; CI-NEXT:    ds_inc_rtn_u32 v3, v0, v1
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
-; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
-; CI-NEXT:    v_mov_b32_e32 v2, s0
-; CI-NEXT:    flat_store_dword v[0:1], v4
-; CI-NEXT:    flat_store_dword v[2:3], v5
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v3
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_inc_shl_base_lds_0_i32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_mul_lo_u32 v5, v0, 4
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mov_b32_e32 v6, 9
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 2, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 8, v0
+; VI-NEXT:    v_mov_b32_e32 v1, 9
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    v_add_u32_e32 v5, vcc, 0, v5
-; VI-NEXT:    v_add_u32_e32 v5, vcc, 8, v5
-; VI-NEXT:    ds_inc_rtn_u32 v5, v5, v6
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
+; VI-NEXT:    ds_inc_rtn_u32 v3, v0, v1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
-; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
-; VI-NEXT:    v_mov_b32_e32 v2, s0
-; VI-NEXT:    flat_store_dword v[0:1], v4
-; VI-NEXT:    flat_store_dword v[2:3], v5
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v3
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: atomic_inc_shl_base_lds_0_i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mul_lo_u32 v1, v0, 4
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT:    v_add_u32_e32 v3, 2, v0
-; GFX9-NEXT:    v_mov_b32_e32 v2, 9
-; GFX9-NEXT:    v_add_u32_e32 v0, 0, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 2, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
 ; GFX9-NEXT:    v_add_u32_e32 v0, 8, v0
-; GFX9-NEXT:    ds_inc_rtn_u32 v2, v0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, 9
+; GFX9-NEXT:    ds_inc_rtn_u32 v3, v0, v1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    global_store_dword v[0:1], v3, off
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    global_store_dword v[0:1], v3, off
 ; GFX9-NEXT:    s_endpgm
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
   %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds0, i32 0, i32 %idx.0
   %val0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %arrayidx0, i32 9, i32 0, i32 0, i1 false)
   store i32 %idx.0, i32 addrspace(1)* %add_use
   store i32 %val0, i32 addrspace(1)* %out
   ret void
; GFX9-NEXT:    s_endpgm
   %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_inc_ret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    v_mov_b32_e32 v5, 0
-; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[4:5] glc
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v4, s3
+; CI-NEXT:    v_add_i32_e32 v5, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v4, vcc, 40, v5
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[4:5], v[2:3] glc
+; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_inc_ret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    v_mov_b32_e32 v5, 0
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[4:5] glc
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s3
+; VI-NEXT:    v_add_u32_e32 v5, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 40, v5
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[4:5], v[2:3] glc
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_inc_ret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 8
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 8
-; GFX9-NEXT:    v_mul_lo_u32 v4, v0, 8
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:    v_mov_b32_e32 v5, s1
-; GFX9-NEXT:    v_add3_u32 v3, v1, v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 40, v1
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v3, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v4, 42
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0
-; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v[0:1], v[4:5], off glc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 42
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, s2, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v4, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 40, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
+; GFX9-NEXT:    global_atomic_inc_x2 v[2:3], v[4:5], v[2:3], off glc
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64 addrspace(1)* %ptr, i32 %id
   %out.gep = getelementptr i64, i64 addrspace(1)* %out, i32 %id
   %gep = getelementptr i64, i64 addrspace(1)* %gep.tid, i32 5
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
   store i64 %result, i64 addrspace(1)* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(i64 addrspace(1)* %ptr) #0 {
 ; CI-LABEL: global_atomic_inc_noret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v4, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v0
-; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: global_atomic_inc_noret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v0
-; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_inc_noret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 8
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 8
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 8
-; GFX9-NEXT:    v_add3_u32 v1, v1, v2, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 42
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 40, v0
-; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v[0:1], v[2:3], off glc
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64 addrspace(1)* %ptr, i32 %id
   %gep = getelementptr i64, i64 addrspace(1)* %gep.tid, i32 5
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
   ret void
 }
; GCN-NEXT:    s_endpgm
   %gep = getelementptr i32, i32* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(i32* %out, i32* %ptr) #0 {
 ; CI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    flat_atomic_inc v0, v[0:1], v4 glc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v2, s3
+; CI-NEXT:    v_add_i32_e32 v3, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 20, v3
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; CI-NEXT:    flat_atomic_inc v2, v[2:3], v4 glc
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dword v[2:3], v0
+; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    flat_atomic_inc v0, v[0:1], v4 glc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, v2, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 20, v3
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; VI-NEXT:    flat_atomic_inc v2, v[2:3], v4 glc
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dword v[2:3], v0
+; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 4
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 4
-; GFX9-NEXT:    v_mul_lo_u32 v4, v0, 4
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:    v_mov_b32_e32 v5, s1
-; GFX9-NEXT:    v_add3_u32 v3, v1, v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 20, v1
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 42
-; GFX9-NEXT:    flat_atomic_inc v0, v[0:1], v4 glc
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v3, vcc
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, s2, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 20, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v5, vcc
+; GFX9-NEXT:    flat_atomic_inc v2, v[2:3], v4 glc
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_store_dword v[2:3], v0
+; GFX9-NEXT:    flat_store_dword v[0:1], v2
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32* %ptr, i32 %id
   %out.gep = getelementptr i32, i32* %out, i32 %id
   %gep = getelementptr i32, i32* %gep.tid, i32 5
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false)
   store i32 %result, i32* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(i32* %ptr) #0 {
 ; CI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 4
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 4
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 4
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 4
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 4
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 4
-; GFX9-NEXT:    v_add3_u32 v1, v1, v2, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 20, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32* %ptr, i32 %id
   %gep = getelementptr i32, i32* %gep.tid, i32 5
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* %gep, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 @lds1 = internal addrspace(3) global [512 x i64] undef, align 8
 
 define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_mul_lo_u32 v7, v0, 8
+; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_add_i32_e32 v6, vcc, 2, v0
+; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 16, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, 9
-; CI-NEXT:    v_add_i32_e32 v7, vcc, 0, v7
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
-; CI-NEXT:    v_add_i32_e32 v7, vcc, 16, v7
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    ds_inc_rtn_u64 v[0:1], v7, v[0:1]
+; CI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
-; CI-NEXT:    v_mov_b32_e32 v5, s1
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
-; CI-NEXT:    v_mov_b32_e32 v4, s0
-; CI-NEXT:    flat_store_dword v[2:3], v6
-; CI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; CI-NEXT:    flat_store_dword v[2:3], v4
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_mul_lo_u32 v7, v0, 8
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 2, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 16, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, 9
-; VI-NEXT:    v_add_u32_e32 v7, vcc, 0, v7
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
-; VI-NEXT:    v_add_u32_e32 v7, vcc, 16, v7
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    ds_inc_rtn_u64 v[0:1], v7, v[0:1]
+; VI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
-; VI-NEXT:    v_mov_b32_e32 v4, s0
-; VI-NEXT:    flat_store_dword v[2:3], v6
-; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    flat_store_dword v[2:3], v4
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mul_lo_u32 v3, v0, 8
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v1, 9
 ; GFX9-NEXT:    v_add_u32_e32 v4, 2, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, 0, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_add_u32_e32 v0, 16, v0
-; GFX9-NEXT:    ds_inc_rtn_u64 v[0:1], v0, v[1:2]
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, 16, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 9
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NEXT:    global_store_dword v[2:3], v4, off
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
 ; GFX9-NEXT:    s_endpgm
; GCN-NEXT:    s_endpgm
   %gep = getelementptr i64, i64* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(i64* %out, i64* %ptr) #0 {
 ; CI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s3
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v4
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v1
-; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CI-NEXT:    v_mov_b32_e32 v5, s1
-; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v4
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; CI-NEXT:    v_mov_b32_e32 v4, 42
-; CI-NEXT:    v_mov_b32_e32 v5, 0
-; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[4:5] glc
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v4, s3
+; CI-NEXT:    v_add_i32_e32 v5, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; CI-NEXT:    v_add_i32_e32 v4, vcc, 40, v5
+; CI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[4:5], v[2:3] glc
+; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v4, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v3, vcc, v1, v3
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s3
-; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v4
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v1
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v4
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; VI-NEXT:    v_mov_b32_e32 v4, 42
-; VI-NEXT:    v_mov_b32_e32 v5, 0
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[4:5] glc
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, s3
+; VI-NEXT:    v_add_u32_e32 v5, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v6, vcc, v4, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 40, v5
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[4:5], v[2:3] glc
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 8
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 8
-; GFX9-NEXT:    v_mul_lo_u32 v4, v0, 8
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:    v_mov_b32_e32 v5, s1
-; GFX9-NEXT:    v_add3_u32 v3, v1, v2, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, s2, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 40, v1
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v3, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v4, 42
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0
-; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[4:5] glc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 42
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, s2, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v4, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 40, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
+; GFX9-NEXT:    flat_atomic_inc_x2 v[2:3], v[4:5], v[2:3] glc
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64* %ptr, i32 %id
   %out.gep = getelementptr i64, i64* %out, i32 %id
   %gep = getelementptr i64, i64* %gep.tid, i32 5
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false)
   store i64 %result, i64* %out.gep
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(i64* %ptr) #0 {
 ; CI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; CI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
+; CI-NEXT:    v_mov_b32_e32 v2, 42
+; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v4, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 40, v0
-; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-NEXT:    v_mov_b32_e32 v3, 0
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; VI-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; VI-NEXT:    v_mul_hi_u32 v3, v0, 8
-; VI-NEXT:    v_mul_lo_u32 v0, v0, 8
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; VI-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v2, 42
+; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 40, v0
-; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 8
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 8
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 8
-; GFX9-NEXT:    v_add3_u32 v1, v1, v2, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 42
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 40, v0
-; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX9-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64* %ptr, i32 %id
   %gep = getelementptr i64, i64* %gep.tid, i32 5
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* %gep, i64 42, i32 0, i32 0, i1 false)
   ret void
 }

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=CI %s
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
 
 ; TODO: Merge with DAG test
 
 define amdgpu_kernel void @is_private_vgpr(i8* addrspace(1)* %ptr.ptr) {
 ; CI-LABEL: is_private_vgpr:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
-; CI-NEXT:    v_mul_lo_u32 v3, v0, 8
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; CI-NEXT:    v_mul_hi_u32 v0, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v0
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; CI-NEXT:    v_mov_b32_e32 v2, s1
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    s_load_dword s0, s[4:5], 0x11
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CI-NEXT:    v_cmp_eq_u32_e32 vcc, s0, v1
 ; CI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; CI-NEXT:    flat_store_dword v[0:1], v0
 ; CI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: is_private_vgpr:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 8
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 8
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 8
-; GFX9-NEXT:    v_add3_u32 v1, v1, v2, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
 ; GFX9-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, 16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=CI %s
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
 
 ; TODO: Merge with DAG test
 
 define amdgpu_kernel void @is_local_vgpr(i8* addrspace(1)* %ptr.ptr) {
 ; CI-LABEL: is_local_vgpr:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; CI-NEXT:    v_mul_lo_u32 v1, v1, 8
-; CI-NEXT:    v_mul_lo_u32 v2, v0, 0
-; CI-NEXT:    v_mul_lo_u32 v3, v0, 8
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; CI-NEXT:    v_mul_hi_u32 v0, v0, 8
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v0
+; CI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; CI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 3
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; CI-NEXT:    v_mov_b32_e32 v2, s1
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    s_load_dword s0, s[4:5], 0x10
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CI-NEXT:    v_cmp_eq_u32_e32 vcc, s0, v1
 ; CI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; CI-NEXT:    flat_store_dword v[0:1], v0
 ; CI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: is_local_vgpr:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 8
-; GFX9-NEXT:    v_mul_lo_u32 v2, v0, 0
-; GFX9-NEXT:    v_mul_hi_u32 v3, v0, 8
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 8
-; GFX9-NEXT:    v_add3_u32 v1, v1, v2, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
 ; GFX9-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 16, 16)
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, 16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll

 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    s_nop 0
 ; GFX8-NEXT:    v_mov_b32_dpp v2, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
-;
 ; GFX10-LABEL: dpp_test:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX10-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX10-NEXT:    v_mov_b32_dpp v2, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX10-NEXT:    s_endpgm
   %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in1, i32 %in2, i32 1, i32 1, i32 1, i1 false)
   store i32 %tmp0, i32 addrspace(1)* %out
   ret void
 }
 define amdgpu_kernel void @update_dpp64_test(i64 addrspace(1)* %arg, i64 %in1, i64 %in2) {
 ; GFX8-LABEL: update_dpp64_test:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mul_lo_u32 v1, v1, 8
-; GFX8-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX8-NEXT:    v_mul_hi_u32 v3, v0, 8
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, 8
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 3, v[0:1]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v4, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s2
 ; GFX8-NEXT:    s_nop 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
 ; GFX8-NEXT:    v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
 ; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
 ; GFX8-NEXT:    s_endpgm
-;
 ; GFX10-LABEL: update_dpp64_test:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_mul_lo_u32 v2, v0, 0
 ; GFX10-NEXT:    v_mul_hi_u32 v3, v0, 8
 ; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 8
 ; GFX10-NEXT:    v_mul_lo_u32 v1, v1, 8

llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll

; GFX7-NEXT:    s_endpgm
   store i32 0, i32 addrspace(1)* %gep
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_sgpr_ptr_sgpr_offset(i32 addrspace(1)* inreg %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_sgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x200000
-; GFX6-NEXT:    v_mul_hi_u32 v0, s4, 4
-; GFX6-NEXT:    s_mov_b32 s1, s3
-; GFX6-NEXT:    s_mul_i32 s3, s4, 4
-; GFX6-NEXT:    s_mul_i32 s5, s5, 4
-; GFX6-NEXT:    s_mul_i32 s4, s4, 0
-; GFX6-NEXT:    s_add_i32 s5, s5, s4
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX6-NEXT:    s_mov_b32 s0, s2
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s5, v0
-; GFX6-NEXT:    v_mov_b32_e32 v0, s3
+; GFX6-NEXT:    s_mov_b32 s1, s3
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_sgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x200000
-; GFX7-NEXT:    v_mul_hi_u32 v0, s4, 4
-; GFX7-NEXT:    s_mov_b32 s1, s3
-; GFX7-NEXT:    s_mul_i32 s3, s4, 4
-; GFX7-NEXT:    s_mul_i32 s5, s5, 4
-; GFX7-NEXT:    s_mul_i32 s4, s4, 0
-; GFX7-NEXT:    s_add_i32 s5, s5, s4
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    s_mov_b32 s0, s2
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s5, v0
-; GFX7-NEXT:    v_mov_b32_e32 v0, s3
+; GFX7-NEXT:    s_mov_b32 s1, s3
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
   store i32 0, i32 addrspace(1)* %gep
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX6-NEXT:    s_mul_i32 s2, s0, 4
-; GFX6-NEXT:    s_mul_i32 s1, s1, 4
-; GFX6-NEXT:    s_mul_i32 s0, s0, 0
-; GFX6-NEXT:    s_add_i32 s1, s1, s0
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX6-NEXT:    s_mov_b32 s6, 0
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[0:1], 2
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX7-NEXT:    s_mul_i32 s2, s0, 4
-; GFX7-NEXT:    s_mul_i32 s1, s1, 4
-; GFX7-NEXT:    s_mul_i32 s0, s0, 0
-; GFX7-NEXT:    s_add_i32 s1, s1, s0
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[0:1], 2
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
   store i32 0, i32 addrspace(1)* %gep
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset_offset256(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX6-NEXT:    s_mul_i32 s2, s0, 4
-; GFX6-NEXT:    s_mul_i32 s1, s1, 4
-; GFX6-NEXT:    s_mul_i32 s0, s0, 0
-; GFX6-NEXT:    s_add_i32 s1, s1, s0
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX6-NEXT:    s_mov_b32 s6, 0
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[0:1], 2
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 offset:1024
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX7-NEXT:    s_mul_i32 s2, s0, 4
-; GFX7-NEXT:    s_mul_i32 s1, s1, 4
-; GFX7-NEXT:    s_mul_i32 s0, s0, 0
-; GFX7-NEXT:    s_add_i32 s1, s1, s0
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[0:1], 2
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 offset:1024
 ; GFX7-NEXT:    s_endpgm
   %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
   %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 256
   store i32 0, i32 addrspace(1)* %gep1
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset256_offset(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_mov_b32 s1, 0
-; GFX6-NEXT:    s_movk_i32 s0, 0x400
-; GFX6-NEXT:    v_mov_b32_e32 v3, s1
-; GFX6-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x200000
-; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_movk_i32 s4, 0x400
+; GFX6-NEXT:    s_mov_b32 s5, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v2, s2, 4
-; GFX6-NEXT:    s_mul_i32 s0, s2, 4
-; GFX6-NEXT:    s_mul_i32 s3, s3, 4
-; GFX6-NEXT:    s_mul_i32 s2, s2, 0
-; GFX6-NEXT:    s_add_i32 s3, s3, s2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s3, v2
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
-; GFX6-NEXT:    s_mov_b32 s2, s1
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    s_mov_b64 s[0:1], 0
+; GFX6-NEXT:    s_mov_b32 s2, s5
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_mov_b32 s1, 0
-; GFX7-NEXT:    s_movk_i32 s0, 0x400
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x200000
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:    s_movk_i32 s4, 0x400
+; GFX7-NEXT:    s_mov_b32 s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_mul_hi_u32 v2, s2, 4
-; GFX7-NEXT:    s_mul_i32 s0, s2, 4
-; GFX7-NEXT:    s_mul_i32 s3, s3, 4
-; GFX7-NEXT:    s_mul_i32 s2, s2, 0
-; GFX7-NEXT:    s_add_i32 s3, s3, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s3, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
-; GFX7-NEXT:    s_mov_b32 s2, s1
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    s_mov_b32 s2, s5
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 256
   %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 %soffset
   store i32 0, i32 addrspace(1)* %gep1
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset(i32 addrspace(1)* inreg %ptr, i32 %voffset) {
 ; GFX6-LABEL: mubuf_store_sgpr_ptr_vgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX6-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX6-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s0, s2
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    s_mov_b32 s1, s3
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX6-NEXT:    s_mov_b32 s2, 0
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_sgpr_ptr_vgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX7-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX7-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s0, s2
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX7-NEXT:    s_mov_b32 s1, s3
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX7-NEXT:    s_mov_b32 s2, 0
-; GFX7-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %voffset
   store i32 0, i32 addrspace(1)* %gep
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset_offset4095(i32 addrspace(1)* inreg %ptr, i32 %voffset) {
 ; GFX6-LABEL: mubuf_store_sgpr_ptr_vgpr_offset_offset4095:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX6-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX6-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s0, s2
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    s_mov_b32 s1, s3
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX6-NEXT:    s_mov_b32 s2, 0
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
-; GFX6-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], s4 addr64
+; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_sgpr_ptr_vgpr_offset_offset4095:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX7-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX7-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s0, s2
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX7-NEXT:    s_mov_b32 s1, s3
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX7-NEXT:    s_mov_b32 s2, 0
-; GFX7-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
-; GFX7-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], s4 addr64
+; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 %voffset
   %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 4095
   store i32 0, i32 addrspace(1)* %gep1
   ret void
 }
 define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4095_vgpr_offset(i32 addrspace(1)* inreg %ptr, i32 %voffset) {
 ; GFX6-LABEL: mubuf_store_sgpr_ptr_offset4095_vgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX6-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX6-NEXT:    v_mul_hi_u32 v0, v0, 4
 ; GFX6-NEXT:    s_add_u32 s4, s2, 0x3ffc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s6, 0
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX6-NEXT:    s_addc_u32 s5, s3, 0
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
+; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_sgpr_ptr_offset4095_vgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX7-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX7-NEXT:    v_mul_hi_u32 v0, v0, 4
 ; GFX7-NEXT:    s_add_u32 s4, s2, 0x3ffc
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s6, 0
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX7-NEXT:    s_addc_u32 s5, s3, 0
-; GFX7-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
+; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 4095
   %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 %voffset
   store i32 0, i32 addrspace(1)* %gep1
   ret void
 }
 
 define amdgpu_ps float @mubuf_load_sgpr_ptr(float addrspace(1)* inreg %ptr) {

 }
 
 define amdgpu_ps float @mubuf_load_sgpr_ptr_sgpr_offset(float addrspace(1)* inreg %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_sgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_mov_b32 s0, s2
 ; GFX6-NEXT:    s_mov_b32 s1, s3
 ; GFX6-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
-; GFX6-NEXT:    v_mul_hi_u32 v0, s2, 4
-; GFX6-NEXT:    s_mul_i32 s4, s2, 0
-; GFX6-NEXT:    s_mul_i32 s3, s3, 4
-; GFX6-NEXT:    s_add_i32 s3, s3, s4
-; GFX6-NEXT:    s_mul_i32 s2, s2, 4
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s3, v0
-; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_sgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_mov_b32 s0, s2
 ; GFX7-NEXT:    s_mov_b32 s1, s3
 ; GFX7-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
-; GFX7-NEXT:    v_mul_hi_u32 v0, s2, 4
-; GFX7-NEXT:    s_mul_i32 s4, s2, 0
-; GFX7-NEXT:    s_mul_i32 s3, s3, 4
-; GFX7-NEXT:    s_add_i32 s3, s3, s4
-; GFX7-NEXT:    s_mul_i32 s2, s2, 4
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s3, v0
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
   %val = load float, float addrspace(1)* %gep
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset(float addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX6-NEXT:    s_mul_i32 s2, s0, 0
-; GFX6-NEXT:    s_mul_i32 s1, s1, 4
-; GFX6-NEXT:    s_add_i32 s1, s1, s2
-; GFX6-NEXT:    s_mul_i32 s0, s0, 4
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX7-NEXT:    s_mul_i32 s2, s0, 0
-; GFX7-NEXT:    s_mul_i32 s1, s1, 4
-; GFX7-NEXT:    s_add_i32 s1, s1, s2
-; GFX7-NEXT:    s_mul_i32 s0, s0, 4
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
   %val = load float, float addrspace(1)* %gep
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(float addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset_offset256:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX6-NEXT:    s_mul_i32 s2, s0, 0
-; GFX6-NEXT:    s_mul_i32 s1, s1, 4
-; GFX6-NEXT:    s_add_i32 s1, s1, s2
-; GFX6-NEXT:    s_mul_i32 s0, s0, 4
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset_offset256:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    v_mul_hi_u32 v2, s0, 4
-; GFX7-NEXT:    s_mul_i32 s2, s0, 0
-; GFX7-NEXT:    s_mul_i32 s1, s1, 4
-; GFX7-NEXT:    s_add_i32 s1, s1, s2
-; GFX7-NEXT:    s_mul_i32 s0, s0, 4
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s1, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
   %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 256
   %val = load float, float addrspace(1)* %gep1
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(float addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset256_offset:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    v_mul_hi_u32 v4, s0, 4
 ; GFX6-NEXT:    s_movk_i32 s4, 0x400
 ; GFX6-NEXT:    s_mov_b32 s5, 0
 ; GFX6-NEXT:    v_mov_b32_e32 v2, s4
-; GFX6-NEXT:    s_mul_i32 s2, s0, 0
-; GFX6-NEXT:    s_mul_i32 s1, s1, 4
-; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
 ; GFX6-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    s_mul_i32 s0, s0, 4
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s1, v4
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, s5
-; GFX6-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset256_offset:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    v_mul_hi_u32 v4, s0, 4
 ; GFX7-NEXT:    s_movk_i32 s4, 0x400
 ; GFX7-NEXT:    s_mov_b32 s5, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    s_mul_i32 s2, s0, 0
-; GFX7-NEXT:    s_mul_i32 s1, s1, 4
-; GFX7-NEXT:    s_add_i32 s1, s1, s2
+; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX7-NEXT:    s_mul_i32 s0, s0, 4
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s1, v4
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, s5
-; GFX7-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 256
   %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 %soffset
   %val = load float, float addrspace(1)* %gep1
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset(float addrspace(1)* inreg %ptr, i32 %voffset) {
 ; GFX6-LABEL: mubuf_load_sgpr_ptr_vgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX6-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX6-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s0, s2
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    s_mov_b32 s1, s3
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    buffer_load_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_sgpr_ptr_vgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX7-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX7-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s0, s2
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX7-NEXT:    s_mov_b32 s1, s3
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    buffer_load_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, float addrspace(1)* %ptr, i32 %voffset
   %val = load float, float addrspace(1)* %gep
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset_offset4095(float addrspace(1)* inreg %ptr, i32 %voffset) {
 ; GFX6-LABEL: mubuf_load_sgpr_ptr_vgpr_offset_offset4095:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX6-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX6-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s0, s2
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    s_mov_b32 s1, s3
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
-; GFX6-NEXT:    buffer_load_dword v0, v[1:2], s[0:3], s4 addr64
+; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_sgpr_ptr_vgpr_offset_offset4095:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX7-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX7-NEXT:    v_mul_hi_u32 v0, v0, 4
+; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s0, s2
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX7-NEXT:    s_mov_b32 s1, s3
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
-; GFX7-NEXT:    buffer_load_dword v0, v[1:2], s[0:3], s4 addr64
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, float addrspace(1)* %ptr, i32 %voffset
   %gep1 = getelementptr float, float addrspace(1)* %gep0, i64 4095
   %val = load float, float addrspace(1)* %gep1
   ret float %val
 }
 define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095_vgpr_offset(float addrspace(1)* inreg %ptr, i32 %voffset) {
 ; GFX6-LABEL: mubuf_load_sgpr_ptr_offset4095_vgpr_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX6-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX6-NEXT:    v_mul_hi_u32 v0, v0, 4
 ; GFX6-NEXT:    s_add_u32 s0, s2, 0x3ffc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX6-NEXT:    s_addc_u32 s1, s3, 0
 ; GFX6-NEXT:    s_mov_b32 s2, 0
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    buffer_load_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_sgpr_ptr_offset4095_vgpr_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v1, 4
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, 0
-; GFX7-NEXT:    v_mul_lo_u32 v1, v0, 4
-; GFX7-NEXT:    v_mul_hi_u32 v0, v0, 4
 ; GFX7-NEXT:    s_add_u32 s0, s2, 0x3ffc
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; GFX7-NEXT:    s_addc_u32 s1, s3, 0
 ; GFX7-NEXT:    s_mov_b32 s2, 0
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    buffer_load_dword v0, v[1:2], s[0:3], 0 addr64
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 4095
   %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 %voffset
   %val = load float, float addrspace(1)* %gep1
   ret float %val
 }
 
 ; define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_offset4095(i32 addrspace(1)* inreg %ptr) {
 ;   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4095
 ;   %result = atomicrmw add i32 addrspace(1)* %gep, i32 2 seq_cst
 ;   %cast = bitcast i32 %result to float
 ;   ret float %cast
 ; }