[GlobalISel] Avoid making G_PTR_ADD with nullptr

[mbrkusanin]

Summary

When the first operand is a null pointer we can avoid making a G_PTR_ADD and
make a G_INTTOPTR with the offset operand.
This helps us avoid making add with 0 later on for targets such as AMDGPU.

[arsenm]

This is probably wrong for non integral pointers. Why not do this as a combine later? We need to start recognizing G_PTRADD (constant) + constant anyway to handle lowered LDS addresses

[foad]

llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll

11 ↗

(On Diff #289930)

Can you avoid generating this unused constant in the first place?

[mbrkusanin]

• Reimplemented as a AMDGPU GICombineRule.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUCombine.td

40 ↗

(On Diff #292192)

ptr_add

40–45 ↗

(On Diff #292192)

This should go in the generic combiner

llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp

45 ↗

(On Diff #292192)

Should not construct a local MachineIRBuilder

llvm/test/CodeGen/AMDGPU/GlobalISel/combine-add-nullptr.mir

53–54

Can simplify these out of the test

57

Should add vector test

[arsenm]

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

2737–2740

Combine the scalar and vector check above?

2743

I have a patch to generalize these checks I haven't posted yet, but I guess having the two paths is fine for now

[mbrkusanin]

• Refactored matchPtrAddZero().

[mbrkusanin]

• Rebase
• Ping

[foad]

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

2728

This is just Ty.getScalarType().

2737

You should be able to assert that it is a vector here.

[mbrkusanin]

• Addressed comments.

[foad]

LGTM.

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

public:
 
   /// \return true if \p MI is a G_SEXT_INREG that can be erased.
   bool matchRedundantSExtInReg(MachineInstr &MI);
 
   /// Combine inverting a result of a compare into the opposite cond code.
   bool matchNotCmp(MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate);
   bool applyNotCmp(MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate);
 
+  /// Combine G_PTR_ADD with nullptr to G_INTTOPTR
+  bool matchPtrAddZero(MachineInstr &MI);
+  bool applyPtrAddZero(MachineInstr &MI);
+
   /// Try to transform \p MI by using all of the above
   /// combine functions. Returns true if changed.
   bool tryCombine(MachineInstr &MI);
 
 private:
   // Memcpy family optimization helpers.
   bool optimizeMemcpy(MachineInstr &MI, Register Dst, Register Src,
                       unsigned KnownLen, Align DstAlign, Align SrcAlign,

llvm/include/llvm/Target/GlobalISel/Combine.td

 // Transform (mul x, -1) -> (sub 0, x)
 def mul_by_neg_one: GICombineRule <
   (defs root:$root),
   (match (wip_match_opcode G_MUL):$root,
          [{ return Helper.matchConstantOp(${root}->getOperand(2), -1); }]),
   (apply [{ return Helper.applyCombineMulByNegativeOne(*${root}); }])
 >;
 
+// Transform (ptr_add 0, x) -> (int_to_ptr x)
+def ptr_add_with_zero: GICombineRule<
+  (defs root:$root),
+  (match (wip_match_opcode G_PTR_ADD):$root,
+         [{ return Helper.matchPtrAddZero(*${root}); }]),
+  (apply [{ return Helper.applyPtrAddZero(*${root}); }])>;
+
 // FIXME: These should use the custom predicate feature once it lands.
 def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
                                      undef_to_negative_one,
                                      binop_left_undef_to_zero,
                                      propagate_undef_any_op,
                                      propagate_undef_all_ops,
                                      propagate_undef_shuffle_mask,
                                      erase_undef_store]>;
def all_combines : GICombineGroup<[trivial_combines, ptr_add_immed_chain,
     identity_combines, simplify_add_to_sub,
     hoist_logic_op_with_same_opcode_hands,
     shl_ashr_to_sext_inreg, sext_inreg_of_load,
     width_reduction_combines, select_combines,
     known_bits_simplifications, ext_ext_fold,
     not_cmp_fold, opt_brcond_by_inverting_cond,
     unmerge_merge, fabs_fabs_fold, unmerge_cst, unmerge_dead_to_trunc,
     unmerge_zext_to_zext, trunc_ext_fold, trunc_shl,
-    constant_fp_op]>;
+    constant_fp_op, ptr_add_with_zero]>;

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

for (Register Reg : RegsToNegate) {
     Observer.changedInstr(*Def);
   }
 
   replaceRegWith(MRI, MI.getOperand(0).getReg(), MI.getOperand(1).getReg());
   MI.eraseFromParent();
   return true;
 }
 
+bool CombinerHelper::matchPtrAddZero(MachineInstr &MI) {
+  Register DstReg = MI.getOperand(0).getReg();
+  LLT Ty = MRI.getType(DstReg);
+  const DataLayout &DL = Builder.getMF().getDataLayout();
+
+  if (Ty.isPointer()) {
+    if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
+      return false;
+
+    auto ConstVal = getConstantVRegVal(MI.getOperand(1).getReg(), MRI);
+    return ConstVal && *ConstVal == 0;
+  }
+
+  if (Ty.isVector()) {
+    LLT ElemTy = Ty.getElementType();
+    if (ElemTy.isPointer()) {
+      if (DL.isNonIntegralAddressSpace(ElemTy.getAddressSpace()))
+        return false;
+
+      const MachineInstr *VecMI = MRI.getVRegDef(MI.getOperand(1).getReg());
+      return isBuildVectorAllZeros(*VecMI, MRI);
+    }
+  }
+
+  return false;
+}
+
+bool CombinerHelper::applyPtrAddZero(MachineInstr &MI) {
+  assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD);
+  Builder.setInstrAndDebugLoc(MI);
+  Builder.buildIntToPtr(MI.getOperand(0), MI.getOperand(2));
+  MI.eraseFromParent();
+  return true;
+}
+
 bool CombinerHelper::tryCombine(MachineInstr &MI) {
   if (tryCombineCopy(MI))
     return true;
   if (tryCombineExtendingLoads(MI))
     return true;
   if (tryCombineIndexedLoadStore(MI))
     return true;
   return false;
 }

llvm/test/CodeGen/AMDGPU/GlobalISel/combine-add-nullptr.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass=amdgpu-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: add_nullptr_shl_add
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0
+
+    ; CHECK-LABEL: name: add_nullptr_shl_add
+    ; CHECK: liveins: $sgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[SHL]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_CONSTANT i32 3
+    %2:_(s32) = G_SHL %0, %1(s32)
+    %3:_(p3) = G_CONSTANT i32 0
+    %4:_(p3) = G_PTR_ADD %3, %2(s32)
+    %5:_(s32) = G_PTRTOINT %4(p3)
+    $vgpr0 = COPY %5(s32)
+
+...
+
+---
+name: add_nullptr_mul_add
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: add_nullptr_mul_add
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[MUL]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p3) = G_CONSTANT i32 0
+    %3:_(s32) = G_MUL %0:_, %1:_
+    %4:_(p3) = G_PTR_ADD %2:_, %3:_(s32)
+    %5:_(s32) = G_PTRTOINT %4:_(p3)
+    $vgpr0 = COPY %5:_(s32)
+
+...
+
+---
+name: add_nullptr_vec_all_zero
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
+
+    ; CHECK-LABEL: name: add_nullptr_vec_all_zero
+    ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(<2 x s32>) = G_SHL [[COPY]], [[BUILD_VECTOR]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SHL]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s32) = COPY $vgpr3
+    %3:_(<2 x s32>) = G_BUILD_VECTOR %1:_(s32), %2:_(s32)
+    %4:_(<2 x s32>) = G_SHL %0, %3(<2 x s32>)
+    %5:_(p3) = G_CONSTANT i32 0
+    %6:_(<2 x p3>) = G_BUILD_VECTOR %5:_(p3), %5:_(p3)
+    %7:_(<2 x p3>) = G_PTR_ADD %6, %4(<2 x s32>)
+    %8:_(<2 x s32>) = G_PTRTOINT %7(<2 x p3>)
+    $vgpr0_vgpr1 = COPY %8(<2 x s32>)
+
+...

llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll

 ; GFX9-NEXT:    s_cselect_b32 s1, s16, s9
 ; GFX9-NEXT:    s_cmp_eq_u32 s7, 2
 ; GFX9-NEXT:    s_cselect_b32 s2, s16, s10
 ; GFX9-NEXT:    s_cmp_eq_u32 s7, 3
 ; GFX9-NEXT:    s_cselect_b32 s3, s16, s11
 ; GFX9-NEXT:    s_cmp_eq_u32 s7, 4
 ; GFX9-NEXT:    s_cselect_b32 s4, s16, s12
 ; GFX9-NEXT:    s_cmp_eq_u32 s7, 5
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    s_cselect_b32 s5, s16, s13
 ; GFX9-NEXT:    s_cmp_eq_u32 s7, 6
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    s_cselect_b32 s6, s16, s14
 ; GFX9-NEXT:    s_cmp_eq_u32 s7, 7
-; GFX9-NEXT:    s_cselect_b32 s7, s16, s15
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
-; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    s_cselect_b32 s7, s16, s15
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v4, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_s_v16i16_s_s:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[8:15], s[2:3], 0x0
 ; GFX8-NEXT:    s_lshr_b32 s7, s5, 1
 ; GFX8-NEXT:    s_cmp_eq_u32 s7, 1

 ; GFX8-NEXT:    s_cselect_b32 s1, s16, s9
 ; GFX8-NEXT:    s_cmp_eq_u32 s7, 2
 ; GFX8-NEXT:    s_cselect_b32 s2, s16, s10
 ; GFX8-NEXT:    s_cmp_eq_u32 s7, 3
 ; GFX8-NEXT:    s_cselect_b32 s3, s16, s11
 ; GFX8-NEXT:    s_cmp_eq_u32 s7, 4
 ; GFX8-NEXT:    s_cselect_b32 s4, s16, s12
 ; GFX8-NEXT:    s_cmp_eq_u32 s7, 5
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    s_cselect_b32 s5, s16, s13
 ; GFX8-NEXT:    s_cmp_eq_u32 s7, 6
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    s_cselect_b32 s6, s16, s14
 ; GFX8-NEXT:    s_cmp_eq_u32 s7, 7
-; GFX8-NEXT:    s_cselect_b32 s7, s16, s15
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v5, 0
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    s_cselect_b32 s7, s16, s15
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v4, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_s_v16i16_s_s:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dwordx8 s[8:15], s[2:3], 0x0
 ; GFX7-NEXT:    s_lshr_b32 s7, s5, 1
 ; GFX7-NEXT:    s_cmp_eq_u32 s7, 1

 ; GFX7-NEXT:    s_cselect_b32 s6, s16, s14
 ; GFX7-NEXT:    s_cmp_eq_u32 s7, 7
 ; GFX7-NEXT:    s_mov_b64 s[8:9], 0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX7-NEXT:    s_mov_b32 s10, -1
 ; GFX7-NEXT:    s_mov_b32 s11, 0xf000
-; GFX7-NEXT:    s_cselect_b32 s7, s16, s15
 ; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
-; GFX7-NEXT:    s_nop 0
+; GFX7-NEXT:    s_cselect_b32 s7, s16, s15
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_mov_b64 s[8:9], 16
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s7
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(4)* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }
 
 define amdgpu_ps void @insertelement_v_v16i16_s_s(<16 x i16> addrspace(1)* %ptr, i16 inreg %val, i32 inreg %idx) {

 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v8, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[10:11]
 ; GFX9-NEXT:    v_and_or_b32 v10, v1, s13, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, v10, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, v10, s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, v10, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v10, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v6, v10, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, v7, v10, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v10, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, v9, v10, s[10:11]
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    v_mov_b32_e32 v11, s1
+; GFX9-NEXT:    v_mov_b32_e32 v10, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, s0
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
 ; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_s_s:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc

 ; GFX8-NEXT:    s_lshl_b32 s13, s2, s1
 ; GFX8-NEXT:    s_not_b32 s14, s0
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[0:1], s12, 2
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[2:3], s12, 3
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], s12, 4
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[6:7], s12, 5
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[8:9], s12, 6
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[10:11], s12, 7
+; GFX8-NEXT:    v_mov_b32_e32 v10, 16
+; GFX8-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v0, v1, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v2, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v3, s[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v4, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v5, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v6, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v8, v8, v7, s[10:11]
 ; GFX8-NEXT:    v_and_b32_e32 v8, s14, v8
 ; GFX8-NEXT:    v_or_b32_e32 v8, s13, v8
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[0:1]
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v8, s[12:13]
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v8, s[2:3]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v8, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[10:11]
 ; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    v_mov_b32_e32 v11, s1
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0
-; GFX8-NEXT:    v_mov_b32_e32 v10, s0
 ; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_v_v16i16_s_s:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_mov_b32 s18, 0
 ; GFX7-NEXT:    s_mov_b32 s19, 0xf000

 ; GFX7-NEXT:    v_or_b32_e32 v10, s13, v0
 ; GFX7-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v2, v10, s[12:13]
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v10, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v4, v10, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v5, v10, s[2:3]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v6, v10, s[4:5]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v7, v10, s[6:7]
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v8, v10, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v9, v10, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:16
+; GFX7-NEXT:    s_mov_b64 s[16:17], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(1 )* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }
 
 define amdgpu_ps void @insertelement_s_v16i16_v_s(<16 x i16> addrspace(4)* inreg %ptr, i16 %val, i32 inreg %idx) {

 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s2, 4
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s15
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 7
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 16
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_s_v16i16_v_s:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[8:15], s[2:3], 0x0
 ; GFX8-NEXT:    s_lshr_b32 s2, s4, 1
 ; GFX8-NEXT:    s_cmp_eq_u32 s2, 1

 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[0:1], s2, 4
 ; GFX8-NEXT:    v_mov_b32_e32 v7, s15
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 7
 ; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
 ; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_s_v16i16_v_s:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dwordx8 s[8:15], s[2:3], 0x0
 ; GFX7-NEXT:    s_lshr_b32 s2, s4, 1
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1

 ; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 6
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
 ; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 7
 ; GFX7-NEXT:    v_mov_b32_e32 v7, s15
 ; GFX7-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
 ; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
+; GFX7-NEXT:    s_mov_b64 s[0:1], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(4)* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }
 
 define amdgpu_ps void @insertelement_s_v16i16_s_v(<16 x i16> addrspace(4)* inreg %ptr, i16 inreg %val, i32 %idx) {

 ; GFX9-NEXT:    v_mov_b32_e32 v1, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s18
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s19
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s20
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s21
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s22
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s23
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[12:13]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[14:15]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[10:11]
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 16
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_s_v16i16_s_v:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[16:23], s[2:3], 0x0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 1, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v8

 ; GFX8-NEXT:    v_mov_b32_e32 v1, s17
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s18
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s19
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s20
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s21
 ; GFX8-NEXT:    v_mov_b32_e32 v6, s22
 ; GFX8-NEXT:    v_mov_b32_e32 v7, s23
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[12:13]
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[2:3]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[14:15]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[10:11]
 ; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_s_v16i16_s_v:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dwordx8 s[16:23], s[2:3], 0x0
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 1, v0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v8

 ; GFX7-NEXT:    v_mov_b32_e32 v7, s23
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[2:3]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[12:13]
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
 ; GFX7-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[14:15]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[6:7]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GFX7-NEXT:    s_mov_b64 s[0:1], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(4)* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }
 
 define amdgpu_ps void @insertelement_s_v16i16_v_v(<16 x i16> addrspace(4)* inreg %ptr, i16 %val, i32 %idx) {

 ; GFX9-NEXT:    v_mov_b32_e32 v1, s13
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s14
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s15
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s16
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s18
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s19
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[12:13]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[10:11]
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 16
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_s_v16i16_v_v:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[12:19], s[2:3], 0x0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 1, v1
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v8

 ; GFX8-NEXT:    v_mov_b32_e32 v1, s13
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s14
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s15
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s16
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s17
 ; GFX8-NEXT:    v_mov_b32_e32 v6, s18
 ; GFX8-NEXT:    v_mov_b32_e32 v7, s19
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[12:13]
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[2:3]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[10:11]
 ; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_s_v16i16_v_v:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dwordx8 s[12:19], s[2:3], 0x0
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 1, v1
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v8

 ; GFX7-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v8
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[2:3]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[12:13]
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
 ; GFX7-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, v9, s[4:5]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v5, v9, s[6:7]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GFX7-NEXT:    s_mov_b64 s[0:1], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(4)* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }
 
 define amdgpu_ps void @insertelement_v_v16i16_s_v(<16 x i16> addrspace(1)* %ptr, i16 inreg %val, i32 %idx) {

 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v5, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v6, s[2:3]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v7, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v8, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v9, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, v11, v10, s[10:11]
 ; GFX9-NEXT:    v_and_or_b32 v11, v11, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v11, s1
+; GFX9-NEXT:    v_mov_b32_e32 v10, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, s0
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
 ; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_s_v:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    v_add_u32_e32 v7, vcc, 16, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc

 ; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v6, s[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v7, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v8, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v9, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v11, v11, v10, s[10:11]
 ; GFX8-NEXT:    v_and_b32_e32 v1, v11, v1
 ; GFX8-NEXT:    v_or_b32_e32 v11, v1, v2
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
 ; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX8-NEXT:    v_mov_b32_e32 v11, s1
+; GFX8-NEXT:    v_mov_b32_e32 v10, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0
-; GFX8-NEXT:    v_mov_b32_e32 v10, s0
+; GFX8-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_v_v16i16_s_v:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_mov_b32 s18, 0
 ; GFX7-NEXT:    s_mov_b32 s19, 0xf000

 ; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v10, s[10:11]
 ; GFX7-NEXT:    v_and_b32_e32 v1, v11, v1
 ; GFX7-NEXT:    v_or_b32_e32 v11, v1, v2
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:16
+; GFX7-NEXT:    s_mov_b64 s[16:17], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(1)* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }
 
 define amdgpu_ps void @insertelement_v_v16i16_v_s(<16 x i16> addrspace(1)* %ptr, i16 %val, i32 inreg %idx) {

 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[2:3]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v8, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v10, s[10:11]
 ; GFX9-NEXT:    v_and_or_b32 v11, v1, s13, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v11, s1
+; GFX9-NEXT:    v_mov_b32_e32 v10, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, s0
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
 ; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_v_s:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    v_add_u32_e32 v7, vcc, 16, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc

 ; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v8, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v10, s[10:11]
 ; GFX8-NEXT:    v_and_b32_e32 v1, s13, v1
 ; GFX8-NEXT:    v_or_b32_e32 v11, v1, v0
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
 ; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX8-NEXT:    v_mov_b32_e32 v11, s1
+; GFX8-NEXT:    v_mov_b32_e32 v10, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0
-; GFX8-NEXT:    v_mov_b32_e32 v10, s0
+; GFX8-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_v_v16i16_v_s:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_mov_b32 s18, 0
 ; GFX7-NEXT:    s_mov_b32 s19, 0xf000

 ; GFX7-NEXT:    v_and_b32_e32 v1, s13, v1
 ; GFX7-NEXT:    v_or_b32_e32 v11, v1, v0
 ; GFX7-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:16
+; GFX7-NEXT:    s_mov_b64 s[16:17], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(1)* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }
 
 define amdgpu_ps void @insertelement_v_v16i16_v_v(<16 x i16> addrspace(1)* %ptr, i16 %val, i32 %idx) {

 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[2:3]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v8, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v10, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
 ; GFX9-NEXT:    v_and_or_b32 v12, v3, v1, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, 0, 16
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v12, s[12:13]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v12, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v12, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v12, s[6:7]
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    s_addc_u32 s1, 0, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v12, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, v12, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v11, s1
+; GFX9-NEXT:    v_mov_b32_e32 v10, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, s0
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
 ; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_v_v:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 16, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc

 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v8, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v10, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
 ; GFX8-NEXT:    v_and_b32_e32 v1, v3, v1
 ; GFX8-NEXT:    v_or_b32_e32 v12, v1, v2
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
-; GFX8-NEXT:    s_add_u32 s0, 0, 16
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v12, s[12:13]
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v12, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v12, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v12, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    s_addc_u32 s1, 0, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v12, s[2:3]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, v12, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s[10:11]
-; GFX8-NEXT:    v_mov_b32_e32 v11, s1
+; GFX8-NEXT:    v_mov_b32_e32 v10, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v9, 0
-; GFX8-NEXT:    v_mov_b32_e32 v10, s0
+; GFX8-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_v_v16i16_v_v:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_mov_b32 s18, 0
 ; GFX7-NEXT:    s_mov_b32 s19, 0xf000

 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v10, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
 ; GFX7-NEXT:    v_and_b32_e32 v1, v3, v1
 ; GFX7-NEXT:    v_or_b32_e32 v12, v1, v2
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v12, s[12:13]
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v12, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v12, s[2:3]
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v12, s[4:5]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v12, s[6:7]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v10, v12, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:16
+; GFX7-NEXT:    s_mov_b64 s[16:17], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0
 ; GFX7-NEXT:    s_endpgm
   %vec = load <16 x i16>, <16 x i16> addrspace(1)* %ptr
   %insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
   store <16 x i16> %insert, <16 x i16> addrspace(1)* null
   ret void
 }

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll

 @lds0 = internal addrspace(3) global [512 x i32] undef
 
 define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_dec_shl_base_lds_0:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
 ; CI-NEXT:    v_mov_b32_e32 v1, 9
 ; CI-NEXT:    s_mov_b32 m0, -1
 ; CI-NEXT:    ds_dec_rtn_u32 v3, v0, v1 offset:8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    flat_store_dword v[0:1], v3
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_dec_shl_base_lds_0:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 2, v0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
 ; VI-NEXT:    v_mov_b32_e32 v1, 9
 ; VI-NEXT:    s_mov_b32 m0, -1
 ; VI-NEXT:    ds_dec_rtn_u32 v3, v0, v1 offset:8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    v_mov_b32_e32 v0, s0

 
 @lds1 = internal addrspace(3) global [512 x i64] undef, align 8
 
 define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_dec_shl_base_lds_0_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
-; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; CI-NEXT:    v_add_i32_e32 v2, vcc, 0, v0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, 9
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    s_mov_b32 m0, -1
 ; CI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_store_dword v[2:3], v4
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_dec_shl_base_lds_0_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
-; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, 9
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    s_mov_b32 m0, -1
 ; VI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dword v[2:3], v4

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll

 @lds0 = internal addrspace(3) global [512 x i32] undef, align 4
 
 define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_inc_shl_base_lds_0_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
 ; CI-NEXT:    v_mov_b32_e32 v1, 9
 ; CI-NEXT:    s_mov_b32 m0, -1
 ; CI-NEXT:    ds_inc_rtn_u32 v3, v0, v1 offset:8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    flat_store_dword v[0:1], v3
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_inc_shl_base_lds_0_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 2, v0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
 ; VI-NEXT:    v_mov_b32_e32 v1, 9
 ; VI-NEXT:    s_mov_b32 m0, -1
 ; VI-NEXT:    ds_inc_rtn_u32 v3, v0, v1 offset:8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v3
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: atomic_inc_shl_base_lds_0_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX9-NEXT:    v_add_u32_e32 v2, 2, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 9
 ; GFX9-NEXT:    ds_inc_rtn_u32 v3, v0, v1 offset:8
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1

 
 @lds1 = internal addrspace(3) global [512 x i64] undef, align 8
 
 define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
-; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; CI-NEXT:    v_add_i32_e32 v2, vcc, 0, v0
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, 9
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    s_mov_b32 m0, -1
 ; CI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_store_dword v[2:3], v4
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
-; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, 9
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    s_mov_b32 m0, -1
 ; VI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dword v[2:3], v4
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX9-NEXT:    v_add_u32_e32 v4, 2, v0
-; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; GFX9-NEXT:    v_add_u32_e32 v2, 0, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 9
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NEXT:    global_store_dword v[2:3], v4, off
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1