AMDGPU/GlobalISel: Stop using NarrowScalar/FewerElements for unaligned splitting

[arsenm]

Summary

These actions should only be used for adjusting the register types
(and the memory type as needed to satisfy the register
type). Unaligned accesses should be split as a type of lowering.

This has the effect of improving the code in many cases since now we
produce zextloads instead of separate loads with ands. The load/store
legality rules still seem far more complicated than necessary though.

[foad]

Looks reasonable to me.

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

1250

Is this case still required for some reason, even though you've removed the corresponding code from the scalar case above?

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

1250

This one you can technically remove, but it results in worse code since it hits the full scalarization path below.

[arsenm]

Rebase and remove another manual check

[arsenm]

c22297244209c6bea27fe54cd5eca15d689cf8ad

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
     .custom();
 
   const auto needToSplitMemOp = [=](const LegalityQuery &Query,
                                     bool IsLoad) -> bool {
     const LLT DstTy = Query.Types[0];
 
     // Split vector extloads.
     unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
-    unsigned AlignBits = Query.MMODescrs[0].AlignInBits;
-
-    if (MemSize < DstTy.getSizeInBits())
-      MemSize = std::max(MemSize, AlignBits);
 
     if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
       return true;
 
     const LLT PtrTy = Query.Types[1];
     unsigned AS = PtrTy.getAddressSpace();
     if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad))
       return true;
 
     // Catch weird sized loads that don't evenly divide into the access sizes
     // TODO: May be able to widen depending on alignment etc.
     unsigned NumRegs = (MemSize + 31) / 32;
     if (NumRegs == 3) {
       if (!ST.hasDwordx3LoadStores())
         return true;
     } else {
       // If the alignment allows, these should have been widened.
       if (!isPowerOf2_32(NumRegs))
         return true;
     }
 
-    if (AlignBits < MemSize) {
-      const SITargetLowering *TLI = ST.getTargetLowering();
-      return !TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS,
-                                                      Align(AlignBits / 8));
-    }
-
     return false;
   };
 
   unsigned GlobalAlign32 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
   unsigned GlobalAlign16 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
   unsigned GlobalAlign8 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;
 
   // TODO: Refine based on subtargets which support unaligned access or 128-bit
Actions
 
               const unsigned DstSize = DstTy.getSizeInBits();
               unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
 
               // Split extloads.
               if (DstSize > MemSize)
                 return std::make_pair(0, LLT::scalar(MemSize));
 
-              if (!isPowerOf2_32(DstSize)) {
-                // We're probably decomposing an odd sized store. Try to split
-                // to the widest type. TODO: Account for alignment. As-is it
-                // should be OK, since the new parts will be further legalized.
-                unsigned FloorSize = PowerOf2Floor(DstSize);
-                return std::make_pair(0, LLT::scalar(FloorSize));
-              }
-
-              if (DstSize > 32 && (DstSize % 32 != 0)) {
-                // FIXME: Need a way to specify non-extload of larger size if
-                // suitably aligned.
-                return std::make_pair(0, LLT::scalar(32 * (DstSize / 32)));
-              }
-
               unsigned MaxSize = maxSizeForAddrSpace(ST,
                                                      PtrTy.getAddressSpace(),
                                                      Op == G_LOAD);
               if (MemSize > MaxSize)
                 return std::make_pair(0, LLT::scalar(MaxSize));
 
               unsigned Align = Query.MMODescrs[0].AlignInBits;
               return std::make_pair(0, LLT::scalar(Align));
Actions
               }
 
               // FIXME: We could probably handle weird extending loads better.
               if (DstTy.getSizeInBits() > MemSize)
                 return std::make_pair(0, EltTy);
 
               unsigned EltSize = EltTy.getSizeInBits();
               unsigned DstSize = DstTy.getSizeInBits();
               if (!isPowerOf2_32(DstSize)) {
                 // We're probably decomposing an odd sized store. Try to split
                 // to the widest type. TODO: Account for alignment. As-is it
                 // should be OK, since the new parts will be further legalized.
                 unsigned FloorSize = PowerOf2Floor(DstSize);
                 return std::make_pair(
                     0, LLT::scalarOrVector(
                            ElementCount::getFixed(FloorSize / EltSize), EltTy));
               }
 
-              // Need to split because of alignment.
-              unsigned Align = Query.MMODescrs[0].AlignInBits;
-              if (EltSize > Align &&
-                  (EltSize / Align < DstTy.getNumElements())) {
-                return std::make_pair(
-                    0, LLT::fixed_vector(EltSize / Align, EltTy));
-              }
-
               // May need relegalization for the scalars.
               return std::make_pair(0, EltTy);
             })
     .minScalar(0, S32)
     .narrowScalarIf(isWideScalarExtLoadTruncStore(0), changeTo(0, S32))
     .widenScalarToNextPow2(0)
     .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
     .lower();

llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll

 ; SI-LABEL: load_v4i8_to_v4f32_unaligned:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[4:7], 0 addr64 offset:1
+; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[4:7], 0 addr64 offset:1
+; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[4:7], 0 addr64 offset:3
 ; SI-NEXT:    buffer_load_ubyte v4, v[0:1], s[4:7], 0 addr64 offset:2
-; SI-NEXT:    buffer_load_ubyte v5, v[0:1], s[4:7], 0 addr64 offset:3
+; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v2
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v3, v5
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_or_b32_e32 v1, v2, v3
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v0
+; SI-NEXT:    v_bfe_u32 v2, v0, 8, 8
+; SI-NEXT:    v_bfe_u32 v4, v0, 16, 8
+; SI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v0
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v1
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v2
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: load_v4i8_to_v4f32_unaligned:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 1, v0
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v0, v[0:1]
-; VI-NEXT:    flat_load_ubyte v1, v[2:3]
-; VI-NEXT:    flat_load_ubyte v2, v[4:5]
+; VI-NEXT:    flat_load_ubyte v2, v[2:3]
 ; VI-NEXT:    flat_load_ubyte v3, v[6:7]
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    flat_load_ubyte v4, v[4:5]
+; VI-NEXT:    flat_load_ubyte v0, v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v1
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v2
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v3, v3
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_or_b32_e32 v3, v1, v0
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; VI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v3
+; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid
   %load = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 1
   %cvt = uitofp <4 x i8> %load to <4 x float>
   store <4 x float> %cvt, <4 x float> addrspace(1)* %out, align 16
   ret void

 ; SI-LABEL: v4i8_zext_v4i32_to_v4f32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[4:7], 0 addr64 offset:1
+; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[4:7], 0 addr64 offset:1
+; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[4:7], 0 addr64 offset:3
 ; SI-NEXT:    buffer_load_ubyte v4, v[0:1], s[4:7], 0 addr64 offset:2
-; SI-NEXT:    buffer_load_ubyte v5, v[0:1], s[4:7], 0 addr64 offset:3
+; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v2
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v3, v5
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_or_b32_e32 v1, v2, v3
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v0
+; SI-NEXT:    v_bfe_u32 v2, v0, 8, 8
+; SI-NEXT:    v_bfe_u32 v4, v0, 16, 8
+; SI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v0
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v1
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v2
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: v4i8_zext_v4i32_to_v4f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 1, v0
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v0, v[0:1]
-; VI-NEXT:    flat_load_ubyte v1, v[2:3]
-; VI-NEXT:    flat_load_ubyte v2, v[4:5]
+; VI-NEXT:    flat_load_ubyte v2, v[2:3]
 ; VI-NEXT:    flat_load_ubyte v3, v[6:7]
-; VI-NEXT:    v_mov_b32_e32 v5, s1
-; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    flat_load_ubyte v4, v[4:5]
+; VI-NEXT:    flat_load_ubyte v0, v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v1
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v2
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v3, v3
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_or_b32_e32 v3, v1, v0
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; VI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v3
+; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid
   %load = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 1
   %ext = zext <4 x i8> %load to <4 x i32>
   %cvt = uitofp <4 x i32> %ext to <4 x float>
   store <4 x float> %cvt, <4 x float> addrspace(1)* %out, align 16

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant-32bit.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=CI %s
 
 ---
 name: test_load_constant32bit_s32_align1
 body: |
   bb.0:
     liveins: $vgpr0
 
     ; CI-LABEL: name: test_load_constant32bit_s32_align1
     ; CI: [[COPY:%[0-9]+]]:_(p6) = COPY $vgpr0
     ; CI-NEXT: [[C:%[0-9]+]]:_(p6) = G_CONSTANT i32 0
     ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[COPY]](p6), [[C]](p6)
-    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[MV]](p4) :: (load (s8), addrspace 6)
+    ; CI-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[MV]](p4) :: (load (s8), addrspace 6)
     ; CI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[MV]], [[C1]](s64)
-    ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (load (s8) from unknown-address + 1, addrspace 6)
-    ; CI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[MV]], [[C2]](s64)
-    ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load (s8) from unknown-address + 2, addrspace 6)
-    ; CI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[MV]], [[C3]](s64)
-    ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load (s8) from unknown-address + 3, addrspace 6)
-    ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
-    ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
-    ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
-    ; CI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
-    ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
-    ; CI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C6]](s32)
-    ; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
-    ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
-    ; CI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
-    ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p4) :: (load (s8) from unknown-address + 1, addrspace 6)
+    ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C2]](s32)
+    ; CI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; CI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[MV]], [[C3]](s64)
+    ; CI-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p4) :: (load (s8) from unknown-address + 2, addrspace 6)
+    ; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[PTR_ADD1]], [[C1]](s64)
+    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load (s8) from unknown-address + 3, addrspace 6)
+    ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C2]](s32)
+    ; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C4]](s32)
+    ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
     ; CI-NEXT: $vgpr0 = COPY [[OR2]](s32)
     %0:_(p6) = COPY $vgpr0
     %1:_(s32) = G_LOAD %0 :: (load (s32), align 1, addrspace 6)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_load_constant32bit_s32_align4

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir

 name: test_truncstore_global_s64_to_s16_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
     ; SI-LABEL: name: test_truncstore_global_s64_to_s16_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C1]]
+    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
-    ; SI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; SI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; SI-NEXT: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
     ; VI-LABEL: name: test_truncstore_global_s64_to_s16_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
-    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s64)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C]](s16)
-    ; VI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C]](s16)
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; VI-NEXT: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (s16), addrspace 1, align 1)
 ...
 
 ---

 name: test_truncstore_global_s64_to_s32_align2
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
     ; SI-LABEL: name: test_truncstore_global_s64_to_s32_align2
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
-    ; SI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; SI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
     ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     ; VI-LABEL: name: test_truncstore_global_s64_to_s32_align2
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
-    ; VI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
     ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (s32), addrspace 1, align 2)
 ...
 
 ---
 name: test_truncstore_global_s64_to_s32_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
     ; SI-LABEL: name: test_truncstore_global_s64_to_s32_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
-    ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
-    ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C2]]
-    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[COPY2]](s32)
-    ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
-    ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C1]](s32)
-    ; SI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; SI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
-    ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
+    ; SI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s32)
+    ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
-    ; SI-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C5]](s64)
+    ; SI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
+    ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[COPY3]](s32)
+    ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD]], [[C4]](s64)
+    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
     ; SI-NEXT: G_STORE [[LSHR2]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
     ; VI-LABEL: name: test_truncstore_global_s64_to_s32_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
-    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
-    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
-    ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
-    ; VI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; VI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
-    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
-    ; VI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
+    ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C2]](s16)
+    ; VI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
-    ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C2]](s16)
+    ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD]], [[C3]](s64)
+    ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
     ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
     ; VI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (s32), addrspace 1, align 1)
 ...
 
 ---

 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
 
     ; SI-LABEL: name: test_store_global_v2s8_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
     ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
-    ; SI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; SI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; SI-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; SI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C2]]
+    ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY1]](s32)
+    ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C1]](s32)
+    ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; SI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
     ; VI-LABEL: name: test_store_global_v2s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
     ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<2 x s32>)
-    ; VI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; VI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; VI-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ANYEXT]](s32)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C1]](s16)
+    ; VI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s8>) = G_IMPLICIT_DEF
     G_STORE %1, %0 :: (store (<2 x s8>), addrspace 1, align 1)
 
 ...
 
 ---
 name: test_store_global_v2s8_align2

 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
 
     ; SI-LABEL: name: test_store_global_v3s8_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; SI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
     ; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
     ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C2]]
     ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY2]](s32)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
     ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
-    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
-    ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[COPY3]](s16)
-    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT]], [[C1]](s32)
-    ; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY3]](s16)
-    ; SI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; SI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
+    ; SI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[DEF]], [[C2]]
+    ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[COPY3]](s32)
+    ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
+    ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
+    ; SI-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY4]], [[C3]](s32)
     ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; SI-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
+    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C5]]
+    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C1]](s32)
+    ; SI-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C6]](s64)
+    ; SI-NEXT: G_STORE [[COPY4]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
     ; VI-LABEL: name: test_store_global_v3s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
     ; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
     ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
     ; VI-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
     ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
-    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
-    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[COPY2]], [[C1]](s16)
-    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY2]](s16)
-    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; VI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
-    ; VI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
+    ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
+    ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
+    ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
+    ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
     ; VI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; VI-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
+    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[OR]], [[C1]](s16)
+    ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     %2:_(<3 x s8>) = G_TRUNC %1
     G_STORE %2, %0 :: (store (<3 x s8>), addrspace 1, align 1)
 
 ...
 
 ---
 name: test_store_global_v3s8_align2
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
 
     ; SI-LABEL: name: test_store_global_v3s8_align2
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; SI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
     ; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
     ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C2]]
-    ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY2]](s32)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
     ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
-    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
-    ; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY2]](s16)
-    ; SI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
-    ; SI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; SI-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
+    ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
+    ; SI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[DEF]], [[C2]]
+    ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+    ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
+    ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
+    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C3]](s32)
+    ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; SI-NEXT: G_STORE [[COPY3]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
     ; VI-LABEL: name: test_store_global_v3s8_align2
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
     ; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
     ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
     ; VI-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
     ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
-    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[OR]](s16)
-    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY2]](s16)
-    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
-    ; VI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; VI-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
+    ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
+    ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
+    ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
+    ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
+    ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; VI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     %2:_(<3 x s8>) = G_TRUNC %1
     G_STORE %2, %0 :: (store (<3 x s8>), addrspace 1, align 2)
 
 ...
 
 ---

 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
 
     ; SI-LABEL: name: test_store_global_v4s8_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
-    ; SI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; SI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; SI-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
-    ; SI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; SI-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
-    ; SI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; SI-NEXT: G_STORE [[UV3]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
+    ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+    ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+    ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
+    ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
+    ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C5]]
+    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C1]](s32)
+    ; SI-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C6]](s64)
+    ; SI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C5]]
+    ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[COPY3]](s32)
+    ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD]], [[C6]](s64)
+    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR2]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
     ; VI-LABEL: name: test_store_global_v4s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
-    ; VI-NEXT: G_STORE [[UV]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
-    ; VI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; VI-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
-    ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; VI-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
-    ; VI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; VI-NEXT: G_STORE [[UV3]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
+    ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+    ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+    ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
+    ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
+    ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; VI-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; VI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C5]]
+    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; VI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C5]]
+    ; VI-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C6]](s16)
+    ; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL3]]
+    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[OR3]], [[C6]](s16)
+    ; VI-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C7]](s64)
+    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
+    ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C6]](s16)
+    ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD]], [[C7]](s64)
+    ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
+    ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; VI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     %2:_(<4 x s8>) = G_TRUNC %1
     G_STORE %2, %0 :: (store (<4 x s8>), addrspace 1, align 1)
 
 ...
 
 ---
 name: test_store_global_v4s8_align2
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
 
     ; SI-LABEL: name: test_store_global_v4s8_align2
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
-    ; SI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
-    ; SI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+    ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
-    ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C2]]
-    ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY2]](s32)
-    ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
-    ; SI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
-    ; SI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
-    ; SI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
-    ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C2]]
-    ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
-    ; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
-    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
-    ; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
-    ; SI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
-    ; SI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; SI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[OR1]](s16)
-    ; SI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
+    ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
+    ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+    ; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
+    ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+    ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; SI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     ; VI-LABEL: name: test_store_global_v4s8_align2
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
-    ; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
-    ; VI-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
-    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
-    ; VI-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
-    ; VI-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
-    ; VI-NEXT: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
-    ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
-    ; VI-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
-    ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[UV3]](s32)
-    ; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
-    ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
-    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
-    ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
-    ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
-    ; VI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[OR1]](s16)
-    ; VI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
+    ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+    ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+    ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
+    ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+    ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
+    ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+    ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     %2:_(<4 x s8>) = G_TRUNC %1
     G_STORE %2, %0 :: (store (<4 x s8>), addrspace 1, align 2)
 
 ...
 
 ---

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -march=amdgcn -verify-machineinstrs -amdgpu-mem-intrinsic-expand-size=19 %s -o - | FileCheck -check-prefix=LOOP %s
 ; RUN: llc -global-isel -march=amdgcn -verify-machineinstrs -amdgpu-mem-intrinsic-expand-size=21 %s -o - | FileCheck -check-prefix=UNROLL %s
 
 declare void @llvm.memcpy.p1i8.p1i8.i32(i8 addrspace(1)*, i8 addrspace(1)*, i32, i1 immarg)
 
 define amdgpu_cs void @memcpy_p1i8(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) {
 ; LOOP-LABEL: memcpy_p1i8:
 ; LOOP:       ; %bb.0:
-; LOOP-NEXT:    s_mov_b32 s6, 0
-; LOOP-NEXT:    s_mov_b32 s7, 0xf000
-; LOOP-NEXT:    s_mov_b64 s[4:5], 0
+; LOOP-NEXT:    s_mov_b32 s2, 0
+; LOOP-NEXT:    s_mov_b32 s3, 0xf000
+; LOOP-NEXT:    s_mov_b64 s[0:1], 0
 ; LOOP-NEXT:    v_mov_b32_e32 v5, v3
 ; LOOP-NEXT:    v_mov_b32_e32 v4, v2
 ; LOOP-NEXT:    v_mov_b32_e32 v7, v1
 ; LOOP-NEXT:    v_mov_b32_e32 v6, v0
-; LOOP-NEXT:    v_mov_b32_e32 v8, s6
+; LOOP-NEXT:    v_mov_b32_e32 v8, s2
 ; LOOP-NEXT:  .LBB0_1: ; %load-store-loop
 ; LOOP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; LOOP-NEXT:    buffer_load_ubyte v9, v[4:5], s[4:7], 0 addr64
-; LOOP-NEXT:    buffer_load_ubyte v10, v[4:5], s[4:7], 0 addr64 offset:1
-; LOOP-NEXT:    buffer_load_ubyte v11, v[4:5], s[4:7], 0 addr64 offset:2
-; LOOP-NEXT:    buffer_load_ubyte v12, v[4:5], s[4:7], 0 addr64 offset:3
-; LOOP-NEXT:    buffer_load_ubyte v13, v[4:5], s[4:7], 0 addr64 offset:4
-; LOOP-NEXT:    buffer_load_ubyte v14, v[4:5], s[4:7], 0 addr64 offset:5
-; LOOP-NEXT:    buffer_load_ubyte v15, v[4:5], s[4:7], 0 addr64 offset:6
-; LOOP-NEXT:    buffer_load_ubyte v16, v[4:5], s[4:7], 0 addr64 offset:7
-; LOOP-NEXT:    buffer_load_ubyte v17, v[4:5], s[4:7], 0 addr64 offset:8
-; LOOP-NEXT:    s_waitcnt expcnt(6)
-; LOOP-NEXT:    buffer_load_ubyte v18, v[4:5], s[4:7], 0 addr64 offset:9
-; LOOP-NEXT:    s_waitcnt expcnt(5)
-; LOOP-NEXT:    buffer_load_ubyte v19, v[4:5], s[4:7], 0 addr64 offset:10
-; LOOP-NEXT:    s_waitcnt expcnt(4)
-; LOOP-NEXT:    buffer_load_ubyte v20, v[4:5], s[4:7], 0 addr64 offset:11
-; LOOP-NEXT:    s_waitcnt expcnt(3)
-; LOOP-NEXT:    buffer_load_ubyte v21, v[4:5], s[4:7], 0 addr64 offset:12
-; LOOP-NEXT:    s_waitcnt expcnt(2)
-; LOOP-NEXT:    buffer_load_ubyte v22, v[4:5], s[4:7], 0 addr64 offset:13
 ; LOOP-NEXT:    s_waitcnt expcnt(1)
-; LOOP-NEXT:    buffer_load_ubyte v23, v[4:5], s[4:7], 0 addr64 offset:14
+; LOOP-NEXT:    buffer_load_ubyte v9, v[4:5], s[0:3], 0 addr64
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    buffer_load_ubyte v10, v[4:5], s[0:3], 0 addr64 offset:1
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v10, 8, v10
+; LOOP-NEXT:    v_or_b32_e32 v9, v10, v9
+; LOOP-NEXT:    buffer_load_ubyte v10, v[4:5], s[0:3], 0 addr64 offset:2
+; LOOP-NEXT:    buffer_load_ubyte v11, v[4:5], s[0:3], 0 addr64 offset:3
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v11, 24, v11
+; LOOP-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; LOOP-NEXT:    v_or_b32_e32 v10, v11, v10
+; LOOP-NEXT:    v_or_b32_e32 v9, v10, v9
+; LOOP-NEXT:    buffer_load_ubyte v10, v[4:5], s[0:3], 0 addr64 offset:4
+; LOOP-NEXT:    buffer_load_ubyte v11, v[4:5], s[0:3], 0 addr64 offset:5
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v11, 8, v11
+; LOOP-NEXT:    v_or_b32_e32 v10, v11, v10
+; LOOP-NEXT:    buffer_load_ubyte v11, v[4:5], s[0:3], 0 addr64 offset:6
+; LOOP-NEXT:    buffer_load_ubyte v12, v[4:5], s[0:3], 0 addr64 offset:7
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v12, 24, v12
+; LOOP-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; LOOP-NEXT:    v_or_b32_e32 v11, v12, v11
+; LOOP-NEXT:    v_or_b32_e32 v10, v11, v10
+; LOOP-NEXT:    buffer_load_ubyte v11, v[4:5], s[0:3], 0 addr64 offset:8
+; LOOP-NEXT:    buffer_load_ubyte v12, v[4:5], s[0:3], 0 addr64 offset:9
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v12, 8, v12
+; LOOP-NEXT:    v_or_b32_e32 v11, v12, v11
+; LOOP-NEXT:    buffer_load_ubyte v12, v[4:5], s[0:3], 0 addr64 offset:10
+; LOOP-NEXT:    buffer_load_ubyte v13, v[4:5], s[0:3], 0 addr64 offset:11
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v13, 24, v13
+; LOOP-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; LOOP-NEXT:    v_or_b32_e32 v12, v13, v12
+; LOOP-NEXT:    v_or_b32_e32 v11, v12, v11
+; LOOP-NEXT:    buffer_load_ubyte v12, v[4:5], s[0:3], 0 addr64 offset:12
+; LOOP-NEXT:    buffer_load_ubyte v13, v[4:5], s[0:3], 0 addr64 offset:13
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v13, 8, v13
+; LOOP-NEXT:    v_or_b32_e32 v12, v13, v12
+; LOOP-NEXT:    buffer_load_ubyte v13, v[4:5], s[0:3], 0 addr64 offset:14
+; LOOP-NEXT:    buffer_load_ubyte v14, v[4:5], s[0:3], 0 addr64 offset:15
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v14, 24, v14
+; LOOP-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; LOOP-NEXT:    v_or_b32_e32 v13, v14, v13
+; LOOP-NEXT:    v_or_b32_e32 v12, v13, v12
+; LOOP-NEXT:    v_lshrrev_b32_e32 v13, 16, v9
+; LOOP-NEXT:    v_bfe_u32 v14, v9, 8, 8
+; LOOP-NEXT:    buffer_store_byte v9, v[6:7], s[0:3], 0 addr64
+; LOOP-NEXT:    buffer_store_byte v14, v[6:7], s[0:3], 0 addr64 offset:1
+; LOOP-NEXT:    s_waitcnt expcnt(1)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v9, 24, v9
+; LOOP-NEXT:    buffer_store_byte v13, v[6:7], s[0:3], 0 addr64 offset:2
+; LOOP-NEXT:    buffer_store_byte v9, v[6:7], s[0:3], 0 addr64 offset:3
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v9, 16, v10
+; LOOP-NEXT:    v_bfe_u32 v13, v10, 8, 8
+; LOOP-NEXT:    buffer_store_byte v10, v[6:7], s[0:3], 0 addr64 offset:4
+; LOOP-NEXT:    buffer_store_byte v13, v[6:7], s[0:3], 0 addr64 offset:5
+; LOOP-NEXT:    s_waitcnt expcnt(1)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v10, 24, v10
+; LOOP-NEXT:    buffer_store_byte v9, v[6:7], s[0:3], 0 addr64 offset:6
+; LOOP-NEXT:    buffer_store_byte v10, v[6:7], s[0:3], 0 addr64 offset:7
+; LOOP-NEXT:    s_waitcnt expcnt(1)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v9, 16, v11
 ; LOOP-NEXT:    s_waitcnt expcnt(0)
-; LOOP-NEXT:    buffer_load_ubyte v24, v[4:5], s[4:7], 0 addr64 offset:15
+; LOOP-NEXT:    v_bfe_u32 v10, v11, 8, 8
+; LOOP-NEXT:    buffer_store_byte v11, v[6:7], s[0:3], 0 addr64 offset:8
+; LOOP-NEXT:    buffer_store_byte v10, v[6:7], s[0:3], 0 addr64 offset:9
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v10, 24, v11
+; LOOP-NEXT:    buffer_store_byte v9, v[6:7], s[0:3], 0 addr64 offset:10
+; LOOP-NEXT:    buffer_store_byte v10, v[6:7], s[0:3], 0 addr64 offset:11
+; LOOP-NEXT:    s_waitcnt expcnt(1)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v9, 16, v12
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_bfe_u32 v10, v12, 8, 8
+; LOOP-NEXT:    buffer_store_byte v12, v[6:7], s[0:3], 0 addr64 offset:12
+; LOOP-NEXT:    buffer_store_byte v10, v[6:7], s[0:3], 0 addr64 offset:13
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v10, 24, v12
+; LOOP-NEXT:    buffer_store_byte v9, v[6:7], s[0:3], 0 addr64 offset:14
+; LOOP-NEXT:    buffer_store_byte v10, v[6:7], s[0:3], 0 addr64 offset:15
 ; LOOP-NEXT:    v_add_i32_e32 v8, vcc, 1, v8
-; LOOP-NEXT:    s_xor_b64 s[0:1], vcc, -1
-; LOOP-NEXT:    s_xor_b64 s[0:1], s[0:1], -1
-; LOOP-NEXT:    s_and_b64 vcc, s[0:1], exec
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v9, v[6:7], s[4:7], 0 addr64
-; LOOP-NEXT:    buffer_store_byte v10, v[6:7], s[4:7], 0 addr64 offset:1
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v11, v[6:7], s[4:7], 0 addr64 offset:2
-; LOOP-NEXT:    buffer_store_byte v12, v[6:7], s[4:7], 0 addr64 offset:3
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v13, v[6:7], s[4:7], 0 addr64 offset:4
-; LOOP-NEXT:    buffer_store_byte v14, v[6:7], s[4:7], 0 addr64 offset:5
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v15, v[6:7], s[4:7], 0 addr64 offset:6
-; LOOP-NEXT:    buffer_store_byte v16, v[6:7], s[4:7], 0 addr64 offset:7
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v17, v[6:7], s[4:7], 0 addr64 offset:8
-; LOOP-NEXT:    buffer_store_byte v18, v[6:7], s[4:7], 0 addr64 offset:9
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v19, v[6:7], s[4:7], 0 addr64 offset:10
-; LOOP-NEXT:    buffer_store_byte v20, v[6:7], s[4:7], 0 addr64 offset:11
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v21, v[6:7], s[4:7], 0 addr64 offset:12
-; LOOP-NEXT:    buffer_store_byte v22, v[6:7], s[4:7], 0 addr64 offset:13
-; LOOP-NEXT:    s_waitcnt vmcnt(14)
-; LOOP-NEXT:    buffer_store_byte v23, v[6:7], s[4:7], 0 addr64 offset:14
-; LOOP-NEXT:    buffer_store_byte v24, v[6:7], s[4:7], 0 addr64 offset:15
-; LOOP-NEXT:    v_add_i32_e64 v6, s[0:1], 16, v6
-; LOOP-NEXT:    v_addc_u32_e64 v7, s[0:1], 0, v7, s[0:1]
-; LOOP-NEXT:    v_add_i32_e64 v4, s[0:1], 16, v4
-; LOOP-NEXT:    v_addc_u32_e64 v5, s[0:1], 0, v5, s[0:1]
+; LOOP-NEXT:    s_xor_b64 s[4:5], vcc, -1
+; LOOP-NEXT:    v_add_i32_e32 v6, vcc, 16, v6
+; LOOP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; LOOP-NEXT:    v_add_i32_e32 v4, vcc, 16, v4
+; LOOP-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; LOOP-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
+; LOOP-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; LOOP-NEXT:    s_cbranch_vccnz .LBB0_1
 ; LOOP-NEXT:  ; %bb.2: ; %memcpy-split
 ; LOOP-NEXT:    s_mov_b32 s2, 0
 ; LOOP-NEXT:    s_mov_b32 s3, 0xf000
 ; LOOP-NEXT:    s_mov_b64 s[0:1], 0
-; LOOP-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:16
-; LOOP-NEXT:    buffer_load_ubyte v5, v[2:3], s[0:3], 0 addr64 offset:17
+; LOOP-NEXT:    buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:17
+; LOOP-NEXT:    buffer_load_ubyte v5, v[2:3], s[0:3], 0 addr64 offset:19
 ; LOOP-NEXT:    buffer_load_ubyte v6, v[2:3], s[0:3], 0 addr64 offset:18
-; LOOP-NEXT:    buffer_load_ubyte v2, v[2:3], s[0:3], 0 addr64 offset:19
-; LOOP-NEXT:    s_waitcnt vmcnt(3)
-; LOOP-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:16
-; LOOP-NEXT:    s_waitcnt vmcnt(3)
-; LOOP-NEXT:    buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:17
-; LOOP-NEXT:    s_waitcnt vmcnt(3)
-; LOOP-NEXT:    buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:18
+; LOOP-NEXT:    buffer_load_ubyte v2, v[2:3], s[0:3], 0 addr64 offset:16
 ; LOOP-NEXT:    s_waitcnt vmcnt(3)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v3, 8, v4
+; LOOP-NEXT:    s_waitcnt vmcnt(2)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v4, 24, v5
+; LOOP-NEXT:    s_waitcnt vmcnt(1)
+; LOOP-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
+; LOOP-NEXT:    s_waitcnt vmcnt(0)
+; LOOP-NEXT:    v_or_b32_e32 v2, v3, v2
+; LOOP-NEXT:    v_or_b32_e32 v3, v4, v5
+; LOOP-NEXT:    v_or_b32_e32 v2, v3, v2
+; LOOP-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; LOOP-NEXT:    v_bfe_u32 v4, v2, 8, 8
+; LOOP-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:16
+; LOOP-NEXT:    s_waitcnt expcnt(0)
+; LOOP-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
+; LOOP-NEXT:    buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:17
+; LOOP-NEXT:    buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:18
 ; LOOP-NEXT:    buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:19
 ; LOOP-NEXT:    s_endpgm
 ;
 ; UNROLL-LABEL: memcpy_p1i8:
 ; UNROLL:       ; %bb.0:
 ; UNROLL-NEXT:    s_mov_b32 s2, 0
 ; UNROLL-NEXT:    s_mov_b32 s3, 0xf000
 ; UNROLL-NEXT:    s_mov_b64 s[0:1], 0

llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll

 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v4, v[0:1], off offset:2
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v5, v[0:1], off offset:3
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v6, v[0:1], off offset:4
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v7, v[0:1], off offset:5
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v8, v[0:1], off offset:6
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v9, v[0:1], off offset:7
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v10, v[0:1], off offset:8
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v11, v[0:1], off offset:9
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v12, v[0:1], off offset:10
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v13, v[0:1], off offset:11
+; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v12, v[0:1], off offset:11
+; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v13, v[0:1], off offset:10
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(10)
 ; GFX9-NOUNALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 8, v2
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 24, v5
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 24, v5
+; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v0, v1, v2, v0
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NOUNALIGNED-NEXT:    v_lshl_or_b32 v3, v7, 8, v6
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 16, v8
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 16, v8
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 24, v9
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v1, v3, v4, v5
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 24, v9
+; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v1, v4, v5, v3
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NOUNALIGNED-NEXT:    v_lshl_or_b32 v6, v11, 8, v10
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 16, v12
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 24, v12
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 24, v13
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v2, v6, v7, v8
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 16, v13
+; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v2, v7, v8, v6
 ; GFX9-NOUNALIGNED-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-UNALIGNED-LABEL: v_load_constant_v3i32_align1:
 ; GFX7-UNALIGNED:       ; %bb.0:
 ; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-UNALIGNED-NEXT:    s_mov_b32 s6, 0
 ; GFX7-UNALIGNED-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-UNALIGNED-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-UNALIGNED-NEXT:    buffer_load_dwordx3 v[0:2], v[0:1], s[4:7], 0 addr64
 ; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-NOUNALIGNED-LABEL: v_load_constant_v3i32_align1:
 ; GFX7-NOUNALIGNED:       ; %bb.0:
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s6, 0
 ; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-NOUNALIGNED-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v2, v[0:1], s[4:7], 0 addr64 offset:1
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v3, v[0:1], s[4:7], 0 addr64 offset:2
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v4, v[0:1], s[4:7], 0 addr64 offset:3
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v3, v[0:1], s[4:7], 0 addr64 offset:3
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v4, v[0:1], s[4:7], 0 addr64 offset:2
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v5, v[0:1], s[4:7], 0 addr64 offset:5
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v6, v[0:1], s[4:7], 0 addr64 offset:6
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v7, v[0:1], s[4:7], 0 addr64 offset:7
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v6, v[0:1], s[4:7], 0 addr64 offset:7
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v7, v[0:1], s[4:7], 0 addr64 offset:6
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v8, v[0:1], s[4:7], 0 addr64 offset:9
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v9, v[0:1], s[4:7], 0 addr64 offset:10
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v10, v[0:1], s[4:7], 0 addr64 offset:11
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v9, v[0:1], s[4:7], 0 addr64 offset:11
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v10, v[0:1], s[4:7], 0 addr64 offset:10
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v11, v[0:1], s[4:7], 0 addr64
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v12, v[0:1], s[4:7], 0 addr64 offset:4
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64 offset:8
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(11)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 8, v2
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(10)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(9)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(8)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 8, v5
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(7)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 24, v6
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v6, 24, v7
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 8, v8
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 24, v9
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v9, 24, v10
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v11, v1
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v1, v11
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v4, v12, v4
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v3, v4, v12
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v4, v5, v6
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v7
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v4, v5
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v4, v0, v8
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v1, v3
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v2, v6
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v4, v9
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v5, v7, v0
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v6, v8, v9
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v2, v1
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v4, v3
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v6, v5
 ; GFX7-NOUNALIGNED-NEXT:    s_setpc_b64 s[30:31]
   %load = load <3 x i32>, <3 x i32> addrspace(4)* %ptr, align 1
   ret <3 x i32> %load
 }
 
 define <3 x i32> @v_load_constant_v3i32_align2(<3 x i32> addrspace(4)* %ptr) {
 ; GFX9-UNALIGNED-LABEL: v_load_constant_v3i32_align2:
 ; GFX9-UNALIGNED:       ; %bb.0:

 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v7, v[0:1], s[4:7], 0 addr64 offset:8
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v5
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v6, v1
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v1, v6
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v7, v2
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v2, v7
 ; GFX7-NOUNALIGNED-NEXT:    s_setpc_b64 s[30:31]
   %load = load <3 x i32>, <3 x i32> addrspace(4)* %ptr, align 2
   ret <3 x i32> %load
 }
 
 define <3 x i32> @v_load_constant_v3i32_align4(<3 x i32> addrspace(4)* %ptr) {
 ; GFX9-LABEL: v_load_constant_v3i32_align4:
 ; GFX9:       ; %bb.0:

 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:2
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:3
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:4
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:5
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:6
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:7
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:8
 ; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:9
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:10
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:11
+; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:11
+; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:10
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(10)
 ; GFX9-NOUNALIGNED-NEXT:    v_lshl_or_b32 v0, v2, 8, v1
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 24, v4
+; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v0, v1, v2, v0
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NOUNALIGNED-NEXT:    v_lshl_or_b32 v3, v6, 8, v5
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 24, v8
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v1, v3, v4, v5
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 24, v8
+; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v1, v4, v5, v3
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NOUNALIGNED-NEXT:    v_lshl_or_b32 v6, v10, 8, v9
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 16, v11
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 24, v11
 ; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 24, v12
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v2, v6, v7, v8
+; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 16, v12
+; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v2, v7, v8, v6
 ; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NOUNALIGNED-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-UNALIGNED-LABEL: s_load_constant_v3i32_align1:
 ; GFX7-UNALIGNED:       ; %bb.0:
 ; GFX7-UNALIGNED-NEXT:    s_mov_b32 s2, -1
 ; GFX7-UNALIGNED-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-UNALIGNED-NEXT:    buffer_load_dwordx3 v[0:2], off, s[0:3], 0
 ; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-UNALIGNED-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX7-UNALIGNED-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX7-UNALIGNED-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX7-UNALIGNED-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-NOUNALIGNED-LABEL: s_load_constant_v3i32_align1:
 ; GFX7-NOUNALIGNED:       ; %bb.0:
 ; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0 offset:1
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v1, off, s[0:3], 0 offset:2
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v2, off, s[0:3], 0 offset:3
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v1, off, s[0:3], 0 offset:3
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v2, off, s[0:3], 0 offset:2
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v3, off, s[0:3], 0 offset:5
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v4, off, s[0:3], 0 offset:6
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v5, off, s[0:3], 0 offset:7
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v4, off, s[0:3], 0 offset:7
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v5, off, s[0:3], 0 offset:6
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v6, off, s[0:3], 0 offset:9
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v7, off, s[0:3], 0 offset:10
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v8, off, s[0:3], 0 offset:11
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v7, off, s[0:3], 0 offset:11
+; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v8, off, s[0:3], 0 offset:10
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v9, off, s[0:3], 0
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v10, off, s[0:3], 0 offset:4
 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ubyte v11, off, s[0:3], 0 offset:8
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(11)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(10)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(9)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(8)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(7)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(6)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 24, v5
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 24, v7
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 24, v8
+; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v9, v0
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v9
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v3, v10
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v6, v11, v6
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v3, v6, v7
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v2
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v1, v5
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v3, v8
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v4, v6, v11
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v5, v7, v8
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v3, v2
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v5, v4
 ; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX7-NOUNALIGNED-NEXT:    ; return to shader part epilog
   %load = load <3 x i32>, <3 x i32> addrspace(4)* %ptr, align 1
   ret <3 x i32> %load
 }
 

 ; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v5, off, s[0:3], 0 offset:8
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
 ; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v3
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v4, v1
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v1, v4
 ; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v5, v2
+; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v2, v5
 ; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX7-NOUNALIGNED-NEXT:    ; return to shader part epilog
   %load = load <3 x i32>, <3 x i32> addrspace(4)* %ptr, align 2
   ret <3 x i32> %load
 }
 

llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll

 ; GFX9-NEXT:    ds_read_u8 v3, v0 offset:2
 ; GFX9-NEXT:    ds_read_u8 v4, v0 offset:3
 ; GFX9-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX9-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX9-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX9-NEXT:    ds_read_u8 v8, v0 offset:7
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
-; GFX9-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; GFX9-NEXT:    v_or3_b32 v4, v1, v2, v3
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_or3_b32 v4, v2, v3, v1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v6, 8, v5
-; GFX9-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 24, v8
-; GFX9-NEXT:    v_or3_b32 v1, v1, v2, v3
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 24, v8
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; GFX9-NEXT:    v_or3_b32 v1, v2, v3, v1
 ; GFX9-NEXT:    ds_read_u8 v2, v0 offset:8
 ; GFX9-NEXT:    ds_read_u8 v3, v0 offset:9
 ; GFX9-NEXT:    ds_read_u8 v5, v0 offset:10
 ; GFX9-NEXT:    ds_read_u8 v6, v0 offset:11
 ; GFX9-NEXT:    ds_read_u8 v7, v0 offset:12
 ; GFX9-NEXT:    ds_read_u8 v8, v0 offset:13
 ; GFX9-NEXT:    ds_read_u8 v9, v0 offset:14
 ; GFX9-NEXT:    ds_read_u8 v0, v0 offset:15
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX9-NEXT:    v_lshl_or_b32 v2, v3, 8, v2
-; GFX9-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v5, 24, v6
-; GFX9-NEXT:    v_or3_b32 v2, v2, v3, v5
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
+; GFX9-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT:    v_or3_b32 v2, v3, v5, v2
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX9-NEXT:    v_lshl_or_b32 v3, v8, 8, v7
-; GFX9-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX9-NEXT:    v_or3_b32 v3, v3, v5, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
+; GFX9-NEXT:    v_or3_b32 v3, v0, v5, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v4i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    ds_read_u8 v1, v0
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v2, v0
 ; GFX7-NEXT:    ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:3
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT:    ds_read_u8 v2, v0 offset:3
 ; GFX7-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX7-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX7-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX7-NEXT:    ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT:    ds_read_u8 v9, v0 offset:8
+; GFX7-NEXT:    ds_read_u8 v10, v0 offset:9
+; GFX7-NEXT:    ds_read_u8 v11, v0 offset:10
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_or_b32_e32 v4, v2, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT:    v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT:    v_or_b32_e32 v1, v5, v1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_waitcnt lgkmcnt(3)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v8
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:8
-; GFX7-NEXT:    ds_read_u8 v3, v0 offset:9
-; GFX7-NEXT:    ds_read_u8 v5, v0 offset:10
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT:    ds_read_u8 v7, v0 offset:12
-; GFX7-NEXT:    ds_read_u8 v8, v0 offset:13
-; GFX7-NEXT:    ds_read_u8 v9, v0 offset:14
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    ds_read_u8 v3, v0 offset:11
+; GFX7-NEXT:    ds_read_u8 v5, v0 offset:12
+; GFX7-NEXT:    ds_read_u8 v6, v0 offset:13
+; GFX7-NEXT:    ds_read_u8 v7, v0 offset:14
 ; GFX7-NEXT:    ds_read_u8 v0, v0 offset:15
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v10
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v9
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v8
+; GFX7-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v8
-; GFX7-NEXT:    v_or_b32_e32 v3, v7, v3
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v6
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v5
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT:    v_or_b32_e32 v3, v3, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT:    v_or_b32_e32 v3, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: load_lds_v4i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_read_u8 v1, v0
 ; GFX10-NEXT:    ds_read_u8 v2, v0 offset:1
 ; GFX10-NEXT:    ds_read_u8 v3, v0 offset:2
 ; GFX10-NEXT:    ds_read_u8 v4, v0 offset:3
 ; GFX10-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX10-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX10-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX10-NEXT:    ds_read_u8 v8, v0 offset:7
 ; GFX10-NEXT:    ds_read_u8 v9, v0 offset:8
 ; GFX10-NEXT:    ds_read_u8 v10, v0 offset:9
 ; GFX10-NEXT:    ds_read_u8 v11, v0 offset:10
 ; GFX10-NEXT:    ds_read_u8 v12, v0 offset:11
 ; GFX10-NEXT:    ds_read_u8 v13, v0 offset:12
 ; GFX10-NEXT:    ds_read_u8 v14, v0 offset:13
-; GFX10-NEXT:    ds_read_u8 v15, v0 offset:14
-; GFX10-NEXT:    ds_read_u8 v0, v0 offset:15
+; GFX10-NEXT:    ds_read_u8 v15, v0 offset:15
+; GFX10-NEXT:    ds_read_u8 v0, v0 offset:14
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(14)
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(13)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(12)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(10)
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(9)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(8)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 24, v8
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX10-NEXT:    v_lshl_or_b32 v7, v10, 8, v9
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v11
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 24, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 24, v12
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX10-NEXT:    v_lshl_or_b32 v10, v14, 8, v13
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 24, v15
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 24, v0
-; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v3
-; GFX10-NEXT:    v_or3_b32 v1, v4, v5, v6
-; GFX10-NEXT:    v_or3_b32 v2, v7, v8, v9
-; GFX10-NEXT:    v_or3_b32 v3, v10, v11, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v0
+; GFX10-NEXT:    v_or3_b32 v0, v2, v3, v1
+; GFX10-NEXT:    v_or3_b32 v1, v5, v6, v4
+; GFX10-NEXT:    v_or3_b32 v2, v8, v9, v7
+; GFX10-NEXT:    v_or3_b32 v3, v11, v12, v10
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 1
   ret <4 x i32> %load
 }
 
 define <4 x i32> @load_lds_v4i32_align2(<4 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v4i32_align2:
 ; GFX9:       ; %bb.0:

 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v4i32_align2:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    ds_read_u16 v1, v0
+; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    ds_read_u16 v2, v0 offset:2
 ; GFX7-NEXT:    ds_read_u16 v3, v0 offset:4
 ; GFX7-NEXT:    ds_read_u16 v4, v0 offset:6
 ; GFX7-NEXT:    ds_read_u16 v5, v0 offset:8
 ; GFX7-NEXT:    ds_read_u16 v6, v0 offset:10
 ; GFX7-NEXT:    ds_read_u16 v7, v0 offset:12
 ; GFX7-NEXT:    ds_read_u16 v8, v0 offset:14
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
-; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v8
-; GFX7-NEXT:    v_or_b32_e32 v2, v5, v2
-; GFX7-NEXT:    v_or_b32_e32 v3, v7, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v5
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: load_lds_v4i32_align2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_read_u16 v1, v0
 ; GFX10-NEXT:    ds_read_u16 v2, v0 offset:2

llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll

 ; GFX9-NEXT:    ds_read_u8 v3, v0 offset:2
 ; GFX9-NEXT:    ds_read_u8 v4, v0 offset:3
 ; GFX9-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX9-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX9-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX9-NEXT:    ds_read_u8 v8, v0 offset:7
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
-; GFX9-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; GFX9-NEXT:    v_or3_b32 v3, v1, v2, v3
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_or3_b32 v3, v2, v3, v1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v6, 8, v5
-; GFX9-NEXT:    ds_read_u8 v4, v0 offset:8
-; GFX9-NEXT:    ds_read_u8 v5, v0 offset:9
-; GFX9-NEXT:    ds_read_u8 v6, v0 offset:10
+; GFX9-NEXT:    ds_read_u8 v2, v0 offset:8
+; GFX9-NEXT:    ds_read_u8 v4, v0 offset:9
+; GFX9-NEXT:    ds_read_u8 v5, v0 offset:10
 ; GFX9-NEXT:    ds_read_u8 v0, v0 offset:11
-; GFX9-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v7, 24, v8
-; GFX9-NEXT:    v_or3_b32 v1, v1, v2, v7
+; GFX9-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
+; GFX9-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX9-NEXT:    v_lshl_or_b32 v2, v5, 8, v4
+; GFX9-NEXT:    v_lshl_or_b32 v2, v4, 8, v2
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX9-NEXT:    v_or3_b32 v2, v2, v4, v0
+; GFX9-NEXT:    v_or3_b32 v1, v6, v7, v1
+; GFX9-NEXT:    v_or3_b32 v2, v0, v4, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v3i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    ds_read_u8 v1, v0
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v2, v0
 ; GFX7-NEXT:    ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT:    ds_read_u8 v5, v0 offset:4
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT:    ds_read_u8 v7, v0 offset:6
-; GFX7-NEXT:    ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT:    v_or_b32_e32 v3, v1, v2
+; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT:    v_or_b32_e32 v1, v5, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:8
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:9
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:10
+; GFX7-NEXT:    ds_read_u8 v2, v0 offset:3
+; GFX7-NEXT:    ds_read_u8 v4, v0 offset:4
+; GFX7-NEXT:    ds_read_u8 v5, v0 offset:5
+; GFX7-NEXT:    ds_read_u8 v6, v0 offset:6
+; GFX7-NEXT:    ds_read_u8 v7, v0 offset:7
+; GFX7-NEXT:    ds_read_u8 v8, v0 offset:8
+; GFX7-NEXT:    ds_read_u8 v9, v0 offset:9
+; GFX7-NEXT:    ds_read_u8 v10, v0 offset:10
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX7-NEXT:    ds_read_u8 v0, v0 offset:11
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_or_b32_e32 v3, v2, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v5
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v4
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 24, v8
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v4
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v7
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
 ; GFX7-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v8
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT:    v_or_b32_e32 v2, v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: load_lds_v3i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_read_u8 v1, v0
 ; GFX10-NEXT:    ds_read_u8 v2, v0 offset:1
 ; GFX10-NEXT:    ds_read_u8 v3, v0 offset:2
 ; GFX10-NEXT:    ds_read_u8 v4, v0 offset:3
 ; GFX10-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX10-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX10-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX10-NEXT:    ds_read_u8 v8, v0 offset:7
 ; GFX10-NEXT:    ds_read_u8 v9, v0 offset:8
 ; GFX10-NEXT:    ds_read_u8 v10, v0 offset:9
-; GFX10-NEXT:    ds_read_u8 v11, v0 offset:10
-; GFX10-NEXT:    ds_read_u8 v0, v0 offset:11
+; GFX10-NEXT:    ds_read_u8 v11, v0 offset:11
+; GFX10-NEXT:    ds_read_u8 v0, v0 offset:10
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(10)
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(9)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(8)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 24, v8
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX10-NEXT:    v_lshl_or_b32 v7, v10, 8, v9
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 24, v11
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 24, v0
-; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v3
-; GFX10-NEXT:    v_or3_b32 v1, v4, v5, v6
-; GFX10-NEXT:    v_or3_b32 v2, v7, v8, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX10-NEXT:    v_or3_b32 v0, v2, v3, v1
+; GFX10-NEXT:    v_or3_b32 v1, v5, v6, v4
+; GFX10-NEXT:    v_or3_b32 v2, v8, v9, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 1
   ret <3 x i32> %load
 }
 
 define <3 x i32> @load_lds_v3i32_align2(<3 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v3i32_align2:
 ; GFX9:       ; %bb.0:

 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_lshl_or_b32 v2, v6, 16, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v3i32_align2:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    ds_read_u16 v1, v0
+; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    ds_read_u16 v2, v0 offset:2
 ; GFX7-NEXT:    ds_read_u16 v3, v0 offset:4
 ; GFX7-NEXT:    ds_read_u16 v4, v0 offset:6
 ; GFX7-NEXT:    ds_read_u16 v5, v0 offset:8
 ; GFX7-NEXT:    ds_read_u16 v6, v0 offset:10
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
-; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT:    v_or_b32_e32 v2, v5, v2
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: load_lds_v3i32_align2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_read_u16 v1, v0
 ; GFX10-NEXT:    ds_read_u16 v2, v0 offset:2
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   ret <3 x i32> %load
 }
 
 define <3 x i32> @load_lds_v3i32_align8(<3 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v3i32_align8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    ds_read_b64 v[0:1], v0
+; GFX9-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1
 ; GFX9-NEXT:    ds_read_b32 v2, v2 offset:8
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v3i32_align8:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX7-NEXT:    s_mov_b32 m0, -1
-; GFX7-NEXT:    ds_read_b64 v[0:1], v0
+; GFX7-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1
 ; GFX7-NEXT:    ds_read_b32 v2, v2 offset:8
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: load_lds_v3i32_align8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0
-; GFX10-NEXT:    ds_read_b64 v[0:1], v0
+; GFX10-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1
 ; GFX10-NEXT:    ds_read_b32 v2, v2 offset:8
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 8
   ret <3 x i32> %load
 }
 
 define <3 x i32> @load_lds_v3i32_align16(<3 x i32> addrspace(3)* %ptr) {

llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll

 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v4i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    ds_read_u8 v1, v0
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v2, v0
 ; GFX7-NEXT:    ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:3
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT:    ds_read_u8 v2, v0 offset:3
 ; GFX7-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX7-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX7-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX7-NEXT:    ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT:    ds_read_u8 v9, v0 offset:8
+; GFX7-NEXT:    ds_read_u8 v10, v0 offset:9
+; GFX7-NEXT:    ds_read_u8 v11, v0 offset:10
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_or_b32_e32 v4, v2, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT:    v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT:    v_or_b32_e32 v1, v5, v1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_waitcnt lgkmcnt(3)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v8
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:8
-; GFX7-NEXT:    ds_read_u8 v3, v0 offset:9
-; GFX7-NEXT:    ds_read_u8 v5, v0 offset:10
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT:    ds_read_u8 v7, v0 offset:12
-; GFX7-NEXT:    ds_read_u8 v8, v0 offset:13
-; GFX7-NEXT:    ds_read_u8 v9, v0 offset:14
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    ds_read_u8 v3, v0 offset:11
+; GFX7-NEXT:    ds_read_u8 v5, v0 offset:12
+; GFX7-NEXT:    ds_read_u8 v6, v0 offset:13
+; GFX7-NEXT:    ds_read_u8 v7, v0 offset:14
 ; GFX7-NEXT:    ds_read_u8 v0, v0 offset:15
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v10
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v9
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v8
+; GFX7-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v8
-; GFX7-NEXT:    v_or_b32_e32 v3, v7, v3
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v6
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v5
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT:    v_or_b32_e32 v3, v3, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT:    v_or_b32_e32 v3, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: load_lds_v4i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_read_u8 v1, v0
 ; GFX10-NEXT:    ds_read_u8 v2, v0 offset:1
 ; GFX10-NEXT:    ds_read_u8 v3, v0 offset:2
 ; GFX10-NEXT:    ds_read_u8 v4, v0 offset:3
 ; GFX10-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX10-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX10-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX10-NEXT:    ds_read_u8 v8, v0 offset:7
 ; GFX10-NEXT:    ds_read_u8 v9, v0 offset:8
 ; GFX10-NEXT:    ds_read_u8 v10, v0 offset:9
 ; GFX10-NEXT:    ds_read_u8 v11, v0 offset:10
 ; GFX10-NEXT:    ds_read_u8 v12, v0 offset:11
 ; GFX10-NEXT:    ds_read_u8 v13, v0 offset:12
 ; GFX10-NEXT:    ds_read_u8 v14, v0 offset:13
-; GFX10-NEXT:    ds_read_u8 v15, v0 offset:14
-; GFX10-NEXT:    ds_read_u8 v0, v0 offset:15
+; GFX10-NEXT:    ds_read_u8 v15, v0 offset:15
+; GFX10-NEXT:    ds_read_u8 v0, v0 offset:14
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(14)
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(13)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(12)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(10)
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(9)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(8)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 24, v8
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX10-NEXT:    v_lshl_or_b32 v7, v10, 8, v9
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v11
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 24, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 24, v12
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX10-NEXT:    v_lshl_or_b32 v10, v14, 8, v13
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 24, v15
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 24, v0
-; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v3
-; GFX10-NEXT:    v_or3_b32 v1, v4, v5, v6
-; GFX10-NEXT:    v_or3_b32 v2, v7, v8, v9
-; GFX10-NEXT:    v_or3_b32 v3, v10, v11, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v0
+; GFX10-NEXT:    v_or3_b32 v0, v2, v3, v1
+; GFX10-NEXT:    v_or3_b32 v1, v5, v6, v4
+; GFX10-NEXT:    v_or3_b32 v2, v8, v9, v7
+; GFX10-NEXT:    v_or3_b32 v3, v11, v12, v10
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 1
   ret <4 x i32> %load
 }
 
 define <3 x i32> @load_lds_v3i32_align1(<3 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v3i32_align1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    ds_read_b96 v[0:2], v0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v3i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    ds_read_u8 v1, v0
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v2, v0
 ; GFX7-NEXT:    ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT:    ds_read_u8 v5, v0 offset:4
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT:    ds_read_u8 v7, v0 offset:6
-; GFX7-NEXT:    ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT:    v_or_b32_e32 v3, v1, v2
+; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT:    v_or_b32_e32 v1, v5, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:8
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:9
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:10
+; GFX7-NEXT:    ds_read_u8 v2, v0 offset:3
+; GFX7-NEXT:    ds_read_u8 v4, v0 offset:4
+; GFX7-NEXT:    ds_read_u8 v5, v0 offset:5
+; GFX7-NEXT:    ds_read_u8 v6, v0 offset:6
+; GFX7-NEXT:    ds_read_u8 v7, v0 offset:7
+; GFX7-NEXT:    ds_read_u8 v8, v0 offset:8
+; GFX7-NEXT:    ds_read_u8 v9, v0 offset:9
+; GFX7-NEXT:    ds_read_u8 v10, v0 offset:10
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX7-NEXT:    ds_read_u8 v0, v0 offset:11
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT:    v_or_b32_e32 v3, v2, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(6)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v5
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v4
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 24, v8
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v4
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 24, v7
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
 ; GFX7-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v8
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT:    v_or_b32_e32 v2, v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: load_lds_v3i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_read_u8 v1, v0
 ; GFX10-NEXT:    ds_read_u8 v2, v0 offset:1
 ; GFX10-NEXT:    ds_read_u8 v3, v0 offset:2
 ; GFX10-NEXT:    ds_read_u8 v4, v0 offset:3
 ; GFX10-NEXT:    ds_read_u8 v5, v0 offset:4
 ; GFX10-NEXT:    ds_read_u8 v6, v0 offset:5
 ; GFX10-NEXT:    ds_read_u8 v7, v0 offset:6
 ; GFX10-NEXT:    ds_read_u8 v8, v0 offset:7
 ; GFX10-NEXT:    ds_read_u8 v9, v0 offset:8
 ; GFX10-NEXT:    ds_read_u8 v10, v0 offset:9
-; GFX10-NEXT:    ds_read_u8 v11, v0 offset:10
-; GFX10-NEXT:    ds_read_u8 v0, v0 offset:11
+; GFX10-NEXT:    ds_read_u8 v11, v0 offset:11
+; GFX10-NEXT:    ds_read_u8 v0, v0 offset:10
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(10)
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(9)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(8)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(6)
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(5)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 24, v8
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX10-NEXT:    v_lshl_or_b32 v7, v10, 8, v9
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 24, v11
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 24, v0
-; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v3
-; GFX10-NEXT:    v_or3_b32 v1, v4, v5, v6
-; GFX10-NEXT:    v_or3_b32 v2, v7, v8, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX10-NEXT:    v_or3_b32 v0, v2, v3, v1
+; GFX10-NEXT:    v_or3_b32 v1, v5, v6, v4
+; GFX10-NEXT:    v_or3_b32 v2, v8, v9, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 1
   ret <3 x i32> %load
 }
 
 define void @store_lds_v4i32_align1(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
 ; GFX9-LABEL: store_lds_v4i32_align1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: store_lds_v4i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-NEXT:    v_bfe_u32 v6, v1, 8, 8
 ; GFX7-NEXT:    ds_write_b8 v0, v1
-; GFX7-NEXT:    ds_write_b8 v0, v5 offset:1
-; GFX7-NEXT:    ds_write_b8 v0, v6 offset:2
-; GFX7-NEXT:    ds_write_b8 v0, v7 offset:3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
+; GFX7-NEXT:    ds_write_b8 v0, v6 offset:1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
+; GFX7-NEXT:    ds_write_b8 v0, v5 offset:2
+; GFX7-NEXT:    ds_write_b8 v0, v1 offset:3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT:    v_bfe_u32 v5, v2, 8, 8
 ; GFX7-NEXT:    ds_write_b8 v0, v2 offset:4
-; GFX7-NEXT:    ds_write_b8 v0, v1 offset:5
-; GFX7-NEXT:    ds_write_b8 v0, v5 offset:6
-; GFX7-NEXT:    ds_write_b8 v0, v6 offset:7
-; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 24, v3
+; GFX7-NEXT:    ds_write_b8 v0, v5 offset:5
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT:    ds_write_b8 v0, v1 offset:6
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:7
+; GFX7-NEXT:    v_bfe_u32 v2, v3, 8, 8
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
 ; GFX7-NEXT:    ds_write_b8 v0, v3 offset:8
-; GFX7-NEXT:    ds_write_b8 v0, v1 offset:9
-; GFX7-NEXT:    ds_write_b8 v0, v2 offset:10
-; GFX7-NEXT:    ds_write_b8 v0, v5 offset:11
-; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v4
-; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
-; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v4
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:9
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; GFX7-NEXT:    ds_write_b8 v0, v1 offset:10
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:11
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v4
+; GFX7-NEXT:    v_bfe_u32 v2, v4, 8, 8
 ; GFX7-NEXT:    ds_write_b8 v0, v4 offset:12
-; GFX7-NEXT:    ds_write_b8 v0, v1 offset:13
-; GFX7-NEXT:    ds_write_b8 v0, v2 offset:14
-; GFX7-NEXT:    ds_write_b8 v0, v3 offset:15
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:13
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v4
+; GFX7-NEXT:    ds_write_b8 v0, v1 offset:14
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:15
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: store_lds_v4i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX10-NEXT:    v_lshrrev_b16 v6, 8, v1
 ; GFX10-NEXT:    ds_write_b8 v0, v1
-; GFX10-NEXT:    ds_write_b8 v0, v5 offset:1
-; GFX10-NEXT:    ds_write_b8 v0, v6 offset:2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
-; GFX10-NEXT:    ds_write_b8 v0, v7 offset:3
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX10-NEXT:    v_lshrrev_b16 v8, 8, v2
+; GFX10-NEXT:    v_lshrrev_b16 v7, 8, v5
 ; GFX10-NEXT:    ds_write_b8 v0, v2 offset:4
-; GFX10-NEXT:    ds_write_b8 v0, v1 offset:5
-; GFX10-NEXT:    ds_write_b8 v0, v5 offset:6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 24, v3
-; GFX10-NEXT:    ds_write_b8 v0, v6 offset:7
+; GFX10-NEXT:    ds_write_b8 v0, v6 offset:1
+; GFX10-NEXT:    ds_write_b8 v0, v5 offset:2
+; GFX10-NEXT:    ds_write_b8 v0, v7 offset:3
+; GFX10-NEXT:    v_lshrrev_b16 v2, 8, v1
+; GFX10-NEXT:    ds_write_b8 v0, v8 offset:5
+; GFX10-NEXT:    ds_write_b8 v0, v1 offset:6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; GFX10-NEXT:    v_lshrrev_b16 v5, 8, v3
+; GFX10-NEXT:    ds_write_b8 v0, v2 offset:7
 ; GFX10-NEXT:    ds_write_b8 v0, v3 offset:8
-; GFX10-NEXT:    ds_write_b8 v0, v1 offset:9
-; GFX10-NEXT:    ds_write_b8 v0, v2 offset:10
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 8, v4
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 24, v4
-; GFX10-NEXT:    ds_write_b8 v0, v5 offset:11
+; GFX10-NEXT:    v_lshrrev_b16 v3, 8, v1
+; GFX10-NEXT:    ds_write_b8 v0, v5 offset:9
+; GFX10-NEXT:    v_lshrrev_b16 v5, 8, v4
+; GFX10-NEXT:    ds_write_b8 v0, v1 offset:10
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v2
+; GFX10-NEXT:    ds_write_b8 v0, v3 offset:11
 ; GFX10-NEXT:    ds_write_b8 v0, v4 offset:12
-; GFX10-NEXT:    ds_write_b8 v0, v1 offset:13
+; GFX10-NEXT:    ds_write_b8 v0, v5 offset:13
 ; GFX10-NEXT:    ds_write_b8 v0, v2 offset:14
-; GFX10-NEXT:    ds_write_b8 v0, v3 offset:15
+; GFX10-NEXT:    ds_write_b8 v0, v1 offset:15
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   store <4 x i32> %x, <4 x i32> addrspace(3)* %out, align 1
   ret void
 }
 
 define void @store_lds_v3i32_align1(<3 x i32> addrspace(3)* %out, <3 x i32> %x) {
 ; GFX9-LABEL: store_lds_v3i32_align1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    ds_write_b96 v0, v[1:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: store_lds_v3i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
 ; GFX7-NEXT:    s_mov_b32 m0, -1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    v_bfe_u32 v5, v1, 8, 8
 ; GFX7-NEXT:    ds_write_b8 v0, v1
-; GFX7-NEXT:    ds_write_b8 v0, v4 offset:1
-; GFX7-NEXT:    ds_write_b8 v0, v5 offset:2
-; GFX7-NEXT:    ds_write_b8 v0, v6 offset:3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
+; GFX7-NEXT:    ds_write_b8 v0, v5 offset:1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
+; GFX7-NEXT:    ds_write_b8 v0, v4 offset:2
+; GFX7-NEXT:    ds_write_b8 v0, v1 offset:3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT:    v_bfe_u32 v4, v2, 8, 8
 ; GFX7-NEXT:    ds_write_b8 v0, v2 offset:4
-; GFX7-NEXT:    ds_write_b8 v0, v1 offset:5
-; GFX7-NEXT:    ds_write_b8 v0, v4 offset:6
-; GFX7-NEXT:    ds_write_b8 v0, v5 offset:7
-; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 24, v3
+; GFX7-NEXT:    ds_write_b8 v0, v4 offset:5
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT:    ds_write_b8 v0, v1 offset:6
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:7
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; GFX7-NEXT:    v_bfe_u32 v2, v3, 8, 8
 ; GFX7-NEXT:    ds_write_b8 v0, v3 offset:8
-; GFX7-NEXT:    ds_write_b8 v0, v1 offset:9
-; GFX7-NEXT:    ds_write_b8 v0, v2 offset:10
-; GFX7-NEXT:    ds_write_b8 v0, v4 offset:11
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:9
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; GFX7-NEXT:    ds_write_b8 v0, v1 offset:10
+; GFX7-NEXT:    ds_write_b8 v0, v2 offset:11
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: store_lds_v3i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX10-NEXT:    v_lshrrev_b16 v5, 8, v1
 ; GFX10-NEXT:    ds_write_b8 v0, v1
-; GFX10-NEXT:    ds_write_b8 v0, v4 offset:1
-; GFX10-NEXT:    ds_write_b8 v0, v5 offset:2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
-; GFX10-NEXT:    ds_write_b8 v0, v6 offset:3
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX10-NEXT:    v_lshrrev_b16 v6, 8, v2
+; GFX10-NEXT:    v_lshrrev_b16 v7, 8, v4
 ; GFX10-NEXT:    ds_write_b8 v0, v2 offset:4
-; GFX10-NEXT:    ds_write_b8 v0, v1 offset:5
-; GFX10-NEXT:    ds_write_b8 v0, v4 offset:6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 24, v3
-; GFX10-NEXT:    ds_write_b8 v0, v5 offset:7
+; GFX10-NEXT:    ds_write_b8 v0, v5 offset:1
+; GFX10-NEXT:    ds_write_b8 v0, v4 offset:2
+; GFX10-NEXT:    ds_write_b8 v0, v7 offset:3
+; GFX10-NEXT:    v_lshrrev_b16 v4, 8, v1
+; GFX10-NEXT:    v_lshrrev_b16 v5, 8, v3
+; GFX10-NEXT:    ds_write_b8 v0, v1 offset:6
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v2
+; GFX10-NEXT:    ds_write_b8 v0, v6 offset:5
+; GFX10-NEXT:    ds_write_b8 v0, v4 offset:7
 ; GFX10-NEXT:    ds_write_b8 v0, v3 offset:8
-; GFX10-NEXT:    ds_write_b8 v0, v1 offset:9
+; GFX10-NEXT:    ds_write_b8 v0, v5 offset:9
 ; GFX10-NEXT:    ds_write_b8 v0, v2 offset:10
-; GFX10-NEXT:    ds_write_b8 v0, v4 offset:11
+; GFX10-NEXT:    ds_write_b8 v0, v1 offset:11
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   store <3 x i32> %x, <3 x i32> addrspace(3)* %out, align 1
   ret void
 }
 
 define amdgpu_ps void @test_s_load_constant_v8i32_align1(<8 x i32> addrspace(4)* inreg %ptr, <8 x i32> addrspace(1)* inreg %out) {
 ; GFX9-LABEL: test_s_load_constant_v8i32_align1:

llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll

; GFX10-NEXT:    s_endpgm
   ret void
 }
 
 define amdgpu_kernel void @store_lds_v4i32_align1(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
 ; GFX9-LABEL: store_lds_v4i32_align1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x24
+; GFX9-NEXT:    s_bfe_u32 s0, 8, 0x100000
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX9-NEXT:    s_bfe_u32 s3, s4, 0x100000
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    s_lshr_b32 s3, s3, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    s_lshr_b32 s1, s4, 16
 ; GFX9-NEXT:    ds_write_b8 v1, v0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    s_lshr_b32 s3, s4, 24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:1
+; GFX9-NEXT:    s_lshr_b32 s2, s1, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:2
-; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_bfe_u32 s2, s5, 0x100000
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:3
-; GFX9-NEXT:    s_lshr_b32 s0, s5, 8
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX9-NEXT:    s_lshr_b32 s1, s5, 16
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:4
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    s_lshr_b32 s2, s5, 24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:5
+; GFX9-NEXT:    s_lshr_b32 s2, s1, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:6
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_bfe_u32 s2, s6, 0x100000
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:7
-; GFX9-NEXT:    s_lshr_b32 s0, s6, 8
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    s_lshr_b32 s1, s6, 16
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    s_lshr_b32 s2, s6, 24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:9
+; GFX9-NEXT:    s_lshr_b32 s2, s1, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:10
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_bfe_u32 s2, s7, 0x100000
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:11
-; GFX9-NEXT:    s_lshr_b32 s0, s7, 8
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s7
 ; GFX9-NEXT:    s_lshr_b32 s1, s7, 16
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:12
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    s_lshr_b32 s2, s7, 24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:13
+; GFX9-NEXT:    s_lshr_b32 s0, s1, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:14
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:15
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v4i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
 ; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x9
+; GFX7-NEXT:    s_mov_b32 s1, 0x80008
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s1, s4, 8
+; GFX7-NEXT:    s_bfe_u32 s3, s4, s1
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX7-NEXT:    s_lshr_b32 s2, s4, 16
 ; GFX7-NEXT:    ds_write_b8 v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
-; GFX7-NEXT:    s_lshr_b32 s3, s4, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s3
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:1
+; GFX7-NEXT:    s_lshr_b32 s0, s4, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:2
-; GFX7-NEXT:    v_mov_b32_e32 v0, s3
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:3
-; GFX7-NEXT:    s_lshr_b32 s0, s5, 8
+; GFX7-NEXT:    s_bfe_u32 s2, s5, s1
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s5
-; GFX7-NEXT:    s_lshr_b32 s1, s5, 16
+; GFX7-NEXT:    s_lshr_b32 s0, s5, 16
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:4
-; GFX7-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-NEXT:    s_lshr_b32 s2, s5, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:5
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    s_lshr_b32 s2, s5, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:6
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:7
-; GFX7-NEXT:    s_lshr_b32 s0, s6, 8
+; GFX7-NEXT:    s_bfe_u32 s2, s6, s1
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    s_lshr_b32 s1, s6, 16
+; GFX7-NEXT:    s_lshr_b32 s0, s6, 16
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-NEXT:    s_lshr_b32 s2, s6, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:9
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    s_lshr_b32 s2, s6, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:10
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:11
-; GFX7-NEXT:    s_lshr_b32 s0, s7, 8
+; GFX7-NEXT:    s_bfe_u32 s1, s7, s1
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s7
-; GFX7-NEXT:    s_lshr_b32 s1, s7, 16
+; GFX7-NEXT:    s_lshr_b32 s0, s7, 16
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:12
-; GFX7-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-NEXT:    s_lshr_b32 s2, s7, 24
-; GFX7-NEXT:    ds_write_b8 v1, v0 offset:13
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    ds_write_b8 v1, v0 offset:13
+; GFX7-NEXT:    s_lshr_b32 s1, s7, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:14
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:15
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: store_lds_v4i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x24
+; GFX10-NEXT:    s_bfe_u32 s0, 8, 0x100000
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX10-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s1, s4, 16
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s2
-; GFX10-NEXT:    s_lshr_b32 s1, s4, 16
-; GFX10-NEXT:    s_lshr_b32 s3, s4, 24
-; GFX10-NEXT:    s_lshr_b32 s2, s5, 8
-; GFX10-NEXT:    s_lshr_b32 s4, s5, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s5, 24
+; GFX10-NEXT:    s_lshr_b32 s2, s5, 16
+; GFX10-NEXT:    s_bfe_u32 s4, s5, 0x100000
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s6, 8
-; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
+; GFX10-NEXT:    s_lshr_b32 s3, s3, s0
+; GFX10-NEXT:    s_lshr_b32 s5, s6, 16
+; GFX10-NEXT:    s_bfe_u32 s8, s6, 0x100000
 ; GFX10-NEXT:    v_mov_b32_e32 v3, s6
-; GFX10-NEXT:    v_mov_b32_e32 v4, s0
-; GFX10-NEXT:    v_mov_b32_e32 v5, s1
-; GFX10-NEXT:    v_mov_b32_e32 v10, s5
-; GFX10-NEXT:    s_lshr_b32 s0, s6, 24
+; GFX10-NEXT:    s_lshr_b32 s6, s1, s0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s1
+; GFX10-NEXT:    s_lshr_b32 s1, s4, s0
+; GFX10-NEXT:    s_lshr_b32 s4, s2, s0
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s3
-; GFX10-NEXT:    v_mov_b32_e32 v11, s9
-; GFX10-NEXT:    v_mov_b32_e32 v7, s2
-; GFX10-NEXT:    v_mov_b32_e32 v8, s4
-; GFX10-NEXT:    v_mov_b32_e32 v9, s8
+; GFX10-NEXT:    v_mov_b32_e32 v7, s6
+; GFX10-NEXT:    v_mov_b32_e32 v5, s2
+; GFX10-NEXT:    s_lshr_b32 s2, s8, s0
+; GFX10-NEXT:    v_mov_b32_e32 v8, s1
+; GFX10-NEXT:    v_mov_b32_e32 v9, s4
 ; GFX10-NEXT:    ds_write_b8 v1, v0
 ; GFX10-NEXT:    ds_write_b8 v1, v2 offset:4
-; GFX10-NEXT:    ds_write_b8 v1, v4 offset:1
-; GFX10-NEXT:    ds_write_b8 v1, v5 offset:2
-; GFX10-NEXT:    ds_write_b8 v1, v6 offset:3
-; GFX10-NEXT:    ds_write_b8 v1, v7 offset:5
-; GFX10-NEXT:    ds_write_b8 v1, v8 offset:6
+; GFX10-NEXT:    ds_write_b8 v1, v4 offset:2
+; GFX10-NEXT:    ds_write_b8 v1, v6 offset:1
+; GFX10-NEXT:    ds_write_b8 v1, v7 offset:3
+; GFX10-NEXT:    ds_write_b8 v1, v8 offset:5
+; GFX10-NEXT:    ds_write_b8 v1, v5 offset:6
+; GFX10-NEXT:    v_mov_b32_e32 v0, s5
+; GFX10-NEXT:    v_mov_b32_e32 v10, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s5, s0
 ; GFX10-NEXT:    ds_write_b8 v1, v9 offset:7
-; GFX10-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-NEXT:    s_lshr_b32 s0, s7, 8
-; GFX10-NEXT:    s_lshr_b32 s1, s7, 16
-; GFX10-NEXT:    v_mov_b32_e32 v2, s7
 ; GFX10-NEXT:    ds_write_b8 v1, v3 offset:8
 ; GFX10-NEXT:    ds_write_b8 v1, v10 offset:9
-; GFX10-NEXT:    ds_write_b8 v1, v11 offset:10
-; GFX10-NEXT:    s_lshr_b32 s2, s7, 24
-; GFX10-NEXT:    v_mov_b32_e32 v3, s0
-; GFX10-NEXT:    v_mov_b32_e32 v4, s1
-; GFX10-NEXT:    v_mov_b32_e32 v5, s2
+; GFX10-NEXT:    ds_write_b8 v1, v0 offset:10
+; GFX10-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-NEXT:    s_bfe_u32 s1, s7, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s7, 16
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s7
+; GFX10-NEXT:    v_mov_b32_e32 v3, s1
+; GFX10-NEXT:    s_lshr_b32 s0, s2, s0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s2
+; GFX10-NEXT:    v_mov_b32_e32 v5, s0
 ; GFX10-NEXT:    ds_write_b8 v1, v0 offset:11
 ; GFX10-NEXT:    ds_write_b8 v1, v2 offset:12
 ; GFX10-NEXT:    ds_write_b8 v1, v3 offset:13
 ; GFX10-NEXT:    ds_write_b8 v1, v4 offset:14
 ; GFX10-NEXT:    ds_write_b8 v1, v5 offset:15
 ; GFX10-NEXT:    s_endpgm
   store <4 x i32> %x, <4 x i32> addrspace(3)* %out, align 1
   ret void

llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll

 ; GFX10-NEXT:    s_endpgm
   store <3 x i32> %x, <3 x i32> addrspace(3)* %out
   ret void
 }
 
 define amdgpu_kernel void @store_lds_v3i32_align1(<3 x i32> addrspace(3)* %out, <3 x i32> %x) {
 ; GFX9-LABEL: store_lds_v3i32_align1:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x24
+; GFX9-NEXT:    s_bfe_u32 s0, 8, 0x100000
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s12, 8
-; GFX9-NEXT:    v_mov_b32_e32 v0, s12
+; GFX9-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    s_lshr_b32 s3, s3, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; GFX9-NEXT:    s_lshr_b32 s1, s12, 16
+; GFX9-NEXT:    s_lshr_b32 s1, s4, 16
 ; GFX9-NEXT:    ds_write_b8 v1, v0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    s_lshr_b32 s3, s12, 24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:1
+; GFX9-NEXT:    s_lshr_b32 s2, s1, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:2
-; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_bfe_u32 s2, s5, 0x100000
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:3
-; GFX9-NEXT:    s_lshr_b32 s0, s13, 8
-; GFX9-NEXT:    v_mov_b32_e32 v0, s13
-; GFX9-NEXT:    s_lshr_b32 s1, s13, 16
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    s_lshr_b32 s1, s5, 16
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:4
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    s_lshr_b32 s2, s13, 24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:5
+; GFX9-NEXT:    s_lshr_b32 s2, s1, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:6
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_bfe_u32 s2, s6, 0x100000
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:7
-; GFX9-NEXT:    s_lshr_b32 s0, s14, 8
-; GFX9-NEXT:    v_mov_b32_e32 v0, s14
-; GFX9-NEXT:    s_lshr_b32 s1, s14, 16
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    s_lshr_b32 s1, s6, 16
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    s_lshr_b32 s2, s14, 24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:9
+; GFX9-NEXT:    s_lshr_b32 s0, s1, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:10
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    ds_write_b8 v1, v0 offset:11
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v3i32_align1:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0xd
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
 ; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x9
+; GFX7-NEXT:    s_mov_b32 s1, 0x80008
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s1, s12, 8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s12
+; GFX7-NEXT:    s_bfe_u32 s3, s4, s1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s0
-; GFX7-NEXT:    s_lshr_b32 s2, s12, 16
+; GFX7-NEXT:    s_lshr_b32 s2, s4, 16
 ; GFX7-NEXT:    ds_write_b8 v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
-; GFX7-NEXT:    s_lshr_b32 s3, s12, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s3
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:1
+; GFX7-NEXT:    s_lshr_b32 s0, s4, 24
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:2
-; GFX7-NEXT:    v_mov_b32_e32 v0, s3
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:3
-; GFX7-NEXT:    s_lshr_b32 s0, s13, 8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s13
-; GFX7-NEXT:    s_lshr_b32 s1, s13, 16
+; GFX7-NEXT:    s_bfe_u32 s2, s5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s5
+; GFX7-NEXT:    s_lshr_b32 s0, s5, 16
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:4
-; GFX7-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-NEXT:    s_lshr_b32 s2, s13, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:5
-; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    s_lshr_b32 s2, s5, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:6
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:7
-; GFX7-NEXT:    s_lshr_b32 s0, s14, 8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s14
-; GFX7-NEXT:    s_lshr_b32 s1, s14, 16
+; GFX7-NEXT:    s_bfe_u32 s1, s6, s1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    s_lshr_b32 s0, s6, 16
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:8
-; GFX7-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-NEXT:    s_lshr_b32 s2, s14, 24
-; GFX7-NEXT:    ds_write_b8 v1, v0 offset:9
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    ds_write_b8 v1, v0 offset:9
+; GFX7-NEXT:    s_lshr_b32 s1, s6, 24
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:10
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX7-NEXT:    ds_write_b8 v1, v0 offset:11
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: store_lds_v3i32_align1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x24
+; GFX10-NEXT:    s_bfe_u32 s0, 8, 0x100000
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_lshr_b32 s0, s12, 8
-; GFX10-NEXT:    v_mov_b32_e32 v0, s12
+; GFX10-NEXT:    s_lshr_b32 s1, s4, 16
+; GFX10-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s2
-; GFX10-NEXT:    s_lshr_b32 s5, s13, 24
-; GFX10-NEXT:    s_lshr_b32 s1, s12, 16
-; GFX10-NEXT:    v_mov_b32_e32 v2, s13
-; GFX10-NEXT:    s_lshr_b32 s3, s12, 24
-; GFX10-NEXT:    s_lshr_b32 s6, s14, 8
-; GFX10-NEXT:    v_mov_b32_e32 v4, s0
-; GFX10-NEXT:    v_mov_b32_e32 v9, s5
-; GFX10-NEXT:    s_lshr_b32 s2, s13, 8
-; GFX10-NEXT:    s_lshr_b32 s4, s13, 16
-; GFX10-NEXT:    s_lshr_b32 s7, s14, 16
-; GFX10-NEXT:    v_mov_b32_e32 v3, s14
-; GFX10-NEXT:    v_mov_b32_e32 v5, s1
-; GFX10-NEXT:    s_lshr_b32 s8, s14, 24
+; GFX10-NEXT:    s_lshr_b32 s2, s5, 16
+; GFX10-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s6, 16
+; GFX10-NEXT:    s_bfe_u32 s7, s6, 0x100000
+; GFX10-NEXT:    v_mov_b32_e32 v3, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s1, s0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s1
+; GFX10-NEXT:    s_lshr_b32 s1, s4, s0
+; GFX10-NEXT:    s_lshr_b32 s4, s2, s0
+; GFX10-NEXT:    s_lshr_b32 s3, s3, s0
+; GFX10-NEXT:    v_mov_b32_e32 v5, s2
+; GFX10-NEXT:    s_lshr_b32 s2, s7, s0
+; GFX10-NEXT:    v_mov_b32_e32 v9, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s3
-; GFX10-NEXT:    v_mov_b32_e32 v10, s6
-; GFX10-NEXT:    v_mov_b32_e32 v7, s2
-; GFX10-NEXT:    v_mov_b32_e32 v8, s4
+; GFX10-NEXT:    v_mov_b32_e32 v7, s6
+; GFX10-NEXT:    v_mov_b32_e32 v8, s1
 ; GFX10-NEXT:    ds_write_b8 v1, v0
 ; GFX10-NEXT:    ds_write_b8 v1, v2 offset:4
-; GFX10-NEXT:    ds_write_b8 v1, v4 offset:1
-; GFX10-NEXT:    ds_write_b8 v1, v5 offset:2
-; GFX10-NEXT:    ds_write_b8 v1, v6 offset:3
-; GFX10-NEXT:    ds_write_b8 v1, v7 offset:5
-; GFX10-NEXT:    ds_write_b8 v1, v8 offset:6
-; GFX10-NEXT:    v_mov_b32_e32 v0, s7
-; GFX10-NEXT:    v_mov_b32_e32 v2, s8
+; GFX10-NEXT:    ds_write_b8 v1, v4 offset:2
+; GFX10-NEXT:    ds_write_b8 v1, v5 offset:6
+; GFX10-NEXT:    ds_write_b8 v1, v6 offset:1
+; GFX10-NEXT:    ds_write_b8 v1, v7 offset:3
+; GFX10-NEXT:    ds_write_b8 v1, v8 offset:5
+; GFX10-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-NEXT:    s_lshr_b32 s0, s5, s0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX10-NEXT:    ds_write_b8 v1, v9 offset:7
 ; GFX10-NEXT:    ds_write_b8 v1, v3 offset:8
-; GFX10-NEXT:    ds_write_b8 v1, v10 offset:9
-; GFX10-NEXT:    ds_write_b8 v1, v0 offset:10
-; GFX10-NEXT:    ds_write_b8 v1, v2 offset:11
+; GFX10-NEXT:    ds_write_b8 v1, v0 offset:9
+; GFX10-NEXT:    ds_write_b8 v1, v2 offset:10
+; GFX10-NEXT:    ds_write_b8 v1, v4 offset:11
 ; GFX10-NEXT:    s_endpgm
   store <3 x i32> %x, <3 x i32> addrspace(3)* %out, align 1
   ret void
 }
 
 define amdgpu_kernel void @store_lds_v3i32_align2(<3 x i32> addrspace(3)* %out, <3 x i32> %x) {
 ; GFX9-LABEL: store_lds_v3i32_align2:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s12, 16
-; GFX9-NEXT:    v_mov_b32_e32 v0, s12
+; GFX9-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    ds_write_b16 v1, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    ds_write_b16 v1, v0 offset:2
-; GFX9-NEXT:    s_lshr_b32 s0, s13, 16
-; GFX9-NEXT:    v_mov_b32_e32 v0, s13
+; GFX9-NEXT:    s_lshr_b32 s0, s5, 16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX9-NEXT:    ds_write_b16 v1, v0 offset:4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    ds_write_b16 v1, v0 offset:6
-; GFX9-NEXT:    s_lshr_b32 s0, s14, 16
-; GFX9-NEXT:    v_mov_b32_e32 v0, s14
+; GFX9-NEXT:    s_lshr_b32 s0, s6, 16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    ds_write_b16 v1, v0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    ds_write_b16 v1, v0 offset:10
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v3i32_align2:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0xd
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
 ; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x9
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshr_b32 s1, s12, 16
-; GFX7-NEXT:    v_mov_b32_e32 v0, s12
+; GFX7-NEXT:    s_lshr_b32 s1, s4, 16
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX7-NEXT:    ds_write_b16 v1, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX7-NEXT:    ds_write_b16 v1, v0 offset:2
-; GFX7-NEXT:    s_lshr_b32 s0, s13, 16
-; GFX7-NEXT:    v_mov_b32_e32 v0, s13
+; GFX7-NEXT:    s_lshr_b32 s0, s5, 16
+; GFX7-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX7-NEXT:    ds_write_b16 v1, v0 offset:4
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b16 v1, v0 offset:6
-; GFX7-NEXT:    s_lshr_b32 s0, s14, 16
-; GFX7-NEXT:    v_mov_b32_e32 v0, s14
+; GFX7-NEXT:    s_lshr_b32 s0, s6, 16
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    ds_write_b16 v1, v0 offset:8
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    ds_write_b16 v1, v0 offset:10
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: store_lds_v3i32_align2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, s12
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s2
-; GFX10-NEXT:    s_lshr_b32 s0, s12, 16
-; GFX10-NEXT:    v_mov_b32_e32 v2, s13
-; GFX10-NEXT:    s_lshr_b32 s1, s13, 16
-; GFX10-NEXT:    v_mov_b32_e32 v3, s14
-; GFX10-NEXT:    s_lshr_b32 s2, s14, 16
+; GFX10-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-NEXT:    s_lshr_b32 s1, s5, 16
+; GFX10-NEXT:    v_mov_b32_e32 v3, s6
+; GFX10-NEXT:    s_lshr_b32 s2, s6, 16
 ; GFX10-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX10-NEXT:    v_mov_b32_e32 v5, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s2
 ; GFX10-NEXT:    ds_write_b16 v1, v0
 ; GFX10-NEXT:    ds_write_b16 v1, v2 offset:4
 ; GFX10-NEXT:    ds_write_b16 v1, v3 offset:8
 ; GFX10-NEXT:    ds_write_b16 v1, v4 offset:2
 ; GFX10-NEXT:    ds_write_b16 v1, v5 offset:6
 ; GFX10-NEXT:    ds_write_b16 v1, v6 offset:10
 ; GFX10-NEXT:    s_endpgm
   store <3 x i32> %x, <3 x i32> addrspace(3)* %out, align 2
   ret void
 }
 
 define amdgpu_kernel void @store_lds_v3i32_align4(<3 x i32> addrspace(3)* %out, <3 x i32> %x) {
 ; GFX9-LABEL: store_lds_v3i32_align4:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s12
-; GFX9-NEXT:    v_mov_b32_e32 v1, s13
-; GFX9-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
-; GFX9-NEXT:    ds_write_b32 v2, v3 offset:8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-NEXT:    ds_write2_b32 v1, v0, v2 offset1:1
+; GFX9-NEXT:    ds_write_b32 v1, v3 offset:8
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v3i32_align4:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0xd
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
 ; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x9
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s12
-; GFX7-NEXT:    v_mov_b32_e32 v1, s13
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s14
-; GFX7-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
-; GFX7-NEXT:    ds_write_b32 v2, v3 offset:8
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s5
+; GFX7-NEXT:    ds_write2_b32 v1, v0, v2 offset1:1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    ds_write_b32 v1, v0 offset:8
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: store_lds_v3i32_align4:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, s12
-; GFX10-NEXT:    v_mov_b32_e32 v1, s13
-; GFX10-NEXT:    v_mov_b32_e32 v2, s2
-; GFX10-NEXT:    v_mov_b32_e32 v3, s14
-; GFX10-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
-; GFX10-NEXT:    ds_write_b32 v2, v3 offset:8
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-NEXT:    v_mov_b32_e32 v3, s6
+; GFX10-NEXT:    ds_write2_b32 v1, v0, v2 offset1:1
+; GFX10-NEXT:    ds_write_b32 v1, v3 offset:8
 ; GFX10-NEXT:    s_endpgm
   store <3 x i32> %x, <3 x i32> addrspace(3)* %out, align 4
   ret void
 }
 
 define amdgpu_kernel void @store_lds_v3i32_align8(<3 x i32> addrspace(3)* %out, <3 x i32> %x) {
 ; GFX9-LABEL: store_lds_v3i32_align8:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s12
-; GFX9-NEXT:    v_mov_b32_e32 v1, s13
-; GFX9-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s14
-; GFX9-NEXT:    ds_write_b64 v2, v[0:1]
-; GFX9-NEXT:    ds_write_b32 v2, v3 offset:8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-NEXT:    ds_write2_b32 v1, v0, v2 offset1:1
+; GFX9-NEXT:    ds_write_b32 v1, v3 offset:8
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v3i32_align8:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0xd
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
 ; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x9
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s12
-; GFX7-NEXT:    v_mov_b32_e32 v1, s13
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s14
-; GFX7-NEXT:    ds_write_b64 v2, v[0:1]
-; GFX7-NEXT:    ds_write_b32 v2, v3 offset:8
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s5
+; GFX7-NEXT:    ds_write2_b32 v1, v0, v2 offset1:1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    ds_write_b32 v1, v0 offset:8
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: store_lds_v3i32_align8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x34
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
 ; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, s12
-; GFX10-NEXT:    v_mov_b32_e32 v1, s13
-; GFX10-NEXT:    v_mov_b32_e32 v2, s2
-; GFX10-NEXT:    v_mov_b32_e32 v3, s14
-; GFX10-NEXT:    ds_write_b64 v2, v[0:1]
-; GFX10-NEXT:    ds_write_b32 v2, v3 offset:8
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-NEXT:    v_mov_b32_e32 v3, s6
+; GFX10-NEXT:    ds_write2_b32 v1, v0, v2 offset1:1
+; GFX10-NEXT:    ds_write_b32 v1, v3 offset:8
 ; GFX10-NEXT:    s_endpgm
   store <3 x i32> %x, <3 x i32> addrspace(3)* %out, align 8
   ret void
 }
 
 define amdgpu_kernel void @store_lds_v3i32_align16(<3 x i32> addrspace(3)* %out, <3 x i32> %x) {
 ; GFX9-LABEL: store_lds_v3i32_align16:
 ; GFX9:       ; %bb.0:

llvm/test/CodeGen/AMDGPU/GlobalISel/widen-i8-i16-scalar-loads.ll

 ; GFX8-LABEL: constant_sextload_i8_align2:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    flat_load_sbyte v2, v[0:1]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_add_u32 s2, s0, 2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_add_u32 s0, s0, 2
-; GFX8-NEXT:    s_addc_u32 s1, s1, 0
+; GFX8-NEXT:    s_addc_u32 s3, s1, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    flat_store_short v[0:1], v3
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: constant_sextload_i8_align2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)

 ; GFX8-LABEL: constant_zextload_i8_align2:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_add_u32 s2, s0, 2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_add_u32 s0, s0, 2
-; GFX8-NEXT:    s_addc_u32 s1, s1, 0
+; GFX8-NEXT:    s_addc_u32 s3, s1, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_store_short v[0:1], v2
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    flat_store_short v[0:1], v3
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: constant_zextload_i8_align2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)

llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll

 ; GFX9-GISEL-LABEL: v_cttz_zero_undef_i32_with_select:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-GISEL-NEXT:    global_load_ubyte v1, v0, s[2:3]
 ; GFX9-GISEL-NEXT:    global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:2
-; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:2
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; GFX9-GISEL-NEXT:    v_or3_b32 v1, v1, v2, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX9-GISEL-NEXT:    v_or3_b32 v1, v2, v3, v1
 ; GFX9-GISEL-NEXT:    v_ffbl_b32_e32 v2, v1
 ; GFX9-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
 ; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, 32, v2, vcc
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
   %val = load i32, i32 addrspace(1)* %arrayidx, align 1
   %cttz = tail call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone
   %cttz_ret = icmp ne i32 %val, 0

 ; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
 ;
 ; GFX9-GISEL-LABEL: v_cttz_zero_undef_i64_with_select:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT:    global_load_ubyte v0, v1, s[2:3] offset:1
-; GFX9-GISEL-NEXT:    global_load_ubyte v2, v1, s[2:3] offset:3
-; GFX9-GISEL-NEXT:    global_load_ubyte v3, v1, s[2:3] offset:5
-; GFX9-GISEL-NEXT:    global_load_ubyte v4, v1, s[2:3] offset:7
-; GFX9-GISEL-NEXT:    global_load_ubyte v5, v1, s[2:3]
-; GFX9-GISEL-NEXT:    global_load_ubyte v6, v1, s[2:3] offset:2
-; GFX9-GISEL-NEXT:    global_load_ubyte v7, v1, s[2:3] offset:4
+; GFX9-GISEL-NEXT:    global_load_ubyte v0, v1, s[2:3]
+; GFX9-GISEL-NEXT:    global_load_ubyte v2, v1, s[2:3] offset:1
+; GFX9-GISEL-NEXT:    global_load_ubyte v3, v1, s[2:3] offset:2
+; GFX9-GISEL-NEXT:    global_load_ubyte v4, v1, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v5, v1, s[2:3] offset:4
+; GFX9-GISEL-NEXT:    global_load_ubyte v6, v1, s[2:3] offset:5
+; GFX9-GISEL-NEXT:    global_load_ubyte v7, v1, s[2:3] offset:7
 ; GFX9-GISEL-NEXT:    global_load_ubyte v8, v1, s[2:3] offset:6
-; GFX9-GISEL-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-GISEL-NEXT:    v_and_b32_e32 v0, s2, v0
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-GISEL-NEXT:    v_and_b32_e32 v2, s2, v2
+; GFX9-GISEL-NEXT:    v_lshl_or_b32 v0, v2, 8, v0
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-GISEL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-GISEL-NEXT:    v_and_b32_e32 v4, s2, v4
-; GFX9-GISEL-NEXT:    v_lshlrev_b16_e32 v3, 8, v3
-; GFX9-GISEL-NEXT:    v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-GISEL-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-GISEL-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-GISEL-NEXT:    v_or3_b32 v2, v2, v3, v0
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-GISEL-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-GISEL-NEXT:    v_or_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v5, 24, v7
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT:    v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT:    v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT:    v_or_b32_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT:    v_bfe_u32 v3, v3, 0, 16
-; GFX9-GISEL-NEXT:    v_bfe_u32 v4, v4, 0, 16
-; GFX9-GISEL-NEXT:    v_bfe_u32 v0, v0, 0, 16
-; GFX9-GISEL-NEXT:    v_bfe_u32 v2, v2, 0, 16
-; GFX9-GISEL-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-GISEL-NEXT:    v_lshl_or_b32 v2, v2, 16, v0
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-GISEL-NEXT:    v_or3_b32 v3, v5, v6, v4
 ; GFX9-GISEL-NEXT:    v_ffbl_b32_e32 v4, v3
 ; GFX9-GISEL-NEXT:    v_ffbl_b32_e32 v0, v2
 ; GFX9-GISEL-NEXT:    v_add_u32_e32 v4, 32, v4
 ; GFX9-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
 ; GFX9-GISEL-NEXT:    v_min_u32_e32 v0, v0, v4
 ; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v0, 32, v0, vcc
 ; GFX9-GISEL-NEXT:    global_store_dwordx2 v1, v[0:1], s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm

 ; GFX9-GISEL-LABEL: v_cttz_i32_sel_eq_neg1:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-GISEL-NEXT:    global_load_ubyte v1, v0, s[2:3]
 ; GFX9-GISEL-NEXT:    global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:2
-; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:2
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; GFX9-GISEL-NEXT:    v_or3_b32 v1, v1, v2, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX9-GISEL-NEXT:    v_or3_b32 v1, v2, v3, v1
 ; GFX9-GISEL-NEXT:    v_ffbl_b32_e32 v2, v1
 ; GFX9-GISEL-NEXT:    v_min_u32_e32 v2, 32, v2
 ; GFX9-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, -1, vcc
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
   %val = load i32, i32 addrspace(1)* %arrayidx, align 1
   %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone

 ; GFX9-GISEL-LABEL: v_cttz_i32_sel_ne_neg1:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-GISEL-NEXT:    global_load_ubyte v1, v0, s[2:3]
 ; GFX9-GISEL-NEXT:    global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:2
-; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:2
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; GFX9-GISEL-NEXT:    v_or3_b32 v1, v1, v2, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX9-GISEL-NEXT:    v_or3_b32 v1, v2, v3, v1
 ; GFX9-GISEL-NEXT:    v_ffbl_b32_e32 v2, v1
 ; GFX9-GISEL-NEXT:    v_min_u32_e32 v2, 32, v2
 ; GFX9-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
 ; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, -1, v2, vcc
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
   %val = load i32, i32 addrspace(1)* %arrayidx, align 1
   %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone

 ; GFX9-GISEL-LABEL: v_cttz_i32_sel_ne_bitwidth:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-GISEL-NEXT:    global_load_ubyte v1, v0, s[2:3]
 ; GFX9-GISEL-NEXT:    global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:2
-; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v3, v0, s[2:3] offset:3
+; GFX9-GISEL-NEXT:    global_load_ubyte v4, v0, s[2:3] offset:2
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; GFX9-GISEL-NEXT:    v_or3_b32 v1, v1, v2, v3
+; GFX9-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX9-GISEL-NEXT:    v_or3_b32 v1, v2, v3, v1
 ; GFX9-GISEL-NEXT:    v_ffbl_b32_e32 v1, v1
 ; GFX9-GISEL-NEXT:    v_min_u32_e32 v1, 32, v1
 ; GFX9-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v1
 ; GFX9-GISEL-NEXT:    v_cndmask_b32_e32 v1, -1, v1, vcc
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
   %val = load i32, i32 addrspace(1)* %arrayidx, align 1
   %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone

llvm/test/CodeGen/AMDGPU/ds-alignment.ll

 ; GCN-NEXT:    ds_write_b8 v1, v0
 ; GCN-NEXT:    s_endpgm
   %val = load i8, i8 addrspace(3)* %in, align 1
   store i8 %val, i8 addrspace(3)* %out, align 1
   ret void
 }
 
 define amdgpu_kernel void @ds2align1(i16 addrspace(3)* %in, i16 addrspace(3)* %out) {
-; ALIGNED-LABEL: ds2align1:
-; ALIGNED:       ; %bb.0:
-; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-NEXT:    ds_read_u8 v1, v0
-; ALIGNED-NEXT:    ds_read_u8 v0, v0 offset:1
-; ALIGNED-NEXT:    v_mov_b32_e32 v2, s1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
-; ALIGNED-NEXT:    ds_write_b8 v2, v1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
-; ALIGNED-NEXT:    ds_write_b8 v2, v0 offset:1
-; ALIGNED-NEXT:    s_endpgm
+; ALIGNED-SDAG-LABEL: ds2align1:
+; ALIGNED-SDAG:       ; %bb.0:
+; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v1, v0
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v0, v0 offset:1
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v2, s1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(1)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v2, v1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(1)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v2, v0 offset:1
+; ALIGNED-SDAG-NEXT:    s_endpgm
+;
+; ALIGNED-GISEL-LABEL: ds2align1:
+; ALIGNED-GISEL:       ; %bb.0:
+; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v0
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v0 offset:1
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 8, v1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v1, 8, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v2, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v2, v1 offset:1
+; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds2align1:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read_u16 v0, v0
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1

 ; GCN-NEXT:    ds_write_b16 v1, v0
 ; GCN-NEXT:    s_endpgm
   %val = load i16, i16 addrspace(3)* %in, align 2
   store i16 %val, i16 addrspace(3)* %out, align 2
   ret void
 }
 
 define amdgpu_kernel void @ds4align1(i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
-; ALIGNED-LABEL: ds4align1:
-; ALIGNED:       ; %bb.0:
-; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-NEXT:    ds_read_u8 v1, v0
-; ALIGNED-NEXT:    ds_read_u8 v2, v0 offset:1
-; ALIGNED-NEXT:    ds_read_u8 v3, v0 offset:2
-; ALIGNED-NEXT:    ds_read_u8 v0, v0 offset:3
-; ALIGNED-NEXT:    v_mov_b32_e32 v4, s1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-NEXT:    ds_write_b8 v4, v1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-NEXT:    ds_write_b8 v4, v2 offset:1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-NEXT:    ds_write_b8 v4, v3 offset:2
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-NEXT:    ds_write_b8 v4, v0 offset:3
-; ALIGNED-NEXT:    s_endpgm
+; ALIGNED-SDAG-LABEL: ds4align1:
+; ALIGNED-SDAG:       ; %bb.0:
+; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v1, v0
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v2, v0 offset:1
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v3, v0 offset:2
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v0, v0 offset:3
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v4, s1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v4, v1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v4, v2 offset:1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v4, v3 offset:2
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v4, v0 offset:3
+; ALIGNED-SDAG-NEXT:    s_endpgm
+;
+; ALIGNED-GISEL-LABEL: ds4align1:
+; ALIGNED-GISEL:       ; %bb.0:
+; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v0
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v0 offset:1
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:3
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v0 offset:2
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v4, s1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v0, v2, v0, v1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v2 offset:1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v0, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v1 offset:2
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v0 offset:3
+; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds4align1:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read_b32 v0, v0
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    ds_write_b32 v1, v0
 ; UNALIGNED-NEXT:    s_endpgm
   %val = load i32, i32 addrspace(3)* %in, align 1
   store i32 %val, i32 addrspace(3)* %out, align 1
   ret void
 }
 
 define amdgpu_kernel void @ds4align2(i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
-; ALIGNED-LABEL: ds4align2:
-; ALIGNED:       ; %bb.0:
-; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-NEXT:    ds_read_u16 v1, v0
-; ALIGNED-NEXT:    ds_read_u16 v0, v0 offset:2
-; ALIGNED-NEXT:    v_mov_b32_e32 v2, s1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
-; ALIGNED-NEXT:    ds_write_b16 v2, v1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(1)
-; ALIGNED-NEXT:    ds_write_b16 v2, v0 offset:2
-; ALIGNED-NEXT:    s_endpgm
+; ALIGNED-SDAG-LABEL: ds4align2:
+; ALIGNED-SDAG:       ; %bb.0:
+; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v1, v0
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v0, v0 offset:2
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v2, s1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(1)
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v2, v1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(1)
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v2, v0 offset:2
+; ALIGNED-SDAG-NEXT:    s_endpgm
+;
+; ALIGNED-GISEL-LABEL: ds4align2:
+; ALIGNED-GISEL:       ; %bb.0:
+; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v1, v0
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v0, v0 offset:2
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v2, v1 offset:2
+; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds4align2:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read_b32 v0, v0
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1

 ; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v7 offset:5
 ; ALIGNED-SDAG-NEXT:    s_endpgm
 ;
 ; ALIGNED-GISEL-LABEL: ds8align1:
 ; ALIGNED-GISEL:       ; %bb.0:
 ; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v0
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:1
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:2
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:3
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v0 offset:4
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v0 offset:5
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v0 offset:6
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v0
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v0 offset:1
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:2
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:3
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:4
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v0 offset:5
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v0 offset:6
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v0 offset:7
-; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v2
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v3 offset:1
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v4 offset:2
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v5 offset:3
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v6 offset:4
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v7 offset:5
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v8 offset:6
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v1, v0 offset:7
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(6)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v1, v2, v3, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v2, v6, 8, v5
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v0, v0, v3, v2
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v4, s1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v3 offset:1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v1, 8, v2
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v2 offset:2
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v1 offset:3
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v0 offset:4
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v2 offset:5
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v0, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v1 offset:6
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v4, v0 offset:7
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds8align1:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1

 ; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v1, v0
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v2, v0 offset:2
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v3, v0 offset:4
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v0, v0 offset:6
 ; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v4, s1
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
 ; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v1
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v0 offset:4
 ; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v2 offset:2
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v3 offset:4
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v0 offset:6
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v4, v1 offset:6
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds8align2:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1

 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(11)
 ; ALIGNED-SDAG-NEXT:    ds_write_b8 v12, v0 offset:11
 ; ALIGNED-SDAG-NEXT:    s_endpgm
 ;
 ; ALIGNED-GISEL-LABEL: ds12align1:
 ; ALIGNED-GISEL:       ; %bb.0:
 ; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s0
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v2
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v2 offset:1
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v2 offset:2
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v2 offset:3
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v2 offset:4
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v2 offset:5
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v2 offset:6
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v2 offset:7
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v0
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v0 offset:1
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:2
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:3
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:4
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v0 offset:5
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v0 offset:6
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v0 offset:7
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(6)
-; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v1, v2, v3, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v2, v6, 8, v5
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:8
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:9
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:10
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v0 offset:11
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; ALIGNED-GISEL-NEXT:    v_or3_b32 v0, v0, v1, v3
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
-; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v6, 8, v5
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v3, v4, 8, v3
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
-; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 24, v8
-; ALIGNED-GISEL-NEXT:    v_or3_b32 v1, v1, v3, v4
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v2 offset:8
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v2 offset:9
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v2 offset:10
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v2 offset:11
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v6, 8, v0
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v9, s1
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v8, 24, v0
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v0
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v6 offset:1
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v7 offset:2
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v8 offset:3
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v1 offset:4
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v0 offset:5
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v6 offset:6
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v7 offset:7
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v3 offset:8
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v4 offset:9
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v5 offset:10
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(11)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v9, v2 offset:11
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v0, v0, v4, v3
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v5, s1
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v2, v6, v7, v2
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v4, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v4 offset:1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v1, 8, v3
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v3 offset:2
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v1 offset:3
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v3, 8, v2
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v2 offset:4
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v3 offset:5
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v1 offset:6
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v2 offset:7
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v0 offset:8
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v2 offset:9
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v0, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v1 offset:10
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v5, v0 offset:11
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds12align1:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read_b96 v[0:2], v0

 ; ALIGNED-GISEL:       ; %bb.0:
 ; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v1, v0
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v2, v0 offset:2
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v3, v0 offset:4
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v4, v0 offset:6
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v6, v0 offset:8
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v7, v0 offset:10
-; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v5, s1
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v5, v0 offset:8
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v0, v0 offset:10
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v6, s1
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v2, 16, v1
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
-; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v0
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v2 offset:2
-; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v1 offset:4
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v0 offset:6
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v6 offset:8
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v7 offset:10
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v2, v4, 16, v3
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 16, v5
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v6, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v6, v3 offset:2
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v6, v2 offset:4
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v6, v1 offset:6
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v6, v0 offset:8
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v6, v1 offset:10
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds12align2:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read_b96 v[0:2], v0

 ; ALIGNED-SDAG-NEXT:    ds_write_b64 v3, v[0:1]
 ; ALIGNED-SDAG-NEXT:    s_endpgm
 ;
 ; ALIGNED-GISEL-LABEL: ds12align8:
 ; ALIGNED-GISEL:       ; %bb.0:
 ; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v2, s0
-; ALIGNED-GISEL-NEXT:    ds_read_b64 v[0:1], v2
+; ALIGNED-GISEL-NEXT:    ds_read2_b32 v[0:1], v2 offset1:1
 ; ALIGNED-GISEL-NEXT:    ds_read_b32 v2, v2 offset:8
 ; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v3, s1
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
-; ALIGNED-GISEL-NEXT:    ds_write_b64 v3, v[0:1]
+; ALIGNED-GISEL-NEXT:    ds_write2_b32 v3, v0, v1 offset1:1
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(1)
 ; ALIGNED-GISEL-NEXT:    ds_write_b32 v3, v2 offset:8
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds12align8:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)

 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v1, v0
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v2, v0 offset:1
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:2
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:3
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:4
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v0 offset:5
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v0 offset:6
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v0 offset:7
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v9, v0 offset:8
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v10, v0 offset:9
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v11, v0 offset:10
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v12, v0 offset:11
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v13, v0 offset:12
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v14, v0 offset:13
-; ALIGNED-GISEL-NEXT:    ds_read_u8 v15, v0 offset:14
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(6)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 8, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v4
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v1, v2, v3, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v2, v6, 8, v5
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 24, v8
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v2, v3, v4, v2
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v3, v0 offset:8
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v4, v0 offset:9
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v5, v0 offset:10
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v6, v0 offset:11
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v7, v0 offset:12
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v8, v0 offset:13
+; ALIGNED-GISEL-NEXT:    ds_read_u8 v9, v0 offset:14
 ; ALIGNED-GISEL-NEXT:    ds_read_u8 v0, v0 offset:15
-; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v16, s1
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v1
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v2 offset:1
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v3 offset:2
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v4 offset:3
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v5 offset:4
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v6 offset:5
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v7 offset:6
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v8 offset:7
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v9 offset:8
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v10 offset:9
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v11 offset:10
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v12 offset:11
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v13 offset:12
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v14 offset:13
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(14)
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v15 offset:14
-; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v0 offset:15
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(6)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v3, v4, 8, v3
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 24, v6
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v3, v4, v5, v3
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v4, v8, 8, v7
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; ALIGNED-GISEL-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
+; ALIGNED-GISEL-NEXT:    v_or3_b32 v0, v0, v5, v4
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v6, s1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v5, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v5 offset:1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v1, 8, v4
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v4 offset:2
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v1 offset:3
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v4, 8, v2
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v2 offset:4
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v4 offset:5
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v1 offset:6
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v2 offset:7
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v3
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v3 offset:8
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v2 offset:9
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v1 offset:10
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v2 offset:11
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v2, 8, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v0 offset:12
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v2 offset:13
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b16_e32 v0, 8, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v1 offset:14
+; ALIGNED-GISEL-NEXT:    ds_write_b8 v6, v0 offset:15
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds16align1:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1

 ; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v0 offset:14
 ; ALIGNED-SDAG-NEXT:    s_endpgm
 ;
 ; ALIGNED-GISEL-LABEL: ds16align2:
 ; ALIGNED-GISEL:       ; %bb.0:
 ; ALIGNED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v2, v0
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v3, v0 offset:2
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v4, v0 offset:4
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v5, v0 offset:6
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v6, v0 offset:8
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v7, v0 offset:10
-; ALIGNED-GISEL-NEXT:    ds_read_u16 v8, v0 offset:12
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v1, v0
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v2, v0 offset:2
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v3, v0 offset:4
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v4, v0 offset:6
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v5, v0 offset:8
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v6, v0 offset:10
+; ALIGNED-GISEL-NEXT:    ds_read_u16 v7, v0 offset:12
 ; ALIGNED-GISEL-NEXT:    ds_read_u16 v0, v0 offset:14
-; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v2
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v3 offset:2
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v4 offset:4
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v5 offset:6
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v6 offset:8
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v7 offset:10
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v8 offset:12
-; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v0 offset:14
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(6)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v2, v4, 16, v3
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v3, v6, 16, v5
+; ALIGNED-GISEL-NEXT:    v_mov_b32_e32 v5, s1
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v1
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v4 offset:2
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; ALIGNED-GISEL-NEXT:    s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 16, v7
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v2 offset:4
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v1 offset:6
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v3 offset:8
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v1 offset:10
+; ALIGNED-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v0 offset:12
+; ALIGNED-GISEL-NEXT:    ds_write_b16 v5, v1 offset:14
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds16align2:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1