[AMDGPU] added SIInstrInfo::getAddNoCarry() helper

[rampitec]

Summary

Addressed rest of post submit comments from D31993.

[rampitec]

SILoadStoreOptimizer itself does not use new helper because that would require an extra SGPR and move for every combined lds operation.

[arsenm]

In D32057#726726, @rampitec wrote:

SILoadStoreOptimizer itself does not use new helper because that would require an extra SGPR and move for every combined lds operation.

What is preventing SIShrinkInstructions from handling it?

lib/Target/AMDGPU/SIInstrInfo.h

771

const DebugLoc&

[rampitec]

Made DL a const reference.

[arsenm]

LGTM

[rampitec]

In D32057#726838, @arsenm wrote:

In D32057#726726, @rampitec wrote:

SILoadStoreOptimizer itself does not use new helper because that would require an extra SGPR and move for every combined lds operation.

What is preventing SIShrinkInstructions from handling it?

Good question. I will need to debug it.

[rampitec]

In D32057#726862, @rampitec wrote:

In D32057#726838, @arsenm wrote:

In D32057#726726, @rampitec wrote:

SILoadStoreOptimizer itself does not use new helper because that would require an extra SGPR and move for every combined lds operation.

What is preventing SIShrinkInstructions from handling it?

Good question. I will need to debug it.

It just does not process it. It ends up here:

if (SDst) {
  if (SDst->getReg() != AMDGPU::VCC) {
    if (TargetRegisterInfo::isVirtualRegister(SDst->getReg()))
      MRI.setRegAllocationHint(SDst->getReg(), 0, AMDGPU::VCC);
    continue;
  }

[rampitec]

In D32057#726887, @rampitec wrote:

In D32057#726862, @rampitec wrote:

In D32057#726838, @arsenm wrote:

In D32057#726726, @rampitec wrote:

SILoadStoreOptimizer itself does not use new helper because that would require an extra SGPR and move for every combined lds operation.

What is preventing SIShrinkInstructions from handling it?

Good question. I will need to debug it.

It just does not process it. It ends up here:

if (SDst) {
  if (SDst->getReg() != AMDGPU::VCC) {
    if (TargetRegisterInfo::isVirtualRegister(SDst->getReg()))
      MRI.setRegAllocationHint(SDst->getReg(), 0, AMDGPU::VCC);
    continue;
  }

It can handle only:

V_MOV_B32_e32
S_NOP
S_ADD_I32
S_MUL_I32
SOPC*
S_MOV_B32
VOPC*
V_CNDMASK_B32_e32

lib/Target/AMDGPU/SIInstrInfo.h

public:
   ScheduleHazardRecognizer *
   CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
                                  const ScheduleDAG *DAG) const override;
 
   ScheduleHazardRecognizer *
   CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const override;
 
   bool isBasicBlockPrologue(const MachineInstr &MI) const override;
+
+  /// \brief Return a partially built integer add instruction without carry.
+  /// Caller must add source operands.
+  /// For pre-GFX9 it will generate unused carry destination operand.
+  /// TODO: After GFX9 it should return a no-carry operation.
+  MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator I,
+                                    const DebugLoc &DL,
+                                    unsigned DestReg) const;
 };
 
 namespace AMDGPU {
   LLVM_READONLY
   int getVOPe64(uint16_t Opcode);
 
   LLVM_READONLY
   int getVOPe32(uint16_t Opcode);

lib/Target/AMDGPU/SIInstrInfo.cpp

 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
   return new GCNHazardRecognizer(MF);
 }
 
 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const {
   return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
          MI.modifiesRegister(AMDGPU::EXEC, &RI);
 }
+
+MachineInstrBuilder
+SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
+                           MachineBasicBlock::iterator I,
+                           const DebugLoc &DL,
+                           unsigned DestReg) const {
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+
+  unsigned UnusedCarry = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
+
+  return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
+           .addReg(UnusedCarry, RegState::Define | RegState::Dead);
+}

lib/Target/AMDGPU/SILoadStoreOptimizer.cpp

MachineBasicBlock::iterator  SILoadStoreOptimizer::mergeRead2Pair(
   DebugLoc DL = CI.I->getDebugLoc();
 
   unsigned BaseReg = AddrReg->getReg();
   unsigned BaseRegFlags = 0;
   if (CI.BaseOff) {
     BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
     BaseRegFlags = RegState::Kill;
     BuildMI(*MBB, CI.Paired, DL, TII->get(AMDGPU::V_ADD_I32_e32), BaseReg)
            .addImm(CI.BaseOff)
            .addReg(AddrReg->getReg());
   }
 
-  MachineInstrBuilder Read2 = BuildMI(*MBB, CI.Paired, DL, Read2Desc, DestReg)
-                                .addReg(BaseReg, BaseRegFlags) // addr
-                                .addImm(NewOffset0)            // offset0
-                                .addImm(NewOffset1)            // offset1
-                                .addImm(0)                     // gds
-                                .addMemOperand(*CI.I->memoperands_begin())
-                                .addMemOperand(*CI.Paired->memoperands_begin());
+  MachineInstrBuilder Read2 =
+    BuildMI(*MBB, CI.Paired, DL, Read2Desc, DestReg)
+      .setMemRefs(CI.I->mergeMemRefsWith(*CI.Paired));
+
   (void)Read2;
 
   const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY);
 
   // Copy to the old destination registers.
   BuildMI(*MBB, CI.Paired, DL, CopyDesc)
       .add(*Dest0) // Copy to same destination including flags and sub reg.
       .addReg(DestReg, 0, SubRegIdx0);
MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
   DebugLoc DL = CI.I->getDebugLoc();
 
   unsigned BaseReg = Addr->getReg();
   unsigned BaseRegFlags = 0;
   if (CI.BaseOff) {
     BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
     BaseRegFlags = RegState::Kill;
     BuildMI(*MBB, CI.Paired, DL, TII->get(AMDGPU::V_ADD_I32_e32), BaseReg)
            .addImm(CI.BaseOff)
            .addReg(Addr->getReg());
   }
 
-  MachineInstrBuilder Write2 = BuildMI(*MBB, CI.Paired, DL, Write2Desc)
-                                .addReg(BaseReg, BaseRegFlags) // addr
-                                .add(*Data0)                   // data0
-                                .add(*Data1)                   // data1
-                                .addImm(NewOffset0)            // offset0
-                                .addImm(NewOffset1)            // offset1
-                                .addImm(0)                     // gds
-                                .addMemOperand(*CI.I->memoperands_begin())
-                                .addMemOperand(*CI.Paired->memoperands_begin());
+  MachineInstrBuilder Write2 =
+    BuildMI(*MBB, CI.Paired, DL, Write2Desc)
+      .setMemRefs(CI.I->mergeMemRefsWith(*CI.Paired));
 
   moveInstsAfter(Write2, CI.InstsToMove);
 
   MachineBasicBlock::iterator Next = std::next(CI.I);
   CI.I->eraseFromParent();
   CI.Paired->eraseFromParent();
 
   DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n');

lib/Target/AMDGPU/SIRegisterInfo.cpp

void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
 
   if (Offset == 0) {
     BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg)
       .addFrameIndex(FrameIdx);
     return;
   }
 
   MachineRegisterInfo &MRI = MF->getRegInfo();
-  unsigned UnusedCarry = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
   unsigned OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
 
   unsigned FIReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
   BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg)
     .addImm(Offset);
   BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), FIReg)
     .addFrameIndex(FrameIdx);
 
-  BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_ADD_I32_e64), BaseReg)
-    .addReg(UnusedCarry, RegState::Define | RegState::Dead)
+  TII->getAddNoCarry(*MBB, Ins, DL, BaseReg)
     .addReg(OffsetReg, RegState::Kill)
     .addReg(FIReg);
 }
 
 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, unsigned BaseReg,
                                        int64_t Offset) const {
 
   MachineBasicBlock *MBB = MI.getParent();

test/CodeGen/AMDGPU/ds-combine-large-stride.ll

-; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
 
-; CHECK-LABEL: ds_read32_combine_stride_400:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:100
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:100
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:100
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:100
-define void @ds_read32_combine_stride_400(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
+; GCN-LABEL: ds_read32_combine_stride_400:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:100
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:100
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:100
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:100
+define amdgpu_kernel void @ds_read32_combine_stride_400(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
 bb:
   %tmp = load float, float addrspace(3)* %arg, align 4
   %tmp2 = fadd float %tmp, 0.000000e+00
   %tmp3 = getelementptr inbounds float, float addrspace(3)* %arg, i32 100
   %tmp4 = load float, float addrspace(3)* %tmp3, align 4
   %tmp5 = fadd float %tmp2, %tmp4
   %tmp6 = getelementptr inbounds float, float addrspace(3)* %arg, i32 200
   %tmp7 = load float, float addrspace(3)* %tmp6, align 4
bb:
   %tmp20 = fadd float %tmp17, %tmp19
   %tmp21 = getelementptr inbounds float, float addrspace(3)* %arg, i32 700
   %tmp22 = load float, float addrspace(3)* %tmp21, align 4
   %tmp23 = fadd float %tmp20, %tmp22
   store float %tmp23, float *%arg1, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_read32_combine_stride_400_back:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:100
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:100
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:100
-; CHECK-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:100
-define void @ds_read32_combine_stride_400_back(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
+; GCN-LABEL: ds_read32_combine_stride_400_back:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:100
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:100
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:100
+; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:100
+define amdgpu_kernel void @ds_read32_combine_stride_400_back(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
 bb:
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 700
   %tmp2 = load float, float addrspace(3)* %tmp, align 4
   %tmp3 = fadd float %tmp2, 0.000000e+00
   %tmp4 = getelementptr inbounds float, float addrspace(3)* %arg, i32 600
   %tmp5 = load float, float addrspace(3)* %tmp4, align 4
   %tmp6 = fadd float %tmp3, %tmp5
   %tmp7 = getelementptr inbounds float, float addrspace(3)* %arg, i32 500
bb:
   %tmp20 = load float, float addrspace(3)* %tmp19, align 4
   %tmp21 = fadd float %tmp18, %tmp20
   %tmp22 = load float, float addrspace(3)* %arg, align 4
   %tmp23 = fadd float %tmp21, %tmp22
   store float %tmp23, float *%arg1, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_read32_combine_stride_8192:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:32
-; CHECK-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:64 offset1:96
-; CHECK-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:128 offset1:160
-; CHECK-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:192 offset1:224
-define void @ds_read32_combine_stride_8192(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
+; GCN-LABEL: ds_read32_combine_stride_8192:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:32
+; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:64 offset1:96
+; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:128 offset1:160
+; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:192 offset1:224
+define amdgpu_kernel void @ds_read32_combine_stride_8192(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
 bb:
   %tmp = load float, float addrspace(3)* %arg, align 4
   %tmp2 = fadd float %tmp, 0.000000e+00
   %tmp3 = getelementptr inbounds float, float addrspace(3)* %arg, i32 2048
   %tmp4 = load float, float addrspace(3)* %tmp3, align 4
   %tmp5 = fadd float %tmp2, %tmp4
   %tmp6 = getelementptr inbounds float, float addrspace(3)* %arg, i32 4096
   %tmp7 = load float, float addrspace(3)* %tmp6, align 4
bb:
   %tmp20 = fadd float %tmp17, %tmp19
   %tmp21 = getelementptr inbounds float, float addrspace(3)* %arg, i32 14336
   %tmp22 = load float, float addrspace(3)* %tmp21, align 4
   %tmp23 = fadd float %tmp20, %tmp22
   store float %tmp23, float *%arg1, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_read32_combine_stride_8192_shifted:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
-; CHECK-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:32
-; CHECK-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:32
-; CHECK-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:32
-define void @ds_read32_combine_stride_8192_shifted(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
+; GCN-LABEL: ds_read32_combine_stride_8192_shifted:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:32
+; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:32
+; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:32
+define amdgpu_kernel void @ds_read32_combine_stride_8192_shifted(float addrspace(3)* nocapture readonly %arg, float *nocapture %arg1) {
 bb:
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 2
   %tmp2 = load float, float addrspace(3)* %tmp, align 4
   %tmp3 = fadd float %tmp2, 0.000000e+00
   %tmp4 = getelementptr inbounds float, float addrspace(3)* %arg, i32 2050
   %tmp5 = load float, float addrspace(3)* %tmp4, align 4
   %tmp6 = fadd float %tmp3, %tmp5
   %tmp7 = getelementptr inbounds float, float addrspace(3)* %arg, i32 4098
   %tmp8 = load float, float addrspace(3)* %tmp7, align 4
   %tmp9 = fadd float %tmp6, %tmp8
   %tmp10 = getelementptr inbounds float, float addrspace(3)* %arg, i32 6146
   %tmp11 = load float, float addrspace(3)* %tmp10, align 4
   %tmp12 = fadd float %tmp9, %tmp11
   %tmp13 = getelementptr inbounds float, float addrspace(3)* %arg, i32 8194
   %tmp14 = load float, float addrspace(3)* %tmp13, align 4
   %tmp15 = fadd float %tmp12, %tmp14
   %tmp16 = getelementptr inbounds float, float addrspace(3)* %arg, i32 10242
   %tmp17 = load float, float addrspace(3)* %tmp16, align 4
   %tmp18 = fadd float %tmp15, %tmp17
   store float %tmp18, float *%arg1, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_read64_combine_stride_400:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
-; CHECK-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:50
-; CHECK-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:100 offset1:150
-; CHECK-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:200 offset1:250
-; CHECK-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:50
-define void @ds_read64_combine_stride_400(double addrspace(3)* nocapture readonly %arg, double *nocapture %arg1) {
+; GCN-LABEL: ds_read64_combine_stride_400:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GCN-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:50
+; GCN-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:100 offset1:150
+; GCN-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:200 offset1:250
+; GCN-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:50
+define amdgpu_kernel void @ds_read64_combine_stride_400(double addrspace(3)* nocapture readonly %arg, double *nocapture %arg1) {
 bb:
   %tmp = load double, double addrspace(3)* %arg, align 8
   %tmp2 = fadd double %tmp, 0.000000e+00
   %tmp3 = getelementptr inbounds double, double addrspace(3)* %arg, i32 50
   %tmp4 = load double, double addrspace(3)* %tmp3, align 8
   %tmp5 = fadd double %tmp2, %tmp4
   %tmp6 = getelementptr inbounds double, double addrspace(3)* %arg, i32 100
   %tmp7 = load double, double addrspace(3)* %tmp6, align 8
bb:
   %tmp20 = fadd double %tmp17, %tmp19
   %tmp21 = getelementptr inbounds double, double addrspace(3)* %arg, i32 350
   %tmp22 = load double, double addrspace(3)* %tmp21, align 8
   %tmp23 = fadd double %tmp20, %tmp22
   store double %tmp23, double *%arg1, align 8
   ret void
 }
 
-; CHECK-LABEL: ds_read64_combine_stride_8192_shifted:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
-; CHECK-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:16
-; CHECK-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:16
-; CHECK-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:16
-define void @ds_read64_combine_stride_8192_shifted(double addrspace(3)* nocapture readonly %arg, double *nocapture %arg1) {
+; GCN-LABEL: ds_read64_combine_stride_8192_shifted:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+; GCN-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:16
+; GCN-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:16
+; GCN-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:16
+define amdgpu_kernel void @ds_read64_combine_stride_8192_shifted(double addrspace(3)* nocapture readonly %arg, double *nocapture %arg1) {
 bb:
   %tmp = getelementptr inbounds double, double addrspace(3)* %arg, i32 1
   %tmp2 = load double, double addrspace(3)* %tmp, align 8
   %tmp3 = fadd double %tmp2, 0.000000e+00
   %tmp4 = getelementptr inbounds double, double addrspace(3)* %arg, i32 1025
   %tmp5 = load double, double addrspace(3)* %tmp4, align 8
   %tmp6 = fadd double %tmp3, %tmp5
   %tmp7 = getelementptr inbounds double, double addrspace(3)* %arg, i32 2049
   %tmp8 = load double, double addrspace(3)* %tmp7, align 8
   %tmp9 = fadd double %tmp6, %tmp8
   %tmp10 = getelementptr inbounds double, double addrspace(3)* %arg, i32 3073
   %tmp11 = load double, double addrspace(3)* %tmp10, align 8
   %tmp12 = fadd double %tmp9, %tmp11
   %tmp13 = getelementptr inbounds double, double addrspace(3)* %arg, i32 4097
   %tmp14 = load double, double addrspace(3)* %tmp13, align 8
   %tmp15 = fadd double %tmp12, %tmp14
   %tmp16 = getelementptr inbounds double, double addrspace(3)* %arg, i32 5121
   %tmp17 = load double, double addrspace(3)* %tmp16, align 8
   %tmp18 = fadd double %tmp15, %tmp17
   store double %tmp18, double *%arg1, align 8
   ret void
 }
 
-; CHECK-LABEL: ds_write32_combine_stride_400:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; CHECK-DAG: ds_write2_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-; CHECK-DAG: ds_write2_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-; CHECK-DAG: ds_write2_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-; CHECK-DAG: ds_write2_b32 [[B3]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-define void @ds_write32_combine_stride_400(float addrspace(3)* nocapture %arg) {
+; GCN-LABEL: ds_write32_combine_stride_400:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GCN-DAG: ds_write2_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+; GCN-DAG: ds_write2_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+; GCN-DAG: ds_write2_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+; GCN-DAG: ds_write2_b32 [[B3]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+define amdgpu_kernel void @ds_write32_combine_stride_400(float addrspace(3)* nocapture %arg) {
 bb:
   store float 1.000000e+00, float addrspace(3)* %arg, align 4
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 100
   store float 1.000000e+00, float addrspace(3)* %tmp, align 4
   %tmp1 = getelementptr inbounds float, float addrspace(3)* %arg, i32 200
   store float 1.000000e+00, float addrspace(3)* %tmp1, align 4
   %tmp2 = getelementptr inbounds float, float addrspace(3)* %arg, i32 300
   store float 1.000000e+00, float addrspace(3)* %tmp2, align 4
   %tmp3 = getelementptr inbounds float, float addrspace(3)* %arg, i32 400
   store float 1.000000e+00, float addrspace(3)* %tmp3, align 4
   %tmp4 = getelementptr inbounds float, float addrspace(3)* %arg, i32 500
   store float 1.000000e+00, float addrspace(3)* %tmp4, align 4
   %tmp5 = getelementptr inbounds float, float addrspace(3)* %arg, i32 600
   store float 1.000000e+00, float addrspace(3)* %tmp5, align 4
   %tmp6 = getelementptr inbounds float, float addrspace(3)* %arg, i32 700
   store float 1.000000e+00, float addrspace(3)* %tmp6, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_write32_combine_stride_400_back:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; CHECK-DAG: ds_write2_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-; CHECK-DAG: ds_write2_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-; CHECK-DAG: ds_write2_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-; CHECK-DAG: ds_write2_b32 [[B3]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
-define void @ds_write32_combine_stride_400_back(float addrspace(3)* nocapture %arg) {
+; GCN-LABEL: ds_write32_combine_stride_400_back:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GCN-DAG: ds_write2_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+; GCN-DAG: ds_write2_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+; GCN-DAG: ds_write2_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+; GCN-DAG: ds_write2_b32 [[B3]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
+define amdgpu_kernel void @ds_write32_combine_stride_400_back(float addrspace(3)* nocapture %arg) {
 bb:
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 700
   store float 1.000000e+00, float addrspace(3)* %tmp, align 4
   %tmp1 = getelementptr inbounds float, float addrspace(3)* %arg, i32 600
   store float 1.000000e+00, float addrspace(3)* %tmp1, align 4
   %tmp2 = getelementptr inbounds float, float addrspace(3)* %arg, i32 500
   store float 1.000000e+00, float addrspace(3)* %tmp2, align 4
   %tmp3 = getelementptr inbounds float, float addrspace(3)* %arg, i32 400
   store float 1.000000e+00, float addrspace(3)* %tmp3, align 4
   %tmp4 = getelementptr inbounds float, float addrspace(3)* %arg, i32 300
   store float 1.000000e+00, float addrspace(3)* %tmp4, align 4
   %tmp5 = getelementptr inbounds float, float addrspace(3)* %arg, i32 200
   store float 1.000000e+00, float addrspace(3)* %tmp5, align 4
   %tmp6 = getelementptr inbounds float, float addrspace(3)* %arg, i32 100
   store float 1.000000e+00, float addrspace(3)* %tmp6, align 4
   store float 1.000000e+00, float addrspace(3)* %arg, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_write32_combine_stride_8192:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
-; CHECK-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset0:64 offset1:96
-; CHECK-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset0:128 offset1:160
-; CHECK-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset0:192 offset1:224
-define void @ds_write32_combine_stride_8192(float addrspace(3)* nocapture %arg) {
+; GCN-LABEL: ds_write32_combine_stride_8192:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
+; GCN-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset0:64 offset1:96
+; GCN-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset0:128 offset1:160
+; GCN-DAG: ds_write2st64_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset0:192 offset1:224
+define amdgpu_kernel void @ds_write32_combine_stride_8192(float addrspace(3)* nocapture %arg) {
 bb:
   store float 1.000000e+00, float addrspace(3)* %arg, align 4
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 2048
   store float 1.000000e+00, float addrspace(3)* %tmp, align 4
   %tmp1 = getelementptr inbounds float, float addrspace(3)* %arg, i32 4096
   store float 1.000000e+00, float addrspace(3)* %tmp1, align 4
   %tmp2 = getelementptr inbounds float, float addrspace(3)* %arg, i32 6144
   store float 1.000000e+00, float addrspace(3)* %tmp2, align 4
   %tmp3 = getelementptr inbounds float, float addrspace(3)* %arg, i32 8192
   store float 1.000000e+00, float addrspace(3)* %tmp3, align 4
   %tmp4 = getelementptr inbounds float, float addrspace(3)* %arg, i32 10240
   store float 1.000000e+00, float addrspace(3)* %tmp4, align 4
   %tmp5 = getelementptr inbounds float, float addrspace(3)* %arg, i32 12288
   store float 1.000000e+00, float addrspace(3)* %tmp5, align 4
   %tmp6 = getelementptr inbounds float, float addrspace(3)* %arg, i32 14336
   store float 1.000000e+00, float addrspace(3)* %tmp6, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_write32_combine_stride_8192_shifted:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 4, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4004, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8004, [[BASE]]
-; CHECK-DAG: ds_write2st64_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
-; CHECK-DAG: ds_write2st64_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
-; CHECK-DAG: ds_write2st64_b32 [[B3]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
-define void @ds_write32_combine_stride_8192_shifted(float addrspace(3)* nocapture %arg) {
+; GCN-LABEL: ds_write32_combine_stride_8192_shifted:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 4, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4004, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8004, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 4, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4004, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8004, [[BASE]]
+; GCN-DAG: ds_write2st64_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
+; GCN-DAG: ds_write2st64_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
+; GCN-DAG: ds_write2st64_b32 [[B3]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
+define amdgpu_kernel void @ds_write32_combine_stride_8192_shifted(float addrspace(3)* nocapture %arg) {
 bb:
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 1
   store float 1.000000e+00, float addrspace(3)* %tmp, align 4
   %tmp1 = getelementptr inbounds float, float addrspace(3)* %arg, i32 2049
   store float 1.000000e+00, float addrspace(3)* %tmp1, align 4
   %tmp2 = getelementptr inbounds float, float addrspace(3)* %arg, i32 4097
   store float 1.000000e+00, float addrspace(3)* %tmp2, align 4
   %tmp3 = getelementptr inbounds float, float addrspace(3)* %arg, i32 6145
   store float 1.000000e+00, float addrspace(3)* %tmp3, align 4
   %tmp4 = getelementptr inbounds float, float addrspace(3)* %arg, i32 8193
   store float 1.000000e+00, float addrspace(3)* %tmp4, align 4
   %tmp5 = getelementptr inbounds float, float addrspace(3)* %arg, i32 10241
   store float 1.000000e+00, float addrspace(3)* %tmp5, align 4
   ret void
 }
 
-; CHECK-LABEL: ds_write64_combine_stride_400:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
-; CHECK-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:50
-; CHECK-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset0:100 offset1:150
-; CHECK-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset0:200 offset1:250
-; CHECK-DAG: ds_write2_b64 [[B1]],   v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:50
-define void @ds_write64_combine_stride_400(double addrspace(3)* nocapture %arg) {
+; GCN-LABEL: ds_write64_combine_stride_400:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GCN-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:50
+; GCN-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset0:100 offset1:150
+; GCN-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset0:200 offset1:250
+; GCN-DAG: ds_write2_b64 [[B1]],   v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:50
+define amdgpu_kernel void @ds_write64_combine_stride_400(double addrspace(3)* nocapture %arg) {
 bb:
   store double 1.000000e+00, double addrspace(3)* %arg, align 8
   %tmp = getelementptr inbounds double, double addrspace(3)* %arg, i32 50
   store double 1.000000e+00, double addrspace(3)* %tmp, align 8
   %tmp1 = getelementptr inbounds double, double addrspace(3)* %arg, i32 100
   store double 1.000000e+00, double addrspace(3)* %tmp1, align 8
   %tmp2 = getelementptr inbounds double, double addrspace(3)* %arg, i32 150
   store double 1.000000e+00, double addrspace(3)* %tmp2, align 8
   %tmp3 = getelementptr inbounds double, double addrspace(3)* %arg, i32 200
   store double 1.000000e+00, double addrspace(3)* %tmp3, align 8
   %tmp4 = getelementptr inbounds double, double addrspace(3)* %arg, i32 250
   store double 1.000000e+00, double addrspace(3)* %tmp4, align 8
   %tmp5 = getelementptr inbounds double, double addrspace(3)* %arg, i32 300
   store double 1.000000e+00, double addrspace(3)* %tmp5, align 8
   %tmp6 = getelementptr inbounds double, double addrspace(3)* %arg, i32 350
   store double 1.000000e+00, double addrspace(3)* %tmp6, align 8
   ret void
 }
 
-; CHECK-LABEL: ds_write64_combine_stride_8192_shifted:
-; CHECK:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
-; CHECK:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; CHECK-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; CHECK-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
-; CHECK-DAG: ds_write2st64_b64 [[B1]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
-; CHECK-DAG: ds_write2st64_b64 [[B2]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
-; CHECK-DAG: ds_write2st64_b64 [[B3]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
-define void @ds_write64_combine_stride_8192_shifted(double addrspace(3)* nocapture %arg) {
+; GCN-LABEL: ds_write64_combine_stride_8192_shifted:
+; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
+; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
+; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+; GCN-DAG: ds_write2st64_b64 [[B1]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
+; GCN-DAG: ds_write2st64_b64 [[B2]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
+; GCN-DAG: ds_write2st64_b64 [[B3]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
+define amdgpu_kernel void @ds_write64_combine_stride_8192_shifted(double addrspace(3)* nocapture %arg) {
 bb:
   %tmp = getelementptr inbounds double, double addrspace(3)* %arg, i32 1
   store double 1.000000e+00, double addrspace(3)* %tmp, align 8
   %tmp1 = getelementptr inbounds double, double addrspace(3)* %arg, i32 1025
   store double 1.000000e+00, double addrspace(3)* %tmp1, align 8
   %tmp2 = getelementptr inbounds double, double addrspace(3)* %arg, i32 2049
   store double 1.000000e+00, double addrspace(3)* %tmp2, align 8
   %tmp3 = getelementptr inbounds double, double addrspace(3)* %arg, i32 3073
   store double 1.000000e+00, double addrspace(3)* %tmp3, align 8
   %tmp4 = getelementptr inbounds double, double addrspace(3)* %arg, i32 4097
   store double 1.000000e+00, double addrspace(3)* %tmp4, align 8
   %tmp5 = getelementptr inbounds double, double addrspace(3)* %arg, i32 5121
   store double 1.000000e+00, double addrspace(3)* %tmp5, align 8
   ret void
 }