[AMDGPU] Eliminate SGPR to VGPR copy when possible

[rampitec]

Summary

SGPRs are generally cheaper, so try to use them over VGPRs.

[arsenm]

Typo in test file name

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

182–187 ↗

(On Diff #102279)

This probably breaks if you have a copy that looks like a sub register extract

192 ↗

(On Diff #102279)

Why do you check the parent?

193 ↗

(On Diff #102279)

Why do you need to check this? Isn't this filtering the global isel pseudos which should never exist here?

194 ↗

(On Diff #102279)

Avoid doing the weird pointer arithmetic?

198 ↗

(On Diff #102279)

I forsee this breaking cases with multiple uses

test/CodeGen/AMDGPU/opt-sgpr-to-vgrp-copy.ll

25 ↗

(On Diff #102279)

Probably need a MIR test to stress multiple uses

[rampitec]

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

182–187 ↗

(On Diff #102279)

The should not be a problem. I'm changing VGPR class to equivalent SGPR class, they support the same set of subregs.

192 ↗

(On Diff #102279)

A copy can be in a divergent control flow. If it is the same block we can be sure that is actually the value we are going to use.

193 ↗

(On Diff #102279)

isOperandLegal does not work with generic opcodes because they have no RC classes assigned to operands.

194 ↗

(On Diff #102279)

How do I get operand number otherwise?

198 ↗

(On Diff #102279)

Why? I'm checking all uses for operand legality. It also does not happen if there is a second def of the same register.

[rampitec]

Fixed typo in test name.

[rampitec]

test/CodeGen/AMDGPU/opt-sgpr-to-vgrp-copy.ll

25 ↗

(On Diff #102279)

Can you be more specific, multiple uses of what? Do you mean multiple uses of the same COPY?

[rampitec]

Replaced test with mir to check multiple use case.

[rampitec]

Removed pointer arithmetic on operands.

[rampitec]

Added subreg-extract like copy test.
Cleaned mir test.

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

182–187 ↗

(On Diff #102279)

I have added test for that.

[rampitec]

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

198 ↗

(On Diff #102279)

I have added test for multiple uses.

[rampitec]

Ping.

[arsenm]

LGTM. It's possible to reduce the mir testcase further though

test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir

102–117 ↗

(On Diff #102698)

You can strip out most of this stuff

[rampitec]

Reduced test.

llvm/trunk/lib/Target/AMDGPU/SIFixSGPRCopies.cpp

 }
 
 static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
                              const TargetRegisterClass *DstRC,
                              const SIRegisterInfo &TRI) {
   return TRI.isSGPRClass(SrcRC) && TRI.hasVGPRs(DstRC);
 }
 
+static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
+                                      const SIRegisterInfo *TRI,
+                                      const SIInstrInfo *TII) {
+  MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
+  auto &Src = MI.getOperand(1);
+  unsigned DstReg = MI.getOperand(0).getReg();
+  unsigned SrcReg = Src.getReg();
+  if (!TargetRegisterInfo::isVirtualRegister(SrcReg) ||
+      !TargetRegisterInfo::isVirtualRegister(DstReg))
+    return false;
+
+  for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
+    const auto *UseMI = MO.getParent();
+    if (UseMI == &MI)
+      continue;
+    if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
+        UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
+        !TII->isOperandLegal(*UseMI, UseMI->getOperandNo(&MO), &Src))
+      return false;
+  }
+  // Change VGPR to SGPR destination.
+  MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
+  return true;
+}
+
 // Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
 //
 // SGPRx = ...
 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
 // VGPRz = COPY SGPRy
 //
 // ==>
 //
if (TargetRegisterInfo::isPhysicalRegister(CopyUse.getOperand(0).getReg()))
     return false;
 
   const TargetRegisterClass *SrcRC, *DstRC;
   std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
 
   if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
     return false;
 
+  if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
+    return true;
+
   // TODO: Could have multiple extracts?
   unsigned SubReg = CopyUse.getOperand(1).getSubReg();
   if (SubReg != AMDGPU::NoSubRegister)
     return false;
 
   MRI.setRegClass(DstReg, DstRC);
 
   // SGPRx = ...
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
           // s_mov_b32.
           if (isSafeToFoldImmIntoCopy(&MI, DefMI, TII, SMovOp, Imm)) {
             MI.getOperand(1).ChangeToImmediate(Imm);
             MI.addImplicitDefUseOperands(MF);
             MI.setDesc(TII->get(SMovOp));
             break;
           }
           TII->moveToVALU(MI);
+        } else if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
+          tryChangeVGPRtoSGPRinCopy(MI, TRI, TII);
         }
 
         break;
       }
       case AMDGPU::PHI: {
         unsigned Reg = MI.getOperand(0).getReg();
         if (!TRI->isSGPRClass(MRI.getRegClass(Reg)))
           break;

llvm/trunk/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 declare i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64, i32, i64) #0
 
 ; GCN-LABEL: {{^}}v_mqsad_pk_u16_u8:
-; GCN: v_mqsad_pk_u16_u8 v[0:1], v[4:5], s{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
+; GCN: v_mqsad_pk_u16_u8 v[0:1], v[4:5], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}]
 ; GCN-DAG: v_mov_b32_e32 v5, v1
 ; GCN-DAG: v_mov_b32_e32 v4, v0
 define amdgpu_kernel void @v_mqsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) {
   %tmp = call i64 asm "v_lsrlrev_b64 $0, $1, 1", "={v[4:5]},v"(i64 %src) #0
   %tmp1 = call i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %tmp, i32 100, i64 100) #0
   %tmp2 = call i64 asm ";; force constraint", "=v,{v[4:5]}"(i64 %tmp1) #0
   store i64 %tmp2, i64 addrspace(1)* %out, align 4
   ret void

llvm/trunk/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 declare i64 @llvm.amdgcn.qsad.pk.u16.u8(i64, i32, i64) #0
 
 ; GCN-LABEL: {{^}}v_qsad_pk_u16_u8:
-; GCN: v_qsad_pk_u16_u8 v[0:1], v[4:5], s{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
+; GCN: v_qsad_pk_u16_u8 v[0:1], v[4:5], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}]
 ; GCN-DAG: v_mov_b32_e32 v5, v1
 ; GCN-DAG: v_mov_b32_e32 v4, v0
 define amdgpu_kernel void @v_qsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) {
   %tmp = call i64 asm "v_lsrlrev_b64 $0, $1, 1", "={v[4:5]},v"(i64 %src) #0
   %tmp1 = call i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %tmp, i32 100, i64 100) #0
   %tmp2 = call i64 asm ";; force constraint", "=v,{v[4:5]}"(i64 %tmp1) #0
   store i64 %tmp2, i64 addrspace(1)* %out, align 4
   ret void

llvm/trunk/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir

+# RUN: llc -march=amdgcn -run-pass si-fix-sgpr-copies,si-fold-operands,dead-mi-elimination -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+
+# Check that constant is in SGPR registers
+
+# GCN-LABEL: {{^}}name: const_to_sgpr{{$}}
+# GCN:        %[[HI:[0-9]+]] = S_MOV_B32 0
+# GCN-NEXT:   %[[LO:[0-9]+]] = S_MOV_B32 1048576
+# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]] = REG_SEQUENCE killed %[[LO]], 1, killed %[[HI]], 2
+# GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
+
+
+# GCN-LABEL: {{^}}name: const_to_sgpr_multiple_use{{$}}
+# GCN:        %[[HI:[0-9]+]] = S_MOV_B32 0
+# GCN-NEXT:   %[[LO:[0-9]+]] = S_MOV_B32 1048576
+# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]] = REG_SEQUENCE killed %[[LO]], 1, killed %[[HI]], 2
+# GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
+# GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
+
+# GCN-LABEL: {{^}}name: const_to_sgpr_subreg{{$}}
+# GCN:       %[[OP0:[0-9]+]] = REG_SEQUENCE killed %{{[0-9]+}}, 1, killed %{{[0-9]+}}, 2
+# GCN-NEXT:  V_CMP_LT_U32_e64 killed %[[OP0]].sub0, 12, implicit %exec
+
+--- |
+  define amdgpu_kernel void @const_to_sgpr(i32 addrspace(1)* nocapture %arg, i64 %id) {
+  bb:
+    br i1 undef, label %bb1, label %bb2
+
+  bb1:                                              ; preds = %bb
+    br label %bb2
+
+  bb2:                                              ; preds = %bb1, %bb
+    ret void
+  }
+
+  define amdgpu_kernel void @const_to_sgpr_multiple_use(i32 addrspace(1)* nocapture %arg, i64 %id1, i64 %id2) {
+  bb:
+    br i1 undef, label %bb1, label %bb2
+
+  bb1:                                              ; preds = %bb
+    br label %bb2
+
+  bb2:                                              ; preds = %bb1, %bb
+    ret void
+  }
+
+  define amdgpu_kernel void @const_to_sgpr_subreg(i32 addrspace(1)* nocapture %arg, i64 %id) {
+  bb:
+    br i1 undef, label %bb1, label %bb2
+
+  bb1:                                              ; preds = %bb
+    br label %bb2
+
+  bb2:                                              ; preds = %bb1, %bb
+    ret void
+  }
+
+...
+---
+name:            const_to_sgpr
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sreg_64 }
+  - { id: 1, class: sreg_64 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_64 }
+  - { id: 4, class: sreg_32_xm0 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sreg_64 }
+  - { id: 7, class: sreg_64_xexec }
+  - { id: 8, class: sreg_64_xexec }
+  - { id: 9, class: sreg_32 }
+  - { id: 10, class: sreg_64 }
+  - { id: 11, class: sreg_32_xm0 }
+  - { id: 12, class: sreg_32_xm0 }
+  - { id: 13, class: sreg_32_xm0 }
+  - { id: 14, class: sreg_32_xm0 }
+  - { id: 15, class: sreg_32_xm0 }
+  - { id: 16, class: sreg_32_xm0 }
+  - { id: 17, class: sreg_64 }
+  - { id: 18, class: sreg_32_xm0 }
+  - { id: 19, class: sreg_32_xm0 }
+  - { id: 20, class: sreg_64 }
+  - { id: 21, class: sreg_64 }
+  - { id: 22, class: vreg_64 }
+  - { id: 23, class: sreg_32_xm0 }
+  - { id: 24, class: sreg_64 }
+  - { id: 25, class: sreg_32_xm0 }
+  - { id: 26, class: sreg_32_xm0 }
+  - { id: 27, class: sgpr_64 }
+  - { id: 28, class: sgpr_128 }
+  - { id: 29, class: vgpr_32 }
+  - { id: 30, class: vreg_64 }
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%2' }
+  - { reg: '%sgpr0_sgpr1', virtual-reg: '%3' }
+body:             |
+  bb.0.bb:
+    successors: %bb.1.bb1(0x40000000), %bb.2.bb2(0x40000000)
+    liveins: %vgpr0, %sgpr0_sgpr1
+
+    %3 = COPY %sgpr0_sgpr1
+    %2 = COPY %vgpr0
+    %7 = S_LOAD_DWORDX2_IMM %3, 9, 0
+    %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
+    %6 = COPY %7
+    %9 = S_MOV_B32 0
+    %10 = REG_SEQUENCE %2, 1, killed %9, 2
+    %0 = COPY %10
+    %11 = COPY %10.sub0
+    %12 = COPY %10.sub1
+    %13 = COPY %8.sub0
+    %14 = COPY %8.sub1
+    %15 = S_ADD_U32 killed %11, killed %13, implicit-def %scc
+    %16 = S_ADDC_U32 killed %12, killed %14, implicit-def dead %scc, implicit %scc
+    %17 = REG_SEQUENCE killed %15, 1, killed %16, 2
+    %18 = S_MOV_B32 0
+    %19 = S_MOV_B32 1048576
+    %20 = REG_SEQUENCE killed %19, 1, killed %18, 2
+    %22 = COPY killed %20
+    %21 = V_CMP_LT_U64_e64 killed %17, %22, implicit %exec
+    %1 = SI_IF killed %21, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_BRANCH %bb.1.bb1
+
+  bb.1.bb1:
+    successors: %bb.2.bb2(0x80000000)
+
+    %23 = S_MOV_B32 2
+    %24 = S_LSHL_B64 %0, killed %23, implicit-def dead %scc
+    %25 = S_MOV_B32 61440
+    %26 = S_MOV_B32 0
+    %27 = REG_SEQUENCE killed %26, 1, killed %25, 2
+    %28 = REG_SEQUENCE %6, 17, killed %27, 18
+    %29 = V_MOV_B32_e32 0, implicit %exec
+    %30 = COPY %24
+    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, implicit %exec
+
+  bb.2.bb2:
+    SI_END_CF %1, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_ENDPGM
+
+...
+---
+name:            const_to_sgpr_multiple_use
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sreg_64 }
+  - { id: 1, class: sreg_64 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_64 }
+  - { id: 4, class: sreg_32_xm0 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sreg_64 }
+  - { id: 7, class: sreg_64_xexec }
+  - { id: 8, class: sreg_64_xexec }
+  - { id: 9, class: sreg_64_xexec }
+  - { id: 10, class: sreg_32 }
+  - { id: 11, class: sreg_64 }
+  - { id: 12, class: sreg_32_xm0 }
+  - { id: 13, class: sreg_32_xm0 }
+  - { id: 14, class: sreg_32_xm0 }
+  - { id: 15, class: sreg_32_xm0 }
+  - { id: 16, class: sreg_32_xm0 }
+  - { id: 17, class: sreg_32_xm0 }
+  - { id: 18, class: sreg_64 }
+  - { id: 19, class: sreg_32_xm0 }
+  - { id: 20, class: sreg_32_xm0 }
+  - { id: 21, class: sreg_32_xm0 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_64 }
+  - { id: 24, class: sreg_32_xm0 }
+  - { id: 25, class: sreg_32_xm0 }
+  - { id: 26, class: sreg_64 }
+  - { id: 27, class: sreg_64 }
+  - { id: 28, class: vreg_64 }
+  - { id: 29, class: sreg_64 }
+  - { id: 30, class: vreg_64 }
+  - { id: 31, class: sreg_64 }
+  - { id: 32, class: sreg_32_xm0 }
+  - { id: 33, class: sreg_64 }
+  - { id: 34, class: sreg_32_xm0 }
+  - { id: 35, class: sreg_32_xm0 }
+  - { id: 36, class: sgpr_64 }
+  - { id: 37, class: sgpr_128 }
+  - { id: 38, class: vgpr_32 }
+  - { id: 39, class: vreg_64 }
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%2' }
+  - { reg: '%sgpr0_sgpr1', virtual-reg: '%3' }
+body:             |
+  bb.0.bb:
+    successors: %bb.1.bb1(0x40000000), %bb.2.bb2(0x40000000)
+    liveins: %vgpr0, %sgpr0_sgpr1
+
+    %3 = COPY %sgpr0_sgpr1
+    %2 = COPY %vgpr0
+    %7 = S_LOAD_DWORDX2_IMM %3, 9, 0
+    %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
+    %9 = S_LOAD_DWORDX2_IMM %3, 13, 0
+    %6 = COPY %7
+    %10 = S_MOV_B32 0
+    %11 = REG_SEQUENCE %2, 1, killed %10, 2
+    %0 = COPY %11
+    %12 = COPY %11.sub0
+    %13 = COPY %11.sub1
+    %14 = COPY %8.sub0
+    %15 = COPY %8.sub1
+    %16 = S_ADD_U32 %12, killed %14, implicit-def %scc
+    %17 = S_ADDC_U32 %13, killed %15, implicit-def dead %scc, implicit %scc
+    %18 = REG_SEQUENCE killed %16, 1, killed %17, 2
+    %19 = COPY %9.sub0
+    %20 = COPY %9.sub1
+    %21 = S_ADD_U32 %12, killed %19, implicit-def %scc
+    %22 = S_ADDC_U32 %13, killed %20, implicit-def dead %scc, implicit %scc
+    %23 = REG_SEQUENCE killed %21, 1, killed %22, 2
+    %24 = S_MOV_B32 0
+    %25 = S_MOV_B32 1048576
+    %26 = REG_SEQUENCE killed %25, 1, killed %24, 2
+    %28 = COPY %26
+    %27 = V_CMP_LT_U64_e64 killed %18, %28, implicit %exec
+    %29 = V_CMP_LT_U64_e64 killed %23, %28, implicit %exec
+    %31 = S_AND_B64 killed %27, killed %29, implicit-def dead %scc
+    %1 = SI_IF killed %31, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_BRANCH %bb.1.bb1
+
+  bb.1.bb1:
+    successors: %bb.2.bb2(0x80000000)
+
+    %32 = S_MOV_B32 2
+    %33 = S_LSHL_B64 %0, killed %32, implicit-def dead %scc
+    %34 = S_MOV_B32 61440
+    %35 = S_MOV_B32 0
+    %36 = REG_SEQUENCE killed %35, 1, killed %34, 2
+    %37 = REG_SEQUENCE %6, 17, killed %36, 18
+    %38 = V_MOV_B32_e32 0, implicit %exec
+    %39 = COPY %33
+    BUFFER_STORE_DWORD_ADDR64 killed %38, killed %39, killed %37, 0, 0, 0, 0, 0, implicit %exec
+
+  bb.2.bb2:
+    SI_END_CF %1, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_ENDPGM
+
+...
+---
+name:            const_to_sgpr_subreg
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sreg_64 }
+  - { id: 1, class: sreg_64 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_64 }
+  - { id: 4, class: sreg_32_xm0 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sreg_64 }
+  - { id: 7, class: sreg_64_xexec }
+  - { id: 8, class: sreg_64_xexec }
+  - { id: 9, class: sreg_32 }
+  - { id: 10, class: sreg_64 }
+  - { id: 11, class: sreg_32_xm0 }
+  - { id: 12, class: sreg_32_xm0 }
+  - { id: 13, class: sreg_32_xm0 }
+  - { id: 14, class: sreg_32_xm0 }
+  - { id: 15, class: sreg_32_xm0 }
+  - { id: 16, class: sreg_32_xm0 }
+  - { id: 17, class: sreg_64 }
+  - { id: 18, class: sreg_32_xm0 }
+  - { id: 19, class: sreg_32_xm0 }
+  - { id: 20, class: sreg_64 }
+  - { id: 21, class: sreg_64 }
+  - { id: 22, class: vgpr_32 }
+  - { id: 23, class: sreg_32_xm0 }
+  - { id: 24, class: sreg_64 }
+  - { id: 25, class: sreg_32_xm0 }
+  - { id: 26, class: sreg_32_xm0 }
+  - { id: 27, class: sgpr_64 }
+  - { id: 28, class: sgpr_128 }
+  - { id: 29, class: vgpr_32 }
+  - { id: 30, class: vreg_64 }
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%2' }
+  - { reg: '%sgpr0_sgpr1', virtual-reg: '%3' }
+body:             |
+  bb.0.bb:
+    successors: %bb.1.bb1(0x40000000), %bb.2.bb2(0x40000000)
+    liveins: %vgpr0, %sgpr0_sgpr1
+
+    %3 = COPY %sgpr0_sgpr1
+    %2 = COPY %vgpr0
+    %7 = S_LOAD_DWORDX2_IMM %3, 9, 0
+    %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
+    %6 = COPY %7
+    %9 = S_MOV_B32 0
+    %10 = REG_SEQUENCE %2, 1, killed %9, 2
+    %0 = COPY %10
+    %11 = COPY %10.sub0
+    %12 = COPY %10.sub1
+    %13 = COPY %8.sub0
+    %14 = COPY %8.sub1
+    %15 = S_ADD_U32 killed %11, killed %13, implicit-def %scc
+    %16 = S_ADDC_U32 killed %12, killed %14, implicit-def dead %scc, implicit %scc
+    %17 = REG_SEQUENCE killed %15, 1, killed %16, 2
+    %18 = S_MOV_B32 12
+    %19 = S_MOV_B32 1048576
+    %20 = REG_SEQUENCE killed %19, 1, killed %18, 2
+    %22 = COPY killed %20.sub1
+    %21 = V_CMP_LT_U32_e64 killed %17.sub0, %22, implicit %exec
+    %1 = SI_IF killed %21, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_BRANCH %bb.1.bb1
+
+  bb.1.bb1:
+    successors: %bb.2.bb2(0x80000000)
+
+    %23 = S_MOV_B32 2
+    %24 = S_LSHL_B64 %0, killed %23, implicit-def dead %scc
+    %25 = S_MOV_B32 61440
+    %26 = S_MOV_B32 0
+    %27 = REG_SEQUENCE killed %26, 1, killed %25, 2
+    %28 = REG_SEQUENCE %6, 17, killed %27, 18
+    %29 = V_MOV_B32_e32 0, implicit %exec
+    %30 = COPY %24
+    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, implicit %exec
+
+  bb.2.bb2:
+    SI_END_CF %1, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_ENDPGM
+
+...

llvm/trunk/test/CodeGen/AMDGPU/sdwa-peephole.ll

 store_label:
   store <2 x i16> %add, <2 x i16> addrspace(1)* %out, align 4
   ret void
 }
 
 
 ; Check that "pulling out" SDWA operands works correctly.
 ; GCN-LABEL: {{^}}pulled_out_test:
-; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_and_b32_sdwa
 ; NOSDWA-NOT: v_or_b32_sdwa
 
 ; SDWA-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; SDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; SDWA-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD

llvm/trunk/test/CodeGen/AMDGPU/sint_to_fp.i64.ll

 ; GCN: v_xor_b32
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}}
 ; GCN: v_cndmask_b32_e{{32|64}} [[SIGN_SEL:v[0-9]+]],
 ; GCN: v_cvt_f16_f32_e32 [[SIGN_SEL_F16:v[0-9]+]], [[SIGN_SEL]]
 ; GCN: {{buffer|flat}}_store_short {{.*}}[[SIGN_SEL_F16]]
 define amdgpu_kernel void @v_sint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid

 ; GCN: v_xor_b32
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}}
 ; GCN: v_cndmask_b32_e{{32|64}} [[SIGN_SEL:v[0-9]+]],
 ; GCN: {{buffer|flat}}_store_dword {{.*}}[[SIGN_SEL]]
 define amdgpu_kernel void @v_sint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid

llvm/trunk/test/CodeGen/AMDGPU/uint_to_fp.i64.ll

 ; GCN: {{buffer|flat}}_load_dwordx2
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_add_i32_e32 [[VR:v[0-9]+]]
 ; GCN: v_cvt_f16_f32_e32 [[VR_F16:v[0-9]+]], [[VR]]
 ; GCN: {{buffer|flat}}_store_short {{.*}}[[VR_F16]]
 define amdgpu_kernel void @v_uint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr half, half addrspace(1)* %out, i32 %tid

 ; GCN: {{buffer|flat}}_load_dwordx2
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_add_i32_e32 [[VR:v[0-9]+]]
 ; GCN: {{buffer|flat}}_store_dword {{.*}}[[VR]]
 define amdgpu_kernel void @v_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
   %val = load i64, i64 addrspace(1)* %in.gep