[AMDGPU] Eliminate SGPR to VGPR copy when possible

[rampitec]

Summary

SGPRs are generally cheaper, so try to use them over VGPRs.

[arsenm]

Typo in test file name

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

182–187

This probably breaks if you have a copy that looks like a sub register extract

192

Why do you check the parent?

193

Why do you need to check this? Isn't this filtering the global isel pseudos which should never exist here?

194

Avoid doing the weird pointer arithmetic?

198

I forsee this breaking cases with multiple uses

test/CodeGen/AMDGPU/opt-sgpr-to-vgrp-copy.ll

25 ↗

(On Diff #102279)

Probably need a MIR test to stress multiple uses

[rampitec]

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

182–187

The should not be a problem. I'm changing VGPR class to equivalent SGPR class, they support the same set of subregs.

192

A copy can be in a divergent control flow. If it is the same block we can be sure that is actually the value we are going to use.

193

isOperandLegal does not work with generic opcodes because they have no RC classes assigned to operands.

194

How do I get operand number otherwise?

198

Why? I'm checking all uses for operand legality. It also does not happen if there is a second def of the same register.

[rampitec]

Fixed typo in test name.

[rampitec]

test/CodeGen/AMDGPU/opt-sgpr-to-vgrp-copy.ll

25 ↗

(On Diff #102279)

Can you be more specific, multiple uses of what? Do you mean multiple uses of the same COPY?

[rampitec]

Replaced test with mir to check multiple use case.

[rampitec]

Removed pointer arithmetic on operands.

[rampitec]

Added subreg-extract like copy test.
Cleaned mir test.

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

182–187

I have added test for that.

[rampitec]

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

198

I have added test for multiple uses.

[rampitec]

Ping.

[arsenm]

LGTM. It's possible to reduce the mir testcase further though

test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir

103–118

You can strip out most of this stuff

[rampitec]

Reduced test.

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

 }
 
 static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
                              const TargetRegisterClass *DstRC,
                              const SIRegisterInfo &TRI) {
   return TRI.isSGPRClass(SrcRC) && TRI.hasVGPRs(DstRC);
 }
 
+static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
+                                      const SIRegisterInfo *TRI,
+                                      const SIInstrInfo *TII) {
+  MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
+  auto &Src = MI.getOperand(1);
+  unsigned DstReg = MI.getOperand(0).getReg();
+  unsigned SrcReg = Src.getReg();
+  if (!TargetRegisterInfo::isVirtualRegister(SrcReg) ||
+      !TargetRegisterInfo::isVirtualRegister(DstReg))
+    return false;
+
+  for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
+    const auto *UseMI = MO.getParent();
+    if (UseMI == &MI)
+      continue;
+    if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
+        UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
+        !TII->isOperandLegal(*UseMI, UseMI->getOperandNo(&MO), &Src))
+      return false;
+  }
+  // Change VGPR to SGPR destination.
+  MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
+  return true;
+}
+
 // Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
 //
 // SGPRx = ...
 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
 // VGPRz = COPY SGPRy
 //
 // ==>
 //
if (TargetRegisterInfo::isPhysicalRegister(CopyUse.getOperand(0).getReg()))
     return false;
 
   const TargetRegisterClass *SrcRC, *DstRC;
   std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
 
   if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
     return false;
 
+  if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
+    return true;
+
   // TODO: Could have multiple extracts?
   unsigned SubReg = CopyUse.getOperand(1).getSubReg();
   if (SubReg != AMDGPU::NoSubRegister)
     return false;
 
   MRI.setRegClass(DstReg, DstRC);
 
   // SGPRx = ...
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
           // s_mov_b32.
           if (isSafeToFoldImmIntoCopy(&MI, DefMI, TII, SMovOp, Imm)) {
             MI.getOperand(1).ChangeToImmediate(Imm);
             MI.addImplicitDefUseOperands(MF);
             MI.setDesc(TII->get(SMovOp));
             break;
           }
           TII->moveToVALU(MI);
+        } else if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
+          tryChangeVGPRtoSGPRinCopy(MI, TRI, TII);
         }
 
         break;
       }
       case AMDGPU::PHI: {
         unsigned Reg = MI.getOperand(0).getReg();
         if (!TRI->isSGPRClass(MRI.getRegClass(Reg)))
           break;

test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 declare i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64, i32, i64) #0
 
 ; GCN-LABEL: {{^}}v_mqsad_pk_u16_u8:
-; GCN: v_mqsad_pk_u16_u8 v[0:1], v[4:5], s{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
+; GCN: v_mqsad_pk_u16_u8 v[0:1], v[4:5], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}]
 ; GCN-DAG: v_mov_b32_e32 v5, v1
 ; GCN-DAG: v_mov_b32_e32 v4, v0
 define amdgpu_kernel void @v_mqsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) {
   %tmp = call i64 asm "v_lsrlrev_b64 $0, $1, 1", "={v[4:5]},v"(i64 %src) #0
   %tmp1 = call i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %tmp, i32 100, i64 100) #0
   %tmp2 = call i64 asm ";; force constraint", "=v,{v[4:5]}"(i64 %tmp1) #0
   store i64 %tmp2, i64 addrspace(1)* %out, align 4
   ret void

test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 declare i64 @llvm.amdgcn.qsad.pk.u16.u8(i64, i32, i64) #0
 
 ; GCN-LABEL: {{^}}v_qsad_pk_u16_u8:
-; GCN: v_qsad_pk_u16_u8 v[0:1], v[4:5], s{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]
+; GCN: v_qsad_pk_u16_u8 v[0:1], v[4:5], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}]
 ; GCN-DAG: v_mov_b32_e32 v5, v1
 ; GCN-DAG: v_mov_b32_e32 v4, v0
 define amdgpu_kernel void @v_qsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) {
   %tmp = call i64 asm "v_lsrlrev_b64 $0, $1, 1", "={v[4:5]},v"(i64 %src) #0
   %tmp1 = call i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %tmp, i32 100, i64 100) #0
   %tmp2 = call i64 asm ";; force constraint", "=v,{v[4:5]}"(i64 %tmp1) #0
   store i64 %tmp2, i64 addrspace(1)* %out, align 4
   ret void

test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir

+# RUN: llc -march=amdgcn -run-pass si-fix-sgpr-copies,si-fold-operands,dead-mi-elimination -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+
+# Check that constant is in SGPR registers
+
+# GCN-LABEL: {{^}}name: const_to_sgpr{{$}}
+# GCN:        %[[HI:[0-9]+]] = S_MOV_B32 0
+# GCN-NEXT:   %[[LO:[0-9]+]] = S_MOV_B32 1048576
+# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]] = REG_SEQUENCE killed %[[LO]], 1, killed %[[HI]], 2
+# GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
+
+
+# GCN-LABEL: {{^}}name: const_to_sgpr_multiple_use{{$}}
+# GCN:        %[[HI:[0-9]+]] = S_MOV_B32 0
+# GCN-NEXT:   %[[LO:[0-9]+]] = S_MOV_B32 1048576
+# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]] = REG_SEQUENCE killed %[[LO]], 1, killed %[[HI]], 2
+# GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
+# GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
+
+# GCN-LABEL: {{^}}name: const_to_sgpr_subreg{{$}}
+# GCN:       %[[OP0:[0-9]+]] = REG_SEQUENCE killed %{{[0-9]+}}, 1, killed %{{[0-9]+}}, 2
+# GCN-NEXT:  V_CMP_LT_U32_e64 killed %[[OP0]].sub0, 12, implicit %exec
+
+--- |
+  define amdgpu_kernel void @const_to_sgpr(i32 addrspace(1)* nocapture %arg, i64 %id) {
+  bb:
+    br i1 undef, label %bb1, label %bb2
+
+  bb1:                                              ; preds = %bb
+    br label %bb2
+
+  bb2:                                              ; preds = %bb1, %bb
+    ret void
+  }
+
+  define amdgpu_kernel void @const_to_sgpr_multiple_use(i32 addrspace(1)* nocapture %arg, i64 %id1, i64 %id2) {
+  bb:
+    br i1 undef, label %bb1, label %bb2
+
+  bb1:                                              ; preds = %bb
+    br label %bb2
+
+  bb2:                                              ; preds = %bb1, %bb
+    ret void
+  }
+
+  define amdgpu_kernel void @const_to_sgpr_subreg(i32 addrspace(1)* nocapture %arg, i64 %id) {
+  bb:
+    br i1 undef, label %bb1, label %bb2
+
+  bb1:                                              ; preds = %bb
+    br label %bb2
+
+  bb2:                                              ; preds = %bb1, %bb
+    ret void
+  }
+
+...
+---
+name:            const_to_sgpr
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sreg_64 }
+  - { id: 1, class: sreg_64 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_64 }
+  - { id: 4, class: sreg_32_xm0 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sreg_64 }
+  - { id: 7, class: sreg_64_xexec }
+  - { id: 8, class: sreg_64_xexec }
+  - { id: 9, class: sreg_32 }
+  - { id: 10, class: sreg_64 }
+  - { id: 11, class: sreg_32_xm0 }
+  - { id: 12, class: sreg_32_xm0 }
+  - { id: 13, class: sreg_32_xm0 }
+  - { id: 14, class: sreg_32_xm0 }
+  - { id: 15, class: sreg_32_xm0 }
+  - { id: 16, class: sreg_32_xm0 }
+  - { id: 17, class: sreg_64 }
+  - { id: 18, class: sreg_32_xm0 }
+  - { id: 19, class: sreg_32_xm0 }
+  - { id: 20, class: sreg_64 }
+  - { id: 21, class: sreg_64 }
+  - { id: 22, class: vreg_64 }
+  - { id: 23, class: sreg_32_xm0 }
+  - { id: 24, class: sreg_64 }
+  - { id: 25, class: sreg_32_xm0 }
+  - { id: 26, class: sreg_32_xm0 }
+  - { id: 27, class: sgpr_64 }
+  - { id: 28, class: sgpr_128 }
+  - { id: 29, class: vgpr_32 }
+  - { id: 30, class: vreg_64 }
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%2' }
+  - { reg: '%sgpr0_sgpr1', virtual-reg: '%3' }
+body:             |
+  bb.0.bb:
+    successors: %bb.1.bb1(0x40000000), %bb.2.bb2(0x40000000)
+    liveins: %vgpr0, %sgpr0_sgpr1
+
+    %3 = COPY %sgpr0_sgpr1
+    %2 = COPY %vgpr0
+    %7 = S_LOAD_DWORDX2_IMM %3, 9, 0
+    %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
+    %6 = COPY %7
+    %9 = S_MOV_B32 0
+    %10 = REG_SEQUENCE %2, 1, killed %9, 2
+    %0 = COPY %10
+    %11 = COPY %10.sub0
+    %12 = COPY %10.sub1
+    %13 = COPY %8.sub0
+    %14 = COPY %8.sub1
+    %15 = S_ADD_U32 killed %11, killed %13, implicit-def %scc
+    %16 = S_ADDC_U32 killed %12, killed %14, implicit-def dead %scc, implicit %scc
+    %17 = REG_SEQUENCE killed %15, 1, killed %16, 2
+    %18 = S_MOV_B32 0
+    %19 = S_MOV_B32 1048576
+    %20 = REG_SEQUENCE killed %19, 1, killed %18, 2
+    %22 = COPY killed %20
+    %21 = V_CMP_LT_U64_e64 killed %17, %22, implicit %exec
+    %1 = SI_IF killed %21, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_BRANCH %bb.1.bb1
+
+  bb.1.bb1:
+    successors: %bb.2.bb2(0x80000000)
+
+    %23 = S_MOV_B32 2
+    %24 = S_LSHL_B64 %0, killed %23, implicit-def dead %scc
+    %25 = S_MOV_B32 61440
+    %26 = S_MOV_B32 0
+    %27 = REG_SEQUENCE killed %26, 1, killed %25, 2
+    %28 = REG_SEQUENCE %6, 17, killed %27, 18
+    %29 = V_MOV_B32_e32 0, implicit %exec
+    %30 = COPY %24
+    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, implicit %exec
+
+  bb.2.bb2:
+    SI_END_CF %1, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_ENDPGM
+
+...
+---
+name:            const_to_sgpr_multiple_use
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sreg_64 }
+  - { id: 1, class: sreg_64 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_64 }
+  - { id: 4, class: sreg_32_xm0 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sreg_64 }
+  - { id: 7, class: sreg_64_xexec }
+  - { id: 8, class: sreg_64_xexec }
+  - { id: 9, class: sreg_64_xexec }
+  - { id: 10, class: sreg_32 }
+  - { id: 11, class: sreg_64 }
+  - { id: 12, class: sreg_32_xm0 }
+  - { id: 13, class: sreg_32_xm0 }
+  - { id: 14, class: sreg_32_xm0 }
+  - { id: 15, class: sreg_32_xm0 }
+  - { id: 16, class: sreg_32_xm0 }
+  - { id: 17, class: sreg_32_xm0 }
+  - { id: 18, class: sreg_64 }
+  - { id: 19, class: sreg_32_xm0 }
+  - { id: 20, class: sreg_32_xm0 }
+  - { id: 21, class: sreg_32_xm0 }
+  - { id: 22, class: sreg_32_xm0 }
+  - { id: 23, class: sreg_64 }
+  - { id: 24, class: sreg_32_xm0 }
+  - { id: 25, class: sreg_32_xm0 }
+  - { id: 26, class: sreg_64 }
+  - { id: 27, class: sreg_64 }
+  - { id: 28, class: vreg_64 }
+  - { id: 29, class: sreg_64 }
+  - { id: 30, class: vreg_64 }
+  - { id: 31, class: sreg_64 }
+  - { id: 32, class: sreg_32_xm0 }
+  - { id: 33, class: sreg_64 }
+  - { id: 34, class: sreg_32_xm0 }
+  - { id: 35, class: sreg_32_xm0 }
+  - { id: 36, class: sgpr_64 }
+  - { id: 37, class: sgpr_128 }
+  - { id: 38, class: vgpr_32 }
+  - { id: 39, class: vreg_64 }
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%2' }
+  - { reg: '%sgpr0_sgpr1', virtual-reg: '%3' }
+body:             |
+  bb.0.bb:
+    successors: %bb.1.bb1(0x40000000), %bb.2.bb2(0x40000000)
+    liveins: %vgpr0, %sgpr0_sgpr1
+
+    %3 = COPY %sgpr0_sgpr1
+    %2 = COPY %vgpr0
+    %7 = S_LOAD_DWORDX2_IMM %3, 9, 0
+    %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
+    %9 = S_LOAD_DWORDX2_IMM %3, 13, 0
+    %6 = COPY %7
+    %10 = S_MOV_B32 0
+    %11 = REG_SEQUENCE %2, 1, killed %10, 2
+    %0 = COPY %11
+    %12 = COPY %11.sub0
+    %13 = COPY %11.sub1
+    %14 = COPY %8.sub0
+    %15 = COPY %8.sub1
+    %16 = S_ADD_U32 %12, killed %14, implicit-def %scc
+    %17 = S_ADDC_U32 %13, killed %15, implicit-def dead %scc, implicit %scc
+    %18 = REG_SEQUENCE killed %16, 1, killed %17, 2
+    %19 = COPY %9.sub0
+    %20 = COPY %9.sub1
+    %21 = S_ADD_U32 %12, killed %19, implicit-def %scc
+    %22 = S_ADDC_U32 %13, killed %20, implicit-def dead %scc, implicit %scc
+    %23 = REG_SEQUENCE killed %21, 1, killed %22, 2
+    %24 = S_MOV_B32 0
+    %25 = S_MOV_B32 1048576
+    %26 = REG_SEQUENCE killed %25, 1, killed %24, 2
+    %28 = COPY %26
+    %27 = V_CMP_LT_U64_e64 killed %18, %28, implicit %exec
+    %29 = V_CMP_LT_U64_e64 killed %23, %28, implicit %exec
+    %31 = S_AND_B64 killed %27, killed %29, implicit-def dead %scc
+    %1 = SI_IF killed %31, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_BRANCH %bb.1.bb1
+
+  bb.1.bb1:
+    successors: %bb.2.bb2(0x80000000)
+
+    %32 = S_MOV_B32 2
+    %33 = S_LSHL_B64 %0, killed %32, implicit-def dead %scc
+    %34 = S_MOV_B32 61440
+    %35 = S_MOV_B32 0
+    %36 = REG_SEQUENCE killed %35, 1, killed %34, 2
+    %37 = REG_SEQUENCE %6, 17, killed %36, 18
+    %38 = V_MOV_B32_e32 0, implicit %exec
+    %39 = COPY %33
+    BUFFER_STORE_DWORD_ADDR64 killed %38, killed %39, killed %37, 0, 0, 0, 0, 0, implicit %exec
+
+  bb.2.bb2:
+    SI_END_CF %1, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_ENDPGM
+
+...
+---
+name:            const_to_sgpr_subreg
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sreg_64 }
+  - { id: 1, class: sreg_64 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_64 }
+  - { id: 4, class: sreg_32_xm0 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sreg_64 }
+  - { id: 7, class: sreg_64_xexec }
+  - { id: 8, class: sreg_64_xexec }
+  - { id: 9, class: sreg_32 }
+  - { id: 10, class: sreg_64 }
+  - { id: 11, class: sreg_32_xm0 }
+  - { id: 12, class: sreg_32_xm0 }
+  - { id: 13, class: sreg_32_xm0 }
+  - { id: 14, class: sreg_32_xm0 }
+  - { id: 15, class: sreg_32_xm0 }
+  - { id: 16, class: sreg_32_xm0 }
+  - { id: 17, class: sreg_64 }
+  - { id: 18, class: sreg_32_xm0 }
+  - { id: 19, class: sreg_32_xm0 }
+  - { id: 20, class: sreg_64 }
+  - { id: 21, class: sreg_64 }
+  - { id: 22, class: vgpr_32 }
+  - { id: 23, class: sreg_32_xm0 }
+  - { id: 24, class: sreg_64 }
+  - { id: 25, class: sreg_32_xm0 }
+  - { id: 26, class: sreg_32_xm0 }
+  - { id: 27, class: sgpr_64 }
+  - { id: 28, class: sgpr_128 }
+  - { id: 29, class: vgpr_32 }
+  - { id: 30, class: vreg_64 }
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%2' }
+  - { reg: '%sgpr0_sgpr1', virtual-reg: '%3' }
+body:             |
+  bb.0.bb:
+    successors: %bb.1.bb1(0x40000000), %bb.2.bb2(0x40000000)
+    liveins: %vgpr0, %sgpr0_sgpr1
+
+    %3 = COPY %sgpr0_sgpr1
+    %2 = COPY %vgpr0
+    %7 = S_LOAD_DWORDX2_IMM %3, 9, 0
+    %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
+    %6 = COPY %7
+    %9 = S_MOV_B32 0
+    %10 = REG_SEQUENCE %2, 1, killed %9, 2
+    %0 = COPY %10
+    %11 = COPY %10.sub0
+    %12 = COPY %10.sub1
+    %13 = COPY %8.sub0
+    %14 = COPY %8.sub1
+    %15 = S_ADD_U32 killed %11, killed %13, implicit-def %scc
+    %16 = S_ADDC_U32 killed %12, killed %14, implicit-def dead %scc, implicit %scc
+    %17 = REG_SEQUENCE killed %15, 1, killed %16, 2
+    %18 = S_MOV_B32 12
+    %19 = S_MOV_B32 1048576
+    %20 = REG_SEQUENCE killed %19, 1, killed %18, 2
+    %22 = COPY killed %20.sub1
+    %21 = V_CMP_LT_U32_e64 killed %17.sub0, %22, implicit %exec
+    %1 = SI_IF killed %21, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_BRANCH %bb.1.bb1
+
+  bb.1.bb1:
+    successors: %bb.2.bb2(0x80000000)
+
+    %23 = S_MOV_B32 2
+    %24 = S_LSHL_B64 %0, killed %23, implicit-def dead %scc
+    %25 = S_MOV_B32 61440
+    %26 = S_MOV_B32 0
+    %27 = REG_SEQUENCE killed %26, 1, killed %25, 2
+    %28 = REG_SEQUENCE %6, 17, killed %27, 18
+    %29 = V_MOV_B32_e32 0, implicit %exec
+    %30 = COPY %24
+    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, implicit %exec
+
+  bb.2.bb2:
+    SI_END_CF %1, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
+    S_ENDPGM
+
+...

test/CodeGen/AMDGPU/sdwa-peephole.ll

 store_label:
   store <2 x i16> %add, <2 x i16> addrspace(1)* %out, align 4
   ret void
 }
 
 
 ; Check that "pulling out" SDWA operands works correctly.
 ; GCN-LABEL: {{^}}pulled_out_test:
-; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_and_b32_sdwa
 ; NOSDWA-NOT: v_or_b32_sdwa
 
 ; SDWA-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; SDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; SDWA-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD

test/CodeGen/AMDGPU/sint_to_fp.i64.ll

 ; GCN: v_xor_b32
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}}
 ; GCN: v_cndmask_b32_e{{32|64}} [[SIGN_SEL:v[0-9]+]],
 ; GCN: v_cvt_f16_f32_e32 [[SIGN_SEL_F16:v[0-9]+]], [[SIGN_SEL]]
 ; GCN: {{buffer|flat}}_store_short {{.*}}[[SIGN_SEL_F16]]
 define amdgpu_kernel void @v_sint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid

 ; GCN: v_xor_b32
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}}
 ; GCN: v_cndmask_b32_e{{32|64}} [[SIGN_SEL:v[0-9]+]],
 ; GCN: {{buffer|flat}}_store_dword {{.*}}[[SIGN_SEL]]
 define amdgpu_kernel void @v_sint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid

test/CodeGen/AMDGPU/uint_to_fp.i64.ll

 ; GCN: {{buffer|flat}}_load_dwordx2
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_add_i32_e32 [[VR:v[0-9]+]]
 ; GCN: v_cvt_f16_f32_e32 [[VR_F16:v[0-9]+]], [[VR]]
 ; GCN: {{buffer|flat}}_store_short {{.*}}[[VR_F16]]
 define amdgpu_kernel void @v_uint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr half, half addrspace(1)* %out, i32 %tid

 ; GCN: {{buffer|flat}}_load_dwordx2
 
 ; GCN: v_ffbh_u32
 ; GCN: v_ffbh_u32
 ; GCN: v_cndmask
 ; GCN: v_cndmask
 
 ; GCN-DAG: v_cmp_eq_u64
-; GCN-DAG: v_cmp_lt_u64
+; GCN-DAG: v_cmp_gt_u64
 
 ; GCN: v_add_i32_e32 [[VR:v[0-9]+]]
 ; GCN: {{buffer|flat}}_store_dword {{.*}}[[VR]]
 define amdgpu_kernel void @v_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
   %val = load i64, i64 addrspace(1)* %in.gep