AMDGPU/GlobalISel: Improve icmp selection coverage.

[arsenm]

Summary

Select s64 eq/ne scalar icmp.

[nhaehnle]

LGTM

[arsenm]

r364765

lib/Target/AMDGPU/AMDGPUInstructionSelector.h

 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
 
 #include "AMDGPU.h"
 #include "AMDGPUArgumentUsageInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
+#include "llvm/IR/InstrTypes.h"
 
 namespace {
 #define GET_GLOBALISEL_PREDICATE_BITSET
 #define AMDGPUSubtarget GCNSubtarget
 #include "AMDGPUGenGlobalISel.inc"
 #undef GET_GLOBALISEL_PREDICATE_BITSET
 #undef AMDGPUSubtarget
 }
private:
   bool selectG_ADD(MachineInstr &I) const;
   bool selectG_EXTRACT(MachineInstr &I) const;
   bool selectG_GEP(MachineInstr &I) const;
   bool selectG_IMPLICIT_DEF(MachineInstr &I) const;
   bool selectG_INSERT(MachineInstr &I) const;
   bool selectG_INTRINSIC(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
   bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I,
                                         CodeGenCoverage &CoverageInfo) const;
+  int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;
   bool selectG_ICMP(MachineInstr &I) const;
   bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const;
   void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI,
                        SmallVectorImpl<GEPInfo> &AddrInfo) const;
   bool selectSMRD(MachineInstr &I, ArrayRef<GEPInfo> AddrInfo) const;
   bool selectG_LOAD(MachineInstr &I) const;
   bool selectG_SELECT(MachineInstr &I) const;
   bool selectG_STORE(MachineInstr &I) const;

lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

BuildMI(*I.getParent(), &I, DL, TII.get(AMDGPU::COPY))
       .addReg(MRI.getLiveInVirtReg(InputPtrReg->getRegister()));
     I.eraseFromParent();
     return true;
   }
   }
   return false;
 }
 
-static unsigned getV_CMPOpcode(CmpInst::Predicate P, unsigned Size) {
-  assert(Size == 32 || Size == 64);
+static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size) {
+  if (Size != 32 && Size != 64)
+    return -1;
   switch (P) {
   default:
     llvm_unreachable("Unknown condition code!");
   case CmpInst::ICMP_NE:
     return Size == 32 ? AMDGPU::V_CMP_NE_U32_e64 : AMDGPU::V_CMP_NE_U64_e64;
   case CmpInst::ICMP_EQ:
     return Size == 32 ? AMDGPU::V_CMP_EQ_U32_e64 : AMDGPU::V_CMP_EQ_U64_e64;
   case CmpInst::ICMP_SGT:
case CmpInst::ICMP_UGE:
     return Size == 32 ? AMDGPU::V_CMP_GE_U32_e64 : AMDGPU::V_CMP_GE_U64_e64;
   case CmpInst::ICMP_ULT:
     return Size == 32 ? AMDGPU::V_CMP_LT_U32_e64 : AMDGPU::V_CMP_LT_U64_e64;
   case CmpInst::ICMP_ULE:
     return Size == 32 ? AMDGPU::V_CMP_LE_U32_e64 : AMDGPU::V_CMP_LE_U64_e64;
   }
 }
 
-static unsigned getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) {
-  // FIXME: VI supports 64-bit comparse.
-  assert(Size == 32);
+int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
+                                              unsigned Size) const {
+  if (Size == 64) {
+    if (!STI.hasScalarCompareEq64())
+      return -1;
+
     switch (P) {
+    case CmpInst::ICMP_NE:
+      return AMDGPU::S_CMP_LG_U64;
+    case CmpInst::ICMP_EQ:
+      return AMDGPU::S_CMP_EQ_U64;
     default:
-    llvm_unreachable("Unknown condition code!");
+      return -1;
+    }
+  }
+
+  if (Size != 32)
+    return -1;
+
+  switch (P) {
   case CmpInst::ICMP_NE:
     return AMDGPU::S_CMP_LG_U32;
   case CmpInst::ICMP_EQ:
     return AMDGPU::S_CMP_EQ_U32;
   case CmpInst::ICMP_SGT:
     return AMDGPU::S_CMP_GT_I32;
   case CmpInst::ICMP_SGE:
     return AMDGPU::S_CMP_GE_I32;
   case CmpInst::ICMP_SLT:
     return AMDGPU::S_CMP_LT_I32;
   case CmpInst::ICMP_SLE:
     return AMDGPU::S_CMP_LE_I32;
   case CmpInst::ICMP_UGT:
     return AMDGPU::S_CMP_GT_U32;
   case CmpInst::ICMP_UGE:
     return AMDGPU::S_CMP_GE_U32;
   case CmpInst::ICMP_ULT:
     return AMDGPU::S_CMP_LT_U32;
   case CmpInst::ICMP_ULE:
     return AMDGPU::S_CMP_LE_U32;
+  default:
+    llvm_unreachable("Unknown condition code!");
   }
 }
 
 bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
   MachineFunction *MF = BB->getParent();
   MachineRegisterInfo &MRI = MF->getRegInfo();
   DebugLoc DL = I.getDebugLoc();
 
   unsigned SrcReg = I.getOperand(2).getReg();
   unsigned Size = RBI.getSizeInBits(SrcReg, MRI, TRI);
-  // FIXME: VI supports 64-bit compares.
-  assert(Size == 32);
+
+  auto Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
 
   unsigned CCReg = I.getOperand(0).getReg();
   if (isSCC(CCReg, MRI)) {
-    unsigned Opcode = getS_CMPOpcode((CmpInst::Predicate)I.getOperand(1).getPredicate(), Size);
+    int Opcode = getS_CMPOpcode(Pred, Size);
+    if (Opcode == -1)
+      return false;
     MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode))
             .add(I.getOperand(2))
             .add(I.getOperand(3));
     BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg)
       .addReg(AMDGPU::SCC);
     bool Ret =
         constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI) &&
         RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, MRI);
     I.eraseFromParent();
     return Ret;
   }
 
-  assert(Size == 32 || Size == 64);
-  unsigned Opcode = getV_CMPOpcode((CmpInst::Predicate)I.getOperand(1).getPredicate(), Size);
+  int Opcode = getV_CMPOpcode(Pred, Size);
+  if (Opcode == -1)
+    return false;
+
   MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode),
             I.getOperand(0).getReg())
             .add(I.getOperand(2))
             .add(I.getOperand(3));
   RBI.constrainGenericRegister(ICmp->getOperand(0).getReg(),
                                AMDGPU::SReg_64RegClass, MRI);
   bool Ret = constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI);
   I.eraseFromParent();
case TargetOpcode::G_IMPLICIT_DEF:
     return selectG_IMPLICIT_DEF(I);
   case TargetOpcode::G_INSERT:
     return selectG_INSERT(I);
   case TargetOpcode::G_INTRINSIC:
     return selectG_INTRINSIC(I, CoverageInfo);
   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
     return selectG_INTRINSIC_W_SIDE_EFFECTS(I, CoverageInfo);
   case TargetOpcode::G_ICMP:
-    return selectG_ICMP(I);
+    if (selectG_ICMP(I))
+      return true;
+    return selectImpl(I, CoverageInfo);
   case TargetOpcode::G_LOAD:
     if (selectImpl(I, CoverageInfo))
       return true;
     return selectG_LOAD(I);
   case TargetOpcode::G_SELECT:
     return selectG_SELECT(I);
   case TargetOpcode::G_STORE:
     return selectG_STORE(I);

test/CodeGen/AMDGPU/GlobalISel/inst-select-icmp.s64.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -pass-remarks-missed='gisel*' -global-isel-abort=2 -o - %s 2> %t | FileCheck -check-prefix=GFX8  %s
+# RUN: FileCheck -check-prefixes=ERR,GFX8-ERR %s < %t
+
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=2 -pass-remarks-missed='gisel*'  -o - %s 2> %t | FileCheck -check-prefix=GFX6  %s
+# RUN: FileCheck -check-prefixes=ERR,GFX6-ERR %s < %t
+
+
+# ERR-NOT: remark
+
+# GFX6-ERR: remark: <unknown>:0:0: cannot select: %2:scc(s1) = G_ICMP intpred(eq), %0:sgpr(s64), %1:sgpr (in function: icmp_eq_s64_ss)
+# GFX6-ERR-NEXT: remark: <unknown>:0:0: cannot select: %2:scc(s1) = G_ICMP intpred(ne), %0:sgpr(s64), %1:sgpr (in function: icmp_ne_s64_ss)
+# GFX6-ERR-NEXT: remark: <unknown>:0:0: cannot select: %2:scc(s1) = G_ICMP intpred(slt), %0:sgpr(s64), %1:sgpr (in function: icmp_slt_s64_ss)
+# GFX6-ERR-NEXT: remark: <unknown>:0:0: cannot select: %2:scc(s1) = G_ICMP intpred(eq), %0:sgpr(p0), %1:sgpr (in function: icmp_eq_p0_ss)
+# GFX6-ERR-NEXT: remark: <unknown>:0:0: cannot select: %2:scc(s1) = G_ICMP intpred(eq), %0:sgpr(p1), %1:sgpr (in function: icmp_eq_p1_ss)
+# GFX6-ERR-NEXT: remark: <unknown>:0:0: cannot select: %2:scc(s1) = G_ICMP intpred(eq), %0:sgpr(p999), %1:sgpr (in function: icmp_eq_p999_ss)
+
+# GFX8-ERR: remark: <unknown>:0:0: cannot select: %2:scc(s1) = G_ICMP intpred(slt), %0:sgpr(s64), %1:sgpr (in function: icmp_slt_s64_ss)
+
+# ERR-NOT: remark
+
+---
+name:            icmp_eq_s64_ss
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_s64_ss
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
+    ; GFX8: S_CMP_EQ_U64 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_CSELECT_B32_]]
+    ; GFX6-LABEL: name: icmp_eq_s64_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; GFX6: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY]](s64), [[COPY1]]
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[ICMP]](s1)
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32(s32) = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_CSELECT_B32_]](s32)
+    %0:sgpr(s64) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = COPY $sgpr2_sgpr3
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_ne_s64_ss
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; GFX8-LABEL: name: icmp_ne_s64_ss
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
+    ; GFX8: S_CMP_LG_U64 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_CSELECT_B32_]]
+    ; GFX6-LABEL: name: icmp_ne_s64_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; GFX6: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s64), [[COPY1]]
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[ICMP]](s1)
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32(s32) = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_CSELECT_B32_]](s32)
+    %0:sgpr(s64) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = COPY $sgpr2_sgpr3
+    %2:scc(s1) = G_ICMP intpred(ne), %0, %1
+    %3:sgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_slt_s64_ss
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; GFX8-LABEL: name: icmp_slt_s64_ss
+    ; GFX8: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; GFX8: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(slt), [[COPY]](s64), [[COPY1]]
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[ICMP]](s1)
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32(s32) = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_CSELECT_B32_]](s32)
+    ; GFX6-LABEL: name: icmp_slt_s64_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; GFX6: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(slt), [[COPY]](s64), [[COPY1]]
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[ICMP]](s1)
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32(s32) = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_CSELECT_B32_]](s32)
+    %0:sgpr(s64) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = COPY $sgpr2_sgpr3
+    %2:scc(s1) = G_ICMP intpred(slt), %0, %1
+    %3:sgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_eq_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_eq_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_ne_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_ne_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_NE_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_ne_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_NE_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(ne), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_sgt_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_sgt_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_GT_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GT_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GT_I64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_sgt_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_GT_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GT_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GT_I64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(sgt), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_sge_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_sge_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_GE_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GE_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GE_I64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_sge_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_GE_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GE_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GE_I64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(sge), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_slt_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_slt_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_LT_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LT_I64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_slt_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_LT_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LT_I64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(slt), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_sle_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_sle_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_LE_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LE_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LE_I64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_sle_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_LE_I64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LE_I64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LE_I64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(sle), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_ugt_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_ugt_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_GT_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GT_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GT_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_ugt_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_GT_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GT_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GT_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(ugt), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_uge_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_uge_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_GE_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GE_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GE_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_uge_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_GE_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_GE_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_GE_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(uge), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_ult_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_ult_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_LT_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LT_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_ult_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_LT_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LT_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(ult), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_ule_s64_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_ule_s64_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_LE_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LE_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LE_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_ule_s64_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_LE_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_LE_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_LE_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s64) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(ule), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_eq_p0_ss
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_p0_ss
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
+    ; GFX8: S_CMP_EQ_U64 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_CSELECT_B32_]]
+    ; GFX6-LABEL: name: icmp_eq_p0_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
+    ; GFX6: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY]](p0), [[COPY1]]
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[ICMP]](s1)
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32(s32) = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_CSELECT_B32_]](s32)
+    %0:sgpr(p0) = COPY $sgpr0_sgpr1
+    %1:sgpr(p0) = COPY $sgpr2_sgpr3
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_eq_p1_ss
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_p1_ss
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
+    ; GFX8: S_CMP_EQ_U64 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_CSELECT_B32_]]
+    ; GFX6-LABEL: name: icmp_eq_p1_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(p1) = COPY $sgpr2_sgpr3
+    ; GFX6: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY]](p1), [[COPY1]]
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[ICMP]](s1)
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32(s32) = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_CSELECT_B32_]](s32)
+    %0:sgpr(p1) = COPY $sgpr0_sgpr1
+    %1:sgpr(p1) = COPY $sgpr2_sgpr3
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_eq_p999_ss
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_p999_ss
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
+    ; GFX8: S_CMP_EQ_U64 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_CSELECT_B32_]]
+    ; GFX6-LABEL: name: icmp_eq_p999_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p999) = COPY $sgpr0_sgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(p999) = COPY $sgpr2_sgpr3
+    ; GFX6: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY]](p999), [[COPY1]]
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[ICMP]](s1)
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32(s32) = S_CSELECT_B32 0, 1, implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_CSELECT_B32_]](s32)
+    %0:sgpr(p999) = COPY $sgpr0_sgpr1
+    %1:sgpr(p999) = COPY $sgpr2_sgpr3
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_eq_p0_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_p0_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_eq_p0_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(p0) = COPY $vgpr0_vgpr1
+    %1:vgpr(p0) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_eq_p1_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_p1_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_eq_p1_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(p1) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...
+
+---
+name:            icmp_eq_p999_vv
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: icmp_eq_p999_vv
+    ; GFX8: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX8: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX6-LABEL: name: icmp_eq_p999_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; GFX6: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U64_e64_]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(p999) = COPY $vgpr0_vgpr1
+    %1:vgpr(p999) = COPY $vgpr2_vgpr3
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s32) = G_ZEXT %2
+    S_ENDPGM 0, implicit %3
+
+...