[AMDGPU] Enable divergence-driven BFE selection

[alex-t]

Summary

Detailed description: This change enables the bit field extract patterns
selection to s_bfe_u32 or v_bfe_u32 dependent on the pattern root node
divergence.

[foad]

There are also patterns in SIInstructions.td that generate S_BFE without checking that the node is uniform.

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

43–44

How about replacing this with a single getBFE32 function, which takes "bool isSigned" and "bool isDivergent" arguments instead of "unsigned Opcode"?

[alex-t]

sign_extend_inreg pattterns added
getS_BFE and getV_BFE replaced with the unified getBFE32 function

[rampitec]

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1968

Cannot IsDivergent be derived from the Val itself? Like SDNode(Val, 0)->isDivergent()?

llvm/test/CodeGen/AMDGPU/divergence-driven-bfe-isel.ll

5

Check constants used in the instructions too.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1968

No else after return

[alex-t]

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1968

I am not sure if this is the right way. The "Val" is N->getOperand(0) here. Formally we should consider the N divergence. On the other hand, operand 0 is the only variable operand in BFE SDNode. Two others (offset and width) are always ConstantSDNode.
Thus, in fact, the divergence property of N is always equal to the divergence of its variable source. From this point of view, we can evolve the N divergence from
N->getOperand(0) divergence. I could save one formal argument, but this would make the code non-obvious and would require some comments to explain.

[rampitec]

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1968

Yes, that's what mean, the rest of arguments are constants, so we can save the argument.

[alex-t]

test updated to check constants, no else after return.

[alex-t]

in getBFE32 IsDivergent argument removed. BFE node divergence evolved from its variable operand.

[rampitec]

LGTM

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h

 //===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 //
private:
 
   void SelectADD_SUB_I64(SDNode *N);
   void SelectAddcSubb(SDNode *N);
   void SelectUADDO_USUBO(SDNode *N);
   void SelectDIV_SCALE(SDNode *N);
   void SelectMAD_64_32(SDNode *N);
   void SelectFMA_W_CHAIN(SDNode *N);
   void SelectFMUL_W_CHAIN(SDNode *N);
-
-  SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
-                   uint32_t Offset, uint32_t Width);
+  SDNode *getBFE32(bool IsSigned, bool IsDivergent, const SDLoc &DL,
+                   SDValue Val, uint32_t Offset, uint32_t Width);
   void SelectS_BFEFromShifts(SDNode *N);
   void SelectS_BFE(SDNode *N);
   bool isCBranchSCC(const SDNode *N) const;
   void SelectBRCOND(SDNode *N);
   void SelectFMAD_FMA(SDNode *N);
   void SelectATOMIC_CMP_SWAP(SDNode *N);
   void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
   void SelectDS_GWS(SDNode *N, unsigned IntrID);

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 //

 
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
 // Instruction Selector Implementation
 //===----------------------------------------------------------------------===//
 
 namespace {
 
 static SDValue stripBitcast(SDValue Val) {
   return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
 }
 
 // Figure out if this is really an extract of the high 16-bits of a dword.
 static bool isExtractHiElt(SDValue In, SDValue &Out) {
   In = stripBitcast(In);
 
   if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
case AMDGPUISD::BFE_U32: {
     if (!Width)
       break;
 
     bool Signed = Opc == AMDGPUISD::BFE_I32;
 
     uint32_t OffsetVal = Offset->getZExtValue();
     uint32_t WidthVal = Width->getZExtValue();
 
-    ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
-                            SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
+    ReplaceNode(N, getBFE32(Signed, N->isDivergent(), SDLoc(N),
+                            N->getOperand(0), OffsetVal, WidthVal));
     return;
   }
   case AMDGPUISD::DIV_SCALE: {
     SelectDIV_SCALE(N);
     return;
   }
   case AMDGPUISD::MAD_I64_I32:
   case AMDGPUISD::MAD_U64_U32: {
bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
   if (isa<ConstantSDNode>(Index))
     return false;
 
   Base = Index;
   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
   return true;
 }
 
-SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
-                                     SDValue Val, uint32_t Offset,
-                                     uint32_t Width) {
+SDNode *AMDGPUDAGToDAGISel::getBFE32(bool IsSigned, bool IsDivergent,
+                                     const SDLoc &DL, SDValue Val,
+                                     uint32_t Offset, uint32_t Width) {
+  if (IsDivergent) {
+    unsigned Opcode = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
+    SDValue Off = CurDAG->getTargetConstant(Offset, DL, MVT::i32);
+    SDValue W = CurDAG->getTargetConstant(Width, DL, MVT::i32);
+
+    return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, Off, W);
+  }
+  unsigned Opcode = IsSigned ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
   // Transformation function, pack the offset and width of a BFE into
   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
   // source, bits [5:0] contain the offset and bits [22:16] the width.
   uint32_t PackedVal = Offset | (Width << 16);
   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
 
   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
 }
 
 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
   // Predicate: 0 < b <= c < 32
 
   const SDValue &Shl = N->getOperand(0);
   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
 
   if (B && C) {
     uint32_t BVal = B->getZExtValue();
     uint32_t CVal = C->getZExtValue();
 
     if (0 < BVal && BVal <= CVal && CVal < 32) {
       bool Signed = N->getOpcode() == ISD::SRA;
-      unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
-
-      ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
-                              32 - CVal));
+      ReplaceNode(N, getBFE32(Signed, N->isDivergent(), SDLoc(N),
+                              Shl.getOperand(0), CVal - BVal, 32 - CVal));
       return;
     }
   }
   SelectCode(N);
 }
 
 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
   switch (N->getOpcode()) {
   case ISD::AND:
     if (N->getOperand(0).getOpcode() == ISD::SRL) {
       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
       // Predicate: isMask(mask)
       const SDValue &Srl = N->getOperand(0);
       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
 
       if (Shift && Mask) {
         uint32_t ShiftVal = Shift->getZExtValue();
         uint32_t MaskVal = Mask->getZExtValue();
 
         if (isMask_32(MaskVal)) {
           uint32_t WidthVal = countPopulation(MaskVal);
-
-          ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
+          ReplaceNode(N, getBFE32(false, N->isDivergent(), SDLoc(N),
                                   Srl.getOperand(0), ShiftVal, WidthVal));
           return;
         }
       }
     }
     break;
   case ISD::SRL:
     if (N->getOperand(0).getOpcode() == ISD::AND) {
       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
       // Predicate: isMask(mask >> b)
       const SDValue &And = N->getOperand(0);
       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
 
       if (Shift && Mask) {
         uint32_t ShiftVal = Shift->getZExtValue();
         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
 
         if (isMask_32(MaskVal)) {
           uint32_t WidthVal = countPopulation(MaskVal);
-
-          ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
+          ReplaceNode(N, getBFE32(false, N->isDivergent(), SDLoc(N),
                                   And.getOperand(0), ShiftVal, WidthVal));
           return;
         }
       }
     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
       SelectS_BFEFromShifts(N);
       return;
     }
case ISD::SIGN_EXTEND_INREG: {
     if (Src.getOpcode() != ISD::SRL)
       break;
 
     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
     if (!Amt)
       break;
 
     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
-    ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
+    ReplaceNode(N, getBFE32(true, N->isDivergent(), SDLoc(N), Src.getOperand(0),
                             Amt->getZExtValue(), Width));
     return;
   }
   }
 
   SelectCode(N);
 }
 

llvm/lib/Target/AMDGPU/SIInstructions.td

(add (select_oneuse (i1 (setugt i32:$src0, i32:$src1)),
        i32:$src2),
   (V_SAD_U32_e64 $src0, $src1, $src2, (i1 0))
 >;
 
 //===----------------------------------------------------------------------===//
 // Conversion Patterns
 //===----------------------------------------------------------------------===//
 
-def : GCNPat<(i32 (sext_inreg i32:$src, i1)),
+class UniformSextInreg<ValueType VT> : PatFrag<
+  (ops node:$src),
+  (sext_inreg $src, VT),
+  [{ return !N->isDivergent(); }]>;
+
+def : GCNPat<(i32 (UniformSextInreg<i1> i32:$src)),
   (S_BFE_I32 i32:$src, (i32 65536))>; // 0 | 1 << 16
 
 // Handle sext_inreg in i64
 def : GCNPat <
-  (i64 (sext_inreg i64:$src, i1)),
+  (i64 (UniformSextInreg<i1> i64:$src)),
   (S_BFE_I64 i64:$src, (i32 0x10000)) // 0 | 1 << 16
 >;
 
 def : GCNPat <
-  (i16 (sext_inreg i16:$src, i1)),
+  (i16 (UniformSextInreg<i1> i16:$src)),
   (S_BFE_I32 $src, (i32 0x00010000)) // 0 | 1 << 16
 >;
 
 def : GCNPat <
-  (i16 (sext_inreg i16:$src, i8)),
+  (i16 (UniformSextInreg<i8> i16:$src)),
   (S_BFE_I32 $src, (i32 0x80000)) // 0 | 8 << 16
 >;
 
 def : GCNPat <
-  (i64 (sext_inreg i64:$src, i8)),
+  (i64 (UniformSextInreg<i8> i64:$src)),
   (S_BFE_I64 i64:$src, (i32 0x80000)) // 0 | 8 << 16
 >;
 
 def : GCNPat <
-  (i64 (sext_inreg i64:$src, i16)),
+  (i64 (UniformSextInreg<i16> i64:$src)),
   (S_BFE_I64 i64:$src, (i32 0x100000)) // 0 | 16 << 16
 >;
 
 def : GCNPat <
-  (i64 (sext_inreg i64:$src, i32)),
+  (i64 (UniformSextInreg<i32> i64:$src)),
   (S_BFE_I64 i64:$src, (i32 0x200000)) // 0 | 32 << 16
 >;
 
+
+class DivergentSextInreg<ValueType VT> : PatFrag<
+  (ops node:$src),
+  (sext_inreg $src, VT),
+  [{ return N->isDivergent(); }]>;
+
+def : GCNPat<(i32 (DivergentSextInreg<i1> i32:$src)),
+  (V_BFE_I32_e64 i32:$src, (i32 0), (i32 1))>;
+
+def : GCNPat <
+  (i16 (DivergentSextInreg<i1> i16:$src)),
+  (V_BFE_I32_e64 $src, (i32 0), (i32 1)) // 0 | 1 << 16
+>;
+
+def : GCNPat <
+  (i16 (DivergentSextInreg<i8> i16:$src)),
+  (V_BFE_I32_e64 $src, (i32 0), (i32 8)) // 0 | 8 << 16
+>;
+
+def : GCNPat <
+  (i64 (DivergentSextInreg<i1> i64:$src)),
+  (REG_SEQUENCE VReg_64,
+    (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1)), sub0,
+    (V_ASHRREV_I32_e32  (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1))), sub1)
+>;
+
+def : GCNPat <
+  (i64 (DivergentSextInreg<i8> i64:$src)),
+  (REG_SEQUENCE VReg_64,
+    (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8)/* 0 | 8 << 16 */), sub0,
+    (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8))), sub1)
+>;
+
+def : GCNPat <
+  (i64 (DivergentSextInreg<i16> i64:$src)),
+  (REG_SEQUENCE VReg_64,
+    (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16)/* 0 | 16 << 16 */), sub0,
+    (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16))), sub1)
+>;
+
+def : GCNPat <
+  (i64 (DivergentSextInreg<i32> i64:$src)),
+  (REG_SEQUENCE VReg_64,
+    (i32 (EXTRACT_SUBREG i64:$src, sub0)), sub0,
+    (V_ASHRREV_I32_e32 (i32 31), (i32 (EXTRACT_SUBREG i64:$src, sub0))), sub1)
+>;
+
 def : GCNPat <
   (i64 (zext i32:$src)),
   (REG_SEQUENCE SReg_64, $src, sub0, (S_MOV_B32 (i32 0)), sub1)
 >;
 
 def : GCNPat <
   (i64 (anyext i32:$src)),
   (REG_SEQUENCE SReg_64, $src, sub0, (i32 (IMPLICIT_DEF)), sub1)

llvm/test/CodeGen/AMDGPU/divergence-driven-bfe-isel.ll

+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GCN_LABEL: @bfe_uniform
+; GCN: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x40010
+define amdgpu_kernel void @bfe_uniform(i32 %val, i32 addrspace(1)* %out) {
+  %hibits = lshr i32 %val, 16
+  %masked = and i32 %hibits, 15
+  store i32 %masked, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN_LABEL: @bfe_divergent
+; GCN: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 16, 4
+define amdgpu_kernel void @bfe_divergent(i32 %val, i32 addrspace(1)* %out) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %divergent = add i32 %val, %tid
+  %hibits = lshr i32 %divergent, 16
+  %masked = and i32 %hibits, 15
+  store i32 %masked, i32 addrspace(1)* %out
+  ret void
+}
+
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+

llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll

 ; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, 0
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    s_movk_i32 s7, 0xfc01
+; SI-NEXT:    s_movk_i32 s6, 0xfc01
 ; SI-NEXT:    s_mov_b32 s0, -1
 ; SI-NEXT:    s_mov_b32 s1, 0xfffff
-; SI-NEXT:    s_brev_b32 s6, -2
+; SI-NEXT:    s_brev_b32 s7, -2
 ; SI-NEXT:    v_mov_b32_e32 v8, 0x3ff00000
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_bfe_u32 v4, v3, 20, 11
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s7, v4
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v4
 ; SI-NEXT:    v_lshr_b64 v[4:5], s[0:1], v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0x80000000, v3
 ; SI-NEXT:    v_not_b32_e32 v4, v4
 ; SI-NEXT:    v_not_b32_e32 v5, v5
 ; SI-NEXT:    v_and_b32_e32 v5, v3, v5
 ; SI-NEXT:    v_and_b32_e32 v4, v2, v4
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v6
 ; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
 ; SI-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
 ; SI-NEXT:    v_cmp_lt_i32_e32 vcc, 51, v6
 ; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
 ; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
 ; SI-NEXT:    v_add_f64 v[6:7], v[2:3], -v[4:5]
-; SI-NEXT:    v_bfi_b32 v2, s6, v8, v3
+; SI-NEXT:    v_bfi_b32 v2, s7, v8, v3
 ; SI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[6:7]|, 0.5
 ; SI-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v2, vcc
 ; SI-NEXT:    v_mov_b32_e32 v2, 0
 ; SI-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
 ; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_endpgm
 ;

llvm/test/CodeGen/AMDGPU/srem-seteq-illegal-types.ll

 
 define <3 x i1> @test_srem_vec(<3 x i31> %X) nounwind {
 ; CHECK-LABEL: test_srem_vec:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_bfe_i32 v3, v2, 0, 31
 ; CHECK-NEXT:    v_bfe_i32 v4, v1, 0, 31
 ; CHECK-NEXT:    v_bfe_i32 v5, v0, 0, 31
-; CHECK-NEXT:    s_mov_b32 s6, 0x38e38e39
-; CHECK-NEXT:    s_mov_b32 s7, 0xc71c71c7
-; CHECK-NEXT:    s_brev_b32 s4, -2
-; CHECK-NEXT:    s_mov_b32 s5, 0x7ffffffd
-; CHECK-NEXT:    v_mul_hi_i32 v5, v5, s6
-; CHECK-NEXT:    v_mul_hi_i32 v4, v4, s6
-; CHECK-NEXT:    v_mul_hi_i32 v3, v3, s7
+; CHECK-NEXT:    s_mov_b32 s4, 0x38e38e39
+; CHECK-NEXT:    s_mov_b32 s5, 0xc71c71c7
+; CHECK-NEXT:    s_brev_b32 s6, -2
+; CHECK-NEXT:    s_mov_b32 s7, 0x7ffffffd
+; CHECK-NEXT:    v_mul_hi_i32 v5, v5, s4
+; CHECK-NEXT:    v_mul_hi_i32 v4, v4, s4
+; CHECK-NEXT:    v_mul_hi_i32 v3, v3, s5
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 31, v5
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 1, v5
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 31, v4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 31, v3
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
 ; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
 ; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
 ; CHECK-NEXT:    v_mul_lo_u32 v5, v5, 9
 ; CHECK-NEXT:    v_mul_lo_u32 v4, v4, 9
 ; CHECK-NEXT:    v_mul_lo_u32 v3, v3, -9
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
 ; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
 ; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
-; CHECK-NEXT:    v_and_b32_e32 v2, s4, v2
-; CHECK-NEXT:    v_and_b32_e32 v1, s4, v1
-; CHECK-NEXT:    v_and_b32_e32 v0, s4, v0
+; CHECK-NEXT:    v_and_b32_e32 v2, s6, v2
+; CHECK-NEXT:    v_and_b32_e32 v1, s6, v1
+; CHECK-NEXT:    v_and_b32_e32 v0, s6, v0
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 3, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v1
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, s7, v1
 ; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 3, v2
 ; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %srem = srem <3 x i31> %X, <i31 9, i31 9, i31 -9>
   %cmp = icmp ne <3 x i31> %srem, <i31 3, i31 -3, i31 3>
   ret <3 x i1> %cmp
 }