[AMDGPU] Packed thread ids in function call ABI

[arsenm]

lib/Target/AMDGPU/SIISelLowering.cpp

1643

This should be derived from the subtarget maximum workgroup size

[rampitec]

lib/Target/AMDGPU/SIISelLowering.cpp

1643

It is derived from the theoretical maximum you can pack into 32 bit register (and agrees with maximum whg size as well of course, which makes it possible).

[arsenm]

LGTM

[rampitec]

lib/Target/AMDGPU/SIISelLowering.cpp

1643

For the record: there is conformation that any given WG dimension can never be more that 1024. This enables this code and allows to add few more AssertZExt I suppose. Probably there is something in clang as well.

lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h

private:
   friend struct AMDGPUFunctionArgInfo;
   friend class AMDGPUArgumentUsageInfo;
 
   union {
     unsigned Register;
     unsigned StackOffset;
   };
 
+  // Bitmask to locate argument within the register.
+  unsigned Mask;
+
   bool IsStack : 1;
   bool IsSet : 1;
 
-  ArgDescriptor(unsigned Val = 0, bool IsStack = false, bool IsSet = false)
-    : Register(Val), IsStack(IsStack), IsSet(IsSet) {}
 public:
-  static ArgDescriptor createRegister(unsigned Reg) {
-    return ArgDescriptor(Reg, false, true);
+  ArgDescriptor(unsigned Val = 0, unsigned Mask = ~0u,
+                bool IsStack = false, bool IsSet = false)
+    : Register(Val), Mask(Mask), IsStack(IsStack), IsSet(IsSet) {}
+
+  static ArgDescriptor createRegister(unsigned Reg, unsigned Mask = ~0u) {
+    return ArgDescriptor(Reg, Mask, false, true);
+  }
+
+  static ArgDescriptor createStack(unsigned Reg, unsigned Mask = ~0u) {
+    return ArgDescriptor(Reg, Mask, true, true);
   }
 
-  static ArgDescriptor createStack(unsigned Reg) {
-    return ArgDescriptor(Reg, true, true);
+  static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask) {
+    return ArgDescriptor(Arg.Register, Mask, Arg.IsStack, Arg.IsSet);
   }
 
   bool isSet() const {
     return IsSet;
   }
 
   explicit operator bool() const {
     return isSet();
   }
 
   bool isRegister() const {
     return !IsStack;
   }
 
   unsigned getRegister() const {
     assert(!IsStack);
     return Register;
   }
 
   unsigned getStackOffset() const {
     assert(IsStack);
     return StackOffset;
   }
 
+  unsigned getMask() const {
+    return Mask;
+  }
+
+  bool isMasked() const {
+    return Mask != ~0u;
+  }
+
   void print(raw_ostream &OS, const TargetRegisterInfo *TRI = nullptr) const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const ArgDescriptor &Arg) {
   Arg.print(OS);
   return OS;
 }
 

lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp

 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
 #include "AMDGPUArgumentUsageInfo.h"
 #include "SIRegisterInfo.h"
+#include "llvm/Support/NativeFormatting.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "amdgpu-argument-reg-usage-info"
 
 INITIALIZE_PASS(AMDGPUArgumentUsageInfo, DEBUG_TYPE,
                 "Argument Register Usage Information Storage", false, true)
 
 void ArgDescriptor::print(raw_ostream &OS,
                           const TargetRegisterInfo *TRI) const {
   if (!isSet()) {
     OS << "<not set>\n";
     return;
   }
 
   if (isRegister())
-    OS << "Reg " << printReg(getRegister(), TRI) << '\n';
+    OS << "Reg " << printReg(getRegister(), TRI);
   else
-    OS << "Stack offset " << getStackOffset() << '\n';
+    OS << "Stack offset " << getStackOffset();
+
+  if (isMasked()) {
+    OS << " & ";
+    llvm::write_hex(OS, Mask, llvm::HexPrintStyle::PrefixLower);
+  }
+
+  OS << '\n';
 }
 
 char AMDGPUArgumentUsageInfo::ID = 0;
 
 const AMDGPUFunctionArgInfo AMDGPUArgumentUsageInfo::ExternFunctionInfo{};
 
 bool AMDGPUArgumentUsageInfo::doInitialization(Module &M) {
   return false;

lib/Target/AMDGPU/AMDGPUISelLowering.cpp

 }
 
 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
                                              const TargetRegisterClass *RC,
                                              EVT VT, const SDLoc &SL,
                                              const ArgDescriptor &Arg) const {
   assert(Arg && "Attempting to load missing argument");
 
-  if (Arg.isRegister())
-    return CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL);
-  return loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
+  SDValue V = Arg.isRegister() ?
+    CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
+    loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
+
+  if (!Arg.isMasked())
+    return V;
+
+  unsigned Mask = Arg.getMask();
+  unsigned Shift = countTrailingZeros<unsigned>(Mask);
+  V = DAG.getNode(ISD::SRL, SL, VT, V,
+                  DAG.getShiftAmountConstant(Shift, VT, SL));
+  return DAG.getNode(ISD::AND, SL, VT, V,
+                     DAG.getConstant(Mask >> Shift, SL, VT));
 }
 
 uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
     const MachineFunction &MF, const ImplicitParameter Param) const {
   const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
   const AMDGPUSubtarget &ST =
       AMDGPUSubtarget::get(getTargetMachine(), MF.getFunction());
   unsigned ExplicitArgOffset = ST.getExplicitKernelArgOffset(MF.getFunction());

lib/Target/AMDGPU/SIISelLowering.cpp

if (Info.hasWorkItemIDZ()) {
 
     CCInfo.AllocateReg(Reg);
     Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
   }
 }
 
 // Try to allocate a VGPR at the end of the argument list, or if no argument
 // VGPRs are left allocating a stack slot.
-static ArgDescriptor allocateVGPR32Input(CCState &CCInfo) {
+// If \p Mask is is given it indicates bitfield position in the register.
+// If \p Arg is given use it with new ]p Mask instead of allocating new.
+static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
+                                         ArgDescriptor Arg = ArgDescriptor()) {
+  if (Arg.isSet())
+    return ArgDescriptor::createArg(Arg, Mask);
+
   ArrayRef<MCPhysReg> ArgVGPRs
     = makeArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
   unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs);
   if (RegIdx == ArgVGPRs.size()) {
     // Spill to stack required.
     int64_t Offset = CCInfo.AllocateStack(4, 4);
 
-    return ArgDescriptor::createStack(Offset);
+    return ArgDescriptor::createStack(Offset, Mask);
   }
 
   unsigned Reg = ArgVGPRs[RegIdx];
   Reg = CCInfo.AllocateReg(Reg);
   assert(Reg != AMDGPU::NoRegister);
 
   MachineFunction &MF = CCInfo.getMachineFunction();
   MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
-  return ArgDescriptor::createRegister(Reg);
+  return ArgDescriptor::createRegister(Reg, Mask);
 }
 
 static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
                                              const TargetRegisterClass *RC,
                                              unsigned NumArgRegs) {
   ArrayRef<MCPhysReg> ArgSGPRs = makeArrayRef(RC->begin(), 32);
   unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
   if (RegIdx == ArgSGPRs.size())

 static ArgDescriptor allocateSGPR64Input(CCState &CCInfo) {
   return allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
 }
 
 static void allocateSpecialInputVGPRs(CCState &CCInfo,
                                       MachineFunction &MF,
                                       const SIRegisterInfo &TRI,
                                       SIMachineFunctionInfo &Info) {
-  if (Info.hasWorkItemIDX())
-    Info.setWorkItemIDX(allocateVGPR32Input(CCInfo));
+  const unsigned Mask = 0x3ff;
+  ArgDescriptor Arg;
 
-  if (Info.hasWorkItemIDY())
-    Info.setWorkItemIDY(allocateVGPR32Input(CCInfo));
+  if (Info.hasWorkItemIDX()) {
+    Arg = allocateVGPR32Input(CCInfo, Mask);
+    Info.setWorkItemIDX(Arg);
+  }
+
+  if (Info.hasWorkItemIDY()) {
+    Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg);
+    Info.setWorkItemIDY(Arg);
+  }
 
   if (Info.hasWorkItemIDZ())
-    Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo));
+    Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg));
 }
 
 static void allocateSpecialInputSGPRs(CCState &CCInfo,
                                       MachineFunction &MF,
                                       const SIRegisterInfo &TRI,
                                       SIMachineFunctionInfo &Info) {
   auto &ArgInfo = Info.getArgInfo();
 
void SITargetLowering::passSpecialInputs(
   AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = {
     AMDGPUFunctionArgInfo::DISPATCH_PTR,
     AMDGPUFunctionArgInfo::QUEUE_PTR,
     AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR,
     AMDGPUFunctionArgInfo::DISPATCH_ID,
     AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
     AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
     AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
-    AMDGPUFunctionArgInfo::WORKITEM_ID_X,
-    AMDGPUFunctionArgInfo::WORKITEM_ID_Y,
-    AMDGPUFunctionArgInfo::WORKITEM_ID_Z,
     AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR
   };
 
   for (auto InputID : InputRegs) {
     const ArgDescriptor *OutgoingArg;
     const TargetRegisterClass *ArgRC;
 
     std::tie(OutgoingArg, ArgRC) = CalleeArgInfo.getPreloadedValue(InputID);
if (OutgoingArg->isRegister()) {
       RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
     } else {
       unsigned SpecialArgOffset = CCInfo.AllocateStack(ArgVT.getStoreSize(), 4);
       SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
                                               SpecialArgOffset);
       MemOpChains.push_back(ArgStore);
     }
   }
+
+  // Pack workitem IDs into a single register or pass it as is if already
+  // packed.
+  const ArgDescriptor *OutgoingArg;
+  const TargetRegisterClass *ArgRC;
+
+  std::tie(OutgoingArg, ArgRC) =
+    CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
+  if (!OutgoingArg)
+    std::tie(OutgoingArg, ArgRC) =
+      CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
+  if (!OutgoingArg)
+    std::tie(OutgoingArg, ArgRC) =
+      CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
+  if (!OutgoingArg)
+    return;
+
+  const ArgDescriptor *IncomingArgX
+    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X).first;
+  const ArgDescriptor *IncomingArgY
+    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y).first;
+  const ArgDescriptor *IncomingArgZ
+    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z).first;
+
+  SDValue InputReg;
+  SDLoc SL;
+
+  // If incoming ids are not packed we need to pack them.
+  if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX)
+    InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
+
+  if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY) {
+    SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
+    Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
+                    DAG.getShiftAmountConstant(10, MVT::i32, SL));
+    InputReg = InputReg.getNode() ?
+                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y;
+  }
+
+  if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ) {
+    SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
+    Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
+                    DAG.getShiftAmountConstant(20, MVT::i32, SL));
+    InputReg = InputReg.getNode() ?
+                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z;
+  }
+
+  if (!InputReg.getNode()) {
+    // Workitem ids are already packed, any of present incoming arguments
+    // will carry all required fields.
+    ArgDescriptor IncomingArg = ArgDescriptor::createArg(
+      IncomingArgX ? *IncomingArgX :
+      IncomingArgY ? *IncomingArgY :
+                     *IncomingArgZ, ~0u);
+    InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
+  }
+
+  if (OutgoingArg->isRegister()) {
+    RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
+  } else {
+    unsigned SpecialArgOffset = CCInfo.AllocateStack(4, 4);
+    SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
+                                            SpecialArgOffset);
+    MemOpChains.push_back(ArgStore);
+  }
 }
 
 static bool canGuaranteeTCO(CallingConv::ID CC) {
   return CC == CallingConv::Fast;
 }
 
 /// Return true if we might ever do TCO for calls with this calling convention.
 static bool mayTailCallThisCC(CallingConv::ID CC) {

test/CodeGen/AMDGPU/call-constexpr.ll

define amdgpu_kernel void @test_bitcast_argument_and_return_types() #0 {
   %val = call float bitcast (i32(i32)* @ident_i32 to float(float)*)(float 2.0)
   %op = fadd float %val, 1.0
   store volatile float %op, float addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_workitem_id_x:
 ; GCN: s_waitcnt
+; GCN-NEXT: v_and_b32_e32 v1, 0x3ff, v1
 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0
 ; GCN-NEXT: s_setpc_b64
 define hidden i32 @use_workitem_id_x(i32 %arg0) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %op = add i32 %id, %arg0
   ret i32 %op
 }
 

test/CodeGen/AMDGPU/callee-special-input-vgprs.ll

-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-code-object-v3 -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-code-object-v3 -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN %s
 
 ; GCN-LABEL: {{^}}use_workitem_id_x:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN: v_and_b32_e32 [[ID:v[0-9]+]], 0x3ff, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_x() #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.x()
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_workitem_id_y:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN: v_bfe_u32 [[ID:v[0-9]+]], v0, 10, 10
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_y() #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.y()
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_workitem_id_z:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN: v_bfe_u32 [[ID:v[0-9]+]], v0, 20, 10
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[ID]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_z() #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.z()
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_workitem_id_xy:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
+; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v0
+; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v0, 10, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_xy() #1 {
   %val0 = call i32 @llvm.amdgcn.workitem.id.x()
   %val1 = call i32 @llvm.amdgcn.workitem.id.y()
   store volatile i32 %val0, i32 addrspace(1)* undef
   store volatile i32 %val1, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_workitem_id_xyz:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v2
+; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v0
+; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v0, 10, 10
+; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v0, 20, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_xyz() #1 {
   %val0 = call i32 @llvm.amdgcn.workitem.id.x()
   %val1 = call i32 @llvm.amdgcn.workitem.id.y()
   %val2 = call i32 @llvm.amdgcn.workitem.id.z()
   store volatile i32 %val0, i32 addrspace(1)* undef
   store volatile i32 %val1, i32 addrspace(1)* undef
   store volatile i32 %val2, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_workitem_id_xz:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
+; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v0
+; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v0, 20, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDX]]
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_xz() #1 {
   %val0 = call i32 @llvm.amdgcn.workitem.id.x()
   %val1 = call i32 @llvm.amdgcn.workitem.id.z()
   store volatile i32 %val0, i32 addrspace(1)* undef
   store volatile i32 %val1, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_workitem_id_yz:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
+; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v0, 10, 10
+; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v0, 20, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDY]]
+; GCN-DAG: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, [[IDZ]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @use_workitem_id_yz() #1 {
   %val0 = call i32 @llvm.amdgcn.workitem.id.y()
   %val1 = call i32 @llvm.amdgcn.workitem.id.z()
   store volatile i32 %val0, i32 addrspace(1)* undef
   store volatile i32 %val1, i32 addrspace(1)* undef
   ret void
define amdgpu_kernel void @kern_indirect_use_workitem_id_x() #1 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_y:
 ; GCN: enable_vgpr_workitem_id = 1
 
 ; GCN-NOT: v0
 ; GCN-NOT: v1
-; GCN: v_mov_b32_e32 v0, v1
+; GCN: v_lshlrev_b32_e32 v0, 10, v1
 ; GCN-NOT: v0
 ; GCN-NOT: v1
 ; GCN: s_swappc_b64
 define amdgpu_kernel void @kern_indirect_use_workitem_id_y() #1 {
   call void @use_workitem_id_y()
   ret void
 }
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_z:
 ; GCN: enable_vgpr_workitem_id = 2
 
 ; GCN-NOT: v0
 ; GCN-NOT: v2
-; GCN: v_mov_b32_e32 v0, v2
+; GCN: v_lshlrev_b32_e32 v0, 20, v2
 ; GCN-NOT: v0
-; GCN-NOT: v2
+; GCN-NOT: v1
 ; GCN: s_swappc_b64
 define amdgpu_kernel void @kern_indirect_use_workitem_id_z() #1 {
   call void @use_workitem_id_z()
   ret void
 }
 
+; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_xy:
+; GCN-NOT: v0
+; GCN-NOT: v1
+; GCN: v_lshlrev_b32_e32 [[IDY:v[0-9]+]], 10, v1
+; GCN: v_or_b32_e32 v0, v0, [[IDY]]
+; GCN-NOT: v0
+; GCN-NOT: v1
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workitem_id_xy() #1 {
+  call void @use_workitem_id_xy()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_xz:
+; GCN-NOT: v0
+; GCN-NOT: v2
+; GCN: v_lshlrev_b32_e32 [[IDZ:v[0-9]+]], 20, v2
+; GCN: v_or_b32_e32 v0, v0, [[IDZ]]
+; GCN-NOT: v0
+; GCN-NOT: v2
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workitem_id_xz() #1 {
+  call void @use_workitem_id_xz()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_yz:
+; GCN-NOT: v1
+; GCN-NOT: v2
+; GCN-DAG: v_lshlrev_b32_e32 [[IDY:v[0-9]+]], 10, v1
+; GCN-DAG: v_lshlrev_b32_e32 [[IDZ:v[0-9]+]], 20, v2
+; GCN: v_or_b32_e32 v0, [[IDY]], [[IDZ]]
+; GCN-NOT: v1
+; GCN-NOT: v2
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workitem_id_yz() #1 {
+  call void @use_workitem_id_yz()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_xyz:
+; GCN-NOT: v0
+; GCN-NOT: v1
+; GCN-NOT: v2
+; GCN-DAG: v_lshlrev_b32_e32 [[IDY:v[0-9]+]], 10, v1
+; GCN-DAG: v_lshlrev_b32_e32 [[IDZ:v[0-9]+]], 20, v2
+; GCN-DAG: v_or_b32_e32 v0, v0, [[IDY]]
+; GCN-DAG: v_or_b32_e32 v0, v0, [[IDZ]]
+; GCN-NOT: v0
+; GCN-NOT: v1
+; GCN-NOT: v2
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workitem_id_xyz() #1 {
+  call void @use_workitem_id_xyz()
+  ret void
+}
+
 ; GCN-LABEL: {{^}}func_indirect_use_workitem_id_x:
 ; GCN-NOT: v0
 ; GCN: s_swappc_b64
 ; GCN-NOT: v0
 define void @func_indirect_use_workitem_id_x() #1 {
   call void @use_workitem_id_x()
   ret void
 }

 ; GCN-NOT: v0
 define void @func_indirect_use_workitem_id_z() #1 {
   call void @use_workitem_id_z()
   ret void
 }
 
 ; GCN-LABEL: {{^}}other_arg_use_workitem_id_x:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GCN-DAG: v_and_b32_e32 [[ID:v[0-9]+]], 0x3ff, v1
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
 define void @other_arg_use_workitem_id_x(i32 %arg0) #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.x()
   store volatile i32 %arg0, i32 addrspace(1)* undef
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}other_arg_use_workitem_id_y:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GCN-DAG: v_bfe_u32 [[ID:v[0-9]+]], v1, 10, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
 define void @other_arg_use_workitem_id_y(i32 %arg0) #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.y()
   store volatile i32 %arg0, i32 addrspace(1)* undef
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}other_arg_use_workitem_id_z:
 ; GCN: s_waitcnt
-; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
-; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GCN-DAG: v_bfe_u32 [[ID:v[0-9]+]], v1, 20, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[ID]]
 define void @other_arg_use_workitem_id_z(i32 %arg0) #1 {
   %val = call i32 @llvm.amdgcn.workitem.id.z()
   store volatile i32 %arg0, i32 addrspace(1)* undef
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 
 ; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workitem_id_x:
 ; GCN: enable_vgpr_workitem_id = 0
 
 ; GCN: v_mov_b32_e32 v1, v0
 ; GCN: v_mov_b32_e32 v0, 0x22b
 ; GCN: s_swappc_b64
 define amdgpu_kernel void @kern_indirect_other_arg_use_workitem_id_x() #1 {
   call void @other_arg_use_workitem_id_x(i32 555)
   ret void
 }
 
 
 ; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workitem_id_y:
 ; GCN: enable_vgpr_workitem_id = 1
 
+; GCN: v_lshlrev_b32_e32 v1, 10, v1
 ; GCN-NOT: v1
 ; GCN: v_mov_b32_e32 v0, 0x22b
 ; GCN-NOT: v1
 ; GCN: s_swappc_b64
 ; GCN-NOT: v0
 define amdgpu_kernel void @kern_indirect_other_arg_use_workitem_id_y() #1 {
   call void @other_arg_use_workitem_id_y(i32 555)
   ret void
 }
 
 ; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workitem_id_z:
 ; GCN: enable_vgpr_workitem_id = 2
 
 ; GCN-DAG: v_mov_b32_e32 v0, 0x22b
-; GCN-DAG: v_mov_b32_e32 v1, v2
+; GCN-DAG: v_lshlrev_b32_e32 v1, 20, v2
 ; GCN: s_swappc_b64
 ; GCN-NOT: v0
 define amdgpu_kernel void @kern_indirect_other_arg_use_workitem_id_z() #1 {
   call void @other_arg_use_workitem_id_z(i32 555)
   ret void
 }
 
 ; GCN-LABEL: {{^}}too_many_args_use_workitem_id_x:
 ; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
+; GCN: v_and_b32_e32 v32, 0x3ff, v32
 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
 
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @too_many_args_use_workitem_id_x(
   i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
   i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,

 ; frame[0] = byval arg32
 ; frame[1] = stack passed workitem ID x
 ; frame[2] = VGPR spill slot
 
 ; GCN-LABEL: {{^}}too_many_args_use_workitem_id_x_byval:
 ; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4
 ; GCN-NEXT: s_waitcnt
+; GCN-NEXT: v_and_b32_e32 v32, 0x3ff, v32
 ; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v32
 ; GCN: buffer_load_dword v0, off, s[0:3], s32{{$}}
 ; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GCN: s_setpc_b64
 define void @too_many_args_use_workitem_id_x_byval(
   i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
   i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
   i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
call void @too_many_args_use_workitem_id_x_byval(
     i32 210, i32 220, i32 230, i32 240,
     i32 250, i32 260, i32 270, i32 280,
     i32 290, i32 300, i32 310, i32 320,
     i32 addrspace(5)* %alloca)
   ret void
 }
 
 ; GCN-LABEL: {{^}}too_many_args_use_workitem_id_xyz:
-; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
+; GCN: v_and_b32_e32 v32, 0x3ff, v32
 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
-; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4{{$}}
+; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
+; GCN: v_bfe_u32 v32, v32, 10, 10
 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
-; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:8{{$}}
+; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
+; GCN: v_bfe_u32 v32, v32, 20, 10
 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
 
-; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @too_many_args_use_workitem_id_xyz(
   i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
   i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
   i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
   i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30, i32 %arg31) #1 {
   %val0 = call i32 @llvm.amdgcn.workitem.id.x()
define void @too_many_args_use_workitem_id_xyz(
   store volatile i32 %arg28, i32 addrspace(1)* undef
   store volatile i32 %arg29, i32 addrspace(1)* undef
   store volatile i32 %arg30, i32 addrspace(1)* undef
   store volatile i32 %arg31, i32 addrspace(1)* undef
 
   ret void
 }
 
-; frame[0] = ID X
-; frame[1] = ID Y
-; frame[2] = ID Z
+; frame[0] = ID { Z, Y, X }
 
 ; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_xyz:
 ; GCN: enable_vgpr_workitem_id = 2
 
-; GCN: s_mov_b32 s33, s7
-; GCN: s_mov_b32 s32, s33
+; GCN-DAG: s_mov_b32 s33, s7
+; GCN-DAG: s_mov_b32 s32, s33
 
-; GCN-DAG: buffer_store_dword v0, off, s[0:3], s32{{$}}
-; GCN-DAG: buffer_store_dword v1, off, s[0:3], s32 offset:4
-; GCN-DAG: buffer_store_dword v2, off, s[0:3], s32 offset:8
+; GCN-DAG: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-DAG: v_or_b32_e32 v0, v0, v1
+; GCN-DAG: v_lshlrev_b32_e32 v2, 20, v2
+; GCN-DAG: v_or_b32_e32 v0, v0, v2
+; GCN: buffer_store_dword v0, off, s[0:3], s32{{$}}
 ; GCN: s_swappc_b64
 define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_xyz() #1 {
   call void @too_many_args_use_workitem_id_xyz(
     i32 10, i32 20, i32 30, i32 40,
     i32 50, i32 60, i32 70, i32 80,
     i32 90, i32 100, i32 110, i32 120,
     i32 130, i32 140, i32 150, i32 160,
     i32 170, i32 180, i32 190, i32 200,
     i32 210, i32 220, i32 230, i32 240,
     i32 250, i32 260, i32 270, i32 280,
     i32 290, i32 300, i32 310, i32 320)
   ret void
 }
 
 ; workitem ID X in register, yz on stack
 ; v31 = workitem ID X
-; frame[0] = workitem Y
-; frame[1] = workitem Z
+; frame[0] = workitem { Z, Y, X }
 
 ; GCN-LABEL: {{^}}too_many_args_use_workitem_id_x_stack_yz:
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v31
-; GCN: buffer_load_dword v31, off, s[0:3], s32{{$}}
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v31
-; GCN: buffer_load_dword v31, off, s[0:3], s32 offset:4{{$}}
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v31
+; GCN-DAG: v_and_b32_e32 [[IDX:v[0-9]+]], 0x3ff, v31
+; GCN-DAG: flat_store_dword v[0:1], [[IDX]]
+; GCN-DAG: v_bfe_u32 [[IDY:v[0-9]+]], v31, 10, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, [[IDY]]
+; GCN-DAG: v_bfe_u32 [[IDZ:v[0-9]+]], v31, 20, 10
+; GCN-DAG: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, [[IDZ]]
 
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
-; GCN: ScratchSize: 12
+; GCN: ScratchSize: 8
 define void @too_many_args_use_workitem_id_x_stack_yz(
   i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
   i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
   i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
   i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30) #1 {
   %val0 = call i32 @llvm.amdgcn.workitem.id.x()
   store volatile i32 %val0, i32 addrspace(1)* undef
   %val1 = call i32 @llvm.amdgcn.workitem.id.y()
define void @too_many_args_use_workitem_id_x_stack_yz(
   store volatile i32 %arg27, i32 addrspace(1)* undef
   store volatile i32 %arg28, i32 addrspace(1)* undef
   store volatile i32 %arg29, i32 addrspace(1)* undef
   store volatile i32 %arg30, i32 addrspace(1)* undef
 
   ret void
 }
 
-; frame[0] = ID Y
-; frame[1] = ID Z
-
 ; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_x_stack_yz:
 ; GCN: enable_vgpr_workitem_id = 2
 
 ; GCN: s_mov_b32 s33, s7
-; GCN: s_mov_b32 s32, s33
 
-; GCN-DAG: v_mov_b32_e32 v31, v0
-; GCN-DAG: buffer_store_dword v1, off, s[0:3], s32{{$}}
-; GCN-DAG: buffer_store_dword v2, off, s[0:3], s32 offset:4
+; GCN-NOT: v0
+; GCN-DAG: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-DAG: v_or_b32_e32 v0, v0, v1
+; GCN-DAG: v_lshlrev_b32_e32 v2, 20, v2
+; GCN-DAG: v_or_b32_e32 v31, v0, v2
+
+; GCN: s_mov_b32 s32, s33
 ; GCN: s_swappc_b64
 define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_x_stack_yz() #1 {
   call void @too_many_args_use_workitem_id_x_stack_yz(
     i32 10, i32 20, i32 30, i32 40,
     i32 50, i32 60, i32 70, i32 80,
     i32 90, i32 100, i32 110, i32 120,
     i32 130, i32 140, i32 150, i32 160,
     i32 170, i32 180, i32 190, i32 200,

test/CodeGen/AMDGPU/zext-lid.ll

-; RUN: llc -march=amdgcn < %s | FileCheck %s
-; RUN: llc -O0 -march=amdgcn < %s | FileCheck %s
+; RUN: llc -march=amdgcn < %s | FileCheck -enable-var-scope -check-prefixes=GCN,O2 %s
+; RUN: llc -O0 -march=amdgcn < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 ; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-intrinsics < %s | FileCheck -check-prefix=OPT %s
 
-; CHECK-NOT: and_b32
+; GCN-LABEL: {{^}}zext_grp_size_128:
+; GCN-NOT: and_b32
 
 ; OPT-LABEL: @zext_grp_size_128
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !0
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.y(), !range !0
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.z(), !range !0
 define amdgpu_kernel void @zext_grp_size_128(i32 addrspace(1)* nocapture %arg) #0 {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = and i32 %tmp, 127
   store i32 %tmp1, i32 addrspace(1)* %arg, align 4
   %tmp2 = tail call i32 @llvm.amdgcn.workitem.id.y()
   %tmp3 = and i32 %tmp2, 127
   %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 1
   store i32 %tmp3, i32 addrspace(1)* %tmp4, align 4
   %tmp5 = tail call i32 @llvm.amdgcn.workitem.id.z()
   %tmp6 = and i32 %tmp5, 127
   %tmp7 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 2
   store i32 %tmp6, i32 addrspace(1)* %tmp7, align 4
   ret void
 }
 
+; GCN-LABEL: {{^}}zext_grp_size_32x4x1:
+; GCN-NOT: and_b32
+
 ; OPT-LABEL: @zext_grp_size_32x4x1
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !2
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.y(), !range !3
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.z(), !range !4
 define amdgpu_kernel void @zext_grp_size_32x4x1(i32 addrspace(1)* nocapture %arg) #0 !reqd_work_group_size !0 {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = and i32 %tmp, 31
   store i32 %tmp1, i32 addrspace(1)* %arg, align 4
   %tmp2 = tail call i32 @llvm.amdgcn.workitem.id.y()
   %tmp3 = and i32 %tmp2, 3
   %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 1
   store i32 %tmp3, i32 addrspace(1)* %tmp4, align 4
   %tmp5 = tail call i32 @llvm.amdgcn.workitem.id.z()
   %tmp6 = and i32 %tmp5, 1
   %tmp7 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 2
   store i32 %tmp6, i32 addrspace(1)* %tmp7, align 4
   ret void
 }
 
+; GCN-LABEL: {{^}}zext_grp_size_1x1x1:
+; GCN-NOT: and_b32
+
 ; When EarlyCSE is not run this call produces a range max with 0 active bits,
 ; which is a special case as an AssertZext from width 0 is invalid.
 ; OPT-LABEL: @zext_grp_size_1x1x1
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !4
 define amdgpu_kernel void @zext_grp_size_1x1x1(i32 addrspace(1)* nocapture %arg) #0 !reqd_work_group_size !1 {
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = and i32 %tmp, 1
   store i32 %tmp1, i32 addrspace(1)* %arg, align 4
   ret void
 }
 
+; GCN-LABEL: {{^}}zext_grp_size_512:
+; GCN-NOT: and_b32
+
 ; OPT-LABEL: @zext_grp_size_512
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !6
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.y(), !range !6
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.z(), !range !6
 define amdgpu_kernel void @zext_grp_size_512(i32 addrspace(1)* nocapture %arg) #1 {
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = and i32 %tmp, 65535
   store i32 %tmp1, i32 addrspace(1)* %arg, align 4
   %tmp2 = tail call i32 @llvm.amdgcn.workitem.id.y()
   %tmp3 = and i32 %tmp2, 65535
   %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 1
   store i32 %tmp3, i32 addrspace(1)* %tmp4, align 4
   %tmp5 = tail call i32 @llvm.amdgcn.workitem.id.z()
   %tmp6 = and i32 %tmp5, 65535
   %tmp7 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 2
   store i32 %tmp6, i32 addrspace(1)* %tmp7, align 4
   ret void
 }
 
+; GCN-LABEL: {{^}}func_test_workitem_id_x_known_max_range:
+; O2-NOT: and_b32
+; O2: v_and_b32_e32 v{{[0-9]+}}, 0x3ff,
+; O2-NOT: and_b32
+
 ; OPT-LABEL: @func_test_workitem_id_x_known_max_range(
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !0
 define void @func_test_workitem_id_x_known_max_range(i32 addrspace(1)* nocapture %out) #0 {
 entry:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %and = and i32 %id, 1023
   store i32 %and, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; GCN-LABEL: {{^}}func_test_workitem_id_x_default_range:
+; O2-NOT: and_b32
+; O2: v_and_b32_e32 v{{[0-9]+}}, 0x3ff,
+; O2-NOT: and_b32
+
 ; OPT-LABEL: @func_test_workitem_id_x_default_range(
 ; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !7
 define void @func_test_workitem_id_x_default_range(i32 addrspace(1)* nocapture %out) #4 {
 entry:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %and = and i32 %id, 1023
   store i32 %and, i32 addrspace(1)* %out, align 4
   ret void