Index: lib/Target/AMDGPU/AMDGPU.h =================================================================== --- lib/Target/AMDGPU/AMDGPU.h +++ lib/Target/AMDGPU/AMDGPU.h @@ -73,6 +73,10 @@ void initializeAMDGPULowerIntrinsicsPass(PassRegistry &); extern char &AMDGPULowerIntrinsicsID; +ModulePass *createAMDGPULowerKernelAttributesPass(); +void initializeAMDGPULowerKernelAttributesPass(PassRegistry &); +extern char &AMDGPULowerKernelAttributesID; + void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &); extern char &AMDGPURewriteOutArgumentsID; Index: lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp =================================================================== --- /dev/null +++ lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp @@ -0,0 +1,270 @@ +//===-- AMDGPULowerKernelAttributes.cpp ------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file This pass does attempts to make use of reqd_work_group_size metadata +/// to eliminate loads from the dispatch packet and to constant fold OpenCL +/// get_local_size-like functions. +// +//===----------------------------------------------------------------------===// + +#include "AMDGPU.h" +#include "AMDGPUTargetMachine.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/PatternMatch.h" +#include "llvm/Pass.h" + +#define DEBUG_TYPE "amdgpu-lower-kernel-attributes" + +using namespace llvm; + +namespace { + +// Field offsets in hsa_kernel_dispatch_packet_t. +enum DispatchPackedOffsets { + WORKGROUP_SIZE_X = 4, + WORKGROUP_SIZE_Y = 6, + WORKGROUP_SIZE_Z = 8, + + GRID_SIZE_X = 12, + GRID_SIZE_Y = 16, + GRID_SIZE_Z = 20 +}; + +class AMDGPULowerKernelAttributes : public ModulePass { + Module *Mod = nullptr; + +public: + static char ID; + + AMDGPULowerKernelAttributes() : ModulePass(ID) {} + + bool processUse(CallInst *CI); + + bool doInitialization(Module &M) override; + bool runOnModule(Module &M) override; + + StringRef getPassName() const override { + return "AMDGPU Kernel Attributes"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } +}; + +} // end anonymous namespace + +bool AMDGPULowerKernelAttributes::doInitialization(Module &M) { + Mod = &M; + return false; +} + +bool AMDGPULowerKernelAttributes::processUse(CallInst *CI) { + Function *F = CI->getParent()->getParent(); + + auto MD = F->getMetadata("reqd_work_group_size"); + const bool HasReqdWorkGroupSize = MD && MD->getNumOperands() == 3; + + const bool HasUniformWorkGroupSize = + F->getFnAttribute("uniform-work-group-size").getValueAsString() == "true"; + + if (!HasReqdWorkGroupSize && !HasUniformWorkGroupSize) + return false; + + Value *WorkGroupSizeX = nullptr; + Value *WorkGroupSizeY = nullptr; + Value *WorkGroupSizeZ = nullptr; + + Value *GridSizeX = nullptr; + Value *GridSizeY = nullptr; + Value *GridSizeZ = nullptr; + + const DataLayout &DL = Mod->getDataLayout(); + + // We expect to see several GEP users, casted to the appropriate type and + // loaded. + for (User *U : CI->users()) { + if (!U->hasOneUse()) + continue; + + int64_t Offset = 0; + if (GetPointerBaseWithConstantOffset(U, Offset, DL) != CI) + continue; + + auto *BCI = dyn_cast(*U->user_begin()); + if (!BCI || !BCI->hasOneUse()) + continue; + + auto *Load = dyn_cast(*BCI->user_begin()); + if (!Load || !Load->isSimple()) + continue; + + unsigned LoadSize = DL.getTypeStoreSize(Load->getType()); + + // TODO: Handle merged loads. + switch (Offset) { + case WORKGROUP_SIZE_X: + if (LoadSize == 2) + WorkGroupSizeX = Load; + break; + case WORKGROUP_SIZE_Y: + if (LoadSize == 2) + WorkGroupSizeY = Load; + break; + case WORKGROUP_SIZE_Z: + if (LoadSize == 2) + WorkGroupSizeZ = Load; + break; + case GRID_SIZE_X: + if (LoadSize == 4) + GridSizeX = Load; + break; + case GRID_SIZE_Y: + if (LoadSize == 4) + GridSizeY = Load; + break; + case GRID_SIZE_Z: + if (LoadSize == 4) + GridSizeZ = Load; + break; + default: + break; + } + } + + // Pattern match the code used to handle partial workgroup dispatches in the + // library implementation of get_local_size, so the entire function can be + // constant folded with a known group size. + // + // uint r = grid_size - group_id * group_size; + // get_local_size = (r < group_size) ? r : group_size; + // + // If we have uniform-work-group-size (which is the default in OpenCL 1.2), + // the grid_size is required to be a multiple of group_size). In this case: + // + // grid_size - (group_id * group_size) < group_size + // -> + // grid_size < group_size + (group_id * group_size) + // + // (grid_size / group_size) < 1 + group_id + // + // grid_size / group_size is at least 1, so we can conclude the select + // condition is false (except for group_id == 0, where the select result is + // the same). + + bool MadeChange = false; + Value *WorkGroupSizes[3] = { WorkGroupSizeX, WorkGroupSizeY, WorkGroupSizeZ }; + Value *GridSizes[3] = { GridSizeX, GridSizeY, GridSizeZ }; + + for (int I = 0; HasUniformWorkGroupSize && I < 3; ++I) { + Value *GroupSize = WorkGroupSizes[I]; + Value *GridSize = GridSizes[I]; + if (!GroupSize || !GridSize) + continue; + + for (User *U : GroupSize->users()) { + auto *ZextGroupSize = dyn_cast(U); + if (!ZextGroupSize) + continue; + + for (User *ZextUser : ZextGroupSize->users()) { + auto *SI = dyn_cast(ZextUser); + if (!SI) + continue; + + using namespace llvm::PatternMatch; + auto GroupIDIntrin = I == 0 ? + m_Intrinsic() : + (I == 1 ? m_Intrinsic() : + m_Intrinsic()); + + auto SubExpr = m_Sub(m_Specific(GridSize), + m_Mul(GroupIDIntrin, m_Specific(ZextGroupSize))); + + ICmpInst::Predicate Pred; + if (match(SI, + m_Select(m_ICmp(Pred, SubExpr, m_Specific(ZextGroupSize)), + SubExpr, + m_Specific(ZextGroupSize))) && + Pred == ICmpInst::ICMP_ULT) { + if (HasReqdWorkGroupSize) { + ConstantInt *KnownSize + = mdconst::extract(MD->getOperand(I)); + SI->replaceAllUsesWith(ConstantExpr::getIntegerCast(KnownSize, + SI->getType(), + false)); + } else { + SI->replaceAllUsesWith(ZextGroupSize); + } + + MadeChange = true; + } + } + } + } + + if (!HasReqdWorkGroupSize) + return MadeChange; + + // Eliminate any other loads we can from the dispatch packet. + for (int I = 0; I < 3; ++I) { + Value *GroupSize = WorkGroupSizes[I]; + if (!GroupSize) + continue; + + ConstantInt *KnownSize = mdconst::extract(MD->getOperand(I)); + GroupSize->replaceAllUsesWith( + ConstantExpr::getIntegerCast(KnownSize, + GroupSize->getType(), + false)); + MadeChange = true; + } + + return MadeChange; +} + +// TODO: Move makeLIDRangeMetadata usage into here. Seem to not get +// TargetPassConfig for subtarget. +bool AMDGPULowerKernelAttributes::runOnModule(Module &M) { + StringRef DispatchPtrName + = Intrinsic::getName(Intrinsic::amdgcn_dispatch_ptr); + + Function *DispatchPtr = Mod->getFunction(DispatchPtrName); + if (!DispatchPtr) // Dispatch ptr not used. + return false; + + bool MadeChange = false; + + SmallPtrSet HandledUses; + for (auto *U : DispatchPtr->users()) { + CallInst *CI = cast(U); + if (HandledUses.insert(CI).second) { + if (processUse(CI)) + MadeChange = true; + } + } + + return MadeChange; +} + +INITIALIZE_PASS_BEGIN(AMDGPULowerKernelAttributes, DEBUG_TYPE, + "AMDGPU IR optimizations", false, false) +INITIALIZE_PASS_END(AMDGPULowerKernelAttributes, DEBUG_TYPE, "AMDGPU IR optimizations", + false, false) + +char AMDGPULowerKernelAttributes::ID = 0; + +ModulePass *llvm::createAMDGPULowerKernelAttributesPass() { + return new AMDGPULowerKernelAttributes(); +} Index: lib/Target/AMDGPU/AMDGPUTargetMachine.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -161,6 +161,7 @@ initializeAMDGPUAnnotateKernelFeaturesPass(*PR); initializeAMDGPUAnnotateUniformValuesPass(*PR); initializeAMDGPUArgumentUsageInfoPass(*PR); + initializeAMDGPULowerKernelAttributesPass(*PR); initializeAMDGPULowerIntrinsicsPass(*PR); initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR); initializeAMDGPUPromoteAllocaPass(*PR); @@ -403,6 +404,10 @@ // Add infer address spaces pass to the opt pipeline after inlining // but before SROA to increase SROA opportunities. PM.add(createInferAddressSpacesPass()); + + // This should run after inlining to have any chance of doing anything, + // and before other cleanup optimizations. + PM.add(createAMDGPULowerKernelAttributesPass()); }); } Index: lib/Target/AMDGPU/CMakeLists.txt =================================================================== --- lib/Target/AMDGPU/CMakeLists.txt +++ lib/Target/AMDGPU/CMakeLists.txt @@ -39,6 +39,7 @@ AMDGPULibCalls.cpp AMDGPULibFunc.cpp AMDGPULowerIntrinsics.cpp + AMDGPULowerKernelAttributes.cpp AMDGPUMachineCFGStructurizer.cpp AMDGPUMachineFunction.cpp AMDGPUMachineModuleInfo.cpp Index: test/CodeGen/AMDGPU/reqd-work-group-size.ll =================================================================== --- /dev/null +++ test/CodeGen/AMDGPU/reqd-work-group-size.ll @@ -0,0 +1,501 @@ +; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -amdgpu-lower-kernel-attributes -instcombine %s | FileCheck -enable-var-scope %s + +; CHECK-LABEL: @invalid_reqd_work_group_size( +; CHECK: load i16, +define amdgpu_kernel void @invalid_reqd_work_group_size(i16 addrspace(1)* %out) #0 !reqd_work_group_size !1 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + store i16 %group.size.x, i16 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @volatile_load_group_size_x( +; CHECK: load volatile i16, +define amdgpu_kernel void @volatile_load_group_size_x(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load volatile i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + store i16 %group.size.x, i16 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @load_group_size_x( +; CHECK-NEXT: store i16 8, +define amdgpu_kernel void @load_group_size_x(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + store i16 %group.size.x, i16 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @load_group_size_y( +; CHECK-NEXT: store i16 16, +define amdgpu_kernel void @load_group_size_y(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 6 + %gep.group.size.y.bc = bitcast i8 addrspace(4)* %gep.group.size.y to i16 addrspace(4)* + %group.size.y = load i16, i16 addrspace(4)* %gep.group.size.y.bc, align 4 + store i16 %group.size.y, i16 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @load_group_size_z( +; CHECK-NEXT: store i16 2, +define amdgpu_kernel void @load_group_size_z(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.z = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 8 + %gep.group.size.z.bc = bitcast i8 addrspace(4)* %gep.group.size.z to i16 addrspace(4)* + %group.size.z = load i16, i16 addrspace(4)* %gep.group.size.z.bc, align 4 + store i16 %group.size.z, i16 addrspace(1)* %out + ret void +} + +; Metadata uses i64 instead of i32 +; CHECK-LABEL: @load_group_size_x_reqd_work_group_size_i64( +; CHECK-NEXT: store i16 8, +define amdgpu_kernel void @load_group_size_x_reqd_work_group_size_i64(i16 addrspace(1)* %out) #0 !reqd_work_group_size !2 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + store i16 %group.size.x, i16 addrspace(1)* %out + ret void +} + +; Metadata uses i16 instead of i32 +; CHECK-LABEL: @load_group_size_x_reqd_work_group_size_i16( +; CHECK-NEXT: store i16 8, +define amdgpu_kernel void @load_group_size_x_reqd_work_group_size_i16(i16 addrspace(1)* %out) #0 !reqd_work_group_size !3 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + store i16 %group.size.x, i16 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @use_local_size_x_8_16_2( +; CHECK-NEXT: store i64 8, +define amdgpu_kernel void @use_local_size_x_8_16_2(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)* + %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + %cmp = icmp ult i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @use_local_size_y_8_16_2( +; CHECK-NEXT: store i64 16, +define amdgpu_kernel void @use_local_size_y_8_16_2(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 6 + %gep.group.size.y.bc = bitcast i8 addrspace(4)* %gep.group.size.y to i16 addrspace(4)* + %group.size.y = load i16, i16 addrspace(4)* %gep.group.size.y.bc, align 4 + %gep.grid.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 16 + %gep.grid.size.y.bc = bitcast i8 addrspace(4)* %gep.grid.size.y to i32 addrspace(4)* + %grid.size.y = load i32, i32 addrspace(4)* %gep.grid.size.y.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.y() + %group.size.y.zext = zext i16 %group.size.y to i32 + %group.id_x_group.size.y = mul i32 %group.id, %group.size.y.zext + %sub = sub i32 %grid.size.y, %group.id_x_group.size.y + %cmp = icmp ult i32 %sub, %group.size.y.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.y.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @use_local_size_z_8_16_2( +; CHECK-NEXT: store i64 2, +define amdgpu_kernel void @use_local_size_z_8_16_2(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.z = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 8 + %gep.group.size.z.bc = bitcast i8 addrspace(4)* %gep.group.size.z to i16 addrspace(4)* + %group.size.z = load i16, i16 addrspace(4)* %gep.group.size.z.bc, align 4 + %gep.grid.size.z = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 20 + %gep.grid.size.z.bc = bitcast i8 addrspace(4)* %gep.grid.size.z to i32 addrspace(4)* + %grid.size.z = load i32, i32 addrspace(4)* %gep.grid.size.z.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.z() + %group.size.z.zext = zext i16 %group.size.z to i32 + %group.id_x_group.size.z = mul i32 %group.id, %group.size.z.zext + %sub = sub i32 %grid.size.z, %group.id_x_group.size.z + %cmp = icmp ult i32 %sub, %group.size.z.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.z.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; Simplification on select is invalid, but we can still eliminate the +; load of the group size. + +; CHECK-LABEL: @local_size_x_8_16_2_wrong_group_id( +; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.y() +; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3 +define amdgpu_kernel void @local_size_x_8_16_2_wrong_group_id(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)* + %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.y() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + %cmp = icmp ult i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @local_size_x_8_16_2_wrong_grid_size( +; CHECK: %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 +; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() +; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3 +; CHECK: %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + define amdgpu_kernel void @local_size_x_8_16_2_wrong_grid_size(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 16 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)* + %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + %cmp = icmp ult i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @local_size_x_8_16_2_wrong_cmp_type( +; CHECK: %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 +; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() +; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3 +; CHECK: %sub = sub i32 %grid.size.x, %group.id_x_group.size.x +; CHECK: %cmp = icmp slt i32 %sub, 8 +; CHECK: %select = select i1 %cmp, i32 %sub, i32 8 +define amdgpu_kernel void @local_size_x_8_16_2_wrong_cmp_type(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)* + %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + %cmp = icmp slt i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @local_size_x_8_16_2_wrong_select( +; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3 +; CHECK: %sub = sub i32 %grid.size.x, %group.id_x_group.size.x +; CHECK: %1 = icmp ugt i32 %sub, 8 +; CHECK: %select = select i1 %1, i32 %sub, i32 8 +; CHECK: %zext = zext i32 %select to i64 +define amdgpu_kernel void @local_size_x_8_16_2_wrong_select(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)* + %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + %cmp = icmp ult i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %group.size.x.zext, i32 %sub + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @use_local_size_x_8_16_2_wrong_grid_load_size( +; CHECK: %grid.size.x = load i16, i16 addrspace(4)* %gep.grid.size.x.bc, align 4 +; CHECK: %grid.size.x.zext = zext i16 %grid.size.x to i32 +; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() +; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3 +; CHECK: %sub = sub i32 %grid.size.x.zext, %group.id_x_group.size.x +define amdgpu_kernel void @use_local_size_x_8_16_2_wrong_grid_load_size(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i16 addrspace(4)* + %grid.size.x = load i16, i16 addrspace(4)* %gep.grid.size.x.bc, align 4 + %grid.size.x.zext = zext i16 %grid.size.x to i32 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x.zext, %group.id_x_group.size.x + %cmp = icmp ult i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @func_group_size_x( +; CHECK-NEXT: ret i32 8 +define i32 @func_group_size_x(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %zext = zext i16 %group.size.x to i32 + ret i32 %zext +} + +; CHECK-LABEL: @__ockl_get_local_size_reqd_size( +; CHECK: %group.size = phi i32 [ 2, %bb17 ], [ 16, %bb9 ], [ 8, %bb1 ], [ 1, %bb ] +define i64 @__ockl_get_local_size_reqd_size(i32 %arg) #1 !reqd_work_group_size !0 { +bb: + %tmp = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #2 + switch i32 %arg, label %bb25 [ + i32 0, label %bb1 + i32 1, label %bb9 + i32 2, label %bb17 + ] + +bb1: ; preds = %bb + %tmp2 = tail call i32 @llvm.amdgcn.workgroup.id.x() + %tmp3 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 12 + %tmp4 = bitcast i8 addrspace(4)* %tmp3 to i32 addrspace(4)* + %tmp5 = load i32, i32 addrspace(4)* %tmp4, align 4 + %tmp6 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 4 + %tmp7 = bitcast i8 addrspace(4)* %tmp6 to i16 addrspace(4)* + %tmp8 = load i16, i16 addrspace(4)* %tmp7, align 4 + br label %bb25 + +bb9: ; preds = %bb + %tmp10 = tail call i32 @llvm.amdgcn.workgroup.id.y() + %tmp11 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 16 + %tmp12 = bitcast i8 addrspace(4)* %tmp11 to i32 addrspace(4)* + %tmp13 = load i32, i32 addrspace(4)* %tmp12, align 8 + %tmp14 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 6 + %tmp15 = bitcast i8 addrspace(4)* %tmp14 to i16 addrspace(4)* + %tmp16 = load i16, i16 addrspace(4)* %tmp15, align 2 + br label %bb25 + +bb17: ; preds = %bb + %tmp18 = tail call i32 @llvm.amdgcn.workgroup.id.z() + %tmp19 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 20 + %tmp20 = bitcast i8 addrspace(4)* %tmp19 to i32 addrspace(4)* + %tmp21 = load i32, i32 addrspace(4)* %tmp20, align 4 + %tmp22 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 8 + %tmp23 = bitcast i8 addrspace(4)* %tmp22 to i16 addrspace(4)* + %tmp24 = load i16, i16 addrspace(4)* %tmp23, align 8 + br label %bb25 + +bb25: ; preds = %bb17, %bb9, %bb1, %bb + %tmp26 = phi i32 [ %tmp21, %bb17 ], [ %tmp13, %bb9 ], [ %tmp5, %bb1 ], [ 0, %bb ] + %group.size = phi i16 [ %tmp24, %bb17 ], [ %tmp16, %bb9 ], [ %tmp8, %bb1 ], [ 1, %bb ] + %tmp28 = phi i32 [ %tmp18, %bb17 ], [ %tmp10, %bb9 ], [ %tmp2, %bb1 ], [ 0, %bb ] + %tmp29 = zext i16 %group.size to i32 + %tmp30 = mul i32 %tmp28, %tmp29 + %tmp31 = sub i32 %tmp26, %tmp30 + %tmp32 = icmp ult i32 %tmp31, %tmp29 + %tmp33 = select i1 %tmp32, i32 %tmp31, i32 %tmp29 + %tmp34 = zext i32 %tmp33 to i64 + ret i64 %tmp34 +} + +; CHECK-LABEL: @all_local_size( +; CHECK-NEXT: store volatile i64 8, i64 addrspace(1)* %out, align 4 +; CHECK-NEXT: store volatile i64 16, i64 addrspace(1)* %out, align 4 +; CHECK-NEXT: store volatile i64 2, i64 addrspace(1)* %out, align 4 +define amdgpu_kernel void @all_local_size(i64 addrspace(1)* nocapture readnone %out) #0 !reqd_work_group_size !0 { + %tmp.i = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #0 + %tmp2.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #0 + %tmp3.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 12 + %tmp4.i = bitcast i8 addrspace(4)* %tmp3.i to i32 addrspace(4)* + %tmp5.i = load i32, i32 addrspace(4)* %tmp4.i, align 4 + %tmp6.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 4 + %tmp7.i = bitcast i8 addrspace(4)* %tmp6.i to i16 addrspace(4)* + %tmp8.i = load i16, i16 addrspace(4)* %tmp7.i, align 4 + %tmp29.i = zext i16 %tmp8.i to i32 + %tmp30.i = mul i32 %tmp2.i, %tmp29.i + %tmp31.i = sub i32 %tmp5.i, %tmp30.i + %tmp32.i = icmp ult i32 %tmp31.i, %tmp29.i + %tmp33.i = select i1 %tmp32.i, i32 %tmp31.i, i32 %tmp29.i + %tmp34.i = zext i32 %tmp33.i to i64 + %tmp10.i = tail call i32 @llvm.amdgcn.workgroup.id.y() #0 + %tmp11.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 16 + %tmp12.i = bitcast i8 addrspace(4)* %tmp11.i to i32 addrspace(4)* + %tmp13.i = load i32, i32 addrspace(4)* %tmp12.i, align 8 + %tmp14.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 6 + %tmp15.i = bitcast i8 addrspace(4)* %tmp14.i to i16 addrspace(4)* + %tmp16.i = load i16, i16 addrspace(4)* %tmp15.i, align 2 + %tmp29.i9 = zext i16 %tmp16.i to i32 + %tmp30.i10 = mul i32 %tmp10.i, %tmp29.i9 + %tmp31.i11 = sub i32 %tmp13.i, %tmp30.i10 + %tmp32.i12 = icmp ult i32 %tmp31.i11, %tmp29.i9 + %tmp33.i13 = select i1 %tmp32.i12, i32 %tmp31.i11, i32 %tmp29.i9 + %tmp34.i14 = zext i32 %tmp33.i13 to i64 + %tmp18.i = tail call i32 @llvm.amdgcn.workgroup.id.z() #0 + %tmp19.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 20 + %tmp20.i = bitcast i8 addrspace(4)* %tmp19.i to i32 addrspace(4)* + %tmp21.i = load i32, i32 addrspace(4)* %tmp20.i, align 4 + %tmp22.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 8 + %tmp23.i = bitcast i8 addrspace(4)* %tmp22.i to i16 addrspace(4)* + %tmp24.i = load i16, i16 addrspace(4)* %tmp23.i, align 8 + %tmp29.i2 = zext i16 %tmp24.i to i32 + %tmp30.i3 = mul i32 %tmp18.i, %tmp29.i2 + %tmp31.i4 = sub i32 %tmp21.i, %tmp30.i3 + %tmp32.i5 = icmp ult i32 %tmp31.i4, %tmp29.i2 + %tmp33.i6 = select i1 %tmp32.i5, i32 %tmp31.i4, i32 %tmp29.i2 + %tmp34.i7 = zext i32 %tmp33.i6 to i64 + store volatile i64 %tmp34.i, i64 addrspace(1)* %out, align 4 + store volatile i64 %tmp34.i14, i64 addrspace(1)* %out, align 4 + store volatile i64 %tmp34.i7, i64 addrspace(1)* %out, align 4 + ret void +} + +; TODO: Should be able to handle this, but not much reason to. +; CHECK-LABEL: @partial_load_group_size_x( +; CHECK-NEXT: %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() +; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 +; CHECK-NEXT: %group.size.x.lo = load i8, i8 addrspace(4)* %gep.group.size.x, align 1 +; CHECK-NEXT: store i8 %group.size.x.lo, i8 addrspace(1)* %out, align 1 +define amdgpu_kernel void @partial_load_group_size_x(i8 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %group.size.x.lo = load i8, i8 addrspace(4)* %gep.group.size.x, align 1 + store i8 %group.size.x.lo, i8 addrspace(1)* %out + ret void +} + +; TODO: Should be able to handle this +; CHECK-LABEL: @load_group_size_xy_i32( +; CHECK: %group.size.xy = load i32, +; CHECK: store i32 %group.size.xy +define amdgpu_kernel void @load_group_size_xy_i32(i32 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i32 addrspace(4)* + %group.size.xy = load i32, i32 addrspace(4)* %gep.group.size.x.bc, align 4 + store i32 %group.size.xy, i32 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @load_group_size_x_y_multiple_dispatch_ptr( +; CHECK-NEXT: store volatile i16 8, i16 addrspace(1)* %out, align 2 +; CHECK-NEXT: store volatile i16 16, i16 addrspace(1)* %out, align 2 +define amdgpu_kernel void @load_group_size_x_y_multiple_dispatch_ptr(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 { + %dispatch.ptr0 = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr0, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + store volatile i16 %group.size.x, i16 addrspace(1)* %out + + %dispatch.ptr1 = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr1, i64 6 + %gep.group.size.y.bc = bitcast i8 addrspace(4)* %gep.group.size.y to i16 addrspace(4)* + %group.size.y = load i16, i16 addrspace(4)* %gep.group.size.y.bc, align 4 + store volatile i16 %group.size.y, i16 addrspace(1)* %out + + ret void +} + +; CHECK-LABEL: @use_local_size_x_uniform_work_group_size( +; CHECK-NEXT: %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() +; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 +; CHECK-NEXT: %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* +; CHECK-NEXT: %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 +; CHECK-NEXT: %zext = zext i16 %group.size.x to i64 +; CHECK-NEXT: store i64 %zext, i64 addrspace(1)* %out, align 4 +define amdgpu_kernel void @use_local_size_x_uniform_work_group_size(i64 addrspace(1)* %out) #2 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)* + %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + %cmp = icmp ult i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @use_local_size_x_uniform_work_group_size_false( +; CHECK: icmp ult +; CHECK: select +define amdgpu_kernel void @use_local_size_x_uniform_work_group_size_false(i64 addrspace(1)* %out) #3 { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4 + %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)* + %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4 + %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12 + %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)* + %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4 + %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x() + %group.size.x.zext = zext i16 %group.size.x to i32 + %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext + %sub = sub i32 %grid.size.x, %group.id_x_group.size.x + %cmp = icmp ult i32 %sub, %group.size.x.zext + %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext + %zext = zext i32 %select to i64 + store i64 %zext, i64 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: @no_use_dispatch_ptr( +; CHECK-NEXT: ret void +define amdgpu_kernel void @no_use_dispatch_ptr() { + %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() + ret void +} + +declare i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #1 +declare i32 @llvm.amdgcn.workgroup.id.x() #1 +declare i32 @llvm.amdgcn.workgroup.id.y() #1 +declare i32 @llvm.amdgcn.workgroup.id.z() #1 + +attributes #0 = { nounwind "uniform-work-group-size"="true" } +attributes #1 = { nounwind readnone speculatable } +attributes #2 = { nounwind "uniform-work-group-size"="true" } +attributes #3 = { nounwind "uniform-work-group-size"="false" } + +!0 = !{i32 8, i32 16, i32 2} +!1 = !{i32 8, i32 16} +!2 = !{i64 8, i64 16, i64 2} +!3 = !{i16 8, i16 16, i16 2}