This is an archive of the discontinued LLVM Phabricator instance.

Differential D74442

AMDGPU: Add option to disable CGP division expansion
ClosedPublic

Authored by arsenm on Feb 11 2020, 2:06 PM.

Details

Reviewers
rampitec
cfang
kerbowa
Summary

The division expansions in AMDGPUCodeGenPrepare can't be relied on for
correctness, since they punt to later optimization and possibly
legalization in some cases. We still need a way to be able to write
tests for the legalizer versions of the expansion. This is mostly for
GlobalISel, since the expected optimzations is expecting aren't
implemented.




The interaction with the flag to expand 64-bit division in the IR is

pretty confusing, but these flags have different purposes.
Diff Detail
Event Timeline
arsenm created this revision.Feb 11 2020, 2:06 PM
Herald added a project: Restricted Project.  ·  View Herald TranscriptFeb 11 2020, 2:06 PM
Herald added subscribers: hiraditya, t-tye, tpr and 6 others.  ·  View Herald Transcript
rampitec accepted this revision.Feb 11 2020, 2:55 PM
This revision is now accepted and ready to land.Feb 11 2020, 2:55 PM
arsenm closed this revision.Feb 14 2020, 11:37 AM
9ec668606b3a8d71ca327d9857844fb9c607a54b
Revision Contents
PathSize
llvm/
lib/
Target/
AMDGPU/
12 lines
test/
CodeGen/
AMDGPU/
GlobalISel/
198 lines
Diff 243995
llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
Show First 20 Lines • Show All 58 Lines • ▼ Show 20 Linesstatic cl::opt<bool> WidenLoads(

  cl::init(true));
  cl::init(true));




static cl::opt<bool> UseMul24Intrin(
static cl::opt<bool> UseMul24Intrin(

  "amdgpu-codegenprepare-mul24",
  "amdgpu-codegenprepare-mul24",

  cl::desc("Introduce mul24 intrinsics in AMDGPUCodeGenPrepare"),
  cl::desc("Introduce mul24 intrinsics in AMDGPUCodeGenPrepare"),

  cl::ReallyHidden,
  cl::ReallyHidden,

  cl::init(true));
  cl::init(true));




// Legalize 64-bit division by using the generic IR expansion.

static cl::opt<bool> ExpandDiv64InIR(
static cl::opt<bool> ExpandDiv64InIR(

  "amdgpu-codegenprepare-expand-div64",
  "amdgpu-codegenprepare-expand-div64",

  cl::desc("Expand 64-bit division in AMDGPUCodeGenPrepare"),
  cl::desc("Expand 64-bit division in AMDGPUCodeGenPrepare"),

  cl::ReallyHidden,
  cl::ReallyHidden,

  cl::init(false));
  cl::init(false));




// Leave all division operations as they are. This supersedes ExpandDiv64InIR

// and is used for testing the legalizer.

static cl::opt<bool> DisableIDivExpand(

  "amdgpu-codegenprepare-disable-idiv-expansion",

  cl::desc("Prevent expanding integer division in AMDGPUCodeGenPrepare"),

  cl::ReallyHidden,

  cl::init(false));



class AMDGPUCodeGenPrepare : public FunctionPass,
class AMDGPUCodeGenPrepare : public FunctionPass,

                             public InstVisitor<AMDGPUCodeGenPrepare, bool> {
                             public InstVisitor<AMDGPUCodeGenPrepare, bool> {

  const GCNSubtarget *ST = nullptr;
  const GCNSubtarget *ST = nullptr;

  AssumptionCache *AC = nullptr;
  AssumptionCache *AC = nullptr;

  DominatorTree *DT = nullptr;
  DominatorTree *DT = nullptr;

  LegacyDivergenceAnalysis *DA = nullptr;
  LegacyDivergenceAnalysis *DA = nullptr;

  Module *Mod = nullptr;
  Module *Mod = nullptr;

  const DataLayout *DL = nullptr;
  const DataLayout *DL = nullptr;

▲ Show 20 Lines • Show All 1,117 Lines • ▼ Show 20 Linesbool AMDGPUCodeGenPrepare::visitBinaryOperator(BinaryOperator &I) {

  Type *Ty = I.getType();
  Type *Ty = I.getType();

  Value *NewDiv = nullptr;
  Value *NewDiv = nullptr;

  unsigned ScalarSize = Ty->getScalarSizeInBits();
  unsigned ScalarSize = Ty->getScalarSizeInBits();




  SmallVector<BinaryOperator *, 8> Div64ToExpand;
  SmallVector<BinaryOperator *, 8> Div64ToExpand;




  if ((Opc == Instruction::URem || Opc == Instruction::UDiv ||
  if ((Opc == Instruction::URem || Opc == Instruction::UDiv ||

       Opc == Instruction::SRem || Opc == Instruction::SDiv) &&
       Opc == Instruction::SRem || Opc == Instruction::SDiv) &&

      ScalarSize <= 64) {
      ScalarSize <= 64 &&

      !DisableIDivExpand) {

    Value *Num = I.getOperand(0);
    Value *Num = I.getOperand(0);

    Value *Den = I.getOperand(1);
    Value *Den = I.getOperand(1);

    IRBuilder<> Builder(&I);
    IRBuilder<> Builder(&I);

    Builder.SetCurrentDebugLocation(I.getDebugLoc());
    Builder.SetCurrentDebugLocation(I.getDebugLoc());




    if (VectorType *VT = dyn_cast<VectorType>(Ty)) {
    if (VectorType *VT = dyn_cast<VectorType>(Ty)) {

      NewDiv = UndefValue::get(VT);
      NewDiv = UndefValue::get(VT);




▲ Show 20 Lines • Show All 203 Lines • Show Last 20 Lines
llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py

; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion -mtriple=amdgcn-amd-amdhsa -stop-after=irtranslator < %s | FileCheck %s



define i32 @udiv_i32(i32 %num, i32 %den) {

  ; CHECK-LABEL: name: udiv_i32

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[COPY]], [[COPY1]]

  ; CHECK:   $vgpr0 = COPY [[UDIV]](s32)

  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]

  ; CHECK:   S_SETPC_B64_return [[COPY3]], implicit $vgpr0

  %result = udiv i32 %num, %den

  ret i32 %result

}



define <2 x i32> @udiv_v2i32(<2 x i32> %num, <2 x i32> %den) {

  ; CHECK-LABEL: name: udiv_v2i32

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2

  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3

  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)

  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)

  ; CHECK:   [[UDIV:%[0-9]+]]:_(<2 x s32>) = G_UDIV [[BUILD_VECTOR]], [[BUILD_VECTOR1]]

  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UDIV]](<2 x s32>)

  ; CHECK:   $vgpr0 = COPY [[UV]](s32)

  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)

  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]

  ; CHECK:   S_SETPC_B64_return [[COPY5]], implicit $vgpr0, implicit $vgpr1

  %result = udiv <2 x i32> %num, %den

  ret <2 x i32> %result

}



define i32 @udiv_i32_pow2k_denom(i32 %num) {

  ; CHECK-LABEL: name: udiv_i32_pow2k_denom

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4096

  ; CHECK:   [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[COPY]], [[C]]

  ; CHECK:   $vgpr0 = COPY [[UDIV]](s32)

  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]

  ; CHECK:   S_SETPC_B64_return [[COPY2]], implicit $vgpr0

  %result = udiv i32 %num, 4096

  ret i32 %result

}



define <2 x i32> @udiv_v2i32_pow2k_denom(<2 x i32> %num) {

  ; CHECK-LABEL: name: udiv_v2i32_pow2k_denom

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4096

  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)

  ; CHECK:   [[UDIV:%[0-9]+]]:_(<2 x s32>) = G_UDIV [[BUILD_VECTOR]], [[BUILD_VECTOR1]]

  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UDIV]](<2 x s32>)

  ; CHECK:   $vgpr0 = COPY [[UV]](s32)

  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)

  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]

  ; CHECK:   S_SETPC_B64_return [[COPY3]], implicit $vgpr0, implicit $vgpr1

  %result = udiv <2 x i32> %num, <i32 4096, i32 4096>

  ret <2 x i32> %result

}



define i32 @udiv_i32_oddk_denom(i32 %num) {

  ; CHECK-LABEL: name: udiv_i32_oddk_denom

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1235195

  ; CHECK:   [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[COPY]], [[C]]

  ; CHECK:   $vgpr0 = COPY [[UDIV]](s32)

  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]

  ; CHECK:   S_SETPC_B64_return [[COPY2]], implicit $vgpr0

  %result = udiv i32 %num, 1235195

  ret i32 %result

}



define <2 x i32> @udiv_v2i32_oddk_denom(<2 x i32> %num) {

  ; CHECK-LABEL: name: udiv_v2i32_oddk_denom

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1235195

  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)

  ; CHECK:   [[UDIV:%[0-9]+]]:_(<2 x s32>) = G_UDIV [[BUILD_VECTOR]], [[BUILD_VECTOR1]]

  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UDIV]](<2 x s32>)

  ; CHECK:   $vgpr0 = COPY [[UV]](s32)

  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)

  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]

  ; CHECK:   S_SETPC_B64_return [[COPY3]], implicit $vgpr0, implicit $vgpr1

  %result = udiv <2 x i32> %num, <i32 1235195, i32 1235195>

  ret <2 x i32> %result

}



define i32 @udiv_i32_pow2_shl_denom(i32 %x, i32 %y) {

  ; CHECK-LABEL: name: udiv_i32_pow2_shl_denom

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4096

  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[COPY1]](s32)

  ; CHECK:   [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[COPY]], [[SHL]]

  ; CHECK:   $vgpr0 = COPY [[UDIV]](s32)

  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]

  ; CHECK:   S_SETPC_B64_return [[COPY3]], implicit $vgpr0

  %shl.y = shl i32 4096, %y

  %r = udiv i32 %x, %shl.y

  ret i32 %r

}



define <2 x i32> @udiv_v2i32_pow2_shl_denom(<2 x i32> %x, <2 x i32> %y) {

  ; CHECK-LABEL: name: udiv_v2i32_pow2_shl_denom

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2

  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3

  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)

  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4096

  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)

  ; CHECK:   [[SHL:%[0-9]+]]:_(<2 x s32>) = G_SHL [[BUILD_VECTOR2]], [[BUILD_VECTOR1]](<2 x s32>)

  ; CHECK:   [[UDIV:%[0-9]+]]:_(<2 x s32>) = G_UDIV [[BUILD_VECTOR]], [[SHL]]

  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UDIV]](<2 x s32>)

  ; CHECK:   $vgpr0 = COPY [[UV]](s32)

  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)

  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]

  ; CHECK:   S_SETPC_B64_return [[COPY5]], implicit $vgpr0, implicit $vgpr1

  %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y

  %r = udiv <2 x i32> %x, %shl.y

  ret <2 x i32> %r

}



define i32 @udiv_i32_24bit(i32 %num, i32 %den) {

  ; CHECK-LABEL: name: udiv_i32_24bit

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215

  ; CHECK:   [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]

  ; CHECK:   [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]

  ; CHECK:   [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]

  ; CHECK:   $vgpr0 = COPY [[UDIV]](s32)

  ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]

  ; CHECK:   S_SETPC_B64_return [[COPY3]], implicit $vgpr0

  %num.mask = and i32 %num, 16777215

  %den.mask = and i32 %den, 16777215

  %result = udiv i32 %num.mask, %den.mask

  ret i32 %result

}



define <2 x i32> @udiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {

  ; CHECK-LABEL: name: udiv_v2i32_24bit

  ; CHECK: bb.1 (%ir-block.0):

  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31

  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0

  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1

  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2

  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3

  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31

  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)

  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)

  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215

  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)

  ; CHECK:   [[AND:%[0-9]+]]:_(<2 x s32>) = G_AND [[BUILD_VECTOR]], [[BUILD_VECTOR2]]

  ; CHECK:   [[AND1:%[0-9]+]]:_(<2 x s32>) = G_AND [[BUILD_VECTOR1]], [[BUILD_VECTOR2]]

  ; CHECK:   [[UDIV:%[0-9]+]]:_(<2 x s32>) = G_UDIV [[AND]], [[AND1]]

  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UDIV]](<2 x s32>)

  ; CHECK:   $vgpr0 = COPY [[UV]](s32)

  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)

  ; CHECK:   [[COPY5:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY4]]

  ; CHECK:   S_SETPC_B64_return [[COPY5]], implicit $vgpr0, implicit $vgpr1

  %num.mask = and <2 x i32> %num, <i32 16777215, i32 16777215>

  %den.mask = and <2 x i32> %den, <i32 16777215, i32 16777215>

  %result = udiv <2 x i32> %num.mask, %den.mask

  ret <2 x i32> %result

}