This is an archive of the discontinued LLVM Phabricator instance.

Differential D30203

AMDGPU: Add another BFE pattern
ClosedPublic

Authored by arsenm on Feb 21 2017, 8:40 AM.

Details

Reviewers
arsenm
mareko
Summary

This is the pattern that falls out of the instruction's
definition if offset == 0.

Diff Detail

Event Timeline

arsenm created this revision.Feb 21 2017, 8:40 AM
Herald added subscribers: tpr, dstuttard, tony-tye and 4 others. · View Herald TranscriptFeb 21 2017, 8:40 AM
mareko edited edge metadata.Feb 22 2017, 1:47 PM

LGTM

arsenm accepted this revision.Feb 22 2017, 4:36 PM

r295912

This revision is now accepted and ready to land.Feb 22 2017, 4:36 PM
arsenm closed this revision.Feb 22 2017, 4:36 PM

Revision Contents

PathSize
lib/
Target/
AMDGPU/
86 lines
2 lines
3 lines
test/
CodeGen/
AMDGPU/
163 lines

Diff 89221

lib/Target/AMDGPU/AMDGPUInstructions.td

Show First 20 Lines • Show All 66 Lines • ▼ Show 20 Lines
} // End OperandType = "OPERAND_IMMEDIATE" } // End OperandType = "OPERAND_IMMEDIATE"
//===--------------------------------------------------------------------===// //===--------------------------------------------------------------------===//
// Custom Operands // Custom Operands
//===--------------------------------------------------------------------===// //===--------------------------------------------------------------------===//
def brtarget : Operand<OtherVT>; def brtarget : Operand<OtherVT>;
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// Misc. PatFrags
//===----------------------------------------------------------------------===//
class HasOneUseBinOp<SDPatternOperator op> : PatFrag<
(ops node:$src0, node:$src1),
(op $src0, $src1),
[{ return N->hasOneUse(); }]
>;
class HasOneUseTernaryOp<SDPatternOperator op> : PatFrag<
(ops node:$src0, node:$src1, node:$src2),
(op $src0, $src1, $src2),
[{ return N->hasOneUse(); }]
>;
let Properties = [SDNPCommutative, SDNPAssociative] in {
def smax_oneuse : HasOneUseBinOp<smax>;
def smin_oneuse : HasOneUseBinOp<smin>;
def umax_oneuse : HasOneUseBinOp<umax>;
def umin_oneuse : HasOneUseBinOp<umin>;
def fminnum_oneuse : HasOneUseBinOp<fminnum>;
def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>;
def and_oneuse : HasOneUseBinOp<and>;
def or_oneuse : HasOneUseBinOp<or>;
def xor_oneuse : HasOneUseBinOp<xor>;
} // Properties = [SDNPCommutative, SDNPAssociative]
def sub_oneuse : HasOneUseBinOp<sub>;
def shl_oneuse : HasOneUseBinOp<shl>;
def select_oneuse : HasOneUseTernaryOp<select>;
//===----------------------------------------------------------------------===//
// PatLeafs for floating-point comparisons // PatLeafs for floating-point comparisons
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
def COND_OEQ : PatLeaf < def COND_OEQ : PatLeaf <
(cond), (cond),
[{return N->get() == ISD::SETOEQ || N->get() == ISD::SETEQ;}] [{return N->get() == ISD::SETOEQ || N->get() == ISD::SETEQ;}]
>; >;
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines
def COND_NULL : PatLeaf < def COND_NULL : PatLeaf <
(cond), (cond),
[{(void)N; return false;}] [{(void)N; return false;}]
>; >;
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// Misc. PatFrags
//===----------------------------------------------------------------------===//
class HasOneUseBinOp<SDPatternOperator op> : PatFrag<
(ops node:$src0, node:$src1),
(op $src0, $src1),
[{ return N->hasOneUse(); }]
>;
class HasOneUseTernaryOp<SDPatternOperator op> : PatFrag<
(ops node:$src0, node:$src1, node:$src2),
(op $src0, $src1, $src2),
[{ return N->hasOneUse(); }]
>;
//===----------------------------------------------------------------------===//
// Load/Store Pattern Fragments // Load/Store Pattern Fragments
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
class PrivateMemOp <dag ops, dag frag> : PatFrag <ops, frag, [{ class PrivateMemOp <dag ops, dag frag> : PatFrag <ops, frag, [{
return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS; return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS;
}]>; }]>;
class PrivateLoad <SDPatternOperator op> : PrivateMemOp < class PrivateLoad <SDPatternOperator op> : PrivateMemOp <
▲ Show 20 Lines • Show All 419 Lines • ▼ Show 20 Linesdef IMMZeroBasedBitfieldMask : PatLeaf <(imm), [{
return isMask_32(N->getZExtValue()); return isMask_32(N->getZExtValue());
}]>; }]>;
def IMMPopCount : SDNodeXForm<imm, [{ def IMMPopCount : SDNodeXForm<imm, [{
return CurDAG->getTargetConstant(countPopulation(N->getZExtValue()), SDLoc(N), return CurDAG->getTargetConstant(countPopulation(N->getZExtValue()), SDLoc(N),
MVT::i32); MVT::i32);
}]>; }]>;
class BFEPattern <Instruction BFE, Instruction MOV> : Pat < multiclass BFEPattern <Instruction UBFE, Instruction SBFE, Instruction MOV> {
def : Pat <
(i32 (and (i32 (srl i32:$src, i32:$rshift)), IMMZeroBasedBitfieldMask:$mask)), (i32 (and (i32 (srl i32:$src, i32:$rshift)), IMMZeroBasedBitfieldMask:$mask)),
(BFE $src, $rshift, (MOV (i32 (IMMPopCount $mask)))) (UBFE $src, $rshift, (MOV (i32 (IMMPopCount $mask))))
>;
def : Pat <
(srl (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)),
(UBFE $src, (i32 0), $width)
>;
def : Pat <
(sra (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)),
(SBFE $src, (i32 0), $width)
>; >;
}
// rotr pattern // rotr pattern
class ROTRPattern <Instruction BIT_ALIGN> : Pat < class ROTRPattern <Instruction BIT_ALIGN> : Pat <
(rotr i32:$src0, i32:$src1), (rotr i32:$src0, i32:$src1),
(BIT_ALIGN $src0, $src0, $src1) (BIT_ALIGN $src0, $src0, $src1)
>; >;
// This matches 16 permutations of // This matches 16 permutations of
// max(min(x, y), min(max(x, y), z)) // max(min(x, y), min(max(x, y), z))
class IntMed3Pat<Instruction med3Inst, class IntMed3Pat<Instruction med3Inst,
SDPatternOperator max, SDPatternOperator max,
SDPatternOperator max_oneuse, SDPatternOperator max_oneuse,
SDPatternOperator min_oneuse> : Pat< SDPatternOperator min_oneuse> : Pat<
(max (min_oneuse i32:$src0, i32:$src1), (max (min_oneuse i32:$src0, i32:$src1),
(min_oneuse (max_oneuse i32:$src0, i32:$src1), i32:$src2)), (min_oneuse (max_oneuse i32:$src0, i32:$src1), i32:$src2)),
(med3Inst $src0, $src1, $src2) (med3Inst $src0, $src1, $src2)
>; >;
let Properties = [SDNPCommutative, SDNPAssociative] in {
def smax_oneuse : HasOneUseBinOp<smax>;
def smin_oneuse : HasOneUseBinOp<smin>;
def umax_oneuse : HasOneUseBinOp<umax>;
def umin_oneuse : HasOneUseBinOp<umin>;
def fminnum_oneuse : HasOneUseBinOp<fminnum>;
def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>;
def and_oneuse : HasOneUseBinOp<and>;
def or_oneuse : HasOneUseBinOp<or>;
def xor_oneuse : HasOneUseBinOp<xor>;
} // Properties = [SDNPCommutative, SDNPAssociative]
def sub_oneuse : HasOneUseBinOp<sub>;
def select_oneuse : HasOneUseTernaryOp<select>;
// Special conversion patterns // Special conversion patterns
def cvt_rpi_i32_f32 : PatFrag < def cvt_rpi_i32_f32 : PatFrag <
(ops node:$src), (ops node:$src),
(fp_to_sint (ffloor (fadd $src, FP_HALF))), (fp_to_sint (ffloor (fadd $src, FP_HALF))),
[{ (void) N; return TM.Options.NoNaNsFPMath; }] [{ (void) N; return TM.Options.NoNaNsFPMath; }]
>; >;
Show All 33 Lines

lib/Target/AMDGPU/EvergreenInstructions.td

Show First 20 Lines • Show All 382 Lines • ▼ Show 20 Linesdef BFE_UINT_eg : R600_3OP <0x4, "BFE_UINT",
VecALU VecALU
>; >;
def BFE_INT_eg : R600_3OP <0x5, "BFE_INT", def BFE_INT_eg : R600_3OP <0x5, "BFE_INT",
[(set i32:$dst, (AMDGPUbfe_i32 i32:$src0, i32:$src1, i32:$src2))], [(set i32:$dst, (AMDGPUbfe_i32 i32:$src0, i32:$src1, i32:$src2))],
VecALU VecALU
>; >;
def : BFEPattern <BFE_UINT_eg, MOV_IMM_I32>; defm : BFEPattern <BFE_UINT_eg, BFE_INT_eg, MOV_IMM_I32>;
def BFI_INT_eg : R600_3OP <0x06, "BFI_INT", def BFI_INT_eg : R600_3OP <0x06, "BFI_INT",
[(set i32:$dst, (AMDGPUbfi i32:$src0, i32:$src1, i32:$src2))], [(set i32:$dst, (AMDGPUbfi i32:$src0, i32:$src1, i32:$src2))],
VecALU VecALU
>; >;
def : Pat<(i32 (sext_inreg i32:$src, i1)), def : Pat<(i32 (sext_inreg i32:$src, i1)),
(BFE_INT_eg i32:$src, (i32 ZERO), (i32 ONE_INT))>; (BFE_INT_eg i32:$src, (i32 ZERO), (i32 ONE_INT))>;
▲ Show 20 Lines • Show All 367 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIInstructions.td

Show First 20 Lines • Show All 1,063 Lines • ▼ Show 20 Linesmulticlass BFMPatterns <ValueType vt, InstSI BFM, InstSI MOV> {
def : Pat < def : Pat <
(vt (add (vt (shl 1, vt:$a)), -1)), (vt (add (vt (shl 1, vt:$a)), -1)),
(BFM $a, (MOV (i32 0))) (BFM $a, (MOV (i32 0)))
>; >;
} }
defm : BFMPatterns <i32, S_BFM_B32, S_MOV_B32>; defm : BFMPatterns <i32, S_BFM_B32, S_MOV_B32>;
// FIXME: defm : BFMPatterns <i64, S_BFM_B64, S_MOV_B64>; // FIXME: defm : BFMPatterns <i64, S_BFM_B64, S_MOV_B64>;
defm : BFEPattern <V_BFE_U32, V_BFE_I32, S_MOV_B32>;
def : BFEPattern <V_BFE_U32, S_MOV_B32>;
def : Pat< def : Pat<
(fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))),
(V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src, 0, 0) (V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src, 0, 0)
>; >;
def : Pat< def : Pat<
(fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))), (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))),
▲ Show 20 Lines • Show All 91 Lines • Show Last 20 Lines

test/CodeGen/AMDGPU/bfe-patterns.ll

  • This file was added.
; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
; GCN-LABEL: {{^}}v_ubfe_sub_i32:
; GCN: {{buffer|flat}}_load_dword [[SRC:v[0-9]+]]
; GCN: {{buffer|flat}}_load_dword [[WIDTH:v[0-9]+]]
; GCN: v_bfe_u32 v{{[0-9]+}}, [[SRC]], 0, [[WIDTH]]
define void @v_ubfe_sub_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in0, i32 addrspace(1)* %in1) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%in0.gep = getelementptr i32, i32 addrspace(1)* %in0, i32 %id.x
%in1.gep = getelementptr i32, i32 addrspace(1)* %in1, i32 %id.x
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%src = load volatile i32, i32 addrspace(1)* %in0.gep
%width = load volatile i32, i32 addrspace(1)* %in0.gep
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = lshr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
ret void
}
; GCN-LABEL: {{^}}v_ubfe_sub_multi_use_shl_i32:
; GCN: {{buffer|flat}}_load_dword [[SRC:v[0-9]+]]
; GCN: {{buffer|flat}}_load_dword [[WIDTH:v[0-9]+]]
; GCN: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 32, [[WIDTH]]
; SI-NEXT: v_lshl_b32_e32 [[SHL:v[0-9]+]], [[SRC]], [[SUB]]
; SI-NEXT: v_lshr_b32_e32 [[BFE:v[0-9]+]], [[SHL]], [[SUB]]
; VI-NEXT: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], [[SUB]], [[SRC]]
; VI-NEXT: v_lshrrev_b32_e32 [[BFE:v[0-9]+]], [[SUB]], [[SHL]]
; GCN: [[BFE]]
; GCN: [[SHL]]
define void @v_ubfe_sub_multi_use_shl_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in0, i32 addrspace(1)* %in1) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%in0.gep = getelementptr i32, i32 addrspace(1)* %in0, i32 %id.x
%in1.gep = getelementptr i32, i32 addrspace(1)* %in1, i32 %id.x
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%src = load volatile i32, i32 addrspace(1)* %in0.gep
%width = load volatile i32, i32 addrspace(1)* %in0.gep
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = lshr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
store volatile i32 %shl, i32 addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}s_ubfe_sub_i32:
; GCN: s_load_dword [[SRC:s[0-9]+]]
; GCN: s_load_dword [[WIDTH:s[0-9]+]]
; GCN: v_mov_b32_e32 [[VWIDTH:v[0-9]+]]
; GCN: v_bfe_u32 v{{[0-9]+}}, [[SRC]], 0, [[VWIDTH]]
define void @s_ubfe_sub_i32(i32 addrspace(1)* %out, i32 %src, i32 %width) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = lshr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
ret void
}
; GCN-LABEL: {{^}}s_ubfe_sub_multi_use_shl_i32:
; GCN: s_load_dword [[SRC:s[0-9]+]]
; GCN: s_load_dword [[WIDTH:s[0-9]+]]
; GCN: s_sub_i32 [[SUB:s[0-9]+]], 32, [[WIDTH]]
; GCN-NEXT: s_lshl_b32 [[SHL:s[0-9]+]], [[SRC]], [[SUB]]
; GCN-NEXT: s_lshr_b32 s{{[0-9]+}}, [[SHL]], [[SUB]]
define void @s_ubfe_sub_multi_use_shl_i32(i32 addrspace(1)* %out, i32 %src, i32 %width) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = lshr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
store volatile i32 %shl, i32 addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}v_sbfe_sub_i32:
; GCN: {{buffer|flat}}_load_dword [[SRC:v[0-9]+]]
; GCN: {{buffer|flat}}_load_dword [[WIDTH:v[0-9]+]]
; GCN: v_bfe_i32 v{{[0-9]+}}, [[SRC]], 0, [[WIDTH]]
define void @v_sbfe_sub_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in0, i32 addrspace(1)* %in1) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%in0.gep = getelementptr i32, i32 addrspace(1)* %in0, i32 %id.x
%in1.gep = getelementptr i32, i32 addrspace(1)* %in1, i32 %id.x
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%src = load volatile i32, i32 addrspace(1)* %in0.gep
%width = load volatile i32, i32 addrspace(1)* %in0.gep
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = ashr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
ret void
}
; GCN-LABEL: {{^}}v_sbfe_sub_multi_use_shl_i32:
; GCN: {{buffer|flat}}_load_dword [[SRC:v[0-9]+]]
; GCN: {{buffer|flat}}_load_dword [[WIDTH:v[0-9]+]]
; GCN: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 32, [[WIDTH]]
; SI-NEXT: v_lshl_b32_e32 [[SHL:v[0-9]+]], [[SRC]], [[SUB]]
; SI-NEXT: v_ashr_i32_e32 [[BFE:v[0-9]+]], [[SHL]], [[SUB]]
; VI-NEXT: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], [[SUB]], [[SRC]]
; VI-NEXT: v_ashrrev_i32_e32 [[BFE:v[0-9]+]], [[SUB]], [[SHL]]
; GCN: [[BFE]]
; GCN: [[SHL]]
define void @v_sbfe_sub_multi_use_shl_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in0, i32 addrspace(1)* %in1) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%in0.gep = getelementptr i32, i32 addrspace(1)* %in0, i32 %id.x
%in1.gep = getelementptr i32, i32 addrspace(1)* %in1, i32 %id.x
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%src = load volatile i32, i32 addrspace(1)* %in0.gep
%width = load volatile i32, i32 addrspace(1)* %in0.gep
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = ashr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
store volatile i32 %shl, i32 addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}s_sbfe_sub_i32:
; GCN: s_load_dword [[SRC:s[0-9]+]]
; GCN: s_load_dword [[WIDTH:s[0-9]+]]
; GCN: v_mov_b32_e32 [[VWIDTH:v[0-9]+]]
; GCN: v_bfe_i32 v{{[0-9]+}}, [[SRC]], 0, [[VWIDTH]]
define void @s_sbfe_sub_i32(i32 addrspace(1)* %out, i32 %src, i32 %width) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = ashr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
ret void
}
; GCN-LABEL: {{^}}s_sbfe_sub_multi_use_shl_i32:
; GCN: s_load_dword [[SRC:s[0-9]+]]
; GCN: s_load_dword [[WIDTH:s[0-9]+]]
; GCN: s_sub_i32 [[SUB:s[0-9]+]], 32, [[WIDTH]]
; GCN-NEXT: s_lshl_b32 [[SHL:s[0-9]+]], [[SRC]], [[SUB]]
; GCN-NEXT: s_ashr_i32 s{{[0-9]+}}, [[SHL]], [[SUB]]
define void @s_sbfe_sub_multi_use_shl_i32(i32 addrspace(1)* %out, i32 %src, i32 %width) #1 {
%id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id.x
%sub = sub i32 32, %width
%shl = shl i32 %src, %sub
%bfe = ashr i32 %shl, %sub
store i32 %bfe, i32 addrspace(1)* %out.gep
store volatile i32 %shl, i32 addrspace(1)* undef
ret void
}
declare i32 @llvm.amdgcn.workitem.id.x() #0
attributes #0 = { nounwind readnone }
attributes #1 = { nounwind }