This is an archive of the discontinued LLVM Phabricator instance.

Differential D32805

[X86][AVX512] Only look at lower bit in constant scalar masks
ClosedPublic

Authored by guyblank on May 3 2017, 7:18 AM.

Details

Reviewers
craig.topper
igorb
zvi
Commits
rG0c42d8c35ba6: VX512] Only look at lower bit in constant scalar masks
rL302546: VX512] Only look at lower bit in constant scalar masks
Summary

for scalar masked instructions only the lower bit of the mask is relevant. so for constant masks we should either do an unmasked operation or no operation, depending on the value of the lower bit.
This patch handles cases where the lower bit is '1'.

Diff Detail

Event Timeline

guyblank created this revision.May 3 2017, 7:18 AM
igorb added inline comments.May 3 2017, 8:07 AM
test/CodeGen/X86/avx512-intrinsics.ll
5204 ↗(On Diff #97633)

This is incorrect behavior. The upper bit of the source should be preserved.
IF k[0]
dst[31:0] := (a[31:0] * b[31:0]) + c[31:0]
ELSE
dst[31:0] := 0
FI
dst[127:32] := a[127:32]

The same for the tests below.

guyblank added inline comments.May 3 2017, 8:51 AM
test/CodeGen/X86/avx512-intrinsics.ll
5204 ↗(On Diff #97633)

ah, i've misread the description.
got confused from the zeroing of DEST[MAX_VL-1:128]

thanks!

guyblank updated this revision to Diff 98137.May 8 2017, 1:49 AM
guyblank retitled this revision from [X86][AVX512] Properly handle scalar masking for constant masks to [X86][AVX512] Only look at lower bit in constant scalar masks.
guyblank edited the summary of this revision. (Show Details)

change the patch to only address cases where the lower bit is '1'

guyblank updated this revision to Diff 98138.May 8 2017, 1:51 AM

this time with the correct patch...

guyblank updated this revision to Diff 98284.May 9 2017, 7:53 AM

rebased

igorb accepted this revision.May 9 2017, 8:16 AM

LGTM

This revision is now accepted and ready to land.May 9 2017, 8:16 AM
Closed by commit rL302546: VX512] Only look at lower bit in constant scalar masks (authored by guyblank). · Explain WhyMay 9 2017, 9:30 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
lib/
Target/
X86/
6 lines
test/
CodeGen/
X86/
107 lines
26 lines

Diff 98138

lib/Target/X86/X86ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 19,015 Lines • ▼ Show 20 Lines
/// to MVT::i1 while lowering masking intrinsics. /// to MVT::i1 while lowering masking intrinsics.
/// The main difference between ScalarMaskingNode and VectorMaskingNode is using /// The main difference between ScalarMaskingNode and VectorMaskingNode is using
/// "X86select" instead of "vselect". We just can't create the "vselect" node /// "X86select" instead of "vselect". We just can't create the "vselect" node
/// for a scalar instruction. /// for a scalar instruction.
static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask, static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
SDValue PreservedSrc, SDValue PreservedSrc,
const X86Subtarget &Subtarget, const X86Subtarget &Subtarget,
SelectionDAG &DAG) { SelectionDAG &DAG) {
if (isAllOnesConstant(Mask))
if (auto *MaskConst = dyn_cast<ConstantSDNode>(Mask))
if (MaskConst->getZExtValue() & 0x1)
return Op; return Op;
MVT VT = Op.getSimpleValueType(); MVT VT = Op.getSimpleValueType();
SDLoc dl(Op); SDLoc dl(Op);
// The mask should be of type MVT::i1 // The mask should be of type MVT::i1
SDValue IMask = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Mask); SDValue IMask = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Mask);
if (Op.getOpcode() == X86ISD::FSETCCM || if (Op.getOpcode() == X86ISD::FSETCCM ||
Op.getOpcode() == X86ISD::FSETCCM_RND) Op.getOpcode() == X86ISD::FSETCCM_RND)
▲ Show 20 Lines • Show All 17,009 Lines • Show Last 20 Lines

test/CodeGen/X86/avx512-scalar_mask.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
declare <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
declare <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
define <4 x float>@test_var_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, i8 %mask) {
; CHECK-LABEL: test_var_mask:
; CHECK: ## BB#0:
; CHECK-NEXT: andl $1, %edi
; CHECK-NEXT: kmovw %edi, %k1
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 %mask, i32 4)
ret < 4 x float> %res
}
define <4 x float>@test_var_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, i8 %mask) {
; CHECK-LABEL: test_var_maskz:
; CHECK: ## BB#0:
; CHECK-NEXT: andl $1, %edi
; CHECK-NEXT: kmovw %edi, %k1
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 %mask, i32 4)
ret < 4 x float> %res
}
; FIXME: we should just return %xmm0 here.
define <4 x float>@test_const0_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const0_mask:
; CHECK: ## BB#0:
; CHECK-NEXT: kxorw %k0, %k0, %k1
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 0, i32 4)
ret < 4 x float> %res
}
; FIXME: we should zero the lower element of xmm0 and return it.
define <4 x float>@test_const0_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const0_maskz:
; CHECK: ## BB#0:
; CHECK-NEXT: kxorw %k0, %k0, %k1
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 0, i32 4)
ret < 4 x float> %res
}
; FIXME: we should just return %xmm0 here.
define <4 x float>@test_const2_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const2_mask:
; CHECK: ## BB#0:
; CHECK-NEXT: kxorw %k0, %k0, %k1
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 2, i32 4)
ret < 4 x float> %res
}
; FIXME: we should zero the lower element of xmm0 and return it.
define <4 x float>@test_const2_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const2_maskz:
; CHECK: ## BB#0:
; CHECK-NEXT: kxorw %k0, %k0, %k1
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 2, i32 4)
ret < 4 x float> %res
}
define <4 x float>@test_const_allone_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const_allone_mask:
; CHECK: ## BB#0:
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 -1, i32 4)
ret < 4 x float> %res
}
define <4 x float>@test_const_allone_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const_allone_maskz:
; CHECK: ## BB#0:
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 -1, i32 4)
ret < 4 x float> %res
}
define <4 x float>@test_const_3_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const_3_mask:
; CHECK: ## BB#0:
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 3, i32 4)
ret < 4 x float> %res
}
define <4 x float>@test_const_3_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
; CHECK-LABEL: test_const_3_maskz:
; CHECK: ## BB#0:
; CHECK-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0
; CHECK-NEXT: retq
%res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %v0,<4 x float> %v1, <4 x float> %v2, i8 3, i32 4)
ret < 4 x float> %res
}

test/CodeGen/X86/avx512er-intrinsics.ll

Show First 20 Lines • Show All 115 Lines • ▼ Show 20 Lines; CHECK-NEXT: retq # encoding: [0xc3]
%res = call <4 x float> @llvm.x86.avx512.rcp28.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 -1, i32 8) ; <<4 x float>> [#uses=1] %res = call <4 x float> @llvm.x86.avx512.rcp28.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 -1, i32 8) ; <<4 x float>> [#uses=1]
ret <4 x float> %res ret <4 x float> %res
} }
declare <4 x float> @llvm.x86.avx512.rcp28.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone declare <4 x float> @llvm.x86.avx512.rcp28.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
define <4 x float> @test_rsqrt28_ss_maskz(<4 x float> %a0) { define <4 x float> @test_rsqrt28_ss_maskz(<4 x float> %a0) {
; CHECK-LABEL: test_rsqrt28_ss_maskz: ; CHECK-LABEL: test_rsqrt28_ss_maskz:
; CHECK: # BB#0: ; CHECK: # BB#0:
; CHECK-NEXT: kxnorw %k0, %k0, %k0 # encoding: [0xc5,0xfc,0x46,0xc0] ; CHECK-NEXT: vrsqrt28ss {sae}, %xmm0, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x18,0xcd,0xc0]
; CHECK-NEXT: kshiftrw $15, %k0, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc8,0x0f]
; CHECK-NEXT: vrsqrt28ss {sae}, %xmm0, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x99,0xcd,0xc0]
; CHECK-NEXT: retq # encoding: [0xc3] ; CHECK-NEXT: retq # encoding: [0xc3]
%res = call <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 7, i32 8) ; %res = call <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 7, i32 8) ;
ret <4 x float> %res ret <4 x float> %res
} }
define <4 x float> @test_rsqrt28_ss_mask(<4 x float> %a0, <4 x float> %b0, <4 x float> %c0) { define <4 x float> @test_rsqrt28_ss_mask(<4 x float> %a0, <4 x float> %b0, <4 x float> %c0) {
; CHECK-LABEL: test_rsqrt28_ss_mask: ; CHECK-LABEL: test_rsqrt28_ss_mask:
; CHECK: # BB#0: ; CHECK: # BB#0:
; CHECK-NEXT: kxnorw %k0, %k0, %k0 # encoding: [0xc5,0xfc,0x46,0xc0] ; CHECK-NEXT: vrsqrt28ss {sae}, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x18,0xcd,0xc1]
; CHECK-NEXT: kshiftrw $15, %k0, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc8,0x0f]
; CHECK-NEXT: vrsqrt28ss {sae}, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x19,0xcd,0xd1]
; CHECK-NEXT: vmovaps %xmm2, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc2]
; CHECK-NEXT: retq # encoding: [0xc3] ; CHECK-NEXT: retq # encoding: [0xc3]
%res = call <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float> %a0, <4 x float> %b0, <4 x float> %c0, i8 7, i32 8) ; %res = call <4 x float> @llvm.x86.avx512.rsqrt28.ss(<4 x float> %a0, <4 x float> %b0, <4 x float> %c0, i8 7, i32 8) ;
ret <4 x float> %res ret <4 x float> %res
} }
define <2 x double> @test_rsqrt28_sd_maskz(<2 x double> %a0) { define <2 x double> @test_rsqrt28_sd_maskz(<2 x double> %a0) {
; CHECK-LABEL: test_rsqrt28_sd_maskz: ; CHECK-LABEL: test_rsqrt28_sd_maskz:
; CHECK: # BB#0: ; CHECK: # BB#0:
; CHECK-NEXT: kxnorw %k0, %k0, %k0 # encoding: [0xc5,0xfc,0x46,0xc0] ; CHECK-NEXT: vrsqrt28sd {sae}, %xmm0, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x18,0xcd,0xc0]
; CHECK-NEXT: kshiftrw $15, %k0, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc8,0x0f]
; CHECK-NEXT: vrsqrt28sd {sae}, %xmm0, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x99,0xcd,0xc0]
; CHECK-NEXT: retq # encoding: [0xc3] ; CHECK-NEXT: retq # encoding: [0xc3]
%res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %a0, <2 x double> zeroinitializer, i8 7, i32 8) ; %res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %a0, <2 x double> zeroinitializer, i8 7, i32 8) ;
ret <2 x double> %res ret <2 x double> %res
} }
define <2 x double> @test_rsqrt28_sd_mask(<2 x double> %a0, <2 x double> %b0, <2 x double> %c0) { define <2 x double> @test_rsqrt28_sd_mask(<2 x double> %a0, <2 x double> %b0, <2 x double> %c0) {
; CHECK-LABEL: test_rsqrt28_sd_mask: ; CHECK-LABEL: test_rsqrt28_sd_mask:
; CHECK: # BB#0: ; CHECK: # BB#0:
; CHECK-NEXT: kxnorw %k0, %k0, %k0 # encoding: [0xc5,0xfc,0x46,0xc0] ; CHECK-NEXT: vrsqrt28sd {sae}, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x18,0xcd,0xc1]
; CHECK-NEXT: kshiftrw $15, %k0, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc8,0x0f]
; CHECK-NEXT: vrsqrt28sd {sae}, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x19,0xcd,0xd1]
; CHECK-NEXT: vmovapd %xmm2, %xmm0 # encoding: [0xc5,0xf9,0x28,0xc2]
; CHECK-NEXT: retq # encoding: [0xc3] ; CHECK-NEXT: retq # encoding: [0xc3]
%res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %b0, <2 x double> %c0, i8 7, i32 8) ; %res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %b0, <2 x double> %c0, i8 7, i32 8) ;
ret <2 x double> %res ret <2 x double> %res
} }
declare <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone declare <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
define <2 x double> @test_rsqrt28_sd_maskz_mem(<2 x double> %a0, double* %ptr ) { define <2 x double> @test_rsqrt28_sd_maskz_mem(<2 x double> %a0, double* %ptr ) {
; CHECK-LABEL: test_rsqrt28_sd_maskz_mem: ; CHECK-LABEL: test_rsqrt28_sd_maskz_mem:
; CHECK: # BB#0: ; CHECK: # BB#0:
; CHECK-NEXT: kxnorw %k0, %k0, %k0 # encoding: [0xc5,0xfc,0x46,0xc0] ; CHECK-NEXT: vrsqrt28sd (%rdi), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0xcd,0x07]
; CHECK-NEXT: kshiftrw $15, %k0, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc8,0x0f]
; CHECK-NEXT: vrsqrt28sd (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0xcd,0x07]
; CHECK-NEXT: retq # encoding: [0xc3] ; CHECK-NEXT: retq # encoding: [0xc3]
%mem = load double , double * %ptr, align 8 %mem = load double , double * %ptr, align 8
%mem_v = insertelement <2 x double> undef, double %mem, i32 0 %mem_v = insertelement <2 x double> undef, double %mem, i32 0
%res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %mem_v, <2 x double> zeroinitializer, i8 7, i32 4) ; %res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %mem_v, <2 x double> zeroinitializer, i8 7, i32 4) ;
ret <2 x double> %res ret <2 x double> %res
} }
define <2 x double> @test_rsqrt28_sd_maskz_mem_offset(<2 x double> %a0, double* %ptr ) { define <2 x double> @test_rsqrt28_sd_maskz_mem_offset(<2 x double> %a0, double* %ptr ) {
; CHECK-LABEL: test_rsqrt28_sd_maskz_mem_offset: ; CHECK-LABEL: test_rsqrt28_sd_maskz_mem_offset:
; CHECK: # BB#0: ; CHECK: # BB#0:
; CHECK-NEXT: kxnorw %k0, %k0, %k0 # encoding: [0xc5,0xfc,0x46,0xc0] ; CHECK-NEXT: vrsqrt28sd 144(%rdi), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0xcd,0x47,0x12]
; CHECK-NEXT: kshiftrw $15, %k0, %k1 # encoding: [0xc4,0xe3,0xf9,0x30,0xc8,0x0f]
; CHECK-NEXT: vrsqrt28sd 144(%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0xcd,0x47,0x12]
; CHECK-NEXT: retq # encoding: [0xc3] ; CHECK-NEXT: retq # encoding: [0xc3]
%ptr1 = getelementptr double, double* %ptr, i32 18 %ptr1 = getelementptr double, double* %ptr, i32 18
%mem = load double , double * %ptr1, align 8 %mem = load double , double * %ptr1, align 8
%mem_v = insertelement <2 x double> undef, double %mem, i32 0 %mem_v = insertelement <2 x double> undef, double %mem, i32 0
%res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %mem_v, <2 x double> zeroinitializer, i8 7, i32 4) ; %res = call <2 x double> @llvm.x86.avx512.rsqrt28.sd(<2 x double> %a0, <2 x double> %mem_v, <2 x double> zeroinitializer, i8 7, i32 4) ;
ret <2 x double> %res ret <2 x double> %res
} }