This is an archive of the discontinued LLVM Phabricator instance.

Differential D65613

[ScalarizeMaskedMemIntrin] Add constant mask support to expandload and compressstore scalarization
ClosedPublic

Authored by craig.topper on Aug 1 2019, 3:27 PM.

Details

Reviewers
RKSimon
spatel
Commits
rGde9b1d791276: [ScalarizeMaskedMemIntrin] Add constant mask support to expandload and…
rL367715: [ScalarizeMaskedMemIntrin] Add constant mask support to expandload and…
Summary

This adds support for generating all the loads or stores for a constant mask into a single basic block with no conditionals.

Diff Detail

Event Timeline

craig.topper created this revision.Aug 1 2019, 3:27 PM
Herald added a project: Restricted Project. · View Herald TranscriptAug 1 2019, 3:27 PM
Herald added a subscriber: hiraditya. · View Herald Transcript
RKSimon accepted this revision.Aug 2 2019, 11:09 AM

LGTM

This revision is now accepted and ready to land.Aug 2 2019, 11:09 AM
Closed by commit rL367715: [ScalarizeMaskedMemIntrin] Add constant mask support to expandload and… (authored by ctopper). · Explain WhyAug 2 2019, 1:05 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
llvm/
lib/
CodeGen/
34 lines
test/
CodeGen/
X86/
8 lines
24 lines
2 lines
2 lines
Transforms/
ScalarizeMaskedMemIntrin/
X86/
49 lines
63 lines

Diff 212917

llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp

Show First 20 Lines • Show All 610 Lines • ▼ Show 20 Linesstatic void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
Builder.SetInsertPoint(InsertPt); Builder.SetInsertPoint(InsertPt);
Builder.SetCurrentDebugLocation(CI->getDebugLoc()); Builder.SetCurrentDebugLocation(CI->getDebugLoc());
unsigned VectorWidth = VecType->getNumElements(); unsigned VectorWidth = VecType->getNumElements();
// The result vector // The result vector
Value *VResult = PassThru; Value *VResult = PassThru;
// Shorten the way if the mask is a vector of constants.
if (isConstantIntVector(Mask)) {
unsigned MemIndex = 0;
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
continue;
Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, NewPtr, 1, "Load" + Twine(Idx));
VResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
++MemIndex;
}
CI->replaceAllUsesWith(VResult);
CI->eraseFromParent();
return;
}
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
// Fill the "else" block, created in the previous iteration // Fill the "else" block, created in the previous iteration
// //
// %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ] // %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ]
// %mask_1 = extractelement <16 x i1> %mask, i32 Idx // %mask_1 = extractelement <16 x i1> %mask, i32 Idx
// br i1 %mask_1, label %cond.load, label %else // br i1 %mask_1, label %cond.load, label %else
// //
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Linesstatic void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
Builder.SetInsertPoint(InsertPt); Builder.SetInsertPoint(InsertPt);
Builder.SetCurrentDebugLocation(CI->getDebugLoc()); Builder.SetCurrentDebugLocation(CI->getDebugLoc());
Type *EltTy = VecType->getVectorElementType(); Type *EltTy = VecType->getVectorElementType();
unsigned VectorWidth = VecType->getNumElements(); unsigned VectorWidth = VecType->getNumElements();
// Shorten the way if the mask is a vector of constants.
if (isConstantIntVector(Mask)) {
unsigned MemIndex = 0;
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
continue;
Value *OneElt =
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
Builder.CreateAlignedStore(OneElt, NewPtr, 1);
++MemIndex;
}
CI->eraseFromParent();
return;
}
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
// Fill the "else" block, created in the previous iteration // Fill the "else" block, created in the previous iteration
// //
// %mask_1 = extractelement <16 x i1> %mask, i32 Idx // %mask_1 = extractelement <16 x i1> %mask, i32 Idx
// br i1 %mask_1, label %cond.store, label %else // br i1 %mask_1, label %cond.store, label %else
// //
Value *Predicate = Builder.CreateExtractElement(Mask, Idx); Value *Predicate = Builder.CreateExtractElement(Mask, Idx);
▲ Show 20 Lines • Show All 122 Lines • Show Last 20 Lines

llvm/test/CodeGen/X86/masked_compressstore.ll

Show First 20 Lines • Show All 1,117 Lines • ▼ Show 20 Lines
; AVX512VLBW-NEXT: vzeroupper ; AVX512VLBW-NEXT: vzeroupper
; AVX512VLBW-NEXT: retq ; AVX512VLBW-NEXT: retq
call void @llvm.masked.compressstore.v8f32(<8 x float> %V, float* %base, <8 x i1> %mask) call void @llvm.masked.compressstore.v8f32(<8 x float> %V, float* %base, <8 x i1> %mask)
ret void ret void
} }
define void @compressstore_v16f32_const(float* %base, <16 x float> %V) { define void @compressstore_v16f32_const(float* %base, <16 x float> %V) {
; SSE2-LABEL: compressstore_v16f32_const: ; SSE2-LABEL: compressstore_v16f32_const:
; SSE2: ## %bb.0: ## %cond.store ; SSE2: ## %bb.0:
; SSE2-NEXT: movss %xmm0, (%rdi) ; SSE2-NEXT: movss %xmm0, (%rdi)
; SSE2-NEXT: movaps %xmm0, %xmm4 ; SSE2-NEXT: movaps %xmm0, %xmm4
; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3] ; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
; SSE2-NEXT: movss %xmm4, 4(%rdi) ; SSE2-NEXT: movss %xmm4, 4(%rdi)
; SSE2-NEXT: movaps %xmm0, %xmm4 ; SSE2-NEXT: movaps %xmm0, %xmm4
; SSE2-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1] ; SSE2-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
; SSE2-NEXT: movss %xmm4, 8(%rdi) ; SSE2-NEXT: movss %xmm4, 8(%rdi)
; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3] ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
Show All 20 Lines
; SSE2-NEXT: movaps %xmm3, %xmm0 ; SSE2-NEXT: movaps %xmm3, %xmm0
; SSE2-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1] ; SSE2-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]
; SSE2-NEXT: movss %xmm0, 52(%rdi) ; SSE2-NEXT: movss %xmm0, 52(%rdi)
; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1,2,3] ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
; SSE2-NEXT: movss %xmm3, 56(%rdi) ; SSE2-NEXT: movss %xmm3, 56(%rdi)
; SSE2-NEXT: retq ; SSE2-NEXT: retq
; ;
; SSE42-LABEL: compressstore_v16f32_const: ; SSE42-LABEL: compressstore_v16f32_const:
; SSE42: ## %bb.0: ## %cond.store ; SSE42: ## %bb.0:
; SSE42-NEXT: movups %xmm0, (%rdi) ; SSE42-NEXT: movups %xmm0, (%rdi)
; SSE42-NEXT: movups %xmm1, 16(%rdi) ; SSE42-NEXT: movups %xmm1, 16(%rdi)
; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0] ; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0]
; SSE42-NEXT: movups %xmm2, 32(%rdi) ; SSE42-NEXT: movups %xmm2, 32(%rdi)
; SSE42-NEXT: extractps $1, %xmm3, 48(%rdi) ; SSE42-NEXT: extractps $1, %xmm3, 48(%rdi)
; SSE42-NEXT: extractps $2, %xmm3, 52(%rdi) ; SSE42-NEXT: extractps $2, %xmm3, 52(%rdi)
; SSE42-NEXT: extractps $3, %xmm3, 56(%rdi) ; SSE42-NEXT: extractps $3, %xmm3, 56(%rdi)
; SSE42-NEXT: retq ; SSE42-NEXT: retq
; ;
; AVX1-LABEL: compressstore_v16f32_const: ; AVX1-LABEL: compressstore_v16f32_const:
; AVX1: ## %bb.0: ## %cond.store ; AVX1: ## %bb.0:
; AVX1-NEXT: vmovups %ymm0, (%rdi) ; AVX1-NEXT: vmovups %ymm0, (%rdi)
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm0 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0] ; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
; AVX1-NEXT: vmovups %xmm1, 32(%rdi) ; AVX1-NEXT: vmovups %xmm1, 32(%rdi)
; AVX1-NEXT: vextractps $1, %xmm0, 48(%rdi) ; AVX1-NEXT: vextractps $1, %xmm0, 48(%rdi)
; AVX1-NEXT: vextractps $2, %xmm0, 52(%rdi) ; AVX1-NEXT: vextractps $2, %xmm0, 52(%rdi)
; AVX1-NEXT: vextractps $3, %xmm0, 56(%rdi) ; AVX1-NEXT: vextractps $3, %xmm0, 56(%rdi)
; AVX1-NEXT: vzeroupper ; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq ; AVX1-NEXT: retq
; ;
; AVX2-LABEL: compressstore_v16f32_const: ; AVX2-LABEL: compressstore_v16f32_const:
; AVX2: ## %bb.0: ## %cond.store ; AVX2: ## %bb.0:
; AVX2-NEXT: vmovups %ymm0, (%rdi) ; AVX2-NEXT: vmovups %ymm0, (%rdi)
; AVX2-NEXT: vmovaps {{.*#+}} xmm0 = [0,1,2,4] ; AVX2-NEXT: vmovaps {{.*#+}} xmm0 = [0,1,2,4]
; AVX2-NEXT: vpermps %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: vpermps %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vmovups %xmm0, 32(%rdi) ; AVX2-NEXT: vmovups %xmm0, 32(%rdi)
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm0 ; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm0
; AVX2-NEXT: vextractps $1, %xmm0, 48(%rdi) ; AVX2-NEXT: vextractps $1, %xmm0, 48(%rdi)
; AVX2-NEXT: vextractps $2, %xmm0, 52(%rdi) ; AVX2-NEXT: vextractps $2, %xmm0, 52(%rdi)
; AVX2-NEXT: vextractps $3, %xmm0, 56(%rdi) ; AVX2-NEXT: vextractps $3, %xmm0, 56(%rdi)
▲ Show 20 Lines • Show All 3,323 Lines • Show Last 20 Lines

llvm/test/CodeGen/X86/masked_expandload.ll

Show First 20 Lines • Show All 1,362 Lines • ▼ Show 20 Lines
; AVX512VL-NEXT: retq ; AVX512VL-NEXT: retq
%mask = icmp eq <2 x i32> %trigger, zeroinitializer %mask = icmp eq <2 x i32> %trigger, zeroinitializer
%res = call <2 x float> @llvm.masked.expandload.v2f32(float* %base, <2 x i1> %mask, <2 x float> %src0) %res = call <2 x float> @llvm.masked.expandload.v2f32(float* %base, <2 x i1> %mask, <2 x float> %src0)
ret <2 x float> %res ret <2 x float> %res
} }
define <4 x float> @expandload_v4f32_const(float* %base, <4 x float> %src0) { define <4 x float> @expandload_v4f32_const(float* %base, <4 x float> %src0) {
; SSE2-LABEL: expandload_v4f32_const: ; SSE2-LABEL: expandload_v4f32_const:
; SSE2: ## %bb.0: ## %cond.load ; SSE2: ## %bb.0:
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0],xmm2[0,0] ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0],xmm2[0,0]
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0] ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[0,2] ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[0,2]
; SSE2-NEXT: movaps %xmm1, %xmm0 ; SSE2-NEXT: movaps %xmm1, %xmm0
; SSE2-NEXT: retq ; SSE2-NEXT: retq
; ;
; SSE42-LABEL: expandload_v4f32_const: ; SSE42-LABEL: expandload_v4f32_const:
; SSE42: ## %bb.0: ## %cond.load ; SSE42: ## %bb.0:
; SSE42-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero ; SSE42-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; SSE42-NEXT: blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] ; SSE42-NEXT: blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3] ; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3] ; SSE42-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
; SSE42-NEXT: retq ; SSE42-NEXT: retq
; ;
; AVX1OR2-LABEL: expandload_v4f32_const: ; AVX1OR2-LABEL: expandload_v4f32_const:
; AVX1OR2: ## %bb.0: ## %cond.load ; AVX1OR2: ## %bb.0:
; AVX1OR2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero ; AVX1OR2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; AVX1OR2-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] ; AVX1OR2-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3] ; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3] ; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
; AVX1OR2-NEXT: retq ; AVX1OR2-NEXT: retq
; ;
; AVX512F-LABEL: expandload_v4f32_const: ; AVX512F-LABEL: expandload_v4f32_const:
; AVX512F: ## %bb.0: ; AVX512F: ## %bb.0:
Show All 19 Lines
; AVX512VLBW-NEXT: vexpandps (%rdi), %xmm0 {%k1} ; AVX512VLBW-NEXT: vexpandps (%rdi), %xmm0 {%k1}
; AVX512VLBW-NEXT: retq ; AVX512VLBW-NEXT: retq
%res = call <4 x float> @llvm.masked.expandload.v4f32(float* %base, <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x float> %src0) %res = call <4 x float> @llvm.masked.expandload.v4f32(float* %base, <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x float> %src0)
ret <4 x float>%res ret <4 x float>%res
} }
define <16 x float> @expandload_v16f32_const(float* %base, <16 x float> %src0) { define <16 x float> @expandload_v16f32_const(float* %base, <16 x float> %src0) {
; SSE2-LABEL: expandload_v16f32_const: ; SSE2-LABEL: expandload_v16f32_const:
; SSE2: ## %bb.0: ## %cond.load ; SSE2: ## %bb.0:
; SSE2-NEXT: movups (%rdi), %xmm0 ; SSE2-NEXT: movups (%rdi), %xmm0
; SSE2-NEXT: movups 16(%rdi), %xmm1 ; SSE2-NEXT: movups 16(%rdi), %xmm1
; SSE2-NEXT: movss {{.*#+}} xmm5 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
; SSE2-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,0],xmm5[0,0] ; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,0],xmm5[0,0]
; SSE2-NEXT: movss {{.*#+}} xmm5 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[3,0] ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm5[0,2] ; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm5[0,2]
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: movss {{.*#+}} xmm5 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm5 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[0,0] ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,0],xmm2[0,0]
; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm3[3,0] ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0],xmm3[3,0]
; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm2[0,2] ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm2[0,2]
; SSE2-NEXT: movaps %xmm4, %xmm2 ; SSE2-NEXT: movaps %xmm4, %xmm2
; SSE2-NEXT: movaps %xmm5, %xmm3 ; SSE2-NEXT: movaps %xmm5, %xmm3
; SSE2-NEXT: retq ; SSE2-NEXT: retq
; ;
; SSE42-LABEL: expandload_v16f32_const: ; SSE42-LABEL: expandload_v16f32_const:
; SSE42: ## %bb.0: ## %cond.load ; SSE42: ## %bb.0:
; SSE42-NEXT: movups (%rdi), %xmm0 ; SSE42-NEXT: movups (%rdi), %xmm0
; SSE42-NEXT: movups 16(%rdi), %xmm1 ; SSE42-NEXT: movups 16(%rdi), %xmm1
; SSE42-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero ; SSE42-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
; SSE42-NEXT: blendps {{.*#+}} xmm2 = xmm4[0],xmm2[1,2,3] ; SSE42-NEXT: blendps {{.*#+}} xmm2 = xmm4[0],xmm2[1,2,3]
; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3] ; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3] ; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
; SSE42-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero ; SSE42-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
; SSE42-NEXT: blendps {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3] ; SSE42-NEXT: blendps {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
; SSE42-NEXT: insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3] ; SSE42-NEXT: insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
; SSE42-NEXT: insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3] ; SSE42-NEXT: insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
; SSE42-NEXT: retq ; SSE42-NEXT: retq
; ;
; AVX1OR2-LABEL: expandload_v16f32_const: ; AVX1OR2-LABEL: expandload_v16f32_const:
; AVX1OR2: ## %bb.0: ## %cond.load ; AVX1OR2: ## %bb.0:
; AVX1OR2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; AVX1OR2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; AVX1OR2-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7] ; AVX1OR2-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7]
; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm0[0],mem[0],xmm0[2,3] ; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm0[0],mem[0],xmm0[2,3]
; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3] ; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0] ; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
; AVX1OR2-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX1OR2-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
; AVX1OR2-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero ; AVX1OR2-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero
; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3] ; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
Show All 33 Lines
; AVX512VLBW-NEXT: vexpandps (%rdi), %zmm0 {%k1} ; AVX512VLBW-NEXT: vexpandps (%rdi), %zmm0 {%k1}
; AVX512VLBW-NEXT: retq ; AVX512VLBW-NEXT: retq
%res = call <16 x float> @llvm.masked.expandload.v16f32(float* %base, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false>, <16 x float> %src0) %res = call <16 x float> @llvm.masked.expandload.v16f32(float* %base, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false>, <16 x float> %src0)
ret <16 x float>%res ret <16 x float>%res
} }
define <16 x float> @expandload_v16f32_const_undef(float* %base) { define <16 x float> @expandload_v16f32_const_undef(float* %base) {
; SSE2-LABEL: expandload_v16f32_const_undef: ; SSE2-LABEL: expandload_v16f32_const_undef:
; SSE2: ## %bb.0: ## %cond.load ; SSE2: ## %bb.0:
; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; SSE2-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero ; SSE2-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero
; SSE2-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0] ; SSE2-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]
; SSE2-NEXT: movups (%rdi), %xmm0 ; SSE2-NEXT: movups (%rdi), %xmm0
; SSE2-NEXT: movups 16(%rdi), %xmm1 ; SSE2-NEXT: movups 16(%rdi), %xmm1
; SSE2-NEXT: movups 44(%rdi), %xmm3 ; SSE2-NEXT: movups 44(%rdi), %xmm3
; SSE2-NEXT: retq ; SSE2-NEXT: retq
; ;
; SSE42-LABEL: expandload_v16f32_const_undef: ; SSE42-LABEL: expandload_v16f32_const_undef:
; SSE42: ## %bb.0: ## %cond.load ; SSE42: ## %bb.0:
; SSE42-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero ; SSE42-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero
; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3] ; SSE42-NEXT: insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
; SSE42-NEXT: movups (%rdi), %xmm0 ; SSE42-NEXT: movups (%rdi), %xmm0
; SSE42-NEXT: movups 16(%rdi), %xmm1 ; SSE42-NEXT: movups 16(%rdi), %xmm1
; SSE42-NEXT: movups 44(%rdi), %xmm3 ; SSE42-NEXT: movups 44(%rdi), %xmm3
; SSE42-NEXT: retq ; SSE42-NEXT: retq
; ;
; AVX1OR2-LABEL: expandload_v16f32_const_undef: ; AVX1OR2-LABEL: expandload_v16f32_const_undef:
; AVX1OR2: ## %bb.0: ## %cond.load ; AVX1OR2: ## %bb.0:
; AVX1OR2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; AVX1OR2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3] ; AVX1OR2-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
; AVX1OR2-NEXT: vinsertf128 $1, 44(%rdi), %ymm0, %ymm1 ; AVX1OR2-NEXT: vinsertf128 $1, 44(%rdi), %ymm0, %ymm1
; AVX1OR2-NEXT: vmovups (%rdi), %ymm0 ; AVX1OR2-NEXT: vmovups (%rdi), %ymm0
; AVX1OR2-NEXT: retq ; AVX1OR2-NEXT: retq
; ;
; AVX512F-LABEL: expandload_v16f32_const_undef: ; AVX512F-LABEL: expandload_v16f32_const_undef:
; AVX512F: ## %bb.0: ; AVX512F: ## %bb.0:
▲ Show 20 Lines • Show All 1,448 Lines • ▼ Show 20 Lines
} }
; ;
; vXi64 ; vXi64
; ;
define <2 x i64> @expandload_v2i64_const(i64* %base, <2 x i64> %src0) { define <2 x i64> @expandload_v2i64_const(i64* %base, <2 x i64> %src0) {
; SSE2-LABEL: expandload_v2i64_const: ; SSE2-LABEL: expandload_v2i64_const:
; SSE2: ## %bb.0: ## %else ; SSE2: ## %bb.0:
; SSE2-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero ; SSE2-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero
; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq ; SSE2-NEXT: retq
; ;
; SSE42-LABEL: expandload_v2i64_const: ; SSE42-LABEL: expandload_v2i64_const:
; SSE42: ## %bb.0: ## %else ; SSE42: ## %bb.0:
; SSE42-NEXT: pinsrq $1, (%rdi), %xmm0 ; SSE42-NEXT: pinsrq $1, (%rdi), %xmm0
; SSE42-NEXT: retq ; SSE42-NEXT: retq
; ;
; AVX1OR2-LABEL: expandload_v2i64_const: ; AVX1OR2-LABEL: expandload_v2i64_const:
; AVX1OR2: ## %bb.0: ## %else ; AVX1OR2: ## %bb.0:
; AVX1OR2-NEXT: vpinsrq $1, (%rdi), %xmm0, %xmm0 ; AVX1OR2-NEXT: vpinsrq $1, (%rdi), %xmm0, %xmm0
; AVX1OR2-NEXT: retq ; AVX1OR2-NEXT: retq
; ;
; AVX512F-LABEL: expandload_v2i64_const: ; AVX512F-LABEL: expandload_v2i64_const:
; AVX512F: ## %bb.0: ; AVX512F: ## %bb.0:
; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0 ; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0
; AVX512F-NEXT: movb $2, %al ; AVX512F-NEXT: movb $2, %al
; AVX512F-NEXT: kmovw %eax, %k1 ; AVX512F-NEXT: kmovw %eax, %k1
▲ Show 20 Lines • Show All 1,590 Lines • Show Last 20 Lines

llvm/test/CodeGen/X86/pr39666.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=btver2 | FileCheck %s ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=btver2 | FileCheck %s
define <2 x i64> @test5(i64* %base, <2 x i64> %src0) { define <2 x i64> @test5(i64* %base, <2 x i64> %src0) {
; CHECK-LABEL: test5: ; CHECK-LABEL: test5:
; CHECK: # %bb.0: # %else ; CHECK: # %bb.0:
; CHECK-NEXT: vpinsrq $1, (%rdi), %xmm0, %xmm0 ; CHECK-NEXT: vpinsrq $1, (%rdi), %xmm0, %xmm0
; CHECK-NEXT: retq ; CHECK-NEXT: retq
%res = call <2 x i64> @llvm.masked.expandload.v2i64(i64* %base, <2 x i1> <i1 false, i1 true>, <2 x i64> %src0) %res = call <2 x i64> @llvm.masked.expandload.v2i64(i64* %base, <2 x i1> <i1 false, i1 true>, <2 x i64> %src0)
ret <2 x i64>%res ret <2 x i64>%res
} }
declare <2 x i64> @llvm.masked.expandload.v2i64(i64*, <2 x i1>, <2 x i64>) declare <2 x i64> @llvm.masked.expandload.v2i64(i64*, <2 x i1>, <2 x i64>)
define void @test11(i64* %base, <2 x i64> %V, <2 x i1> %mask) { define void @test11(i64* %base, <2 x i64> %V, <2 x i1> %mask) {
Show All 20 Lines

llvm/test/CodeGen/X86/pr40994.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2 | FileCheck %s ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.2 | FileCheck %s
define <8 x i8> @foo(<16 x i8> %a) { define <8 x i8> @foo(<16 x i8> %a) {
; CHECK-LABEL: foo: ; CHECK-LABEL: foo:
; CHECK: # %bb.0: # %cond.store ; CHECK: # %bb.0:
; CHECK-NEXT: pextrb $0, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $0, %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pextrb $2, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $2, %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pextrb $4, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $4, %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pextrb $6, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $6, %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pextrb $8, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $8, %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pextrb $10, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $10, %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pextrb $12, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $12, %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pextrb $14, %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: pextrb $14, %xmm0, -{{[0-9]+}}(%rsp)
Show All 9 Lines

llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll

Show All 21 Lines
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> %mask) call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> %mask)
ret void ret void
} }
define void @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %data) { define void @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %data) {
; CHECK-LABEL: @scalarize_v2i64_ones_mask( ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
; CHECK-NEXT: br i1 true, label [[COND_STORE:%.*]], label [[ELSE:%.*]] ; CHECK-NEXT: [[ELT0:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
; CHECK: cond.store: ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0 ; CHECK-NEXT: store i64 [[ELT0]], i64* [[TMP1]], align 1
; CHECK-NEXT: store i64 [[TMP1]], i64* [[P:%.*]], align 1 ; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]] ; CHECK-NEXT: store i64 [[ELT1]], i64* [[TMP2]], align 1
; CHECK: else:
; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
; CHECK-NEXT: br i1 true, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.store1:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
; CHECK-NEXT: store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 true, i1 true>) call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 true, i1 true>)
ret void ret void
} }
define void @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %data) { define void @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %data) {
; CHECK-LABEL: @scalarize_v2i64_zero_mask( ; CHECK-LABEL: @scalarize_v2i64_zero_mask(
; CHECK-NEXT: br i1 false, label [[COND_STORE:%.*]], label [[ELSE:%.*]]
; CHECK: cond.store:
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
; CHECK-NEXT: store i64 [[TMP1]], i64* [[P:%.*]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
; CHECK-NEXT: br i1 false, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.store1:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
; CHECK-NEXT: store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 false>) call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 false>)
ret void ret void
} }
define void @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %data) { define void @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %data) {
; CHECK-LABEL: @scalarize_v2i64_const_mask( ; CHECK-LABEL: @scalarize_v2i64_const_mask(
; CHECK-NEXT: br i1 false, label [[COND_STORE:%.*]], label [[ELSE:%.*]] ; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 1
; CHECK: cond.store: ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0 ; CHECK-NEXT: store i64 [[ELT1]], i64* [[TMP1]], align 1
; CHECK-NEXT: store i64 [[TMP1]], i64* [[P:%.*]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
; CHECK-NEXT: br i1 true, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.store1:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
; CHECK-NEXT: store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 true>) call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 true>)
ret void ret void
} }
declare void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64>, i64*, <2 x i1>) declare void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64>, i64*, <2 x i1>)

llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll

Show All 23 Lines
; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]] ; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
; ;
%ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %passthru) %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %passthru)
ret <2 x i64> %ret ret <2 x i64> %ret
} }
define <2 x i64> @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %passthru) { define <2 x i64> @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %passthru) {
; CHECK-LABEL: @scalarize_v2i64_ones_mask( ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
; CHECK-NEXT: br i1 true, label [[COND_LOAD:%.*]], label [[ELSE:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
; CHECK: cond.load: ; CHECK-NEXT: [[LOAD0:%.*]] = load i64, i64* [[TMP1]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1 ; CHECK-NEXT: [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1 ; CHECK-NEXT: [[LOAD1:%.*]] = load i64, i64* [[TMP2]], align 1
; CHECK-NEXT: br label [[ELSE]] ; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[RES0]], i64 [[LOAD1]], i64 1
; CHECK: else: ; CHECK-NEXT: ret <2 x i64> [[RES1]]
; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
; CHECK-NEXT: br i1 true, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
; ;
%ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru) %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
ret <2 x i64> %ret ret <2 x i64> %ret
} }
define <2 x i64> @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %passthru) { define <2 x i64> @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %passthru) {
; CHECK-LABEL: @scalarize_v2i64_zero_mask( ; CHECK-LABEL: @scalarize_v2i64_zero_mask(
; CHECK-NEXT: br i1 false, label [[COND_LOAD:%.*]], label [[ELSE:%.*]] ; CHECK-NEXT: ret <2 x i64> [[PASSTHRU:%.*]]
; CHECK: cond.load:
; CHECK-NEXT: [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
; CHECK-NEXT: br i1 false, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
; ;
%ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru) %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru)
ret <2 x i64> %ret ret <2 x i64> %ret
} }
define <2 x i64> @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %passthru) { define <2 x i64> @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %passthru) {
; CHECK-LABEL: @scalarize_v2i64_const_mask( ; CHECK-LABEL: @scalarize_v2i64_const_mask(
; CHECK-NEXT: br i1 false, label [[COND_LOAD:%.*]], label [[ELSE:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i32 0
; CHECK: cond.load: ; CHECK-NEXT: [[LOAD1:%.*]] = load i64, i64* [[TMP1]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1 ; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD1]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0 ; CHECK-NEXT: ret <2 x i64> [[RES1]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
; CHECK-NEXT: br i1 true, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
; ;
%ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru) %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)
ret <2 x i64> %ret ret <2 x i64> %ret
} }
declare <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64*, <2 x i1>, <2 x i64>) declare <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64*, <2 x i1>, <2 x i64>)