[LLVM][X86] Converting X86 intrinisics movmsk{ps|pd}{|256} and pmovmskb{128|256}

[m_zuckerman]

Summary

In this patch, I am converting the above intrinsic into generic IR. This allows us to do further optimization on the above intrinsic. Besides of the above, we added also a new instcombine that combines ExtractElementInst with bitcast into shuffle instruction. This folding fixes an issue of inserting an avx2's intrinsics as input into avx512's intrinsics.

The patch is a part of two patches: clang side and this LLVM side.
clang side: https://reviews.llvm.org/D40465

[guyblank]

lib/IR/AutoUpgrade.cpp

206 ↗

(On Diff #124275)

what's this one? i don't see it in the removed intrinsics

208 ↗

(On Diff #124275)

this one also

921 ↗

(On Diff #124275)

typo EmitX86Msak -> EmitX86Mask

934 ↗

(On Diff #124275)

typo, same as above

1316 ↗

(On Diff #124275)

same one from above

lib/Target/X86/X86ISelLowering.cpp

35708 ↗

(On Diff #124275)

should the combines be separate commits?
also they need tests

lib/Transforms/InstCombine/InstCombineCasts.cpp

2044 ↗

(On Diff #124275)

separate patch + tests

test/CodeGen/X86/avx2-intrinsics-x86.ll

1 ↗

(On Diff #124275)

can you regenerate and commit this test, there's a lot of noise here.

test/CodeGen/X86/movmsk.ll

136 ↗

(On Diff #124275)

this doesn't seem right, what happened here?

test/Transforms/InstCombine/X86/x86-movmsk.ll

1 ↗

(On Diff #124275)

no reason to keep this test anymore, you've removed the combines it is testing.

[m_zuckerman]

lib/Target/X86/X86ISelLowering.cpp

35708 ↗

(On Diff #124275)

All the combines are important for the test. Since we deleted intrinsics name, the front end IR doesn't aware of names. And so instcombinecall cant works for them. For satisfying the tests, we had to add new combines that replace the old (instcombinecall).

All the combines are already covered by original tests.

lib/Transforms/InstCombine/InstCombineCasts.cpp

2044 ↗

(On Diff #124275)

I agree with you

[m_zuckerman]

test/Transforms/InstCombine/X86/x86-movmsk.ll

1 ↗

(On Diff #124275)

The first case is not cover it (MMX)

[guyblank]

you've uploaded the clang patch instead of the llvm one.

[m_zuckerman]

SORRY

lib/CodeGen/CGBuiltin.cpp

if (Ops.size() == 2)
     return Res;
 
   assert(Ops.size() == 4);
   return EmitX86Select(CGF, Ops[3], Res, Ops[2]);
 }
 
 static Value *EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, 
                               llvm::Type *DstTy) {
-  unsigned NumberOfElements = DstTy->getVectorNumElements();
+  unsigned NumberOfElements = DstTy->getVectorNumElements(); 
   Value *Mask = getMaskVecValue(CGF, Op, NumberOfElements);
   return CGF.Builder.CreateSExt(Mask, DstTy, "vpmovm2");
 }
 
+static Value *EmitX86Mask(CodeGenFunction &CGF, ArrayRef<Value *> Ops) {
+  llvm::Type *Typ1 = Ops[0]->getType();
+  Value *Cmp = CGF.Builder.CreateICmp(CmpInst::Predicate::ICMP_SLT, Ops[0],
+                                      llvm::ConstantVector::getNullValue(Typ1));
+  Value *BitCast = CGF.Builder.CreateBitCast(
+      Cmp, llvm::Type::getIntNTy(CGF.Builder.getContext(),
+                                 Typ1->getVectorNumElements()));
+  return (Typ1->getVectorNumElements() < 32)
+             ? CGF.Builder.CreateZExt(
+                   BitCast, llvm::Type::getInt32Ty(CGF.getLLVMContext()))
+             : BitCast;
+}
+
+static Value *EmitX86MaskFloat(CodeGenFunction &CGF, ArrayRef<Value *> Ops) {
+  Value *FloatOp = Ops[0];
+  unsigned DstTypEle = FloatOp->getType()->getVectorNumElements();
+  llvm::Type *IntTy = llvm::Type::getIntNTy(
+      CGF.Builder.getContext(), FloatOp->getType()->getScalarSizeInBits());
+  llvm::Type *DstTyp = llvm::VectorType::get(IntTy, DstTypEle);
+  Value *BitCastFloatToInt = CGF.Builder.CreateBitCast(FloatOp, DstTyp);
+  return EmitX86Mask(CGF, BitCastFloatToInt);
+}
+
 Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
   const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
   StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
   return EmitX86CpuIs(CPUStr);
 }
 
 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
 
#undef INTRINSIC_X86_XSAVE_ID
   case X86::BI__builtin_ia32_cvtmask2d128:
   case X86::BI__builtin_ia32_cvtmask2d256:
   case X86::BI__builtin_ia32_cvtmask2d512:
   case X86::BI__builtin_ia32_cvtmask2q128:
   case X86::BI__builtin_ia32_cvtmask2q256:
   case X86::BI__builtin_ia32_cvtmask2q512:
     return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType()));
 
+  case X86::BI__builtin_ia32_pmovmskb128:
+  case X86::BI__builtin_ia32_pmovmskb256:
+    return EmitX86Mask(*this, Ops);
+  case X86::BI__builtin_ia32_movmskps256:
+  case X86::BI__builtin_ia32_movmskpd256:
+  case X86::BI__builtin_ia32_movmskps:
+  case X86::BI__builtin_ia32_movmskpd:
+    return EmitX86MaskFloat(*this, Ops);
   case X86::BI__builtin_ia32_movdqa32store128_mask:
   case X86::BI__builtin_ia32_movdqa64store128_mask:
   case X86::BI__builtin_ia32_storeaps128_mask:
   case X86::BI__builtin_ia32_storeapd128_mask:
   case X86::BI__builtin_ia32_movdqa32store256_mask:
   case X86::BI__builtin_ia32_movdqa64store256_mask:
   case X86::BI__builtin_ia32_storeaps256_mask:
   case X86::BI__builtin_ia32_storeapd256_mask:

test/CodeGen/avx-builtins.c

 __m256 test_mm256_moveldup_ps(__m256 A) {
   // CHECK-LABEL: test_mm256_moveldup_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   return _mm256_moveldup_ps(A);
 }
 
 int test_mm256_movemask_pd(__m256d A) {
   // CHECK-LABEL: test_mm256_movemask_pd
-  // CHECK: call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %{{.*}})
+  // CHECK: bitcast <4 x double> %1 to <4 x i64>
+  // CHECK: icmp slt <4 x i64> %2, zeroinitializer
+  // CHECK: bitcast <4 x i1> %3 to i4
+  // CHECK: zext i4 %4 to i32
   return _mm256_movemask_pd(A);
 }
 
 int test_mm256_movemask_ps(__m256 A) {
   // CHECK-LABEL: test_mm256_movemask_ps
-  // CHECK: call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %{{.*}})
+  // CHECK: load <8 x float>, <8 x float>* %__a.addr.i, align 32
+  // CHECK: bitcast <8 x float> %1 to <8 x i32>
+  // CHECK: icmp slt <8 x i32> %2, zeroinitializer
+  // CHECK: bitcast <8 x i1> %3 to i8
+  // CHECK: zext i8 %4 to i32
   return _mm256_movemask_ps(A);
 }
 
 __m256d test_mm256_mul_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_mul_pd
   // CHECK: fmul <4 x double>
   return _mm256_mul_pd(A, B);
 }

test/CodeGen/avx2-builtins.c

__m256i test_mm256_min_epu32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_min_epu32
   // CHECK:       [[CMP:%.*]] = icmp ult <8 x i32> [[X:%.*]], [[Y:%.*]]
   // CHECK-NEXT:  select <8 x i1> [[CMP]], <8 x i32> [[X]], <8 x i32> [[Y]]
   return _mm256_min_epu32(a, b);
 }
 
 int test_mm256_movemask_epi8(__m256i a) {
   // CHECK-LABEL: test_mm256_movemask_epi8
-  // CHECK: call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> %{{.*}})
+  // CHECK: bitcast <4 x i64> %1 to <32 x i8>
+  // CHECK: icmp slt <32 x i8> %2, zeroinitializer
+  // CHECK: bitcast <32 x i1> %3 to i32
   return _mm256_movemask_epi8(a);
 }
 
 __m256i test_mm256_mpsadbw_epu8(__m256i x, __m256i y) {
   // CHECK-LABEL: test_mm256_mpsadbw_epu8
   // CHECK: call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, i8 3)
   return _mm256_mpsadbw_epu8(x, y, 3);
 }

test/CodeGen/sse-builtins.c

 __m128 test_mm_movelh_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_movelh_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   return _mm_movelh_ps(A, B);
 }
 
 int test_mm_movemask_ps(__m128 A) {
   // CHECK-LABEL: test_mm_movemask_ps
-  // CHECK: call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %{{.*}})
+  //CHECK: bitcast <4 x float> %1 to <4 x i32>
+  //CHECK: icmp slt <4 x i32> %2, zeroinitializer
+  //CHECK: bitcast <4 x i1> %3 to i4
+  //CHECK: zext i4 %4 to i32
   return _mm_movemask_ps(A);
 }
 
 __m128 test_mm_mul_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_mul_ps
   // CHECK: fmul <4 x float>
   return _mm_mul_ps(A, B);
 }

test/CodeGen/sse2-builtins.c

__m128d test_mm_move_sd(__m128d A, __m128d B) {
   // CHECK: insertelement <2 x double> undef, double %{{.*}}, i32 0
   // CHECK: extractelement <2 x double> %{{.*}}, i32 1
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 1
   return _mm_move_sd(A, B);
 }
 
 int test_mm_movemask_epi8(__m128i A) {
   // CHECK-LABEL: test_mm_movemask_epi8
-  // CHECK: call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %{{.*}})
+  // CHECK: bitcast <2 x i64> %1 to <16 x i8>
+  // CHECK: icmp slt <16 x i8> %2, zeroinitializer
+  // CHECK: bitcast <16 x i1> %3 to i16
+  // CHECK: zext i16 %4 to i32
   return _mm_movemask_epi8(A);
 }
 
 int test_mm_movemask_pd(__m128d A) {
   // CHECK-LABEL: test_mm_movemask_pd
-  // CHECK: call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %{{.*}})
+  // CHECK: bitcast <2 x double> %1 to <2 x i64>
+  // CHECK: icmp slt <2 x i64> %2, zeroinitializer
+  // CHECK: bitcast <2 x i1> %3 to i2
+  // CHECK: zext i2 %4 to i32
   return _mm_movemask_pd(A);
 }
 
 __m128i test_mm_mul_epu32(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_mul_epu32
   // CHECK: call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_mul_epu32(A, B);
 }