[X86] Lowering SAD (sum of absolute differences) intrinsics to native IR (clang side)

[mike.dvoretsky]

Summary

This patch lowers the SAD intrinsics to native LLVM IR. Comes with an LLVM patch (D45723).

[craig.topper]

lib/CodeGen/CGBuiltin.cpp

8427

This clear isn't needed.

8432

You shouldn't need to explicitly create an ArrayRef here. It should automatically convert. And if it doesn't makeArrayRef is what you should use. It will automatically infer the uint32_t from the vector.

[mike.dvoretsky]

Updated per comments.

[craig.topper]

clang/lib/CodeGen/CGBuiltin.cpp

8426 ↗

(On Diff #142914)

Size the ShuffleMask to N when it's created. Then you can use just direct assign each array entry in the loops. This will remove the need for the clear() in the later loop. It will also remove the hidden code that checks if we need to grow on every call to push_back.

8431 ↗

(On Diff #142914)

You can just pass AD twice. You don't need to create an Undef value. It will get optimized later.

[mike.dvoretsky]

Updated per comments.

[craig.topper]

clang/lib/CodeGen/CGBuiltin.cpp

8425 ↗

(On Diff #143715)

Is this not a dead variable now?

8426 ↗

(On Diff #143715)

SmallVector<uint32_t, 8> ShuffleMask(N);

Then you don't need the resize call.

[mike.dvoretsky]

Updated per comments.

[craig.topper]

LGTM

[mike.dvoretsky]

Closing this due to failure of D45723.

lib/CodeGen/CGBuiltin.cpp

 
 static Value *EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, 
                               llvm::Type *DstTy) {
   unsigned NumberOfElements = DstTy->getVectorNumElements();
   Value *Mask = getMaskVecValue(CGF, Op, NumberOfElements);
   return CGF.Builder.CreateSExt(Mask, DstTy, "vpmovm2");
 }
 
+// Lowers PSAD intrinsics to native IR.
+static Value *EmitX86SAD(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
+                         unsigned N) {
+  // The operands arrive already bitcast to byte vectors, the N parameter
+  // showing the corresponding number of qwords.
+  Value *A = Ops[0];
+  Value *B = Ops[1];
+
+  Value *Cmp = CGF.Builder.CreateICmp(ICmpInst::ICMP_UGT, A, B);
+  Value *AD = CGF.Builder.CreateSelect(Cmp, CGF.Builder.CreateSub(A, B),
+                                       CGF.Builder.CreateSub(B, A));
+
+  llvm::Type *QTy = CGF.Builder.getInt64Ty();
+  llvm::Type *VTy = llvm::VectorType::get(QTy, N);
+  llvm::Type *BTy = llvm::VectorType::get(CGF.Builder.getInt8Ty(), N * 8);
+  SmallVector<uint32_t, 8> ShuffleMask;
+  ShuffleMask.clear();
+  for (unsigned i = 0; i < N; ++i)
+    ShuffleMask.push_back(i * 8);
+  Value *Res = CGF.Builder.CreateZExt(
+      CGF.Builder.CreateShuffleVector(AD, llvm::UndefValue::get(BTy),
+                                      ArrayRef<uint32_t>(ShuffleMask)),
+      VTy);
+  for (unsigned i = 1; i < 8; ++i) {
+    ShuffleMask.clear();
+    for (unsigned j = 0; j < N; ++j)
+      ShuffleMask.push_back(i + j * 8);
+    Value *Sum = CGF.Builder.CreateShuffleVector(
+        AD, llvm::UndefValue::get(BTy), ArrayRef<uint32_t>(ShuffleMask));
+    Res = CGF.Builder.CreateAdd(Res, CGF.Builder.CreateZExt(Sum, VTy));
+  }
+  return Res;
+}
+
 Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
   const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
   StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
   return EmitX86CpuIs(CPUStr);
 }
 
 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
 
#undef INTRINSIC_X86_XSAVE_ID
   case X86::BI__builtin_ia32_pabsq128_mask:
   case X86::BI__builtin_ia32_pabsq256_mask:
   case X86::BI__builtin_ia32_pabsb512_mask:
   case X86::BI__builtin_ia32_pabsw512_mask:
   case X86::BI__builtin_ia32_pabsd512_mask:
   case X86::BI__builtin_ia32_pabsq512_mask:
     return EmitX86Abs(*this, Ops);
 
+  case X86::BI__builtin_ia32_psadbw128:
+    return EmitX86SAD(*this, Ops, 2);
+  case X86::BI__builtin_ia32_psadbw256:
+    return EmitX86SAD(*this, Ops, 4);
+  case X86::BI__builtin_ia32_psadbw512:
+    return EmitX86SAD(*this, Ops, 8);
+
   case X86::BI__builtin_ia32_pmaxsb128:
   case X86::BI__builtin_ia32_pmaxsw128:
   case X86::BI__builtin_ia32_pmaxsd128:
   case X86::BI__builtin_ia32_pmaxsq128_mask:
   case X86::BI__builtin_ia32_pmaxsb256:
   case X86::BI__builtin_ia32_pmaxsw256:
   case X86::BI__builtin_ia32_pmaxsd256:
   case X86::BI__builtin_ia32_pmaxsq256_mask:

test/CodeGen/avx2-builtins.c

 __m256 test_mm256_permutevar8x32_ps(__m256 a, __m256i b) {
   // CHECK-LABEL: test_mm256_permutevar8x32_ps
   // CHECK: call <8 x float> @llvm.x86.avx2.permps(<8 x float> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_permutevar8x32_ps(a, b);
 }
 
 __m256i test_mm256_sad_epu8(__m256i x, __m256i y) {
   // CHECK-LABEL: test_mm256_sad_epu8
-  // CHECK: call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
+  // CHECK: %{{.*}} = icmp ugt <32 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = sub <32 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = sub <32 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 1, i32 9, i32 17, i32 25>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = add <4 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 2, i32 10, i32 18, i32 26>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = add <4 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = add <4 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 4, i32 12, i32 20, i32 28>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = add <4 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 5, i32 13, i32 21, i32 29>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = add <4 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 6, i32 14, i32 22, i32 30>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = add <4 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <32 x i8> %{{.*}}, <32 x i8> undef, <4 x i32> <i32 7, i32 15, i32 23, i32 31>
+  // CHECK: %{{.*}} = zext <4 x i8> %{{.*}} to <4 x i64>
+  // CHECK: %{{.*}} = add <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_sad_epu8(x, y);
 }
 
 __m256i test_mm256_shuffle_epi8(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_shuffle_epi8
   // CHECK: call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_shuffle_epi8(a, b);
 }

test/CodeGen/avx512bw-builtins.c

 __m512i test_mm512_mm_maskz_dbsad_epu8(__mmask32 __U, __m512i __A, __m512i __B) {
   // CHECK-LABEL: @test_mm512_mm_maskz_dbsad_epu8
   // CHECK: @llvm.x86.avx512.mask.dbpsadbw.512
   return _mm512_maskz_dbsad_epu8(__U, __A, __B, 170); 
 }
 
 __m512i test_mm512_sad_epu8(__m512i __A, __m512i __B) {
   // CHECK-LABEL: @test_mm512_sad_epu8
-  // CHECK: @llvm.x86.avx512.psad.bw.512
+  // CHECK: %{{.*}} = icmp ugt <64 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = sub <64 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = sub <64 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = add <8 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = add <8 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = add <8 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = add <8 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = add <8 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = add <8 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <64 x i8> %{{.*}}, <64 x i8> undef, <8 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63>
+  // CHECK: %{{.*}} = zext <8 x i8> %{{.*}} to <8 x i64>
+  // CHECK: %{{.*}} = add <8 x i64> %{{.*}}, %{{.*}}
   return _mm512_sad_epu8(__A, __B); 
 }
 
 __mmask32 test_mm512_movepi16_mask(__m512i __A) {
   // CHECK-LABEL: @test_mm512_movepi16_mask
   // CHECK: [[CMP:%.*]] = icmp slt <32 x i16> %{{.*}}, zeroinitializer
   // CHECK: bitcast <32 x i1> [[CMP]] to i32
   return _mm512_movepi16_mask(__A); 

test/CodeGen/sse2-builtins.c

 void test_mm_pause() {
   // CHECK-LABEL: test_mm_pause
   // CHECK: call void @llvm.x86.sse2.pause()
   return _mm_pause();
 }
 
 __m128i test_mm_sad_epu8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_sad_epu8
-  // CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
+  // CHECK: %{{.*}} = icmp ugt <16 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = sub <16 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = sub <16 x i8> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 0, i32 8>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 1, i32 9>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = add <2 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 2, i32 10>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = add <2 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 3, i32 11>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = add <2 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 4, i32 12>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = add <2 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 5, i32 13>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = add <2 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 6, i32 14>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = add <2 x i64> %{{.*}}, %{{.*}}
+  // CHECK: %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <2 x i32> <i32 7, i32 15>
+  // CHECK: %{{.*}} = zext <2 x i8> %{{.*}} to <2 x i64>
+  // CHECK: %{{.*}} = add <2 x i64> %{{.*}}, %{{.*}}
   return _mm_sad_epu8(A, B);
 }
 
 __m128i test_mm_set_epi8(char A, char B, char C, char D,
                          char E, char F, char G, char H,
                          char I, char J, char K, char L,
                          char M, char N, char O, char P) {
   // CHECK-LABEL: test_mm_set_epi8