Allow vectorization of few missed llvm intrinsic calls in BBVectorizor

[karthikthecool]

Summary

Hi Hal,
I'm currently going through and trying to understand BB Vectorizer while doing so i found that few llvm intrinsic function calls which can be vectorized are missed in the switch case of isVectorizableIntrinsic as a result these intrinsic functions are not getting vectorized. This patch modifies isVectorizableIntrinsic function and covers these intrinsic functions in the switch case so that they get vectorized.
Does this patch look good to commit?
Thanks
Karthik Bhat

[echristo]

I think bswap can also be vectorized on some architectures?

-eric

[karthikthecool]

Hi Eric,
Yes i think we can vectorize bswap as well as other bitmap manipulation intrinsics such as cttz,ctlz etc. But i think these cannot be classified as VectorizeMath as this flag is for floating-point math intrinsics.
To handle bitmap manipulation intrinsics (e.g. bsap,cttz,ctlz) we might have to add a new flag and support to enable/disable it from command line in BB Vectorizor.
Some flag like *VectorizeBitManipulations* and a command line support to enable/disable it such as -

static cl::opt<bool>
  NoBitManipulation("bb-vectorize-no-bitmanip", cl::init(false), cl::Hidden,
  cl::desc("Don't try to vectorize BitManipulation intrinsics"));

Does it look like the correct approach? I will update the patch to support bitmap manipulation intrinsics if you feel this approach is ok.

Thanks
Karthik Bhat

[hfinkel]

LGTM, thanks!

Please feel free to submit a follow-up patch for the bit intrinsics.

[karthikthecool]

Thanks Hal, Committed as r207085.
Hal,Eric i will shortly follow up with a patch vectorizing bit intrinsics.
Thanks
Karthik Bhat

[echristo]

Awesome. Thanks!

-eric

lib/Transforms/Vectorize/BBVectorize.cpp

bool isVectorizableIntrinsic(CallInst* I) {
       case Intrinsic::sin:
       case Intrinsic::cos:
       case Intrinsic::log:
       case Intrinsic::log2:
       case Intrinsic::log10:
       case Intrinsic::exp:
       case Intrinsic::exp2:
       case Intrinsic::pow:
+      case Intrinsic::round:
+      case Intrinsic::copysign:
+      case Intrinsic::ceil:
+      case Intrinsic::nearbyint:
+      case Intrinsic::rint:
+      case Intrinsic::trunc:
+      case Intrinsic::floor:
+      case Intrinsic::fabs:
         return Config.VectorizeMath;
       case Intrinsic::fma:
       case Intrinsic::fmuladd:
         return Config.VectorizeFMA;
       }
     }
 
     bool isPureIEChain(InsertElementInst *IE) {

test/Transforms/BBVectorize/simple-int.ll

 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 ; RUN: opt < %s -bb-vectorize -bb-vectorize-req-chain-depth=3 -bb-vectorize-ignore-target-info -instcombine -gvn -S | FileCheck %s
 
 declare double @llvm.fma.f64(double, double, double)
 declare double @llvm.fmuladd.f64(double, double, double)
 declare double @llvm.cos.f64(double)
 declare double @llvm.powi.f64(double, i32)
+declare double @llvm.round.f64(double)
+declare double @llvm.copysign.f64(double, double)
+declare double @llvm.ceil.f64(double)
+declare double @llvm.nearbyint.f64(double)
+declare double @llvm.rint.f64(double)
+declare double @llvm.trunc.f64(double)
+declare double @llvm.floor.f64(double)
+declare double @llvm.fabs.f64(double)
 
 ; Basic depth-3 chain with fma
 define double @test1(double %A1, double %A2, double %B1, double %B2, double %C1, double %C2) {
   %X1 = fsub double %A1, %B1
   %X2 = fsub double %A2, %B2
   %Y1 = call double @llvm.fma.f64(double %X1, double %A1, double %C1)
   %Y2 = call double @llvm.fma.f64(double %X2, double %A2, double %C2)
   %Z1 = fadd double %Y1, %B1
define double @test4(double %A1, double %A2, double %B1, double %B2, i32 %P) {
   %Z2 = fadd double %Y2, %B2
   %R  = fmul double %Z1, %Z2
   ret double %R
 ; CHECK-LABEL: @test4(
 ; CHECK-NOT: <2 x double>
 ; CHECK: ret double %R
 }
 
+; Basic depth-3 chain with round
+define double @testround(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.round.f64(double %X1)
+  %Y2 = call double @llvm.round.f64(double %X2)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testround
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1 = call <2 x double> @llvm.round.v2f64(<2 x double> %X1)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+; Basic depth-3 chain with copysign
+define double @testcopysign(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.copysign.f64(double %X1, double %A1)
+  %Y2 = call double @llvm.copysign.f64(double %X2, double %A1)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testcopysign
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1.v.i1.2 = insertelement <2 x double> %X1.v.i0.1, double %A1, i32 1
+; CHECK: %Y1 = call <2 x double> @llvm.copysign.v2f64(<2 x double> %X1, <2 x double> %Y1.v.i1.2)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+; Basic depth-3 chain with ceil
+define double @testceil(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.ceil.f64(double %X1)
+  %Y2 = call double @llvm.ceil.f64(double %X2)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testceil
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1 = call <2 x double> @llvm.ceil.v2f64(<2 x double> %X1)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+; Basic depth-3 chain with nearbyint
+define double @testnearbyint(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.nearbyint.f64(double %X1)
+  %Y2 = call double @llvm.nearbyint.f64(double %X2)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testnearbyint
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1 = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> %X1)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+; Basic depth-3 chain with rint
+define double @testrint(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.rint.f64(double %X1)
+  %Y2 = call double @llvm.rint.f64(double %X2)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testrint
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1 = call <2 x double> @llvm.rint.v2f64(<2 x double> %X1)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+; Basic depth-3 chain with trunc
+define double @testtrunc(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.trunc.f64(double %X1)
+  %Y2 = call double @llvm.trunc.f64(double %X2)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testtrunc
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1 = call <2 x double> @llvm.trunc.v2f64(<2 x double> %X1)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+; Basic depth-3 chain with floor
+define double @testfloor(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.floor.f64(double %X1)
+  %Y2 = call double @llvm.floor.f64(double %X2)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testfloor
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1 = call <2 x double> @llvm.floor.v2f64(<2 x double> %X1)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+; Basic depth-3 chain with fabs
+define double @testfabs(double %A1, double %A2, double %B1, double %B2) {
+  %X1 = fsub double %A1, %B1
+  %X2 = fsub double %A2, %B2
+  %Y1 = call double @llvm.fabs.f64(double %X1)
+  %Y2 = call double @llvm.fabs.f64(double %X2)
+  %Z1 = fadd double %Y1, %B1
+  %Z2 = fadd double %Y2, %B2
+  %R  = fmul double %Z1, %Z2
+  ret double %R
+; CHECK: @testfabs
+; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0
+; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1
+; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0
+; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1
+; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2
+; CHECK: %Y1 = call <2 x double> @llvm.fabs.v2f64(<2 x double> %X1)
+; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2
+; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0
+; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1
+; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2
+; CHECK: ret double %R
+
+}
+
+
 ; CHECK: declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) #0
 ; CHECK: declare <2 x double> @llvm.fmuladd.v2f64(<2 x double>, <2 x double>, <2 x double>) #0
 ; CHECK: declare <2 x double> @llvm.cos.v2f64(<2 x double>) #0
 ; CHECK: declare <2 x double> @llvm.powi.v2f64(<2 x double>, i32) #0
-
+; CHECK: declare <2 x double> @llvm.round.v2f64(<2 x double>) #0
+; CHECK: declare <2 x double> @llvm.copysign.v2f64(<2 x double>, <2 x double>) #0
+; CHECK: declare <2 x double> @llvm.ceil.v2f64(<2 x double>) #0
+; CHECK: declare <2 x double> @llvm.nearbyint.v2f64(<2 x double>) #0
+; CHECK: declare <2 x double> @llvm.rint.v2f64(<2 x double>) #0
+; CHECK: declare <2 x double> @llvm.trunc.v2f64(<2 x double>) #0
+; CHECK: declare <2 x double> @llvm.floor.v2f64(<2 x double>) #0
+; CHECK: declare <2 x double> @llvm.fabs.v2f64(<2 x double>) #0
 ; CHECK: attributes #0 = { nounwind readnone }