[PowerPC] Provide fastmath sqrt and div functions in altivec.h

[nemanjai]

Summary

This adds the long overdue implementations of these functions that have been part of the ABI document and are now part of the "Power Vector Intrinsic Programming Reference" (PVIPR).

The approach is to add new builtins and to emit code with the fast flag regardless of whether fastmath was specified on the command line.

[qiucf]

clang/lib/CodeGen/CGBuiltin.cpp

15122

Seems FMF will be automatically restored without the three lines.

vector float test_recipdivd(vector float a, vector float b) {
  vector float x = vec_recipdiv(a, b);
  vector float y = x + b;
  return y;
}

define dso_local <4 x float> @test_recipdivd(<4 x float> %a, <4 x float> %b) {
entry:
  %recipdiv.i = fdiv fast <4 x float> %a, %b
  %add = fadd <4 x float> %recipdiv.i, %b
  ret <4 x float> %add
}

See https://reviews.llvm.org/D96231#inline-901337.

[nemanjai]

clang/lib/CodeGen/CGBuiltin.cpp

15122

Thanks for finding that. I did notice that and was wondering how the FMF flags return to what they were in the X86 code. So I added the reset of the flags just to be on the safe side. Now that I see that, I'll get rid of those.

[bmahjour]

clang/lib/CodeGen/CGBuiltin.cpp

15130

I wonder if we can do better than "fdiv fast"... does the current lowering of "fdiv fast" employ an estimation algorithm via iterative refinement on POWER?

15134

This doesn't implement a reciprocal square root, it just performs a square root! At the very least we need a divide instruction following the call to the intrinsic, but I'm not sure if that'll result in the most optimal codegen at the end. Perhaps we need a new builtin?

clang/test/CodeGen/builtins-ppc-vsx.c

2297

See my comment above about the missing reciprocal operation.

[nemanjai]

clang/lib/CodeGen/CGBuiltin.cpp

15130

Yes. This fast includes arcp which will trigger the estimation+refinement algorithm in the back end.

15134

Oh, I misread the documentation. This really seems like a bizarre thing to offer a user. I will change this to 1/sqrt().
In terms of providing optimal performance, with fast-math, the optimizer should get rid of the divide. If compiled at -O0, it isn't reasonable to expect optimal performance to begin with.

[nemanjai]

Changed rsqrt to be an actual reciprocal rather than just a refined sqrt estimate.

I have verified that the code generated is equivalent to gcc's and the results produced are the same.

[bmahjour]

LGTM

clang/include/clang/Basic/BuiltinsPPC.def

 BUILTIN(__builtin_subf128_round_to_odd, "LLdLLdLLd", "")
 BUILTIN(__builtin_mulf128_round_to_odd, "LLdLLdLLd", "")
 BUILTIN(__builtin_divf128_round_to_odd, "LLdLLdLLd", "")
 BUILTIN(__builtin_fmaf128_round_to_odd, "LLdLLdLLdLLd", "")
 BUILTIN(__builtin_truncf128_round_to_odd, "dLLd", "")
 BUILTIN(__builtin_vsx_scalar_extract_expq, "ULLiLLd", "")
 BUILTIN(__builtin_vsx_scalar_insert_exp_qp, "LLdLLdULLi", "")
 
+// Fastmath by default builtins
+BUILTIN(__builtin_ppc_rsqrtf, "V4fV4f", "")
+BUILTIN(__builtin_ppc_rsqrtd, "V2dV2d", "")
+BUILTIN(__builtin_ppc_recipdivf, "V4fV4fV4f", "")
+BUILTIN(__builtin_ppc_recipdivd, "V2dV2dV2d", "")
+
 // HTM builtins
 BUILTIN(__builtin_tbegin, "UiUIi", "")
 BUILTIN(__builtin_tend, "UiUIi", "")
 
 BUILTIN(__builtin_tabort, "UiUi", "")
 BUILTIN(__builtin_tabortdc, "UiUiUiUi", "")
 BUILTIN(__builtin_tabortdci, "UiUiUii", "")
 BUILTIN(__builtin_tabortwc, "UiUiUiUi", "")

clang/lib/CodeGen/CGBuiltin.cpp

Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
   case PPC::BI__builtin_vsx_xvabsdp:
   case PPC::BI__builtin_vsx_xvabssp: {
     llvm::Type *ResultType = ConvertType(E->getType());
     Value *X = EmitScalarExpr(E->getArg(0));
     llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
     return Builder.CreateCall(F, X);
   }
 
+  // Fastmath by default
+  case PPC::BI__builtin_ppc_recipdivf:
+  case PPC::BI__builtin_ppc_recipdivd:
+  case PPC::BI__builtin_ppc_rsqrtf:
+  case PPC::BI__builtin_ppc_rsqrtd: {
+    auto FMF = Builder.getFastMathFlags();
+    Builder.getFastMathFlags().setFast();
+    llvm::Type *ResultType = ConvertType(E->getType());
+    Value *X = EmitScalarExpr(E->getArg(0));
+
+    if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
+        BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
+      Value *Y = EmitScalarExpr(E->getArg(1));
+      auto Ret = Builder.CreateFDiv(X, Y, "recipdiv");
+      Builder.setFastMathFlags(FMF);
+      return Ret;
+    }
+    llvm::Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
+    auto Ret = Builder.CreateCall(F, X);
+    Builder.setFastMathFlags(FMF);
+    return Ret;
+  }
+
   // FMA variations
   case PPC::BI__builtin_vsx_xvmaddadp:
   case PPC::BI__builtin_vsx_xvmaddasp:
   case PPC::BI__builtin_vsx_xvnmaddadp:
   case PPC::BI__builtin_vsx_xvnmaddasp:
   case PPC::BI__builtin_vsx_xvmsubadp:
   case PPC::BI__builtin_vsx_xvmsubasp:
   case PPC::BI__builtin_vsx_xvnmsubadp:

clang/lib/Headers/altivec.h

 /*===---- altivec.h - Standard header for type generic math ---------------===*\
  *
  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  * See https://llvm.org/LICENSE.txt for license information.
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
 \*===----------------------------------------------------------------------===*/
 
 #ifndef __ALTIVEC_H
 #define __ALTIVEC_H
 
 #ifndef __ALTIVEC__
 #error "AltiVec support not enabled"
 #endif
 
 /* Constants for mapping CR6 bits to predicate result. */
 
 #define __CR6_EQ 0
 #define __CR6_EQ_REV 1
 #define __CR6_LT 2
 #define __CR6_LT_REV 3
#define __VEC_CLASS_FP_NOT_NORMAL (__VEC_CLASS_FP_NAN        | \
                                    __VEC_CLASS_FP_SUBNORMAL  | \
                                    __VEC_CLASS_FP_ZERO       | \
                                    __VEC_CLASS_FP_INFINITY)
 
 #define __ATTRS_o_ai __attribute__((__overloadable__, __always_inline__))
 
 #include <stddef.h>
 
 static __inline__ vector signed char __ATTRS_o_ai vec_perm(
     vector signed char __a, vector signed char __b, vector unsigned char __c);
 
 static __inline__ vector unsigned char __ATTRS_o_ai
 vec_perm(vector unsigned char __a, vector unsigned char __b,
          vector unsigned char __c);
 
 static __inline__ vector bool char __ATTRS_o_ai
 vec_perm(vector bool char __a, vector bool char __b, vector unsigned char __c);
 
 static __inline__ vector short __ATTRS_o_ai vec_perm(vector signed short __a,
                                                      vector signed short __b,
                                                      vector unsigned char __c);
 
 static __inline__ vector unsigned short __ATTRS_o_ai
 vec_perm(vector unsigned short __a, vector unsigned short __b,
          vector unsigned char __c);
 
 static __inline__ vector bool short __ATTRS_o_ai vec_perm(
     vector bool short __a, vector bool short __b, vector unsigned char __c);
 
 static __inline__ vector pixel __ATTRS_o_ai vec_perm(vector pixel __a,
                                                      vector pixel __b,
                                                      vector unsigned char __c);

 }
 
 #ifdef __VSX__
 static __inline__ vector double __ATTRS_o_ai vec_rsqrte(vector double __a) {
   return __builtin_vsx_xvrsqrtedp(__a);
 }
 #endif
 
+static vector float __ATTRS_o_ai vec_rsqrt(vector float __a) {
+  return __builtin_ppc_rsqrtf(__a);
+}
+
+#ifdef __VSX__
+static vector double __ATTRS_o_ai vec_rsqrt(vector double __a) {
+  return __builtin_ppc_rsqrtd(__a);
+}
+#endif
+
 /* vec_vrsqrtefp */
 
 static __inline__ __vector float __attribute__((__always_inline__))
 vec_vrsqrtefp(vector float __a) {
   return __builtin_altivec_vrsqrtefp(__a);
 }
 
 /* vec_xvtsqrt */

 static vector signed short __ATTRS_o_ai vec_nabs(vector signed short __a) {
   return __builtin_altivec_vminsh(__a, -__a);
 }
 
 static vector signed char __ATTRS_o_ai vec_nabs(vector signed char __a) {
   return __builtin_altivec_vminsb(__a, -__a);
 }
 
+static vector float __ATTRS_o_ai vec_recipdiv(vector float __a,
+                                              vector float __b) {
+  return __builtin_ppc_recipdivf(__a, __b);
+}
+
+#ifdef __VSX__
+static vector double __ATTRS_o_ai vec_recipdiv(vector double __a,
+                                               vector double __b) {
+  return __builtin_ppc_recipdivd(__a, __b);
+}
+#endif
+
 #ifdef __POWER10_VECTOR__
 
 /* vec_extractm */
 
 static __inline__ unsigned int __ATTRS_o_ai
 vec_extractm(vector unsigned char __a) {
   return __builtin_altivec_vextractbm(__a);
 }

clang/test/CodeGen/builtins-ppc-altivec.c

void test12() {
   vec_xst_be(vui, param_sll, &param_ui);
   // CHECK: store <4 x i32> %{{[0-9]+}}, <4 x i32>* %{{[0-9]+}}, align 1
   // CHECK-LE: call void @llvm.ppc.vsx.stxvw4x.be(<4 x i32> %{{[0-9]+}}, i8* %{{[0-9]+}})
 
   vec_xst_be(vf, param_sll, &param_f);
   // CHECK: store <4 x float> %{{[0-9]+}}, <4 x float>* %{{[0-9]+}}, align 1
   // CHECK-LE: call void @llvm.ppc.vsx.stxvw4x.be(<4 x i32> %{{[0-9]+}}, i8* %{{[0-9]+}})
 }
+
+vector float test_rsqrtf(vector float a, vector float b) {
+  // CHECK-LABEL: test_rsqrtf
+  // CHECK: call fast <4 x float> @llvm.sqrt.v4f32
+  // CHECK-LE-LABEL: test_rsqrtf
+  // CHECK-LE: call fast <4 x float> @llvm.sqrt.v4f32
+  return vec_rsqrt(a);
+}
+
+vector float test_recipdivf(vector float a, vector float b) {
+  // CHECK-LABEL: test_recipdivf
+  // CHECK: fdiv fast <4 x float>
+  // CHECK-LE-LABEL: test_recipdivf
+  // CHECK-LE: fdiv fast <4 x float>
+  return vec_recipdiv(a, b);
+}

clang/test/CodeGen/builtins-ppc-vsx.c

 // CHECK-DAG: load{{.*}}%a
 // CHECK-DAG: load{{.*}}%b
 // CHECK-NOT: SEPARATOR
 // CHECK-DAG: [[RA:%[0-9]+]] = load <2 x double>, <2 x double>* %a.addr
 // CHECK-DAG: [[RB:%[0-9]+]] = load <2 x double>, <2 x double>* %b.addr
 // CHECK-NEXT: call <2 x double> @llvm.copysign.v2f64(<2 x double> [[RA]], <2 x double> [[RB]])
   __builtin_vsx_xvcpsgndp(a, b);
 }
+
+vector double test_recipdivd(vector double a, vector double b) {
+  // CHECK-LABEL: test_recipdivd
+  // CHECK: fdiv fast <2 x double>
+  // CHECK-LE-LABEL: test_recipdivd
+  // CHECK-LE: fdiv fast <2 x double>
+  return vec_recipdiv(a, b);
+}
+
+vector double test_rsqrtd(vector double a, vector double b) {
+  // CHECK-LABEL: test_rsqrtd
+  // CHECK: call fast <2 x double> @llvm.sqrt.v2f64
+  // CHECK-LE-LABEL: test_rsqrtd
+  // CHECK-LE: call fast <2 x double> @llvm.sqrt.v2f64
+  return vec_rsqrt(a);
+}