[PowerPC] Implement Vector Multiply High/Divide Extended Builtins in LLVM/Clang

[amyk]

Summary

This patch implements the function prototypes vec_mulh and vec_dive in order to utilize
the vector multiply high (vmulh[s|u][w|d]) and vector divide extended (vdive[s|u][w|d])
instructions introduced in Power10.

Depends on D82576.

[anil9]

clang/test/CodeGen/builtins-ppc-p10vector.c

92

I may be wrong but where are the variables declared ? I do not see the variables delclared above in the file, i mean many of them.

llvm/test/CodeGen/PowerPC/p10-vector-divide.ll

52

nit : you put a comment right at this position in the multiply.ll file, for consistency you could add one here or remove the one there.

[amyk]

clang/test/CodeGen/builtins-ppc-p10vector.c

92

I accidentally uploaded the wrong diff, I will update this.

llvm/test/CodeGen/PowerPC/p10-vector-divide.ll

52

Will fix.

[amyk]

Addressed Anil's comments.

[amyk]

Rebase patch, remove MC tests from this patch.

[lei]

clang/test/CodeGen/builtins-ppc-p10vector.c

89

why does the ck stops matching at the first param? Shouldn't we check the remaining param type and number of param are correct as well?

llvm/lib/Target/PowerPC/PPCInstrPrefix.td

1049

nit: indent to match up with v4i32 on the previous line.

[amyk]

clang/test/CodeGen/builtins-ppc-p10vector.c

89

Yes, thanks for pointing that out. Will be fixing the CHECKs.

llvm/lib/Target/PowerPC/PPCInstrPrefix.td

1049

I will update with the proper indentation.

[amyk]

Addressed the following comments:

• updated CHECK lines to check for the full intrinsic call
• updated indentation

[amyk]

Ping.

[NeHuang]

llvm/test/CodeGen/PowerPC/p10-vector-divide.ll

59

nit: do we also need _intrinsic in the name as the test cases for the vector multiply high intrinsics.

[amyk]

llvm/test/CodeGen/PowerPC/p10-vector-divide.ll

59

I am thinking that it may be OK to leave the vector divide extended tests without the _intrinsic suffix.

The reason why I added _intrinsic to vector multiply high is because the vector multiply high instructions can be produced in two ways:

• by the mulhs node (which is present in the test, I have named them test_vmulh...)
• by the @llvm.ppc.altivec.vmulh[s|u][w|d] intrinsic (these tests have the _intrinsic suffix)

So far for the vector divide extended, we produce the instructions only if we have the intrinsic.

However I do not feel too strongly either way on this so if there is a preference to add _intrinsic to these tests, I can definitely do that.

[lei]

LGTM
Thx!

clang/include/clang/Basic/BuiltinsPPC.def

 BUILTIN(__builtin_altivec_vsrv, "V16UcV16UcV16Uc", "")
 
 // P9 Vector rotate built-ins
 BUILTIN(__builtin_altivec_vrlwmi, "V4UiV4UiV4UiV4Ui", "")
 BUILTIN(__builtin_altivec_vrldmi, "V2ULLiV2ULLiV2ULLiV2ULLi", "")
 BUILTIN(__builtin_altivec_vrlwnm, "V4UiV4UiV4Ui", "")
 BUILTIN(__builtin_altivec_vrldnm, "V2ULLiV2ULLiV2ULLi", "")
 
+// P10 Vector Divide Extended built-ins.
+BUILTIN(__builtin_altivec_vdivesw, "V4SiV4SiV4Si", "")
+BUILTIN(__builtin_altivec_vdiveuw, "V4UiV4UiV4Ui", "")
+BUILTIN(__builtin_altivec_vdivesd, "V2LLiV2LLiV2LLi", "")
+BUILTIN(__builtin_altivec_vdiveud, "V2ULLiV2ULLiV2ULLi", "")
+
+// P10 Vector Multiply High built-ins.
+BUILTIN(__builtin_altivec_vmulhsw, "V4SiV4SiV4Si", "")
+BUILTIN(__builtin_altivec_vmulhuw, "V4UiV4UiV4Ui", "")
+BUILTIN(__builtin_altivec_vmulhsd, "V2LLiV2LLiV2LLi", "")
+BUILTIN(__builtin_altivec_vmulhud, "V2ULLiV2ULLiV2ULLi", "")
+
 // P10 Vector Parallel Bits built-ins.
 BUILTIN(__builtin_altivec_vpdepd, "V2ULLiV2ULLiV2ULLi", "")
 BUILTIN(__builtin_altivec_vpextd, "V2ULLiV2ULLiV2ULLi", "")
 
 // P10 Vector Centrifuge built-in.
 BUILTIN(__builtin_altivec_vcfuged, "V2ULLiV2ULLiV2ULLi", "")
 
 // P10 Vector Gather Every N-th Bit built-in.

clang/lib/Headers/altivec.h

 }
 
 static __inline__ vector double __ATTRS_o_ai vec_div(vector double __a,
                                                      vector double __b) {
   return __a / __b;
 }
 #endif
 
+/* vec_dive */
+
+#ifdef __POWER10_VECTOR__
+static __inline__ vector signed int __ATTRS_o_ai
+vec_dive(vector signed int __a, vector signed int __b) {
+  return __builtin_altivec_vdivesw(__a, __b);
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_dive(vector unsigned int __a, vector unsigned int __b) {
+  return __builtin_altivec_vdiveuw(__a, __b);
+}
+
+static __inline__ vector signed long long __ATTRS_o_ai
+vec_dive(vector signed long long __a, vector signed long long __b) {
+  return __builtin_altivec_vdivesd(__a, __b);
+}
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_dive(vector unsigned long long __a, vector unsigned long long __b) {
+  return __builtin_altivec_vdiveud(__a, __b);
+}
+#endif
+
 /* vec_dss */
 
 #define vec_dss __builtin_altivec_dss
 
 /* vec_dssall */
 
 static __inline__ void __attribute__((__always_inline__)) vec_dssall(void) {
   __builtin_altivec_dssall();

 vec_vmuleuh(vector unsigned short __a, vector unsigned short __b) {
 #ifdef __LITTLE_ENDIAN__
   return __builtin_altivec_vmulouh(__a, __b);
 #else
   return __builtin_altivec_vmuleuh(__a, __b);
 #endif
 }
 
+/* vec_mulh */
+
+#ifdef __POWER10_VECTOR__
+static __inline__ vector signed int __ATTRS_o_ai
+vec_mulh(vector signed int __a, vector signed int __b) {
+  return __builtin_altivec_vmulhsw(__a, __b);
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_mulh(vector unsigned int __a, vector unsigned int __b) {
+  return __builtin_altivec_vmulhuw(__a, __b);
+}
+
+static __inline__ vector signed long long __ATTRS_o_ai
+vec_mulh(vector signed long long __a, vector signed long long __b) {
+  return __builtin_altivec_vmulhsd(__a, __b);
+}
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_mulh(vector unsigned long long __a, vector unsigned long long __b) {
+  return __builtin_altivec_vmulhud(__a, __b);
+}
+#endif
+
 /* vec_mulo */
 
 static __inline__ vector short __ATTRS_o_ai vec_mulo(vector signed char __a,
                                                      vector signed char __b) {
 #ifdef __LITTLE_ENDIAN__
   return __builtin_altivec_vmulesb(__a, __b);
 #else
   return __builtin_altivec_vmulosb(__a, __b);

clang/test/CodeGen/builtins-ppc-p10vector.c

 
 vector unsigned long long test_vec_mod_ull(void) {
   // CHECK: urem <2 x i64>
   // CHECK-NEXT: ret <2 x i64>
   return vec_mod(vulla, vullb);
 }
 
 
+vector signed int test_vec_dive_si(void) {
+  // CHECK: @llvm.ppc.altivec.vdivesw(<4 x i32> %{{.+}}, <4 x i32> %{{.+}})
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_dive(vsia, vsib);
+}
+
+vector unsigned int test_vec_dive_ui(void) {
+  // CHECK: @llvm.ppc.altivec.vdiveuw(<4 x i32> %{{.+}}, <4 x i32> %{{.+}})
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_dive(vuia, vuib);
+}
+
+vector signed long long test_vec_dive_sll(void) {
+  // CHECK: @llvm.ppc.altivec.vdivesd(<2 x i64> %{{.+}}, <2 x i64> %{{.+}})
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_dive(vslla, vsllb);
+}
+
+vector unsigned long long test_vec_dive_ull(void) {
+  // CHECK: @llvm.ppc.altivec.vdiveud(<2 x i64> %{{.+}}, <2 x i64> %{{.+}})
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_dive(vulla, vullb);
+}
+
+vector signed int test_vec_mulh_si(void) {
+  // CHECK: @llvm.ppc.altivec.vmulhsw(<4 x i32> %{{.+}}, <4 x i32> %{{.+}})
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_mulh(vsia, vsib);
+}
+
+vector unsigned int test_vec_mulh_ui(void) {
+  // CHECK: @llvm.ppc.altivec.vmulhuw(<4 x i32> %{{.+}}, <4 x i32> %{{.+}})
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_mulh(vuia, vuib);
+}
+
+vector signed long long test_vec_mulh_sll(void) {
+  // CHECK: @llvm.ppc.altivec.vmulhsd(<2 x i64> %{{.+}}, <2 x i64> %{{.+}})
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_mulh(vslla, vsllb);
+}
+
+vector unsigned long long test_vec_mulh_ull(void) {
+  // CHECK: @llvm.ppc.altivec.vmulhud(<2 x i64> %{{.+}}, <2 x i64> %{{.+}})
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_mulh(vulla, vullb);
+}
+
 vector unsigned long long test_vpdepd(void) {
   // CHECK: @llvm.ppc.altivec.vpdepd(<2 x i64>
   // CHECK-NEXT: ret <2 x i64>
   return vec_pdep(vulla, vullb);
 }
 
 vector unsigned long long test_vpextd(void) {
   // CHECK: @llvm.ppc.altivec.vpextd(<2 x i64>

llvm/include/llvm/IR/IntrinsicsPowerPC.td

 def int_ppc_altivec_vrldnm :
       PowerPC_Vec_Intrinsic<"vrldnm", [llvm_v2i64_ty],
                             [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>;
 def int_ppc_altivec_vrldmi :
       PowerPC_Vec_Intrinsic<"vrldmi", [llvm_v2i64_ty],
                             [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty],
                             [IntrNoMem]>;
 
+// Vector Divide Extended Intrinsics.
+def int_ppc_altivec_vdivesw : PowerPC_Vec_WWW_Intrinsic<"vdivesw">;
+def int_ppc_altivec_vdiveuw : PowerPC_Vec_WWW_Intrinsic<"vdiveuw">;
+def int_ppc_altivec_vdivesd : PowerPC_Vec_DDD_Intrinsic<"vdivesd">;
+def int_ppc_altivec_vdiveud : PowerPC_Vec_DDD_Intrinsic<"vdiveud">;
+
+// Vector Multiply High Intrinsics.
+def int_ppc_altivec_vmulhsw : PowerPC_Vec_WWW_Intrinsic<"vmulhsw">;
+def int_ppc_altivec_vmulhuw : PowerPC_Vec_WWW_Intrinsic<"vmulhuw">;
+def int_ppc_altivec_vmulhsd : PowerPC_Vec_DDD_Intrinsic<"vmulhsd">;
+def int_ppc_altivec_vmulhud : PowerPC_Vec_DDD_Intrinsic<"vmulhud">;
+
 //===----------------------------------------------------------------------===//
 // PowerPC VSX Intrinsic Definitions.
 
 let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
 
 // Vector load.
 def int_ppc_vsx_lxvw4x :
       Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>;

llvm/lib/Target/PowerPC/PPCInstrPrefix.td

def VDIVUW : VXForm_1<139, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                         [(set v4i32:$vD, (udiv v4i32:$vA, v4i32:$vB))]>;
   def VDIVSD : VXForm_1<459, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                         "vdivsd $vD, $vA, $vB", IIC_VecGeneral,
                         [(set v2i64:$vD, (sdiv v2i64:$vA, v2i64:$vB))]>;
   def VDIVUD : VXForm_1<203, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                         "vdivud $vD, $vA, $vB", IIC_VecGeneral,
                         [(set v2i64:$vD, (udiv v2i64:$vA, v2i64:$vB))]>;
   def VDIVESW : VXForm_1<907, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vdivesw $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vdivesw $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v4i32:$vD, (int_ppc_altivec_vdivesw v4i32:$vA,
+                               v4i32:$vB))]>;
   def VDIVEUW : VXForm_1<651, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vdiveuw $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vdiveuw $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v4i32:$vD, (int_ppc_altivec_vdiveuw v4i32:$vA,
+                               v4i32:$vB))]>;
   def VDIVESD : VXForm_1<971, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vdivesd $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vdivesd $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v2i64:$vD, (int_ppc_altivec_vdivesd v2i64:$vA,
+                               v2i64:$vB))]>;
   def VDIVEUD : VXForm_1<715, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vdiveud $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vdiveud $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v2i64:$vD, (int_ppc_altivec_vdiveud v2i64:$vA,
+                               v2i64:$vB))]>;
   def XVTLSBB : XX2_BF3_XO5_XB6_XO9<60, 2, 475, (outs crrc:$BF), (ins vsrc:$XB),
                                     "xvtlsbb $BF, $XB", IIC_VecGeneral, []>;
 
   // The XFormMemOp flag for the following 8 instructions is set on
   // the instruction format.
   let mayLoad = 1, mayStore = 0 in {
     def LXVRBX : X_XT6_RA5_RB5<31, 13, "lxvrbx", vsrc, []>;
     def LXVRHX : X_XT6_RA5_RB5<31, 45, "lxvrhx", vsrc, []>;
let Predicates = [IsISA3_1] in {
   def : Pat<(v4i32 (int_ppc_vsx_xxgenpcvwm v4i32:$VRB, imm:$IMM)),
             (v4i32 (COPY_TO_REGCLASS (XXGENPCVWM $VRB, imm:$IMM), VRRC))>;
   def : Pat<(v2i64 (int_ppc_vsx_xxgenpcvdm v2i64:$VRB, imm:$IMM)),
             (v2i64 (COPY_TO_REGCLASS (XXGENPCVDM $VRB, imm:$IMM), VRRC))>;
   def : Pat<(i32 (int_ppc_vsx_xvtlsbb v16i8:$XB, 1)),
             (EXTRACT_SUBREG (XVTLSBB (COPY_TO_REGCLASS $XB, VSRC)), sub_lt)>;
   def : Pat<(i32 (int_ppc_vsx_xvtlsbb v16i8:$XB, 0)),
             (EXTRACT_SUBREG (XVTLSBB (COPY_TO_REGCLASS $XB, VSRC)), sub_eq)>;
+  // Exploit the vector multiply high instructions using intrinsics.
+  def : Pat<(v4i32 (int_ppc_altivec_vmulhsw v4i32:$vA, v4i32:$vB)),
+            (v4i32 (VMULHSW $vA, $vB))>;
+  def : Pat<(v4i32 (int_ppc_altivec_vmulhuw v4i32:$vA, v4i32:$vB)),
+            (v4i32 (VMULHUW $vA, $vB))>;
+  def : Pat<(v2i64 (int_ppc_altivec_vmulhsd v2i64:$vA, v2i64:$vB)),
+            (v2i64 (VMULHSD $vA, $vB))>;
+  def : Pat<(v2i64 (int_ppc_altivec_vmulhud v2i64:$vA, v2i64:$vB)),
+            (v2i64 (VMULHUD $vA, $vB))>;
 }
 
 let AddedComplexity = 400, Predicates = [IsISA3_1] in {
   def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$rS, 0)), xoaddr:$src),
             (STXVRBX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$src)>;
   def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$rS, 0)), xoaddr:$src),
             (STXVRHX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$src)>;
   def : Pat<(store (i32 (vector_extract v4i32:$rS, 0)), xoaddr:$src),

llvm/test/CodeGen/PowerPC/p10-vector-divide.ll

 ; CHECK-LABEL: test_vdivsw:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vdivsw v2, v2, v3
 ; CHECK-NEXT:    blr
 entry:
   %div = sdiv <4 x i32> %a, %b
   ret <4 x i32> %div
 }
+
+; Test the vector divide extended intrinsics.
+declare <4 x i32> @llvm.ppc.altivec.vdivesw(<4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.ppc.altivec.vdiveuw(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.ppc.altivec.vdivesd(<2 x i64>, <2 x i64>)
+declare <2 x i64> @llvm.ppc.altivec.vdiveud(<2 x i64>, <2 x i64>)
+
+define <4 x i32> @test_vdivesw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vdivesw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdivesw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = tail call <4 x i32> @llvm.ppc.altivec.vdivesw(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %div
+}
+
+define <4 x i32> @test_vdiveuw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vdiveuw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdiveuw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = tail call <4 x i32> @llvm.ppc.altivec.vdiveuw(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %div
+}
+
+define <2 x i64> @test_vdivesd(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vdivesd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdivesd v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = tail call <2 x i64> @llvm.ppc.altivec.vdivesd(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %div
+}
+
+define <2 x i64> @test_vdiveud(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vdiveud:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdiveud v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = tail call <2 x i64> @llvm.ppc.altivec.vdiveud(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %div
+}

llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll

 entry:
   %0 = zext <4 x i32> %a to <4 x i64>
   %1 = zext <4 x i32> %b to <4 x i64>
   %mul = mul <4 x i64> %1, %0
   %shr = lshr <4 x i64> %mul, <i64 32, i64 32, i64 32, i64 32>
   %tr = trunc <4 x i64> %shr to <4 x i32>
   ret <4 x i32> %tr
 }
+
+; Test the vector multiply high intrinsics.
+declare <4 x i32> @llvm.ppc.altivec.vmulhsw(<4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.ppc.altivec.vmulhuw(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.ppc.altivec.vmulhsd(<2 x i64>, <2 x i64>)
+declare <2 x i64> @llvm.ppc.altivec.vmulhud(<2 x i64>, <2 x i64>)
+
+define <4 x i32> @test_vmulhsw_intrinsic(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmulhsw_intrinsic:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhsw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %mulh = tail call <4 x i32> @llvm.ppc.altivec.vmulhsw(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %mulh
+}
+
+define <4 x i32> @test_vmulhuw_intrinsic(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmulhuw_intrinsic:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhuw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %mulh = tail call <4 x i32> @llvm.ppc.altivec.vmulhuw(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %mulh
+}
+
+define <2 x i64> @test_vmulhsd_intrinsic(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmulhsd_intrinsic:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhsd v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %mulh = tail call <2 x i64> @llvm.ppc.altivec.vmulhsd(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %mulh
+}
+
+define <2 x i64> @test_vmulhud_intrinsic(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmulhud_intrinsic:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhud v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %mulh = tail call <2 x i64> @llvm.ppc.altivec.vmulhud(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %mulh
+}