[ARM] Lower CTTZ on MVE

[oliverlars]

Summary

Lower CTTZ on MVE using VBRSR and VCLS which will reverse the bits and count the leading zeros, equivalent to a count trailing zeros (CTTZ).

[oliverlars]

Added verify-machineinstrs, a test for 2i64 as well as tests for the 'isundef' parameter

[oliverlars]

fixed vbrsr not taking a register operand

[dmgreen]

Like it. LGTM.

[oliverlars]

rG5f799ef1627f

llvm/lib/Target/ARM/ARMISelLowering.cpp

for (auto VT : IntTypes) {
     setOperationAction(ISD::SRL, VT, Custom);
     setOperationAction(ISD::SMIN, VT, Legal);
     setOperationAction(ISD::SMAX, VT, Legal);
     setOperationAction(ISD::UMIN, VT, Legal);
     setOperationAction(ISD::UMAX, VT, Legal);
     setOperationAction(ISD::ABS, VT, Legal);
     setOperationAction(ISD::SETCC, VT, Custom);
     setOperationAction(ISD::CTLZ, VT, Legal);
+    setOperationAction(ISD::CTTZ, VT, Custom);
 
     // No native support for these.
     setOperationAction(ISD::UDIV, VT, Expand);
     setOperationAction(ISD::SDIV, VT, Expand);
     setOperationAction(ISD::UREM, VT, Expand);
     setOperationAction(ISD::SREM, VT, Expand);
     setOperationAction(ISD::CTPOP, VT, Expand);
 
SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
                      DAG.getConstant(3, dl, MVT::i32));
 }
 
 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
                          const ARMSubtarget *ST) {
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
-  if (VT.isVector()) {
-    assert(ST->hasNEON());
-
+  if (VT.isVector() && ST->hasNEON()) {
     // Compute the least significant set bit: LSB = X & -X
     SDValue X = N->getOperand(0);
     SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
     SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
 
     EVT ElemTy = VT.getVectorElementType();
 
     if (ElemTy == MVT::i8) {

llvm/test/CodeGen/Thumb2/mve-cttz.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x i64> @cttz_2i64_0_t(<2 x i64> %src){
+; CHECK-LABEL: cttz_2i64_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    rbit r2, r2
+; CHECK-NEXT:    clz r2, r2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    add.w r2, r2, #32
+; CHECK-NEXT:    rbit r1, r0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r2, r1
+; CHECK-NEXT:    vmov s6, r2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    rbit r2, r2
+; CHECK-NEXT:    clz r2, r2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    add.w r2, r2, #32
+; CHECK-NEXT:    rbit r1, r0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r2, r1
+; CHECK-NEXT:    vmov s4, r2
+; CHECK-NEXT:    vldr s5, .LCPI0_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %src, i1 0)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cttz_4i32_0_t(<4 x i32> %src){
+; CHECK-LABEL: cttz_4i32_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %src, i1 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cttz_8i16_0_t(<8 x i16> %src){
+; CHECK-LABEL: cttz_8i16_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %src, i1 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cttz_16i8_0_t(<16 x i8> %src) {
+; CHECK-LABEL: cttz_16i8_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %src, i1 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cttz_2i64_1_t(<2 x i64> %src){
+; CHECK-LABEL: cttz_2i64_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    rbit r2, r2
+; CHECK-NEXT:    clz r2, r2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    add.w r2, r2, #32
+; CHECK-NEXT:    rbit r1, r0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r2, r1
+; CHECK-NEXT:    vmov s6, r2
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    rbit r2, r2
+; CHECK-NEXT:    clz r2, r2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    add.w r2, r2, #32
+; CHECK-NEXT:    rbit r1, r0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r2, r1
+; CHECK-NEXT:    vmov s4, r2
+; CHECK-NEXT:    vldr s5, .LCPI4_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %src, i1 1)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cttz_4i32_1_t(<4 x i32> %src){
+; CHECK-LABEL: cttz_4i32_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %src, i1 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cttz_8i16_1_t(<8 x i16> %src){
+; CHECK-LABEL: cttz_8i16_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %src, i1 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cttz_16i8_1_t(<16 x i8> %src) {
+; CHECK-LABEL: cttz_16i8_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %src, i1 1)
+  ret <16 x i8> %0
+}
+
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)