[AArch64] use CNT for ISD::popcnt and ISD::parity if available

[stuij]

Summary

These are the two places where we explicitly want to use cnt in
SelectionDAG when feature CSSC is available: ISD::popcnt and ISD::parity

For both, we need to make sure we're emitting optimized code for i32 (and
lower), i64 and i128. The most optimal way is of course using the GPR CNT
instruction. If we don't have CSSC, but we do have neon, we'll use floating
point CNT. If all fails, we'll fall back on the general GPR popcnt and parity
implementations.

spec:
https://developer.arm.com/documentation/ddi0602/2022-09/Base-Instructions/CNT--Count-bits-

[tschuett]

You are probably less interested in:
https://github.com/llvm/llvm-project/blob/311801a0179be9dc86895e08e3bba66cea2c0724/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp#L1235

[lenary]

LGTM.

In D138808#3953436, @tschuett wrote:

You are probably less interested in:
https://github.com/llvm/llvm-project/blob/311801a0179be9dc86895e08e3bba66cea2c0724/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp#L1235

This is bypassed when we mark the ISD operations as legal, because we want to directly select the ISD operations into instructions, or do the generic expansion. That implementation is still needed when we don't have these instructions though, when we mark the ISD operations for custom legalisation.

[tschuett]

No worries, but this is GlobalIsel and not SDAG.

[stuij]

In D138808#3953436, @tschuett wrote:

You are probably less interested in:
https://github.com/llvm/llvm-project/blob/311801a0179be9dc86895e08e3bba66cea2c0724/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp#L1235

Oh no we are. I've got GIsel patches ready for CNT and the other CSSC instructions, which will amend amongst others that piece of code. I'm a bit busy right now, but they'll be put up for review somewhere next week.

[lenary]

In D138808#3953650, @tschuett wrote:

No worries, but this is GlobalIsel and not SDAG.

Sorry, my mistake. Yeah, as Ties says, we do have patches for doing the relevant GlobalISel changes to add support for these instructions.

[tschuett]

Thanks!

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   // AArch64 doesn't have i32 MULH{S|U}.
   setOperationAction(ISD::MULHU, MVT::i32, Expand);
   setOperationAction(ISD::MULHS, MVT::i32, Expand);
 
   // AArch64 doesn't have {U|S}MUL_LOHI.
   setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
   setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
 
+  if (Subtarget->hasCSSC()) {
+    setOperationAction(ISD::CTPOP, MVT::i32, Legal);
+    setOperationAction(ISD::CTPOP, MVT::i64, Legal);
+    setOperationAction(ISD::CTPOP, MVT::i128, Expand);
+    setOperationAction(ISD::PARITY, MVT::i128, Expand);
+  } else {
     setOperationAction(ISD::CTPOP, MVT::i32, Custom);
     setOperationAction(ISD::CTPOP, MVT::i64, Custom);
     setOperationAction(ISD::CTPOP, MVT::i128, Custom);
 
     setOperationAction(ISD::PARITY, MVT::i64, Custom);
     setOperationAction(ISD::PARITY, MVT::i128, Custom);
+  }
 
   setOperationAction(ISD::ABS, MVT::i32, Custom);
   setOperationAction(ISD::ABS, MVT::i64, Custom);
 
   setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
   setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     setOperationAction(ISD::SDIVREM, VT, Expand);
SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
           Attribute::NoImplicitFloat))
     return SDValue();
 
   if (!Subtarget->hasNEON())
     return SDValue();
 
   bool IsParity = Op.getOpcode() == ISD::PARITY;
+  SDValue Val = Op.getOperand(0);
+  SDLoc DL(Op);
+  EVT VT = Op.getValueType();
 
-  // While there is no integer popcount instruction, it can
+  // for i32, general parity function using EORs is more efficient compared to
+  // using floating point
+  if (VT == MVT::i32 && IsParity)
+    return SDValue();
+
+  // If there is no CNT instruction available, GPR popcount can
   // be more efficiently lowered to the following sequence that uses
   // AdvSIMD registers/instructions as long as the copies to/from
   // the AdvSIMD registers are cheap.
   //  FMOV    D0, X0        // copy 64-bit int to vector, high bits zero'd
   //  CNT     V0.8B, V0.8B  // 8xbyte pop-counts
   //  ADDV    B0, V0.8B     // sum 8xbyte pop-counts
   //  UMOV    X0, V0.B[0]   // copy byte result back to integer reg
-  SDValue Val = Op.getOperand(0);
-  SDLoc DL(Op);
-  EVT VT = Op.getValueType();
-
   if (VT == MVT::i32 || VT == MVT::i64) {
     if (VT == MVT::i32)
       Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val);
     Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
 
     SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
     SDValue UaddLV = DAG.getNode(
         ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 defm RCWCLR  : ReadCheckWriteOperation<0b001, "clr">;
 defm RCWSET  : ReadCheckWriteOperation<0b011, "set">;
 defm RCWSWP  : ReadCheckWriteOperation<0b010, "swp">;
 
 //===----------------------------------------------------------------------===//
 // General Data-Processing Instructions (FEAT_V94_DP)
 //===----------------------------------------------------------------------===//
 defm ABS : OneOperandData<0b001000, "abs">, Requires<[HasCSSC]>;
-defm CNT : OneOperandData<0b000111, "cnt">, Requires<[HasCSSC]>;
+defm CNT : OneOperandData<0b000111, "cnt", ctpop>, Requires<[HasCSSC]>;
 defm CTZ : OneOperandData<0b000110, "ctz">, Requires<[HasCSSC]>;
 
 defm SMAX : ComparisonOp<0, 0, "smax">, Requires<[HasCSSC]>;
 defm SMIN : ComparisonOp<0, 1, "smin">, Requires<[HasCSSC]>;
 defm UMAX : ComparisonOp<1, 0, "umax">, Requires<[HasCSSC]>;
 defm UMIN : ComparisonOp<1, 1, "umin">, Requires<[HasCSSC]>;
 
 def RPRFM:

llvm/test/CodeGen/AArch64/arm64-popcnt.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
 ; RUN: llc < %s -mtriple=aarch64-eabi -mattr -neon -aarch64-neon-syntax=apple | FileCheck -check-prefix=CHECK-NONEON %s
+; RUN: llc < %s -mtriple=aarch64-eabi -mattr +cssc -aarch64-neon-syntax=apple | FileCheck -check-prefix=CHECK-CSSC %s
 
 define i32 @cnt32_advsimd(i32 %x) nounwind readnone {
 ; CHECK-LABEL: cnt32_advsimd:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    cnt.8b v0, v0
 ; CHECK-NEXT:    uaddlv.8b h0, v0

 ; CHECK-NONEON-NEXT:    and w9, w9, #0x33333333
 ; CHECK-NONEON-NEXT:    and w10, w10, #0x33333333
 ; CHECK-NONEON-NEXT:    add w9, w9, w10
 ; CHECK-NONEON-NEXT:    add w9, w9, w9, lsr #4
 ; CHECK-NONEON-NEXT:    and w9, w9, #0xf0f0f0f
 ; CHECK-NONEON-NEXT:    mul w8, w9, w8
 ; CHECK-NONEON-NEXT:    lsr w0, w8, #24
 ; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: cnt32_advsimd:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt w0, w0
+; CHECK-CSSC-NEXT:    ret
   %cnt = tail call i32 @llvm.ctpop.i32(i32 %x)
   ret i32 %cnt
 }
 
 define i32 @cnt32_advsimd_2(<2 x i32> %x) {
 ; CHECK-LABEL: cnt32_advsimd_2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0

 ; CHECK-NONEON-NEXT:    and w9, w9, #0x33333333
 ; CHECK-NONEON-NEXT:    and w10, w10, #0x33333333
 ; CHECK-NONEON-NEXT:    add w9, w9, w10
 ; CHECK-NONEON-NEXT:    add w9, w9, w9, lsr #4
 ; CHECK-NONEON-NEXT:    and w9, w9, #0xf0f0f0f
 ; CHECK-NONEON-NEXT:    mul w8, w9, w8
 ; CHECK-NONEON-NEXT:    lsr w0, w8, #24
 ; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: cnt32_advsimd_2:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CSSC-NEXT:    fmov w8, s0
+; CHECK-CSSC-NEXT:    cnt w0, w8
+; CHECK-CSSC-NEXT:    ret
   %1 = extractelement <2 x i32> %x, i64 0
   %2 = tail call i32 @llvm.ctpop.i32(i32 %1)
   ret i32 %2
 }
 
 define i64 @cnt64_advsimd(i64 %x) nounwind readnone {
 ; CHECK-LABEL: cnt64_advsimd:
 ; CHECK:       // %bb.0:

 ; CHECK-NONEON-NEXT:    and x9, x9, #0x3333333333333333
 ; CHECK-NONEON-NEXT:    and x10, x10, #0x3333333333333333
 ; CHECK-NONEON-NEXT:    add x9, x9, x10
 ; CHECK-NONEON-NEXT:    add x9, x9, x9, lsr #4
 ; CHECK-NONEON-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
 ; CHECK-NONEON-NEXT:    mul x8, x9, x8
 ; CHECK-NONEON-NEXT:    lsr x0, x8, #56
 ; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: cnt64_advsimd:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt x0, x0
+; CHECK-CSSC-NEXT:    ret
   %cnt = tail call i64 @llvm.ctpop.i64(i64 %x)
   ret i64 %cnt
 }
 
 ; Do not use AdvSIMD when -mno-implicit-float is specified.
 ; rdar://9473858
 
 define i32 @cnt32(i32 %x) nounwind readnone noimplicitfloat {

 ; CHECK-NONEON-NEXT:    and w9, w9, #0x33333333
 ; CHECK-NONEON-NEXT:    and w10, w10, #0x33333333
 ; CHECK-NONEON-NEXT:    add w9, w9, w10
 ; CHECK-NONEON-NEXT:    add w9, w9, w9, lsr #4
 ; CHECK-NONEON-NEXT:    and w9, w9, #0xf0f0f0f
 ; CHECK-NONEON-NEXT:    mul w8, w9, w8
 ; CHECK-NONEON-NEXT:    lsr w0, w8, #24
 ; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: cnt32:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt w0, w0
+; CHECK-CSSC-NEXT:    ret
   %cnt = tail call i32 @llvm.ctpop.i32(i32 %x)
   ret i32 %cnt
 }
 
 define i64 @cnt64(i64 %x) nounwind readnone noimplicitfloat {
 ; CHECK-LABEL: cnt64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr x9, x0, #1

 ; CHECK-NONEON-NEXT:    and x9, x9, #0x3333333333333333
 ; CHECK-NONEON-NEXT:    and x10, x10, #0x3333333333333333
 ; CHECK-NONEON-NEXT:    add x9, x9, x10
 ; CHECK-NONEON-NEXT:    add x9, x9, x9, lsr #4
 ; CHECK-NONEON-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
 ; CHECK-NONEON-NEXT:    mul x8, x9, x8
 ; CHECK-NONEON-NEXT:    lsr x0, x8, #56
 ; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: cnt64:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt x0, x0
+; CHECK-CSSC-NEXT:    ret
   %cnt = tail call i64 @llvm.ctpop.i64(i64 %x)
   ret i64 %cnt
 }
 
 define i32 @ctpop_eq_one(i64 %x) nounwind readnone {
 ; CHECK-LABEL: ctpop_eq_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub x8, x0, #1
 ; CHECK-NEXT:    tst x0, x8
 ; CHECK-NEXT:    ccmp x0, #0, #4, eq
 ; CHECK-NEXT:    cset w0, ne
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-NONEON-LABEL: ctpop_eq_one:
 ; CHECK-NONEON:       // %bb.0:
 ; CHECK-NONEON-NEXT:    sub x8, x0, #1
 ; CHECK-NONEON-NEXT:    tst x0, x8
 ; CHECK-NONEON-NEXT:    ccmp x0, #0, #4, eq
 ; CHECK-NONEON-NEXT:    cset w0, ne
 ; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: ctpop_eq_one:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt x8, x0
+; CHECK-CSSC-NEXT:    cmp x8, #1
+; CHECK-CSSC-NEXT:    cset w0, eq
+; CHECK-CSSC-NEXT:    ret
   %count = tail call i64 @llvm.ctpop.i64(i64 %x)
   %cmp = icmp eq i64 %count, 1
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @ctpop_ne_one(i64 %x) nounwind readnone {
 ; CHECK-LABEL: ctpop_ne_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub x8, x0, #1
 ; CHECK-NEXT:    tst x0, x8
 ; CHECK-NEXT:    ccmp x0, #0, #4, eq
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-NONEON-LABEL: ctpop_ne_one:
 ; CHECK-NONEON:       // %bb.0:
 ; CHECK-NONEON-NEXT:    sub x8, x0, #1
 ; CHECK-NONEON-NEXT:    tst x0, x8
 ; CHECK-NONEON-NEXT:    ccmp x0, #0, #4, eq
 ; CHECK-NONEON-NEXT:    cset w0, eq
 ; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: ctpop_ne_one:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt x8, x0
+; CHECK-CSSC-NEXT:    cmp x8, #1
+; CHECK-CSSC-NEXT:    cset w0, ne
+; CHECK-CSSC-NEXT:    ret
   %count = tail call i64 @llvm.ctpop.i64(i64 %x)
   %cmp = icmp ne i64 %count, 1
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i1 @ctpop32_ne_one(i32 %x) nounwind readnone {
 ; CHECK-LABEL: ctpop32_ne_one:

llvm/test/CodeGen/AArch64/ctpop-nonean.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=-neon < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=-neon -mattr=+cssc < %s | FileCheck %s -check-prefix=CHECK-CSSC
 
 declare i128 @llvm.ctpop.i128(i128)
 
 define i128 @ctpop_i128(i128 %i) {
 ; CHECK-LABEL: ctpop_i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr x8, x1, #1
 ; CHECK-NEXT:    lsr x9, x0, #1

 ; CHECK-NEXT:    add x9, x9, x9, lsr #4
 ; CHECK-NEXT:    and x8, x8, #0xf0f0f0f0f0f0f0f
 ; CHECK-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
 ; CHECK-NEXT:    mul x8, x8, x10
 ; CHECK-NEXT:    mul x9, x9, x10
 ; CHECK-NEXT:    lsr x9, x9, #56
 ; CHECK-NEXT:    add x0, x9, x8, lsr #56
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: ctpop_i128:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt x8, x1
+; CHECK-CSSC-NEXT:    cnt x9, x0
+; CHECK-CSSC-NEXT:    add x0, x9, x8
+; CHECK-CSSC-NEXT:    mov x1, xzr
+; CHECK-CSSC-NEXT:    ret
   %c = call i128 @llvm.ctpop.i128(i128 %i)
   ret i128 %c
 }

llvm/test/CodeGen/AArch64/parity.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -o - %s -mtriple=aarch64-linux-gnu | FileCheck %s
+; RUN: llc -verify-machineinstrs -o - %s -mtriple=aarch64-linux-gnu -mattr=+cssc | FileCheck %s -check-prefix=CHECK-CSSC
 
 define i4 @parity_4(i4 %x) {
 ; CHECK-LABEL: parity_4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xf
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_4:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    and w8, w0, #0xf
+; CHECK-CSSC-NEXT:    cnt w8, w8
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i4 @llvm.ctpop.i4(i4 %x)
   %2 = and i4 %1, 1
   ret i4 %2
 }
 
 define i8 @parity_8(i8 %x) {
 ; CHECK-LABEL: parity_8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #4
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_8:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    and w8, w0, #0xff
+; CHECK-CSSC-NEXT:    cnt w8, w8
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i8 @llvm.ctpop.i8(i8 %x)
   %2 = and i8 %1, 1
   ret i8 %2
 }
 
 define i16 @parity_16(i16 %x) {
 ; CHECK-LABEL: parity_16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xffff
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #8
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #4
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_16:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    and w8, w0, #0xffff
+; CHECK-CSSC-NEXT:    cnt w8, w8
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i16 @llvm.ctpop.i16(i16 %x)
   %2 = and i16 %1, 1
   ret i16 %2
 }
 
 define i17 @parity_17(i17 %x) {
 ; CHECK-LABEL: parity_17:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0x1ffff
 ; CHECK-NEXT:    eor w9, w8, w8, lsr #16
 ; CHECK-NEXT:    eor w8, w9, w8, lsr #8
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #4
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_17:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    and w8, w0, #0x1ffff
+; CHECK-CSSC-NEXT:    cnt w8, w8
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i17 @llvm.ctpop.i17(i17 %x)
   %2 = and i17 %1, 1
   ret i17 %2
 }
 
 define i32 @parity_32(i32 %x) {
 ; CHECK-LABEL: parity_32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor w8, w0, w0, lsr #16
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #8
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #4
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_32:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt w8, w0
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i32 @llvm.ctpop.i32(i32 %x)
   %2 = and i32 %1, 1
   ret i32 %2
 }
 
 define i64 @parity_64(i64 %x) {
 ; CHECK-LABEL: parity_64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov d0, x0
 ; CHECK-NEXT:    cnt v0.8b, v0.8b
 ; CHECK-NEXT:    uaddlv h0, v0.8b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_64:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt x8, x0
+; CHECK-CSSC-NEXT:    and x0, x8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i64 @llvm.ctpop.i64(i64 %x)
   %2 = and i64 %1, 1
   ret i64 %2
 }
 
 define i128 @parity_128(i128 %x) {
 ; CHECK-LABEL: parity_128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov d0, x0
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    mov x1, xzr
 ; CHECK-NEXT:    cnt v0.16b, v0.16b
 ; CHECK-NEXT:    uaddlv h0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_128:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    eor x8, x0, x1
+; CHECK-CSSC-NEXT:    mov x1, xzr
+; CHECK-CSSC-NEXT:    cnt x8, x8
+; CHECK-CSSC-NEXT:    and x0, x8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i128 @llvm.ctpop.i128(i128 %x)
   %2 = and i128 %1, 1
   ret i128 %2
 }
 
 define i32 @parity_64_trunc(i64 %x) {
 ; CHECK-LABEL: parity_64_trunc:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov d0, x0
 ; CHECK-NEXT:    cnt v0.8b, v0.8b
 ; CHECK-NEXT:    uaddlv h0, v0.8b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_64_trunc:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt x8, x0
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i64 @llvm.ctpop.i64(i64 %x)
   %2 = trunc i64 %1 to i32
   %3 = and i32 %2, 1
   ret i32 %3
 }
 
 define i8 @parity_32_trunc(i32 %x) {
 ; CHECK-LABEL: parity_32_trunc:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor w8, w0, w0, lsr #16
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #8
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #4
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_32_trunc:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    cnt w8, w0
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %1 = tail call i32 @llvm.ctpop.i32(i32 %x)
   %2 = trunc i32 %1 to i8
   %3 = and i8 %2, 1
   ret i8 %3
 }
 
 define i32 @parity_8_zext(i8 %x) {
 ; CHECK-LABEL: parity_8_zext:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #4
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_8_zext:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    and w8, w0, #0xff
+; CHECK-CSSC-NEXT:    cnt w8, w8
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %a = zext i8 %x to i32
   %b = tail call i32 @llvm.ctpop.i32(i32 %a)
   %c = and i32 %b, 1
   ret i32 %c
 }
 
 define i32 @parity_8_mask(i32 %x) {
 ; CHECK-LABEL: parity_8_mask:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0xff
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #4
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #2
 ; CHECK-NEXT:    eor w8, w8, w8, lsr #1
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: parity_8_mask:
+; CHECK-CSSC:       // %bb.0:
+; CHECK-CSSC-NEXT:    and w8, w0, #0xff
+; CHECK-CSSC-NEXT:    cnt w8, w8
+; CHECK-CSSC-NEXT:    and w0, w8, #0x1
+; CHECK-CSSC-NEXT:    ret
   %a = and i32 %x, 255
   %b = tail call i32 @llvm.ctpop.i32(i32 %a)
   %c = and i32 %b, 1
   ret i32 %c
 }
 
 declare i4 @llvm.ctpop.i4(i4 %x)
 declare i8 @llvm.ctpop.i8(i8 %x)
 declare i16 @llvm.ctpop.i16(i16 %x)
 declare i17 @llvm.ctpop.i17(i17 %x)
 declare i32 @llvm.ctpop.i32(i32 %x)
 declare i64 @llvm.ctpop.i64(i64 %x)
 declare i128 @llvm.ctpop.i128(i128 %x)