[PowerPC] custom lowering for CNTPOP(i128)

[shawnl]

Summary

There are many instructions getting generated here that do nothing,
but that will have to be cleaned up elsewhere.

[steven.zhang]

llvm/test/CodeGen/PowerPC/popcount.ll

13

deg.

44

deg.

llvm/lib/Target/PowerPC/PPCISelLowering.h

private:
     SDValue LowerBSWAP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVectorStore(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
                             CallingConv::ID CallConv, bool isVarArg,
                             const SmallVectorImpl<ISD::InputArg> &Ins,
                             const SDLoc &dl, SelectionDAG &DAG,

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   } else {
     setOperationAction(ISD::CTTZ , MVT::i32  , Expand);
     setOperationAction(ISD::CTTZ , MVT::i64  , Expand);
   }
 
   if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
     setOperationAction(ISD::CTPOP, MVT::i32  , Legal);
     setOperationAction(ISD::CTPOP, MVT::i64  , Legal);
+    setOperationAction(ISD::CTPOP, MVT::i128 , Custom);
   } else {
     setOperationAction(ISD::CTPOP, MVT::i32  , Expand);
     setOperationAction(ISD::CTPOP, MVT::i64  , Expand);
   }
 
   // PowerPC does not have ROTR
   setOperationAction(ISD::ROTR, MVT::i32   , Expand);
   setOperationAction(ISD::ROTR, MVT::i64   , Expand);
if (ShiftElts) {
     return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
   }
   SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2);
   SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2,
                             DAG.getConstant(InsertAtByte, dl, MVT::i32));
   return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
 }
 
+
+/// LowerCTPOP - Custom lowering for CTPOP(i128) to vector.reduce.add(CTPOP(v2i64))
+SDValue PPCTargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
+  assert(Op.getOpcode() == ISD::CTPOP && "Should only be called for ISD::CTPOP");
+  assert((Op.getValueType() == MVT::i128) &&
+         "Only set i128 as custom, other type shouldn't reach here!");
+  SDLoc dl(Op);
+  SDValue N0 = peekThroughBitcasts(Op.getOperand(0));
+  SDValue BitCast = DAG.getBitcast(MVT::v2i64, N0);
+  SDValue CTPOP = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, BitCast);
+  SDValue Add = SDValue(DAG.getMachineNode(PPC::VADDUDM, dl, MVT::v2i64, CTPOP, CTPOP), 0);
+  return DAG.getBitcast(MVT::i128, Add);
+}
+
 /// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8).
 /// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is
 /// a multiple of 8. Otherwise convert it to a scalar rotation(i128)
 /// i.e (or (shl x, C1), (srl x, 128-C1)).
 SDValue PPCTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
   assert(Op.getOpcode() == ISD::ROTL && "Should only be called for ISD::ROTL");
   assert(Op.getValueType() == MVT::v1i128 &&
          "Only set v1i128 as custom, other type shouldn't reach here!");
SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::MUL:                return LowerMUL(Op, DAG);
   case ISD::ABS:                return LowerABS(Op, DAG);
   case ISD::FP_EXTEND:          return LowerFP_EXTEND(Op, DAG);
   case ISD::ROTL:               return LowerROTL(Op, DAG);
+  case ISD::CTPOP:              return LowerCTPOP(Op, DAG);
 
   // For counter-based loop handling.
   case ISD::INTRINSIC_W_CHAIN:  return SDValue();
 
   case ISD::BITCAST:            return LowerBITCAST(Op, DAG);
 
   // Frame & Return address.
   case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
if (TrgVT.isVector() &&
         OpVT.getSizeInBits() <= 128 &&
         isPowerOf2_32(OpVT.getVectorElementType().getSizeInBits()))
       Results.push_back(LowerTRUNCATEVector(SDValue(N, 0), DAG));
     return;
   }
   case ISD::BITCAST:
     // Don't handle bitcast here.
     return;
+  case ISD::CTPOP: {
+    SDValue Lowered = LowerCTPOP(SDValue(N, 0), DAG);
+    if (Lowered)
+      Results.push_back(Lowered);
+    return;
+  }
   case ISD::FP_EXTEND:
     SDValue Lowered = LowerFP_EXTEND(SDValue(N, 0), DAG);
     if (Lowered)
       Results.push_back(Lowered);
     return;
   }
 }
 

llvm/test/CodeGen/PowerPC/popcount.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -O0 -mtriple=powerpc64le-unknown-unknown | FileCheck %s
 
 ; Function Attrs: nobuiltin nounwind readonly
 define i8 @popcount128(i128* nocapture nonnull readonly %0) {
 ; CHECK-LABEL: popcount128:
 ; CHECK:       # %bb.0: # %Entry
-; CHECK-NEXT:    ld 4, 0(3)
-; CHECK-NEXT:    ld 3, 8(3)
-; CHECK-NEXT:    popcntd 3, 3
-; CHECK-NEXT:    popcntd 4, 4
-; CHECK-NEXT:    add 3, 4, 3
-; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    xxswapd 0, 0
+; CHECK-NEXT:    xxlor 34, 0, 0
+; CHECK-NEXT:    vpopcntd 2, 2
+; CHECK-NEXT:    vaddudm 2, 2, 2
+; CHECK-NEXT:    xxswapd 0, 34
+; CHECK-NEXT:    # kill: def $f0 killed $f0 killed $vsl0
+; CHECK-NEXT:    mffprwz 3, 0
 ; CHECK-NEXT:    clrldi 3, 3, 56
 ; CHECK-NEXT:    blr
 Entry:
   %1 = load i128, i128* %0, align 16
   %2 = tail call i128 @llvm.ctpop.i128(i128 %1)
   %3 = trunc i128 %2 to i8
   ret i8 %3
 }
 
 ; Function Attrs: nounwind readnone speculatable willreturn
 declare i128 @llvm.ctpop.i128(i128)
 
 ; Function Attrs: nobuiltin nounwind readonly
 define i16 @popcount256(i256* nocapture nonnull readonly %0) {
 ; CHECK-LABEL: popcount256:
 ; CHECK:       # %bb.0: # %Entry
-; CHECK-NEXT:    ld 4, 0(3)
-; CHECK-NEXT:    ld 5, 8(3)
-; CHECK-NEXT:    ld 6, 16(3)
-; CHECK-NEXT:    ld 3, 24(3)
-; CHECK-NEXT:    popcntd 3, 3
-; CHECK-NEXT:    popcntd 6, 6
-; CHECK-NEXT:    add 3, 6, 3
-; CHECK-NEXT:    popcntd 5, 5
-; CHECK-NEXT:    popcntd 4, 4
-; CHECK-NEXT:    add 4, 4, 5
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    lxvd2x 0, 3, 4
+; CHECK-NEXT:    xxswapd 0, 0
+; CHECK-NEXT:    xxlor 34, 0, 0
+; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    xxswapd 0, 0
+; CHECK-NEXT:    xxlor 35, 0, 0
+; CHECK-NEXT:    vpopcntd 2, 2
+; CHECK-NEXT:    vaddudm 2, 2, 2
+; CHECK-NEXT:    xxswapd 0, 34
+; CHECK-NEXT:    # kill: def $f0 killed $f0 killed $vsl0
+; CHECK-NEXT:    mffprd 3, 0
+; CHECK-NEXT:    vpopcntd 2, 3
+; CHECK-NEXT:    vaddudm 2, 2, 2
+; CHECK-NEXT:    xxswapd 0, 34
+; CHECK-NEXT:    # kill: def $f0 killed $f0 killed $vsl0
+; CHECK-NEXT:    mffprd 4, 0
 ; CHECK-NEXT:    add 3, 4, 3
 ; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $x3
 ; CHECK-NEXT:    clrldi 3, 3, 48
 ; CHECK-NEXT:    blr
 Entry:
   %1 = load i256, i256* %0, align 16
   %2 = tail call i256 @llvm.ctpop.i256(i256 %1)
   %3 = trunc i256 %2 to i16
   ret i16 %3
 }
 
 ; Function Attrs: nounwind readnone speculatable willreturn
 declare i256 @llvm.ctpop.i256(i256)
 
 define <1 x i128> @popcount1x128(<1 x i128> %0) {
 ; CHECK-LABEL: popcount1x128:
 ; CHECK:       # %bb.0: # %Entry
-; CHECK-NEXT:    xxlor 0, 34, 34
-; CHECK-NEXT:    # kill: def $f0 killed $f0 killed $vsl0
-; CHECK-NEXT:    mffprd 3, 0
-; CHECK-NEXT:    popcntd 3, 3
-; CHECK-NEXT:    xxswapd 0, 34
-; CHECK-NEXT:    # kill: def $f0 killed $f0 killed $vsl0
-; CHECK-NEXT:    mffprd 4, 0
-; CHECK-NEXT:    popcntd 4, 4
-; CHECK-NEXT:    add 3, 4, 3
-; CHECK-NEXT:    mtfprd 0, 3
-; CHECK-NEXT:    # kill: def $vsl0 killed $f0
-; CHECK-NEXT:    li 3, 0
-; CHECK-NEXT:    mtfprd 1, 3
-; CHECK-NEXT:    # kill: def $vsl1 killed $f1
-; CHECK-NEXT:    xxmrghd 34, 1, 0
+; CHECK-NEXT:    vpopcntd 2, 2
+; CHECK-NEXT:    vaddudm 2, 2, 2
 ; CHECK-NEXT:    blr
 Entry:
   %1 = tail call <1 x i128> @llvm.ctpop.v1.i128(<1 x i128> %0)
   ret <1 x i128> %1
 }
 
 declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>)