[PowerPC] Exploit test data class instruction for isinf/iszero

[qiucf]

Summary

Since ISA 3.0, we have test data class instruction (x(s|v)tstdc(s|d|q)p) to classify a floating point value, which can be used to combine common FP classify operations like isinf or zero comparison.

[qiucf]

Ping

[shchenz]

Thanks for working on this, I think it should be workable. However seems adding a new PPCISD would be preferable, like PPCISD::FP_CLASS? You can find same handling in AMDGPU arch FP_CLASS node.
With this new ISD:
1: we can leverage the table-gen to select the instruction.
2: we can do further combine
3: maybe this node can also be used for the fp class intrinsics like int_ppc_test_data_class_f and int_ppc_test_data_class_d?

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

14145

Can we make these enums be ordered?

14159

Any reason we don't handle NAN and denormal inputs?

[qiucf]

Will change to a combine to IS_FPCLASS (based on D140381)

[qiucf]

Rebased on D140381

[shchenz]

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

14353

I have same concern as previous patch: any reason for the nan and negdenormal/posdenormal are excluded here?

[qiucf]

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

14353

isnan uses cond code SETUO against arbitary non-nan number, whose codegen is also a single fcmpu.

C std or LLVM intrinsics don't have issubnormal/isdenormal. For isnormal, the pattern is (abs(x) uge CONSTANT1) and (abs(x) olt CONSTANT2). Optimizing it in backend may be profitable, but (1) is out of scope of this patch; (2) clang will directly generate is_fpclass call in the future

[qiucf]

Gentle ping...

[nemanjai]

Actually, now that I've looked over the whole patch, it is not clear to me why this is in the PPC back end? There doesn't seem to be any PPC-specific requirements here. Why is it not OK to put the combine in DAGCombiner.cpp and if we don't want the combine to fire on all targets/subtargets, we can just add TargetLoweringInfo::shouldCombineToIsFPClass().

I've requested changes to require that either this be moved to target independent code or a justification be provided as to why this is PPC specific.

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

14337

Nit: name this something like IsFABS or IsAbsVal (my preference would be the first one).

[qiucf]

I'd like to abandon this one since frontend change will eliminate this problem.

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

       DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst),
       ShiftCst);
 }
 
 SDValue PPCTargetLowering::combineSetCC(SDNode *N,
                                         DAGCombinerInfo &DCI) const {
   assert(N->getOpcode() == ISD::SETCC &&
          "Should be called with a SETCC node");
+  EVT VT = N->getValueType(0);
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+  EVT OpVT = LHS.getValueType();
+  SDLoc DL(N);
+  SelectionDAG &DAG = DCI.DAG;
 
   ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
   if (CC == ISD::SETNE || CC == ISD::SETEQ) {
-    SDValue LHS = N->getOperand(0);
-    SDValue RHS = N->getOperand(1);
 
     // If there is a '0 - y' pattern, canonicalize the pattern to the RHS.
     if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) &&
         LHS.hasOneUse())
       std::swap(LHS, RHS);
 
     // x == 0-y --> x+y == 0
     // x != 0-y --> x+y != 0
     if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) &&
         RHS.hasOneUse()) {
-      SDLoc DL(N);
-      SelectionDAG &DAG = DCI.DAG;
-      EVT VT = N->getValueType(0);
       EVT OpVT = LHS.getValueType();
       SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1));
       return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC);
     }
+  } else if (CC == ISD::SETOEQ && Subtarget.hasP9Vector()) {
+    unsigned Opcode = 0;
+    if (OpVT == MVT::f128)
+      Opcode = PPC::XSTSTDCQP;
+    else if (OpVT.getScalarType() == MVT::f64)
+      Opcode = OpVT.isVector() ? PPC::XVTSTDCDP : PPC::XSTSTDCDP;
+    else
+      Opcode = OpVT.isVector() ? PPC::XVTSTDCSP : PPC::XSTSTDCSP;
+
+    // Masks defined for ISA 3.0 test data class instructions.
+    enum DataClassMask {
+      DC_NAN = 1 << 6,
+      DC_NEG_INF = 1 << 4,
+      DC_POS_INF = 1 << 5,
+      DC_NEG_ZERO = 1 << 2,
+      DC_POS_ZERO = 1 << 3,
+      DC_NEG_SUBNORM = 1,
+      DC_POS_SUBNORM = 1 << 1,
+    };
+
+    auto GetMask = [](const SDValue &Op, bool IsAbsolute) -> unsigned {
+      unsigned Mask = 0;
+      if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Op.getNode())) {
+        const APFloat &APF = CFP->getValueAPF();
+        if (APF.isInfinity() && !APF.isNegative())
+          Mask = IsAbsolute ? (DC_POS_INF | DC_NEG_INF) : DC_POS_INF;
+        else if (APF.isInfinity() && APF.isNegative())
+          Mask = IsAbsolute ? 0 : DC_NEG_INF;
+        else if (APF.isPosZero())
+          Mask = IsAbsolute ? (DC_POS_ZERO | DC_NEG_ZERO) : DC_POS_ZERO;
+        else if (APF.isNegZero())
+          Mask = IsAbsolute ? 0 : DC_NEG_ZERO;
+      }
+      return Mask;
+    };
+
+    SDValue TestOp = LHS;
+    bool IsAbsolute = LHS.getOpcode() == ISD::FABS;
+    if (IsAbsolute)
+      TestOp = LHS.getOperand(0);
+    unsigned Mask = 0;
+    if (RHS.getOpcode() == ISD::BUILD_VECTOR && DAG.isSplatValue(RHS, true)) {
+      Mask = GetMask(RHS.getOperand(0), IsAbsolute);
+    } else if (!OpVT.isVector()) {
+      Mask = GetMask(RHS, IsAbsolute);
+    }
+
+    if (Opcode != 0 && Mask != 0) {
+      SDValue TestClass{
+          DAG.getMachineNode(Opcode, DL,
+                             OpVT == MVT::v2f64
+                                 ? MVT::v2i64
+                                 : (OpVT == MVT::v4f32 ? MVT::v4i32 : MVT::i32),
+                             DAG.getTargetConstant(Mask, DL, MVT::i32), TestOp),
+          0};
+      if (OpVT.isVector())
+        return TestClass;
+      return SDValue(
+          DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, VT, TestClass,
+                             DAG.getTargetConstant(PPC::sub_eq, DL, MVT::i32)),
+          0);
+    }
   }
 
   return DAGCombineTruncBoolExt(N, DCI);
 }
 
 // Is this an extending load from an f32 to an f64?
 static bool isFPExtLoad(SDValue Op) {
   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode()))

       Opcode = ISD::FP_TO_SINT;
     else
       Opcode = ISD::FP_TO_UINT;
 
     EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32;
     SDValue BV = DAG.getBuildVector(NewVT, dl, Ops);
     return DAG.getNode(Opcode, dl, TargetVT, BV);
   }
   return SDValue();

llvm/test/CodeGen/PowerPC/fp-classify.ll

 ; P8-NEXT:    lfs 1, .LCPI0_0@toc@l(3)
 ; P8-NEXT:    li 3, 0
 ; P8-NEXT:    fcmpu 0, 0, 1
 ; P8-NEXT:    iseleq 3, 4, 3
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: abs_isinff:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
-; P9-NEXT:    xsabsdp 0, 1
-; P9-NEXT:    li 4, 1
-; P9-NEXT:    lfs 1, .LCPI0_0@toc@l(3)
+; P9-NEXT:    xststdcsp 0, 1, 48
 ; P9-NEXT:    li 3, 0
-; P9-NEXT:    fcmpu 0, 0, 1
+; P9-NEXT:    li 4, 1
 ; P9-NEXT:    iseleq 3, 4, 3
 ; P9-NEXT:    blr
 entry:
   %0 = tail call float @llvm.fabs.f32(float %x)
   %cmpinf = fcmp oeq float %0, 0x7FF0000000000000
   ret i1 %cmpinf
 }
 
 define zeroext i1 @abs_isinf(double %x) {
 ; P8-LABEL: abs_isinf:
 ; P8:       # %bb.0: # %entry
 ; P8-NEXT:    xsabsdp 0, 1
 ; P8-NEXT:    addis 3, 2, .LCPI1_0@toc@ha
 ; P8-NEXT:    li 4, 1
 ; P8-NEXT:    lfs 1, .LCPI1_0@toc@l(3)
 ; P8-NEXT:    li 3, 0
 ; P8-NEXT:    fcmpu 0, 0, 1
 ; P8-NEXT:    iseleq 3, 4, 3
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: abs_isinf:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    addis 3, 2, .LCPI1_0@toc@ha
-; P9-NEXT:    xsabsdp 0, 1
-; P9-NEXT:    li 4, 1
-; P9-NEXT:    lfs 1, .LCPI1_0@toc@l(3)
+; P9-NEXT:    xststdcdp 0, 1, 48
 ; P9-NEXT:    li 3, 0
-; P9-NEXT:    fcmpu 0, 0, 1
+; P9-NEXT:    li 4, 1
 ; P9-NEXT:    iseleq 3, 4, 3
 ; P9-NEXT:    blr
 entry:
   %0 = tail call double @llvm.fabs.f64(double %x)
   %cmpinf = fcmp oeq double %0, 0x7FF0000000000000
   ret i1 %cmpinf
 }
 

 ; P8-NEXT:    srwi 3, 3, 5
 ; P8-NEXT:    addi 1, 1, 48
 ; P8-NEXT:    ld 0, 16(1)
 ; P8-NEXT:    mtlr 0
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: abs_isinfq:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
-; P9-NEXT:    xsabsqp 2, 2
-; P9-NEXT:    li 4, 1
-; P9-NEXT:    addi 3, 3, .LCPI2_0@toc@l
-; P9-NEXT:    lxv 35, 0(3)
+; P9-NEXT:    xststdcqp 0, 2, 48
 ; P9-NEXT:    li 3, 0
-; P9-NEXT:    xscmpuqp 0, 2, 3
+; P9-NEXT:    li 4, 1
 ; P9-NEXT:    iseleq 3, 4, 3
 ; P9-NEXT:    blr
 entry:
   %0 = tail call fp128 @llvm.fabs.f128(fp128 %x)
   %cmpinf = fcmp oeq fp128 %0, 0xL00000000000000007FFF000000000000
   ret i1 %cmpinf
 }
 
 define <4 x i1> @abs_isinfv4f32(<4 x float> %x) {
 ; P8-LABEL: abs_isinfv4f32:
 ; P8:       # %bb.0: # %entry
 ; P8-NEXT:    addis 3, 2, .LCPI3_0@toc@ha
 ; P8-NEXT:    xvabssp 0, 34
 ; P8-NEXT:    addi 3, 3, .LCPI3_0@toc@l
 ; P8-NEXT:    lxvd2x 1, 0, 3
 ; P8-NEXT:    xxswapd 1, 1
 ; P8-NEXT:    xvcmpeqsp 34, 0, 1
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: abs_isinfv4f32:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    addis 3, 2, .LCPI3_0@toc@ha
-; P9-NEXT:    xvabssp 0, 34
-; P9-NEXT:    addi 3, 3, .LCPI3_0@toc@l
-; P9-NEXT:    lxv 1, 0(3)
-; P9-NEXT:    xvcmpeqsp 34, 0, 1
+; P9-NEXT:    xvtstdcsp 34, 34, 48
 ; P9-NEXT:    blr
 entry:
   %0 = tail call <4 x float> @llvm.fabs.v4f32(<4 x float> %x)
   %cmpinf = fcmp oeq <4 x float> %0, <float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000>
   ret <4 x i1> %cmpinf
 }
 
 define <2 x i1> @abs_isinfv2f64(<2 x double> %x) {
 ; P8-LABEL: abs_isinfv2f64:
 ; P8:       # %bb.0: # %entry
 ; P8-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
 ; P8-NEXT:    xvabsdp 0, 34
 ; P8-NEXT:    addi 3, 3, .LCPI4_0@toc@l
 ; P8-NEXT:    lxvd2x 1, 0, 3
 ; P8-NEXT:    xxswapd 1, 1
 ; P8-NEXT:    xvcmpeqdp 34, 0, 1
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: abs_isinfv2f64:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
-; P9-NEXT:    xvabsdp 0, 34
-; P9-NEXT:    addi 3, 3, .LCPI4_0@toc@l
-; P9-NEXT:    lxv 1, 0(3)
-; P9-NEXT:    xvcmpeqdp 34, 0, 1
+; P9-NEXT:    xvtstdcdp 34, 34, 48
 ; P9-NEXT:    blr
 entry:
   %0 = tail call <2 x double> @llvm.fabs.v2f64(<2 x double> %x)
   %cmpinf = fcmp oeq <2 x double> %0, <double 0x7FF0000000000000, double 0x7FF0000000000000>
   ret <2 x i1> %cmpinf
 }
 
 define zeroext i1 @iszerof(float %x) {
 ; P8-LABEL: iszerof:
 ; P8:       # %bb.0: # %entry
 ; P8-NEXT:    xxlxor 0, 0, 0
 ; P8-NEXT:    li 3, 0
 ; P8-NEXT:    li 4, 1
 ; P8-NEXT:    fcmpu 0, 1, 0
 ; P8-NEXT:    iseleq 3, 4, 3
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: iszerof:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    xxlxor 0, 0, 0
+; P9-NEXT:    xststdcsp 0, 1, 8
 ; P9-NEXT:    li 3, 0
 ; P9-NEXT:    li 4, 1
-; P9-NEXT:    fcmpu 0, 1, 0
 ; P9-NEXT:    iseleq 3, 4, 3
 ; P9-NEXT:    blr
 entry:
   %cmp = fcmp oeq float %x, 0.000000e+00
   ret i1 %cmp
 }
 
 define zeroext i1 @iszero(double %x) {
 ; P8-LABEL: iszero:
 ; P8:       # %bb.0: # %entry
 ; P8-NEXT:    xxlxor 0, 0, 0
 ; P8-NEXT:    li 3, 0
 ; P8-NEXT:    li 4, 1
 ; P8-NEXT:    fcmpu 0, 1, 0
 ; P8-NEXT:    iseleq 3, 4, 3
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: iszero:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    xxlxor 0, 0, 0
+; P9-NEXT:    xststdcdp 0, 1, 8
 ; P9-NEXT:    li 3, 0
 ; P9-NEXT:    li 4, 1
-; P9-NEXT:    fcmpu 0, 1, 0
 ; P9-NEXT:    iseleq 3, 4, 3
 ; P9-NEXT:    blr
 entry:
   %cmp = fcmp oeq double %x, 0.000000e+00
   ret i1 %cmp
 }
 
 define zeroext i1 @iszeroq(fp128 %x) {

 ; P8-NEXT:    srwi 3, 3, 5
 ; P8-NEXT:    addi 1, 1, 32
 ; P8-NEXT:    ld 0, 16(1)
 ; P8-NEXT:    mtlr 0
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: iszeroq:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    addis 3, 2, .LCPI7_0@toc@ha
-; P9-NEXT:    li 4, 1
-; P9-NEXT:    addi 3, 3, .LCPI7_0@toc@l
-; P9-NEXT:    lxv 35, 0(3)
+; P9-NEXT:    xststdcqp 0, 2, 8
 ; P9-NEXT:    li 3, 0
-; P9-NEXT:    xscmpuqp 0, 2, 3
+; P9-NEXT:    li 4, 1
 ; P9-NEXT:    iseleq 3, 4, 3
 ; P9-NEXT:    blr
 entry:
   %cmp = fcmp oeq fp128 %x, 0xL00000000000000000000000000000000
   ret i1 %cmp
 }
 
 define <4 x i1> @iszerov4f32(<4 x float> %x) {
 ; P8-LABEL: iszerov4f32:
 ; P8:       # %bb.0: # %entry
 ; P8-NEXT:    xxlxor 0, 0, 0
 ; P8-NEXT:    xvcmpeqsp 34, 34, 0
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: iszerov4f32:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    xxlxor 0, 0, 0
-; P9-NEXT:    xvcmpeqsp 34, 34, 0
+; P9-NEXT:    xvtstdcsp 34, 34, 8
 ; P9-NEXT:    blr
 entry:
   %cmp = fcmp oeq <4 x float> %x, <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>
   ret <4 x i1> %cmp
 }
 
 define <2 x i1> @iszerov2f64(<2 x double> %x) {
 ; P8-LABEL: iszerov2f64:
 ; P8:       # %bb.0: # %entry
 ; P8-NEXT:    xxlxor 0, 0, 0
 ; P8-NEXT:    xvcmpeqdp 34, 34, 0
 ; P8-NEXT:    blr
 ;
 ; P9-LABEL: iszerov2f64:
 ; P9:       # %bb.0: # %entry
-; P9-NEXT:    xxlxor 0, 0, 0
-; P9-NEXT:    xvcmpeqdp 34, 34, 0
+; P9-NEXT:    xvtstdcdp 34, 34, 8
 ; P9-NEXT:    blr
 entry:
   %cmp = fcmp oeq <2 x double> %x, <double 0.000000e+00, double 0.000000e+00>
   ret <2 x i1> %cmp
 }
 
 declare float @llvm.fabs.f32(float)
 declare double @llvm.fabs.f64(double)
 declare fp128 @llvm.fabs.f128(fp128)
 declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
 declare <2 x double> @llvm.fabs.v2f64(<2 x double>)

llvm/test/CodeGen/PowerPC/is_fpclass.ll

 ; CHECK-NEXT:    blr
   %1 = call i1 @llvm.is.fpclass.f128(fp128 %x, i32 144)  ; 0x90 = "subnormal"
   ret i1 %1
 }
 
 define i1 @iszero_float(float %x) nounwind {
 ; CHECK-LABEL: iszero_float:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xxlxor 0, 0, 0
+; CHECK-NEXT:    xststdcsp 0, 1, 8
 ; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    li 4, 1
-; CHECK-NEXT:    fcmpu 0, 1, 0
 ; CHECK-NEXT:    iseleq 3, 4, 3
 ; CHECK-NEXT:    blr
   %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 96)  ; 0x60 = "zero"
   ret i1 %1
 }
 
 define i1 @iszero_f128(fp128 %x) nounwind {
 ; CHECK-LABEL: iszero_f128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addis 3, 2, .LCPI18_0@toc@ha
-; CHECK-NEXT:    li 4, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI18_0@toc@l
-; CHECK-NEXT:    lxv 35, 0(3)
+; CHECK-NEXT:    xststdcqp 0, 2, 8
 ; CHECK-NEXT:    li 3, 0
-; CHECK-NEXT:    xscmpuqp 0, 2, 3
+; CHECK-NEXT:    li 4, 1
 ; CHECK-NEXT:    iseleq 3, 4, 3
 ; CHECK-NEXT:    blr
   %1 = call i1 @llvm.is.fpclass.f128(fp128 %x, i32 96)  ; 0x60 = "zero"
   ret i1 %1
 }
 
 declare i1 @llvm.is.fpclass.f32(float, i32)
 declare i1 @llvm.is.fpclass.f64(double, i32)
 declare i1 @llvm.is.fpclass.ppcf128(ppc_fp128, i32)
 declare i1 @llvm.is.fpclass.f128(fp128, i32)