[AArch64][SVE] Improve sve.convert.to.svbool lowering

[peterwaller-arm]

Summary

The sve.convert.to.svbool lowering has the effect of widening a logical
<M x i1> vector representing lanes into a physical <16 x i1> vector
representing bits in a predicate register.

In general, if converting to svbool, the contents of lanes in the
physical register might not be known. For sve.convert.to.svbool the new
lanes are specified to be zeroed, requiring 'and' instructions to mask
off the new lanes. For lanes coming from a ptrue or a comparison,
however, they are known to be zero.

CodeGen Before:

ptrue p0.s, vl16
ptrue p1.s
ptrue p2.b
and   p0.b, p2/z, p0.b, p1.b
ret

After:

ptrue	p0.s, vl16
ret

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

3684

This is used five times within the function so perhaps worth having SDValue InOp?

3701

Can you use getConstantOperandVal here?

llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll

87

If this required? Same comment for reinterpret_cmpgt.

88

I think you can be more precise here as we are expecting ptrue p0.h

99

Is this required? I would have thought we can check for the precise output. If you pass the mask as a parameter I think we'd effectively have:

CHECK: cmpgt p0/z, z0.h, z1.h
CHECK-NEXT: ret

In fact there's a good shout here for just using llvm/utils/update_llc_test_checks.py to generate the check lines for this file.

[peterwaller-arm]

• Address review comments.
• Use update_llc_test_checks to generate tests.

[peterwaller-arm]

Push updates missed out due to an error of mine on the last run.

• Address review comments.
• Use update_llc_test_checks to generate tests.

[peterwaller-arm]

Fix wrong indentation.

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll

86

Personally I see this as a function comment rather than belonging with the CHECK lines.

103

You can simplify the test by passing cmpgt's predicate as a function parameter.

[peterwaller-arm]

Update for review comments.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 }
 
 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
                                int Pattern) {
   return DAG.getNode(AArch64ISD::PTRUE, DL, VT,
                      DAG.getTargetConstant(Pattern, DL, MVT::i32));
 }
 
+static SDValue lowerConvertToSVBool(SDValue Op, SelectionDAG &DAG) {
+    SDLoc DL(Op);
+    EVT OutVT = Op.getValueType();
+    SDValue InOp = Op.getOperand(1);
+    EVT InVT = InOp.getValueType();
+
+    // Return the operand if the cast isn't changing type,
+    // i.e. <n x 16 x i1> -> <n x 16 x i1>
+    if (InVT == OutVT)
+      return InOp;
+
+    SDValue Reinterpret =
+        DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, OutVT, InOp);
+
+    // If the argument converted to an svbool is a ptrue or a comparison, the
+    // lanes introduced by the widening are zero by construction.
+    switch (InOp.getOpcode()) {
+      case AArch64ISD::SETCC_MERGE_ZERO:
+        return Reinterpret;
+      case ISD::INTRINSIC_WO_CHAIN:
+        if (InOp.getConstantOperandVal(0) == Intrinsic::aarch64_sve_ptrue)
+          return Reinterpret;
+    }
+
+    // Otherwise, zero the newly introduced lanes.
+    SDValue Mask = getPTrue(DAG, DL, InVT, AArch64SVEPredPattern::all);
+    SDValue MaskReinterpret =
+        DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, OutVT, Mask);
+    return DAG.getNode(ISD::AND, DL, OutVT, Reinterpret, MaskReinterpret);
+}
+
 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                      SelectionDAG &DAG) const {
   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   SDLoc dl(Op);
   switch (IntNo) {
   default: return SDValue();    // Don't custom lower most intrinsics.
   case Intrinsic::thread_pointer: {
     EVT PtrVT = getPointerTy(DAG.getDataLayout());
case Intrinsic::aarch64_sve_cnt: {
     return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Data, Op.getOperand(1));
   }
   case Intrinsic::aarch64_sve_dupq_lane:
     return LowerDUPQLane(Op, DAG);
   case Intrinsic::aarch64_sve_convert_from_svbool:
     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, Op.getValueType(),
                        Op.getOperand(1));
+  case Intrinsic::aarch64_sve_convert_to_svbool:
+    return lowerConvertToSVBool(Op, DAG);
   case Intrinsic::aarch64_sve_fneg:
     return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintp:
     return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintm:
     return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, dl, Op.getValueType(),
case Intrinsic::aarch64_sve_fabs:
     return DAG.getNode(AArch64ISD::FABS_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_abs:
     return DAG.getNode(AArch64ISD::ABS_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_neg:
     return DAG.getNode(AArch64ISD::NEG_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
-  case Intrinsic::aarch64_sve_convert_to_svbool: {
-    EVT OutVT = Op.getValueType();
-    EVT InVT = Op.getOperand(1).getValueType();
-    // Return the operand if the cast isn't changing type,
-    // i.e. <n x 16 x i1> -> <n x 16 x i1>
-    if (InVT == OutVT)
-      return Op.getOperand(1);
-    // Otherwise, zero the newly introduced lanes.
-    SDValue Reinterpret =
-        DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, OutVT, Op.getOperand(1));
-    SDValue Mask = getPTrue(DAG, dl, InVT, AArch64SVEPredPattern::all);
-    SDValue MaskReinterpret =
-        DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, OutVT, Mask);
-    return DAG.getNode(ISD::AND, dl, OutVT, Reinterpret, MaskReinterpret);
-  }
-
   case Intrinsic::aarch64_sve_insr: {
     SDValue Scalar = Op.getOperand(2);
     EVT ScalarTy = Scalar.getValueType();
     if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
       Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
 
     return DAG.getNode(AArch64ISD::INSR, dl, Op.getValueType(),
                        Op.getOperand(1), Scalar);

llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 ;
 ; Converting to svbool_t (<vscale x 16 x i1>)
 ;
 
 define <vscale x 16 x i1> @reinterpret_bool_from_b(<vscale x 16 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_from_b:
-; CHECK: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv16i1(<vscale x 16 x i1> %pg)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @reinterpret_bool_from_h(<vscale x 8 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_from_h:
-; CHECK: ptrue p1.h
-; CHECK-NEXT: ptrue p2.b
-; CHECK-NEXT: and p0.b, p2/z, p0.b, p1.b
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    and p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %pg)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @reinterpret_bool_from_s(<vscale x 4 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_from_s:
-; CHECK: ptrue p1.s
-; CHECK-NEXT: ptrue p2.b
-; CHECK-NEXT: and p0.b, p2/z, p0.b, p1.b
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    and p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %pg)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @reinterpret_bool_from_d(<vscale x 2 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_from_d:
-; CHECK: ptrue p1.d
-; CHECK-NEXT: ptrue p2.b
-; CHECK-NEXT: and p0.b, p2/z, p0.b, p1.b
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    and p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %pg)
   ret <vscale x 16 x i1> %out
 }
 
 ;
 ; Converting from svbool_t
 ;
 
 define <vscale x 16 x i1> @reinterpret_bool_to_b(<vscale x 16 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_to_b:
-; CHECK: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv16i1(<vscale x 16 x i1> %pg)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 8 x i1> @reinterpret_bool_to_h(<vscale x 16 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_to_h:
-; CHECK: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %pg)
   ret <vscale x 8 x i1> %out
 }
 
 define <vscale x 4 x i1> @reinterpret_bool_to_s(<vscale x 16 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_to_s:
-; CHECK: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %pg)
   ret <vscale x 4 x i1> %out
 }
 
 define <vscale x 2 x i1> @reinterpret_bool_to_d(<vscale x 16 x i1> %pg) {
 ; CHECK-LABEL: reinterpret_bool_to_d:
-; CHECK: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> %pg)
   ret <vscale x 2 x i1> %out
 }
 
+define <vscale x 16 x i1> @reinterpret_ptrue() {
+; Reinterpreting a ptrue should not introduce an `and` instruction.
+; CHECK-LABEL: reinterpret_ptrue:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ret
+  %in = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %out = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %in)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @reinterpret_cmpgt(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; Reinterpreting a comparison not introduce an `and` instruction.
+; CHECK-LABEL: reinterpret_cmpgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %1 = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  %2 = tail call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %1, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+  %3 = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %2)
+  ret <vscale x 16 x i1> %3
+}
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 immarg)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv16i1(<vscale x 16 x i1>)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1>)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1>)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1>)
 
 declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv16i1(<vscale x 16 x i1>)
 declare <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1>)
 declare <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1>)
 declare <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1>)