[CodeGen][SVE] Add patterns for whole vector predicate select

[sdesmalen]

Summary

Added patterns to implement select i1 %p, <vty> %a, <vty> %b

[efriedma]

We use whilelo to lower VECTOR_SPLAT; I think that ends up being one fewer vector instruction. What's the tradeoff between that vs. dup+cmpne?

[sdesmalen]

In D79356#2018893, @efriedma wrote:

We use whilelo to lower VECTOR_SPLAT; I think that ends up being one fewer vector instruction. What's the tradeoff between that vs. dup+cmpne?

There's no specific reason to choose dup+cmpne, other than that in these patterns the GPR32 can be any non-zero value, where to use whilelo we'd need a sign-extended i1 value.
I'm happy to change it to use whilelo, but I think it then needs to be implemented in ISelLowering code where we can do analysis on the value of the predicate using computeKnownBits, rather than using patterns. Is that correct?

[efriedma]

It's possible to write a TableGen pattern that explicitly generates an SBFM. In general that's what we would generate anyway. But you'd want to do custom lowering to give DAGCombine a chance to combine it away, yes.

[sdesmalen]

• select now lowers to splat_vector for the predicate (which codegens to whilelo) plus vselect.
• Updated tests and rebased patch.

[efriedma]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   if (Subtarget->hasSVE()) {
     // FIXME: Add custom lowering of MLOAD to handle different passthrus (not a
     // splat of 0 or undef) once vector selects supported in SVE codegen. See
     // D68877 for more details.
     for (MVT VT : MVT::integer_scalable_vector_valuetypes()) {
       if (isTypeLegal(VT)) {
         setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
+        setOperationAction(ISD::SELECT, VT, Custom);
         setOperationAction(ISD::SDIV, VT, Custom);
         setOperationAction(ISD::UDIV, VT, Custom);
         setOperationAction(ISD::SMIN, VT, Custom);
         setOperationAction(ISD::UMIN, VT, Custom);
         setOperationAction(ISD::SMAX, VT, Custom);
         setOperationAction(ISD::UMAX, VT, Custom);
         setOperationAction(ISD::SHL, VT, Custom);
         setOperationAction(ISD::SRL, VT, Custom);
         setOperationAction(ISD::SRA, VT, Custom);
       }
     }
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
 
     for (MVT VT : MVT::fp_scalable_vector_valuetypes()) {
       if (isTypeLegal(VT)) {
         setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
+        setOperationAction(ISD::SELECT, VT, Custom);
       }
     }
   }
 
   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
 }
 
 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {

 
 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDValue CCVal = Op->getOperand(0);
   SDValue TVal = Op->getOperand(1);
   SDValue FVal = Op->getOperand(2);
   SDLoc DL(Op);
 
+  EVT Ty = Op.getValueType();
+  if (Ty.isScalableVector()) {
+    SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, CCVal);
+    MVT PredVT = MVT::getVectorVT(MVT::i1, Ty.getVectorElementCount());
+    SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, TruncCC);
+    return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal);
+  }
+
   // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select
   // instruction.
   if (ISD::isOverflowIntrOpRes(CCVal)) {
     // Only lower legal XALUO ops.
     if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0)))
       return SDValue();
 
     AArch64CC::CondCode OFCC;

llvm/test/CodeGen/AArch64/select-sve.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -verify-machineinstrs < %s | FileCheck %s
+
+define <vscale x  16 x i8> @select_nxv16i8(i1 %cond, <vscale x  16 x i8> %a, <vscale x  16 x i8> %b) {
+; CHECK-LABEL: select_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p0.b, xzr, x8
+; CHECK-NEXT:    sel z0.b, p0, z0.b, z1.b
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  16 x i8> %a, <vscale x  16 x i8> %b
+  ret <vscale x  16 x i8> %res
+}
+
+define <vscale x  8 x i16> @select_nxv8i16(i1 %cond, <vscale x  8 x i16> %a, <vscale x  8 x i16> %b) {
+; CHECK-LABEL: select_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p0.h, xzr, x8
+; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  8 x i16> %a, <vscale x  8 x i16> %b
+  ret <vscale x  8 x i16> %res
+}
+
+define <vscale x  4 x i32> @select_nxv4i32(i1 %cond, <vscale x  4 x i32> %a, <vscale x  4 x i32> %b) {
+; CHECK-LABEL: select_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p0.s, xzr, x8
+; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  4 x i32> %a, <vscale x  4 x i32> %b
+  ret <vscale x  4 x i32> %res
+}
+
+define <vscale x  2 x i64> @select_nxv2i64(i1 %cond, <vscale x  2 x i64> %a, <vscale x  2 x i64> %b) {
+; CHECK-LABEL: select_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p0.d, xzr, x8
+; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  2 x i64> %a, <vscale x  2 x i64> %b
+  ret <vscale x  2 x i64> %res
+}
+
+define <vscale x  8 x half> @select_nxv8f16(i1 %cond, <vscale x  8 x half> %a, <vscale x  8 x half> %b) {
+; CHECK-LABEL: select_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p0.h, xzr, x8
+; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  8 x half> %a, <vscale x  8 x half> %b
+  ret <vscale x  8 x half> %res
+}
+
+define <vscale x  4 x float> @select_nxv4f32(i1 %cond, <vscale x  4 x float> %a, <vscale x  4 x float> %b) {
+; CHECK-LABEL: select_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p0.s, xzr, x8
+; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  4 x float> %a, <vscale x  4 x float> %b
+  ret <vscale x  4 x float> %res
+}
+
+define <vscale x  2 x double> @select_nxv2f64(i1 %cond, <vscale x  2 x double> %a, <vscale x  2 x double> %b) {
+; CHECK-LABEL: select_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p0.d, xzr, x8
+; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  2 x double> %a, <vscale x  2 x double> %b
+  ret <vscale x  2 x double> %res
+}
+
+define <vscale x  16 x i1> @select_nxv16i1(i1 %cond, <vscale x  16 x i1> %a, <vscale x  16 x i1> %b) {
+; CHECK-LABEL: select_nxv16i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p2.b, xzr, x8
+; CHECK-NEXT:    sel p0.b, p2, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  16 x i1> %a, <vscale x  16 x i1> %b
+  ret <vscale x  16 x i1> %res
+}
+
+define <vscale x  8 x i1> @select_nxv8i1(i1 %cond, <vscale x  8 x i1> %a, <vscale x  8 x i1> %b) {
+; CHECK-LABEL: select_nxv8i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p2.h, xzr, x8
+; CHECK-NEXT:    sel p0.b, p2, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  8 x i1> %a, <vscale x  8 x i1> %b
+  ret <vscale x  8 x i1> %res
+}
+
+define <vscale x  4 x i1> @select_nxv4i1(i1 %cond, <vscale x  4 x i1> %a, <vscale x  4 x i1> %b) {
+; CHECK-LABEL: select_nxv4i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p2.s, xzr, x8
+; CHECK-NEXT:    sel p0.b, p2, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  4 x i1> %a, <vscale x  4 x i1> %b
+  ret <vscale x  4 x i1> %res
+}
+
+define <vscale x  2 x i1> @select_nxv2i1(i1 %cond, <vscale x  2 x i1> %a, <vscale x  2 x i1> %b) {
+; CHECK-LABEL: select_nxv2i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sbfx x8, x0, #0, #1
+; CHECK-NEXT:    whilelo p2.d, xzr, x8
+; CHECK-NEXT:    sel p0.b, p2, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = select i1 %cond, <vscale x  2 x i1> %a, <vscale x  2 x i1> %b
+  ret <vscale x  2 x i1> %res
+}