[AArch64][SVE] Create AArch64ISD node for DUPQLANE128

[MattDevereau]

Summary

Create an AArch64ISD node instead of emitting machine node DUP_ZZI_Q. This allows a simpler DAG combine for work previously attempted in https://reviews.llvm.org/D128503

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.h

162

Given the existing naming perhaps DUPLANE128 is more consistent?

llvm/lib/Target/AArch64/AArch64InstrGISel.td

114–118

Do you need any of the AArch64InstrGISel.td changes? I suspect not.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

283

There's nothing different about SDT_AArch64DupQLane compared to SDT_AArch64DupLane so you can just reuse SDT_AArch64DupLane?

[MattDevereau]

removed redundant changes and made naming more consistent with previously added patterns

[paulwalker-arm]

A potential code placement improvement but otherwise looks good.

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

675–690

Sorry I didn't have access to code or would have suggested this before but I think these can be moved into the multiclass that defines sve_int_perm_dup_i. I can see it already has patterns for the B,H,S and D forms.

[MattDevereau]

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

675–690

Moving them there works fine, I'll push that change without updating the phabricator review

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
 
   // Scalar-to-vector duplication
   DUP,
   DUPLANE8,
   DUPLANE16,
   DUPLANE32,
   DUPLANE64,
 
+  DUPQLANE,
+
   // Vector immedate moves
   MOVI,
   MOVIshift,
   MOVIedit,
   MOVImsl,
   FMOV,
   MVNIshift,
   MVNImsl,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::FCMP)
     MAKE_CASE(AArch64ISD::STRICT_FCMP)
     MAKE_CASE(AArch64ISD::STRICT_FCMPE)
     MAKE_CASE(AArch64ISD::DUP)
     MAKE_CASE(AArch64ISD::DUPLANE8)
     MAKE_CASE(AArch64ISD::DUPLANE16)
     MAKE_CASE(AArch64ISD::DUPLANE32)
     MAKE_CASE(AArch64ISD::DUPLANE64)
+    MAKE_CASE(AArch64ISD::DUPQLANE)
     MAKE_CASE(AArch64ISD::MOVI)
     MAKE_CASE(AArch64ISD::MOVIshift)
     MAKE_CASE(AArch64ISD::MOVIedit)
     MAKE_CASE(AArch64ISD::MOVImsl)
     MAKE_CASE(AArch64ISD::FMOV)
     MAKE_CASE(AArch64ISD::MVNIshift)
     MAKE_CASE(AArch64ISD::MVNImsl)
     MAKE_CASE(AArch64ISD::BICi)
SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
   if (!isTypeLegal(VT) || !VT.isScalableVector())
     return SDValue();
 
   // Current lowering only supports the SVE-ACLE types.
   if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock)
     return SDValue();
 
   // The DUPQ operation is indepedent of element type so normalise to i64s.
-  SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
   SDValue Idx128 = Op.getOperand(2);
 
   // DUPQ can be used when idx is in range.
   auto *CIdx = dyn_cast<ConstantSDNode>(Idx128);
   if (CIdx && (CIdx->getZExtValue() <= 3)) {
     SDValue CI = DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i64);
-    SDNode *DUPQ =
-        DAG.getMachineNode(AArch64::DUP_ZZI_Q, DL, MVT::nxv2i64, V, CI);
-    return DAG.getNode(ISD::BITCAST, DL, VT, SDValue(DUPQ, 0));
+    return DAG.getNode(AArch64ISD::DUPQLANE, DL, VT, Op.getOperand(1), CI);
   }
 
+  SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
+
   // The ACLE says this must produce the same result as:
   //   svtbl(data, svadd_x(svptrue_b64(),
   //                       svand_x(svptrue_b64(), svindex_u64(0, 1), 1),
   //                       index * 2))
   SDValue One = DAG.getConstant(1, DL, MVT::i64);
   SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One);
 
   // create the vector 0,1,0,1,...

llvm/lib/Target/AArch64/AArch64InstrGISel.td

def G_DUPLANE32 : AArch64GenericInstruction {
   let hasSideEffects = 0;
 }
 def G_DUPLANE64 : AArch64GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins type0:$src, type1:$lane);
   let hasSideEffects = 0;
 }
 
+def G_DUPQLANE : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src, type1:$quadword);
+  let hasSideEffects = 0;
+}
+
 // Represents a trn1 instruction. Produced post-legalization from
 // G_SHUFFLE_VECTORs with appropriate masks.
 def G_TRN1 : AArch64GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins type0:$v1, type0:$v2);
   let hasSideEffects = 0;
 }
 

 def : GINodeEquiv<G_UZP2, AArch64uzp2>;
 def : GINodeEquiv<G_ZIP1, AArch64zip1>;
 def : GINodeEquiv<G_ZIP2, AArch64zip2>;
 def : GINodeEquiv<G_DUP, AArch64dup>;
 def : GINodeEquiv<G_DUPLANE8, AArch64duplane8>;
 def : GINodeEquiv<G_DUPLANE16, AArch64duplane16>;
 def : GINodeEquiv<G_DUPLANE32, AArch64duplane32>;
 def : GINodeEquiv<G_DUPLANE64, AArch64duplane64>;
+def : GINodeEquiv<G_DUPQLANE, AArch64dupqlane>;
 def : GINodeEquiv<G_TRN1, AArch64trn1>;
 def : GINodeEquiv<G_TRN2, AArch64trn2>;
 def : GINodeEquiv<G_EXT, AArch64ext>;
 def : GINodeEquiv<G_VASHR, AArch64vashr>;
 def : GINodeEquiv<G_VLSHR, AArch64vlshr>;
 def : GINodeEquiv<G_SITOF, AArch64sitof>;
 def : GINodeEquiv<G_UITOF, AArch64uitof>;
 

llvm/lib/Target/AArch64/AArch64InstrInfo.td

def SDT_AArch64FCCMP : SDTypeProfile<1, 5,
                                       SDTCisInt<3>,
                                       SDTCisInt<4>,
                                       SDTCisVT<5, i32>]>;
 def SDT_AArch64FCmp   : SDTypeProfile<0, 2,
                                    [SDTCisFP<0>,
                                     SDTCisSameAs<0, 1>]>;
 def SDT_AArch64Dup   : SDTypeProfile<1, 1, [SDTCisVec<0>]>;
 def SDT_AArch64DupLane   : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<2>]>;
+def SDT_AArch64DupQLane   : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<2>]>;
 def SDT_AArch64Insr  : SDTypeProfile<1, 2, [SDTCisVec<0>]>;
 def SDT_AArch64Zip   : SDTypeProfile<1, 2, [SDTCisVec<0>,
                                           SDTCisSameAs<0, 1>,
                                           SDTCisSameAs<0, 2>]>;
 def SDT_AArch64MOVIedit : SDTypeProfile<1, 1, [SDTCisInt<1>]>;
 def SDT_AArch64MOVIshift : SDTypeProfile<1, 2, [SDTCisInt<1>, SDTCisInt<2>]>;
 def SDT_AArch64vecimm : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                            SDTCisInt<2>, SDTCisInt<3>]>;
def AArch64any_fcmp     : PatFrags<(ops node:$lhs, node:$rhs),
                                     (AArch64fcmp node:$lhs, node:$rhs)]>;
 
 def AArch64dup       : SDNode<"AArch64ISD::DUP", SDT_AArch64Dup>;
 def AArch64duplane8  : SDNode<"AArch64ISD::DUPLANE8", SDT_AArch64DupLane>;
 def AArch64duplane16 : SDNode<"AArch64ISD::DUPLANE16", SDT_AArch64DupLane>;
 def AArch64duplane32 : SDNode<"AArch64ISD::DUPLANE32", SDT_AArch64DupLane>;
 def AArch64duplane64 : SDNode<"AArch64ISD::DUPLANE64", SDT_AArch64DupLane>;
 
+def AArch64dupqlane  : SDNode<"AArch64ISD::DUPQLANE", SDT_AArch64DupQLane>;
+
 def AArch64insr      : SDNode<"AArch64ISD::INSR", SDT_AArch64Insr>;
 
 def AArch64zip1      : SDNode<"AArch64ISD::ZIP1", SDT_AArch64Zip>;
 def AArch64zip2      : SDNode<"AArch64ISD::ZIP2", SDT_AArch64Zip>;
 def AArch64uzp1      : SDNode<"AArch64ISD::UZP1", SDT_AArch64Zip>;
 def AArch64uzp2      : SDNode<"AArch64ISD::UZP2", SDT_AArch64Zip>;
 def AArch64trn1      : SDNode<"AArch64ISD::TRN1", SDT_AArch64Zip>;
 def AArch64trn2      : SDNode<"AArch64ISD::TRN2", SDT_AArch64Zip>;

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

def : Pat<(nxv2f64 (splat_vector (f64 FPR64:$src))),
             (DUP_ZZI_D (INSERT_SUBREG (IMPLICIT_DEF), FPR64:$src, dsub), 0)>;
   def : Pat<(nxv8bf16 (splat_vector (bf16 FPR16:$src))),
             (DUP_ZZI_H (INSERT_SUBREG (IMPLICIT_DEF), FPR16:$src, hsub), 0)>;
   def : Pat<(nxv4bf16 (splat_vector (bf16 FPR16:$src))),
             (DUP_ZZI_H (INSERT_SUBREG (IMPLICIT_DEF), FPR16:$src, hsub), 0)>;
   def : Pat<(nxv2bf16 (splat_vector (bf16 FPR16:$src))),
             (DUP_ZZI_H (INSERT_SUBREG (IMPLICIT_DEF), FPR16:$src, hsub), 0)>;
 
+  def : Pat<(nxv16i8 (AArch64dupqlane nxv16i8:$Op1, i64:$imm)),
+            (nxv16i8 (DUP_ZZI_Q $Op1, $imm))>;
+  def : Pat<(nxv8i16 (AArch64dupqlane nxv8i16:$Op1, i64:$imm)),
+            (nxv8i16 (DUP_ZZI_Q $Op1, $imm))>;
+  def : Pat<(nxv4i32 (AArch64dupqlane nxv4i32:$Op1, i64:$imm)),
+            (nxv4i32 (DUP_ZZI_Q $Op1, $imm))>;
+  def : Pat<(nxv2i64 (AArch64dupqlane nxv2i64:$Op1, i64:$imm)),
+            (nxv2i64 (DUP_ZZI_Q $Op1, $imm))>;
+  def : Pat<(nxv8f16 (AArch64dupqlane nxv8f16:$Op1, i64:$imm)),
+            (nxv8f16 (DUP_ZZI_Q $Op1, $imm))>;
+  def : Pat<(nxv4f32 (AArch64dupqlane nxv4f32:$Op1, i64:$imm)),
+            (nxv4f32 (DUP_ZZI_Q $Op1, $imm))>;
+  def : Pat<(nxv2f64 (AArch64dupqlane nxv2f64:$Op1, i64:$imm)),
+            (nxv2f64 (DUP_ZZI_Q $Op1, $imm))>;
+  def : Pat<(nxv8bf16 (AArch64dupqlane nxv8bf16:$Op1, i64:$imm)),
+            (nxv8bf16 (DUP_ZZI_Q $Op1, $imm))>;
+
   // Duplicate +0.0 into all vector elements
   def : Pat<(nxv8f16 (splat_vector (f16 fpimm0))), (DUP_ZI_H 0, 0)>;
   def : Pat<(nxv4f16 (splat_vector (f16 fpimm0))), (DUP_ZI_H 0, 0)>;
   def : Pat<(nxv2f16 (splat_vector (f16 fpimm0))), (DUP_ZI_H 0, 0)>;
   def : Pat<(nxv4f32 (splat_vector (f32 fpimm0))), (DUP_ZI_S 0, 0)>;
   def : Pat<(nxv2f32 (splat_vector (f32 fpimm0))), (DUP_ZI_S 0, 0)>;
   def : Pat<(nxv2f64 (splat_vector (f64 fpimm0))), (DUP_ZI_D 0, 0)>;
   def : Pat<(nxv8bf16 (splat_vector (bf16 fpimm0))), (DUP_ZI_H 0, 0)>;