[AArch64][SVE] Add DAG-Combine to push bitcasts from floating point loads after DUPLANE128

[MattDevereau]

Summary

This patch lowers

duplane128(insert_subvector(undef, bitcast(op(128bitsubvec)), 0), 0)

to

bitcast(duplane128(insert_subvector(undef, op(128bitsubvec), 0), 0)).

This enables floating-point loads to match patterns added in https://reviews.llvm.org/D130010

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

19212–19213

You can use getConstantOperandVal here.

19217

You need to check this is in fact a ISD::BITCAST before accessing its operands.

19218

I guess this is a legacy from the previous patch but can you pick a different name.

19220

Specially this wants to be is128BitVector() because that's the only fixed length vector type supported for the DUPLANE128 operation.

llvm/test/CodeGen/AArch64/sve-ld1r.ll

822–835

As per our conclusion on the precursor patch, these new tests don't offer any value?

[MattDevereau]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

19217

I'm confused as in the old patch we removed it: https://reviews.llvm.org/D129758?vs=on&id=444647#inline-1248238

llvm/test/CodeGen/AArch64/sve-ld1r.ll

822–835

Correct, i was waiting for the parent revision before update this.

[MattDevereau]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

19217

Never mind, I read that wrong, sorry

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

19226

SubvecVT? Which might have the side effect of fitting on the same line.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static SDValue performBSPExpandForSVE(SDNode *N, SelectionDAG &DAG,
   SDValue In2 = N->getOperand(2);
 
   SDValue InvMask = DAG.getNOT(DL, Mask, VT);
   SDValue Sel = DAG.getNode(ISD::AND, DL, VT, Mask, In1);
   SDValue SelInv = DAG.getNode(ISD::AND, DL, VT, InvMask, In2);
   return DAG.getNode(ISD::OR, DL, VT, Sel, SelInv);
 }
 
+static SDValue performDupLane128Combine(SDNode *N, SelectionDAG &DAG) {
+  EVT VT = N->getValueType(0);
+
+  SDValue Insert = N->getOperand(0);
+  if (Insert.getOpcode() != ISD::INSERT_SUBVECTOR)
+    return SDValue();
+
+  if (!Insert.getOperand(0).isUndef())
+    return SDValue();
+
+  auto IdxInsert = cast<ConstantSDNode>(Insert.getOperand(2))->getZExtValue();
+  auto IdxDupLane = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
+  if (IdxInsert != IdxDupLane)
+    return SDValue();
+
+  SDValue Bitcast = Insert.getOperand(1);
+  SDValue Load = Bitcast.getOperand(0);
+  EVT LoadVT = Load.getValueType();
+  if (!LoadVT.isFixedLengthVector())
+    return SDValue();
+  EVT NewVT = getPackedSVEVectorVT(Load.getValueType().getVectorElementType());
+
+  SDLoc DL(N);
+  SDValue NewInsert =
+      DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, DAG.getUNDEF(NewVT), Load,
+                  Insert->getOperand(2));
+  SDValue NewDuplane128 = DAG.getNode(AArch64ISD::DUPLANE128, DL, NewVT,
+                                      NewInsert, N->getOperand(1));
+  return DAG.getNode(ISD::BITCAST, DL, VT, NewDuplane128);
+}
+
 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   switch (N->getOpcode()) {
   default:
     LLVM_DEBUG(dbgs() << "Custom combining: skipping\n");
     break;
   case ISD::ADD:
case AArch64ISD::BRCOND:
     return performBRCONDCombine(N, DCI, DAG);
   case AArch64ISD::TBNZ:
   case AArch64ISD::TBZ:
     return performTBZCombine(N, DCI, DAG);
   case AArch64ISD::CSEL:
     return performCSELCombine(N, DCI, DAG);
   case AArch64ISD::DUP:
     return performDUPCombine(N, DCI);
+  case AArch64ISD::DUPLANE128:
+    return performDupLane128Combine(N, DAG);
   case AArch64ISD::NVCAST:
     return performNVCASTCombine(N);
   case AArch64ISD::SPLICE:
     return performSpliceCombine(N, DAG);
   case AArch64ISD::UUNPKLO:
   case AArch64ISD::UUNPKHI:
     return performUnpackCombine(N, DAG);
   case AArch64ISD::UZP1:

llvm/test/CodeGen/AArch64/sve-ld1r.ll

; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 2 x double> undef, double %val, i32 0
   %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   ret <vscale x 2 x double> %shf
 }
 
 define <vscale x 2 x double> @dupq_ld1rqd_f64_ptr(<2 x double>* %a) {
 ; CHECK-LABEL: dupq_ld1rqd_f64_ptr:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1rqd { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %1 = load <2 x double>, <2 x double>* %a
   %2 = tail call fast <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> %1, i64 0)
   %3 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double> %2, i64 0)
   ret <vscale x 2 x double> %3
 }
 
 define <vscale x 4 x float> @dupq_ld1rqw_f32_ptr(<4 x float>* %a) {
 ; CHECK-LABEL: dupq_ld1rqw_f32_ptr:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1rqw { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %1 = load <4 x float>, <4 x float>* %a
   %2 = tail call fast <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %1, i64 0)
   %3 = tail call fast <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float> %2, i64 0)
   ret <vscale x 4 x float> %3
 }
 
 define <vscale x 8 x half> @dupq_ld1rqh_f16_ptr(<8 x half>* %a) {
 ; CHECK-LABEL: dupq_ld1rqh_f16_ptr:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %1 = load <8 x half>, <8 x half>* %a
   %2 = tail call fast <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half> undef, <8 x half> %1, i64 0)
   %3 = tail call fast <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half> %2, i64 0)
   ret <vscale x 8 x half> %3
 }
 
 define <vscale x 8 x bfloat> @dupq_ld1rqh_bf16_ptr(<8 x bfloat>* %a) #0 {
 ; CHECK-LABEL: dupq_ld1rqh_bf16_ptr:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %1 = load <8 x bfloat>, <8 x bfloat>* %a
   %2 = tail call fast <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> undef, <8 x bfloat> %1, i64 0)
   %3 = tail call fast <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %2, i64 0)
   ret <vscale x 8 x bfloat> %3
 }
 
 define <vscale x 2 x i64> @dupq_ld1rqd_i64_ptr(<2 x i64>* %a) #0 {

 ; CHECK-NEXT:    ld1rqb { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %1 = load <16 x i8>, <16 x i8>* %a
   %2 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> %1, i64 0)
   %3 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8> %2, i64 0)
   ret <vscale x 16 x i8> %3
 }
 
+define dso_local <vscale x 2 x double> @dupq_ld1rqd_f64() {
+; CHECK-LABEL: dupq_ld1rqd_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI61_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI61_0
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1rqd { z0.d }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = tail call fast <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> <double 1.000000e+00, double 2.000000e+00>, i64 0)
+  %2 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double> %1, i64 0)
+  ret <vscale x 2 x double> %2
+}
+
+define dso_local <vscale x 4 x float> @dupq_ld1rqw_f32() {
+; CHECK-LABEL: dupq_ld1rqw_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI62_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI62_0
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1rqw { z0.s }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = tail call fast <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>, i64 0)
+  %2 = tail call fast <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float> %1, i64 0)
+  ret <vscale x 4 x float> %2
+}
+
+define dso_local <vscale x 8 x half> @dupq_ld1rqh_f16() {
+; CHECK-LABEL: dupq_ld1rqh_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI63_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI63_0
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = tail call fast <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half> undef, <8 x half> <half 0xH3C00, half 0xH4000, half 0xH4200, half 0xH4400, half 0xH4500, half 0xH4600, half 0xH4700, half 0xH4800>, i64 0)
+  %2 = tail call fast <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half> %1, i64 0)
+  ret <vscale x 8 x half> %2
+}
+
+define dso_local <vscale x 8 x bfloat> @dupq_ld1rqh_bf16() #0 {
+; CHECK-LABEL: dupq_ld1rqh_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI64_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI64_0
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> undef, <8 x bfloat> <bfloat 1.000e+00, bfloat 2.000e+00, bfloat 3.000e+00, bfloat 4.000e+00, bfloat 5.000e+00, bfloat 6.000e+00, bfloat 7.000e+00, bfloat 8.000e+00>, i64 0)
+  %2 = call <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %1, i64 0)
+  ret <vscale x 8 x bfloat> %2
+}
+
+define dso_local <vscale x 2 x i64> @dupq_ld1rqd_i64() {
+; CHECK-LABEL: dupq_ld1rqd_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI65_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI65_0
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1rqd { z0.d }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> <i64 1, i64 2>, i64 0)
+  %2 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64> %1, i64 0)
+  ret <vscale x 2 x i64> %2
+}
+
+define dso_local <vscale x 4 x i32> @dupq_ld1rqw_i32() {
+; CHECK-LABEL: dupq_ld1rqw_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI66_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI66_0
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1rqw { z0.s }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>, i64 0)
+  %2 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32> %1, i64 0)
+  ret <vscale x 4 x i32> %2
+}
+
+define dso_local <vscale x 8 x i16> @dupq_ld1rqh_i16() {
+; CHECK-LABEL: dupq_ld1rqh_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI67_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI67_0
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = tail call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, i64 0)
+  %2 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16> %1, i64 0)
+  ret <vscale x 8 x i16> %2
+}
+
+define dso_local <vscale x 16 x i8> @dupq_ld1rqb_i8() {
+; CHECK-LABEL: dupq_ld1rqb_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI68_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI68_0
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ld1rqb { z0.b }, p0/z, [x8]
+; CHECK-NEXT:    ret
+  %1 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, i64 0)
+  %2 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8> %1, i64 0)
+  ret <vscale x 16 x i8> %2
+}
+
 declare <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8>, i64)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16>, i64)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32>, i64)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64>, i64)
 declare <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half>, i64)
 declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat>, i64)
 declare <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float>, i64)
 declare <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double>, i64)