[SVE] Fix VLS selection error from performPostLD1Combine(...)

[cameron.mcinally]

Summary

This is a patch for a VLS selection error from performPostLD1Combine(...). That peep is currently specific to NEON, so bail out if fixed length SVE vectors are found.

Notice that performPostLD1Combine(...) is static, so I don't have access to the usual useSVEForFixedLengthVectorVT(...) and friends that would usually solve a problem like this. Is there a better way to handle this situation?

Also notice that the test case was reduced by Bugpoint from a real code. I tried to target the performPostLD1Combine(...) peep directly, but was not able to match the same code paths that trigger the optimization. Maybe someone else sees what I missed.

[cameron.mcinally]

Further reduced test case, but still not great.

[efriedma]

Notice that performPostLD1Combine(...) is static, so I don't have access to the usual useSVEForFixedLengthVectorVT(...) and friends that would usually solve a problem like this. Is there a better way to handle this situation?

You can make performPostLD1Combine a member of AArch64TargetLowering if that would help.

[paulwalker-arm]

I believe this is a duplicate of D117674, which had not been reviewed yet so I've just pushed it along.

[cameron.mcinally]

I believe this is a duplicate of D117674, which had not been reviewed yet so I've just pushed it along.

I believe it is, with a better test case. I'll abandon this patch. Thanks, Paul and Eli.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static SDValue performSunpkloCombine(SDNode *N, SelectionDAG &DAG) {
 
   return SDValue();
 }
 
 /// Target-specific DAG combine function for post-increment LD1 (lane) and
 /// post-increment LD1R.
 static SDValue performPostLD1Combine(SDNode *N,
                                      TargetLowering::DAGCombinerInfo &DCI,
-                                     bool IsLaneOp) {
+                                     bool IsLaneOp,
+                                     const AArch64Subtarget *Subtarget) {
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
 
   if (VT.isScalableVector())
     return SDValue();
 
+  if (VT.isFixedLengthVector() && VT.getFixedSizeInBits() > 128 &&
+      Subtarget->useSVEForFixedLengthVectors())
+    return SDValue();
+
   unsigned LoadIdx = IsLaneOp ? 1 : 0;
   SDNode *LD = N->getOperand(LoadIdx).getNode();
   // If it is not LOAD, can not do such combine.
   if (LD->getOpcode() != ISD::LOAD)
     return SDValue();
 
   // The vector lane must be a constant in the LD1LANE opcode.
   SDValue Lane;
static SDValue removeRedundantInsertVectorElt(SDNode *N) {
   if (!isLanes1toNKnownZero(ExtractVec))
     return SDValue();
 
   // The explicit zeroing is redundant.
   return ExtractVec;
 }
 
 static SDValue
-performInsertVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
+performInsertVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+                              const AArch64Subtarget *Subtarget) {
   if (SDValue Res = removeRedundantInsertVectorElt(N))
     return Res;
 
-  return performPostLD1Combine(N, DCI, true);
+  return performPostLD1Combine(N, DCI, true, Subtarget);
 }
 
 SDValue performSVESpliceCombine(SDNode *N, SelectionDAG &DAG) {
   EVT Ty = N->getValueType(0);
   if (Ty.isInteger())
     return SDValue();
 
   EVT IntTy = Ty.changeVectorElementTypeToInteger();
SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case AArch64ISD::BRCOND:
     return performBRCONDCombine(N, DCI, DAG);
   case AArch64ISD::TBNZ:
   case AArch64ISD::TBZ:
     return performTBZCombine(N, DCI, DAG);
   case AArch64ISD::CSEL:
     return performCSELCombine(N, DCI, DAG);
   case AArch64ISD::DUP:
-    return performPostLD1Combine(N, DCI, false);
+    return performPostLD1Combine(N, DCI, false, Subtarget);
   case AArch64ISD::NVCAST:
     return performNVCASTCombine(N);
   case AArch64ISD::SPLICE:
     return performSpliceCombine(N, DAG);
   case AArch64ISD::UUNPKLO:
   case AArch64ISD::UUNPKHI:
     return performUnpackCombine(N, DAG);
   case AArch64ISD::UZP1:
SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case AArch64ISD::GLD1S_IMM_MERGE_ZERO:
     return performGLD1Combine(N, DAG);
   case AArch64ISD::VASHR:
   case AArch64ISD::VLSHR:
     return performVectorShiftCombine(N, *this, DCI);
   case AArch64ISD::SUNPKLO:
     return performSunpkloCombine(N, DAG);
   case ISD::INSERT_VECTOR_ELT:
-    return performInsertVectorEltCombine(N, DCI);
+    return performInsertVectorEltCombine(N, DCI, Subtarget);
   case ISD::EXTRACT_VECTOR_ELT:
     return performExtractVectorEltCombine(N, DAG);
   case ISD::VECREDUCE_ADD:
     return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
   case ISD::INTRINSIC_VOID:
   case ISD::INTRINSIC_W_CHAIN:
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:

llvm/test/CodeGen/AArch64/sve-fixed-length-ld1-post-inc.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -mcpu=a64fx -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s
+
+; This test is here to ensure we don't get a selection error caused
+; by performPostLD1Combine, which should bail out if it sees fixed length
+; vectors.
+
+define dso_local void @test_fixed_ld1_post_inc(i8** %a, double* %b, i8* %c) {
+; CHECK-LABEL: test_fixed_ld1_post_inc:
+; CHECK:       // %bb.0: // %L.entry
+; CHECK-NEXT:    index z3.d, #0, #1
+; CHECK-NEXT:    ldr d0, [x8]
+; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    mov z2.d, x8
+; CHECK-NEXT:    cmpeq p0.d, p0/z, z3.d, z2.d
+; CHECK-NEXT:    mov z0.d, p0/m, d1
+; CHECK-NEXT:    ptrue p0.d, vl8
+; CHECK-NEXT:    fmad z0.d, p0/m, z0.d, z0.d
+; CHECK-NEXT:    mov z0.d, z0.d[5]
+; CHECK-NEXT:    stur d0, [x2, #-8]
+; CHECK-NEXT:    ldr d0, [x1, #8]
+; CHECK-NEXT:    str d0, [x1]
+; CHECK-NEXT:    ret
+L.entry:
+  %0 = getelementptr i8, i8* %c, i64 -8
+  %1 = bitcast i8* %0 to double*
+  %2 = load <1 x double>, <1 x double>* undef
+  %r = load double, double* %b
+  %3 = shufflevector <1 x double> %2, <1 x double> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %4 = insertelement <8 x double> %3, double %r, i32 1
+  %5 = shufflevector <8 x double> %4, <8 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+  %6 = call <8 x double> @llvm.fma.v8f64(<8 x double> undef, <8 x double> %5, <8 x double> undef) #2
+  %7 = extractelement <8 x double> %6, i32 5
+  store double %7, double* %1
+  %8 = getelementptr double, double* %b, i64 1
+  %9 = load double, double* %8
+  store double %9, double* %b
+  ret void
+}
+
+; Function Attrs: nofree nosync nounwind readnone speculatable willreturn
+declare dso_local <8 x double> @llvm.fma.v8f64(<8 x double>, <8 x double>, <8 x double>)