Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
===================================================================
--- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -317,11 +317,13 @@
 
     if (Subtarget->useSVEForFixedLengthVectors()) {
       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
-        if (useSVEForFixedLengthVectorVT(VT))
+        if (Subtarget->useSVEForFixedLengthVectors() &&
+            useSVEForFixedLengthVectorVT(VT))
           addRegisterClass(VT, &AArch64::ZPRRegClass);
 
       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
-        if (useSVEForFixedLengthVectorVT(VT))
+        if (Subtarget->useSVEForFixedLengthVectors() &&
+            useSVEForFixedLengthVectorVT(VT))
           addRegisterClass(VT, &AArch64::ZPRRegClass);
     }
   }
@@ -1342,10 +1344,12 @@
     // than the preferred option of combining it with the addRegisterClass call.
     if (Subtarget->useSVEForFixedLengthVectors()) {
       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
-        if (useSVEForFixedLengthVectorVT(VT))
+        if (Subtarget->useSVEForFixedLengthVectors() &&
+            useSVEForFixedLengthVectorVT(VT))
           addTypeForFixedLengthSVE(VT);
       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
-        if (useSVEForFixedLengthVectorVT(VT))
+        if (Subtarget->useSVEForFixedLengthVectors() &&
+            useSVEForFixedLengthVectorVT(VT))
           addTypeForFixedLengthSVE(VT);
 
       // 64bit results can mean a bigger than NEON input.
@@ -3183,7 +3187,8 @@
 }
 
 SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
-  if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+  if (Subtarget->useSVEForFixedLengthVectors() &
+      useSVEForFixedLengthVectorVT(Op.getValueType()))
     return LowerToScalableOp(Op, DAG);
 
   SDValue Sel = Op.getOperand(0);
@@ -3364,7 +3369,8 @@
   if (VT.isScalableVector())
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_EXTEND_MERGE_PASSTHRU);
 
-  if (useSVEForFixedLengthVectorVT(VT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(VT))
     return LowerFixedLengthFPExtendToSVE(Op, DAG);
 
   assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
@@ -3380,12 +3386,14 @@
   SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
   EVT SrcVT = SrcVal.getValueType();
 
-  if (useSVEForFixedLengthVectorVT(SrcVT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(SrcVT))
     return LowerFixedLengthFPRoundToSVE(Op, DAG);
 
   if (SrcVT != MVT::f128) {
     // Expand cases where the input is a vector bigger than NEON.
-    if (useSVEForFixedLengthVectorVT(SrcVT))
+    if (Subtarget->useSVEForFixedLengthVectors() &&
+        useSVEForFixedLengthVectorVT(SrcVT))
       return SDValue();
 
     // It's legal except when f128 is involved
@@ -3410,7 +3418,8 @@
     return LowerToPredicatedOp(Op, DAG, Opcode);
   }
 
-  if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT)))
     return LowerFixedLengthFPToIntToSVE(Op, DAG);
 
   unsigned NumElts = InVT.getVectorNumElements();
@@ -3627,7 +3636,8 @@
     return LowerToPredicatedOp(Op, DAG, Opcode);
   }
 
-  if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT)))
     return LowerFixedLengthIntToFPToSVE(Op, DAG);
 
   uint64_t VTSize = VT.getFixedSizeInBits();
@@ -3721,7 +3731,8 @@
   EVT OpVT = Op.getValueType();
   EVT ArgVT = Op.getOperand(0).getValueType();
 
-  if (useSVEForFixedLengthVectorVT(OpVT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(OpVT))
     return LowerFixedLengthBitcastToSVE(Op, DAG);
 
   if (OpVT.isScalableVector()) {
@@ -3946,7 +3957,8 @@
   // If SVE is available then i64 vector multiplications can also be made legal.
   bool OverrideNEON = VT == MVT::v2i64 || VT == MVT::v1i64;
 
-  if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
+  if (VT.isScalableVector() || (Subtarget->useSVEForFixedLengthVectors() &&
+                                useSVEForFixedLengthVectorVT(VT, OverrideNEON)))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED);
 
   // Multiplications are only custom-lowered for 128-bit vectors so that
@@ -4422,8 +4434,9 @@
 
 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
   return ExtVal.getValueType().isScalableVector() ||
-         useSVEForFixedLengthVectorVT(ExtVal.getValueType(),
-                                      /*OverrideNEON=*/true);
+         (Subtarget->useSVEForFixedLengthVectors() &&
+          useSVEForFixedLengthVectorVT(ExtVal.getValueType(),
+                                       /*OverrideNEON=*/true));
 }
 
 unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
@@ -4763,7 +4776,8 @@
   assert(LoadNode && "Expected custom lowering of a masked load node");
   EVT VT = Op->getValueType(0);
 
-  if (useSVEForFixedLengthVectorVT(VT, true))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(VT, true))
     return LowerFixedLengthVectorMLoadToSVE(Op, DAG);
 
   SDValue PassThru = LoadNode->getPassThru();
@@ -4830,7 +4844,8 @@
   EVT MemVT = StoreNode->getMemoryVT();
 
   if (VT.isVector()) {
-    if (useSVEForFixedLengthVectorVT(VT, true))
+    if (Subtarget->useSVEForFixedLengthVectors() &&
+        useSVEForFixedLengthVectorVT(VT, true))
       return LowerFixedLengthVectorStoreToSVE(Op, DAG);
 
     unsigned AS = StoreNode->getAddressSpace();
@@ -5211,7 +5226,8 @@
   case ISD::MLOAD:
     return LowerMLOAD(Op, DAG);
   case ISD::LOAD:
-    if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+    if (Subtarget->useSVEForFixedLengthVectors() &&
+        useSVEForFixedLengthVectorVT(Op.getValueType()))
       return LowerFixedLengthVectorLoadToSVE(Op, DAG);
     return LowerLOAD(Op, DAG);
   case ISD::ADD:
@@ -5255,10 +5271,7 @@
 
 bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
     EVT VT, bool OverrideNEON) const {
-  if (!Subtarget->useSVEForFixedLengthVectors())
-    return false;
-
-  if (!VT.isFixedLengthVector())
+  if (!VT.isFixedLengthVector() || !Subtarget->hasSVE())
     return false;
 
   // Don't use SVE for vectors we cannot scalarize if required.
@@ -7436,7 +7449,8 @@
     return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV);
   }
 
-  if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
+  if (VT.isScalableVector() || (Subtarget->useSVEForFixedLengthVectors() &&
+                                useSVEForFixedLengthVectorVT(VT)))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);
 
   assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
@@ -7465,7 +7479,8 @@
 SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   assert(VT.isScalableVector() ||
-         useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true));
+         (Subtarget->useSVEForFixedLengthVectors() &&
+          useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true)));
 
   SDLoc DL(Op);
   SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
@@ -7497,7 +7512,8 @@
   }
 
   if (VT.isScalableVector() ||
-      useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true)) {
+      (Subtarget->useSVEForFixedLengthVectors() &&
+       useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))) {
     switch (Opcode) {
     default:
       llvm_unreachable("Wrong instruction");
@@ -7523,7 +7539,8 @@
   EVT VT = Op.getValueType();
 
   if (VT.isScalableVector() ||
-      useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
+      (Subtarget->useSVEForFixedLengthVectors() &&
+       useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true)))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::BITREVERSE_MERGE_PASSTHRU);
 
   SDLoc DL(Op);
@@ -7923,7 +7940,8 @@
     return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal);
   }
 
-  if (useSVEForFixedLengthVectorVT(Ty)) {
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(Ty)) {
     // FIXME: Ideally this would be the same as above using i1 types, however
     // for the moment we can't deal with fixed i1 vector types properly, so
     // instead extend the predicate to a result type sized integer vector.
@@ -9830,7 +9848,8 @@
 
   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
 
-  if (useSVEForFixedLengthVectorVT(VT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(VT))
     return LowerFixedLengthVECTOR_SHUFFLEToSVE(Op, DAG);
 
   // Convert shuffles that are directly supported on NEON to target-specific
@@ -10005,7 +10024,8 @@
   EVT ElemVT = VT.getScalarType();
   SDValue SplatVal = Op.getOperand(0);
 
-  if (useSVEForFixedLengthVectorVT(VT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(VT))
     return LowerToScalableOp(Op, DAG);
 
   // Extend input splat value where needed to fit into a GPR (32b or 64b only)
@@ -10437,7 +10457,8 @@
 
 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
                                              SelectionDAG &DAG) const {
-  if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(Op.getValueType()))
     return LowerToScalableOp(Op, DAG);
 
   // Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2))
@@ -10867,7 +10888,8 @@
 
 SDValue AArch64TargetLowering::LowerCONCAT_VECTORS(SDValue Op,
                                                    SelectionDAG &DAG) const {
-  if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(Op.getValueType()))
     return LowerFixedLengthConcatVectorsToSVE(Op, DAG);
 
   assert(Op.getValueType().isScalableVector() &&
@@ -10905,7 +10927,8 @@
                                                       SelectionDAG &DAG) const {
   assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!");
 
-  if (useSVEForFixedLengthVectorVT(Op.getValueType()))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(Op.getValueType()))
     return LowerFixedLengthInsertVectorElt(Op, DAG);
 
   // Check for non-constant or out of range lane.
@@ -10973,7 +10996,8 @@
     return DAG.getAnyExtOrTrunc(Extract, DL, Op.getValueType());
   }
 
-  if (useSVEForFixedLengthVectorVT(VT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(VT))
     return LowerFixedLengthExtractVectorElt(Op, DAG);
 
   // Check for non-constant or out of range lane.
@@ -11038,7 +11062,8 @@
       InVT.getSizeInBits() == 128)
     return Op;
 
-  if (useSVEForFixedLengthVectorVT(InVT)) {
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(InVT)) {
     SDLoc DL(Op);
 
     EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
@@ -11219,7 +11244,8 @@
 
 bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
   // Currently no fixed length shuffles that require SVE are legal.
-  if (useSVEForFixedLengthVectorVT(VT))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(VT))
     return false;
 
   if (VT.getVectorNumElements() == 4 &&
@@ -11318,7 +11344,8 @@
   if (!VT.isVector() || VT.isScalableVector())
     return SDValue();
 
-  if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
     return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
 
   return SDValue();
@@ -11339,7 +11366,8 @@
     llvm_unreachable("unexpected shift opcode");
 
   case ISD::SHL:
-    if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
+    if (VT.isScalableVector() || (Subtarget->useSVEForFixedLengthVectors() &&
+                                  useSVEForFixedLengthVectorVT(VT)))
       return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_PRED);
 
     if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
@@ -11351,7 +11379,8 @@
                        Op.getOperand(0), Op.getOperand(1));
   case ISD::SRA:
   case ISD::SRL:
-    if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) {
+    if (VT.isScalableVector() || (Subtarget->useSVEForFixedLengthVectors() &&
+                                  useSVEForFixedLengthVectorVT(VT))) {
       unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED
                                                 : AArch64ISD::SRL_PRED;
       return LowerToPredicatedOp(Op, DAG, Opc);
@@ -11483,7 +11512,8 @@
   if (Op.getValueType().isScalableVector())
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SETCC_MERGE_ZERO);
 
-  if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
     return LowerFixedLengthVectorSetccToSVE(Op, DAG);
 
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
@@ -11569,7 +11599,8 @@
                       (Op.getOpcode() != ISD::VECREDUCE_ADD &&
                        SrcVT.getVectorElementType() == MVT::i64);
   if (SrcVT.isScalableVector() ||
-      useSVEForFixedLengthVectorVT(SrcVT, OverrideNEON)) {
+      (Subtarget->useSVEForFixedLengthVectors() &&
+       useSVEForFixedLengthVectorVT(SrcVT, OverrideNEON))) {
 
     if (SrcVT.getVectorElementType() == MVT::i1)
       return LowerPredReductionToSVE(Op, DAG);
@@ -12968,7 +12999,8 @@
 bool AArch64TargetLowering::generateFMAsInMachineCombiner(
     EVT VT, CodeGenOpt::Level OptLevel) const {
   return (OptLevel >= CodeGenOpt::Aggressive) && !VT.isScalableVector() &&
-         !useSVEForFixedLengthVectorVT(VT);
+         !(Subtarget->useSVEForFixedLengthVectors() &&
+           useSVEForFixedLengthVectorVT(VT));
 }
 
 const MCPhysReg *
@@ -19422,7 +19454,8 @@
 SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
                                                  SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
-  assert(useSVEForFixedLengthVectorVT(VT) &&
+  assert(Subtarget->useSVEForFixedLengthVectors() &&
+         useSVEForFixedLengthVectorVT(VT) &&
          "Only expected to lower fixed length vector operation!");
   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
 
@@ -19438,7 +19471,8 @@
     }
 
     // "cast" fixed length vector to a scalable vector.
-    assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
+    assert(Subtarget->useSVEForFixedLengthVectors() &&
+           useSVEForFixedLengthVectorVT(V.getValueType()) &&
            "Only fixed length vectors are supported!");
     Ops.push_back(convertToScalableVector(DAG, ContainerVT, V));
   }
@@ -19520,7 +19554,8 @@
   SDValue VecOp = ScalarOp.getOperand(0);
   EVT SrcVT = VecOp.getValueType();
 
-  if (useSVEForFixedLengthVectorVT(SrcVT, true)) {
+  if (Subtarget->useSVEForFixedLengthVectors() &&
+      useSVEForFixedLengthVectorVT(SrcVT, true)) {
     EVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT);
     VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
   }
@@ -19575,7 +19610,8 @@
   EVT InVT = Op.getOperand(0).getValueType();
   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
 
-  assert(useSVEForFixedLengthVectorVT(InVT) &&
+  assert(Subtarget->useSVEForFixedLengthVectors() &&
+         useSVEForFixedLengthVectorVT(InVT) &&
          "Only expected to lower fixed length vector operation!");
   assert(Op.getValueType() == InVT.changeTypeToInteger() &&
          "Expected integer result of the same bit length as the inputs!");