Index: llvm/include/llvm/CodeGen/TargetLowering.h =================================================================== --- llvm/include/llvm/CodeGen/TargetLowering.h +++ llvm/include/llvm/CodeGen/TargetLowering.h @@ -412,7 +412,7 @@ virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const { // The default action for one element vectors is to scalarize - if (VT.getVectorNumElements() == 1) + if (VT.getVectorElementCount() == 1) return TypeScalarizeVector; // The default action for an odd-width vector is to widen. if (!VT.isPow2VectorType()) Index: llvm/include/llvm/Support/TypeSize.h =================================================================== --- llvm/include/llvm/Support/TypeSize.h +++ llvm/include/llvm/Support/TypeSize.h @@ -15,6 +15,7 @@ #ifndef LLVM_SUPPORT_TYPESIZE_H #define LLVM_SUPPORT_TYPESIZE_H +#include "llvm/Support/MathExtras.h" #include "llvm/Support/WithColor.h" #include @@ -49,6 +50,16 @@ bool operator!=(const ElementCount& RHS) const { return !(*this == RHS); } + bool operator==(unsigned RHS) const { + return Min == RHS && !Scalable; + } + bool operator!=(unsigned RHS) const { + return !(*this == RHS); + } + + ElementCount NextPowerOf2() const { + return ElementCount(llvm::NextPowerOf2(Min), Scalable); + } }; // This class is used to represent the size of types. If the type is of fixed Index: llvm/lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- llvm/lib/CodeGen/TargetLoweringBase.cpp +++ llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -823,9 +823,7 @@ "Promote may not follow Expand or Promote"); if (LA == TypeSplitVector) - return LegalizeKind(LA, - EVT::getVectorVT(Context, SVT.getVectorElementType(), - SVT.getVectorNumElements() / 2)); + return LegalizeKind(LA, SVT.getHalfNumVectorElementsVT()); if (LA == TypeScalarizeVector) return LegalizeKind(LA, SVT.getVectorElementType()); return LegalizeKind(LA, NVT); @@ -852,13 +850,16 @@ } // Handle vector types. - unsigned NumElts = VT.getVectorNumElements(); + ElementCount NumElts = VT.getVectorElementCount(); EVT EltVT = VT.getVectorElementType(); // Vectors with only one element are always scalarized. if (NumElts == 1) return LegalizeKind(TypeScalarizeVector, EltVT); + if (VT.getVectorElementCount() == ElementCount(1, true)) + report_fatal_error("Cannot legalize this vector"); + // Try to widen vector elements until the element type is a power of two and // promote it to a legal type later on, for example: // <3 x i8> -> <4 x i8> -> <4 x i32> @@ -866,7 +867,7 @@ // Vectors with a number of elements that is not a power of two are always // widened, for example <3 x i8> -> <4 x i8>. if (!VT.isPow2VectorType()) { - NumElts = (unsigned)NextPowerOf2(NumElts); + NumElts = NumElts.NextPowerOf2(); EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts); return LegalizeKind(TypeWidenVector, NVT); } @@ -915,7 +916,7 @@ // If there is no wider legal type, split the vector. while (true) { // Round up to the next power of 2. - NumElts = (unsigned)NextPowerOf2(NumElts); + NumElts = NumElts.NextPowerOf2(); // If there is no simple vector type with this many elements then there // cannot be a larger legal vector type. Note that this assumes that @@ -938,7 +939,7 @@ } // Vectors with illegal element types are expanded. - EVT NVT = EVT::getVectorVT(Context, EltVT, VT.getVectorNumElements() / 2); + EVT NVT = EVT::getVectorVT(Context, EltVT, VT.getVectorElementCount() / 2); return LegalizeKind(TypeSplitVector, NVT); } @@ -1261,7 +1262,7 @@ continue; MVT EltVT = VT.getVectorElementType(); - unsigned NElts = VT.getVectorNumElements(); + ElementCount EC = VT.getVectorElementCount(); bool IsLegalWiderType = false; bool IsScalable = VT.isScalableVector(); LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT); @@ -1278,8 +1279,7 @@ // Promote vectors of integers to vectors with the same number // of elements, with a wider element type. if (SVT.getScalarSizeInBits() > EltVT.getSizeInBits() && - SVT.getVectorNumElements() == NElts && - SVT.isScalableVector() == IsScalable && isTypeLegal(SVT)) { + SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) { TransformToType[i] = SVT; RegisterTypeForVT[i] = SVT; NumRegistersForVT[i] = 1; @@ -1294,13 +1294,13 @@ } case TypeWidenVector: - if (isPowerOf2_32(NElts)) { + if (isPowerOf2_32(EC.Min)) { // Try to widen the vector. for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { MVT SVT = (MVT::SimpleValueType) nVT; - if (SVT.getVectorElementType() == EltVT - && SVT.getVectorNumElements() > NElts - && SVT.isScalableVector() == IsScalable && isTypeLegal(SVT)) { + if (SVT.getVectorElementType() == EltVT && + SVT.isScalableVector() == IsScalable && + SVT.getVectorElementCount().Min > EC.Min && isTypeLegal(SVT)) { TransformToType[i] = SVT; RegisterTypeForVT[i] = SVT; NumRegistersForVT[i] = 1; @@ -1344,10 +1344,16 @@ ValueTypeActions.setTypeAction(VT, TypeScalarizeVector); else if (PreferredAction == TypeSplitVector) ValueTypeActions.setTypeAction(VT, TypeSplitVector); + else if (EC.Min > 1) + ValueTypeActions.setTypeAction(VT, TypeSplitVector); + else if (EC == 1) + ValueTypeActions.setTypeAction(VT, TypeScalarizeVector); else - // Set type action according to the number of elements. - ValueTypeActions.setTypeAction(VT, NElts == 1 ? TypeScalarizeVector - : TypeSplitVector); + ; // Can't split or scalarize `vscale x 1 x ``, but cannot use + // llvm_unreachable here because targets that support scalable + // vectors can address this with widening/promotion, whereas targets + // that don't support scalable vectors can't lower them at all and + // end up here. } else { TransformToType[i] = NVT; ValueTypeActions.setTypeAction(VT, TypeWidenVector); Index: llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp =================================================================== --- llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp +++ llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp @@ -17,9 +17,7 @@ #include "llvm/Target/TargetMachine.h" #include "gtest/gtest.h" -using namespace llvm; - -namespace { +namespace llvm { class AArch64SelectionDAGTest : public testing::Test { protected: @@ -42,7 +40,7 @@ TargetOptions Options; TM = std::unique_ptr(static_cast( - T->createTargetMachine("AArch64", "", "", Options, None, None, + T->createTargetMachine("AArch64", "", "+sve", Options, None, None, CodeGenOpt::Aggressive))); if (!TM) return; @@ -69,6 +67,14 @@ DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr); } + TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) { + return DAG->getTargetLoweringInfo().getTypeAction(Context, VT); + } + + EVT getTypeToTransformTo(EVT VT) { + return DAG->getTargetLoweringInfo().getTypeToTransformTo(Context, VT); + } + LLVMContext Context; std::unique_ptr TM; std::unique_ptr M; @@ -377,4 +383,60 @@ EXPECT_EQ(SplatIdx, 0); } -} // end anonymous namespace +TEST_F(AArch64SelectionDAGTest, getTypeConversion_SplitScalableMVT) { + if (!TM) + return; + + MVT VT = MVT::nxv4i64; + EXPECT_EQ(getTypeAction(VT), TargetLoweringBase::TypeSplitVector); + ASSERT_TRUE(getTypeToTransformTo(VT).isScalableVector()); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_PromoteScalableMVT) { + if (!TM) + return; + + MVT VT = MVT::nxv2i32; + EXPECT_EQ(getTypeAction(VT), TargetLoweringBase::TypePromoteInteger); + ASSERT_TRUE(getTypeToTransformTo(VT).isScalableVector()); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_NoScalarizeMVT_nxv1f32) { + if (!TM) + return; + + MVT VT = MVT::nxv1f32; + EXPECT_NE(getTypeAction(VT), TargetLoweringBase::TypeScalarizeVector); + ASSERT_TRUE(getTypeToTransformTo(VT).isScalableVector()); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_SplitScalableEVT) { + if (!TM) + return; + + EVT VT = EVT::getVectorVT(Context, MVT::i64, 256, true); + EXPECT_EQ(getTypeAction(VT), TargetLoweringBase::TypeSplitVector); + EXPECT_EQ(getTypeToTransformTo(VT), VT.getHalfNumVectorElementsVT(Context)); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_WidenScalableEVT) { + if (!TM) + return; + + EVT FromVT = EVT::getVectorVT(Context, MVT::i64, 6, true); + EVT ToVT = EVT::getVectorVT(Context, MVT::i64, 8, true); + + EXPECT_EQ(getTypeAction(FromVT), TargetLoweringBase::TypeWidenVector); + EXPECT_EQ(getTypeToTransformTo(FromVT), ToVT); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_NoScalarizeEVT_nxv1f128) { + if (!TM) + return; + + EVT FromVT = EVT::getVectorVT(Context, MVT::f128, 1, true); + EXPECT_DEATH(getTypeAction(FromVT), "Cannot legalize this vector"); +} + + +} // end namespace llvm