Index: llvm/include/llvm/CodeGen/TargetLowering.h =================================================================== --- llvm/include/llvm/CodeGen/TargetLowering.h +++ llvm/include/llvm/CodeGen/TargetLowering.h @@ -186,6 +186,7 @@ /// This base class for TargetLowering contains the SelectionDAG-independent /// parts that can be used from the rest of CodeGen. class TargetLoweringBase { + friend class AArch64SelectionDAGTest; public: /// This enum indicates whether operations are valid for a target, and if not, /// what action should be used to make them valid. @@ -412,7 +413,7 @@ virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const { // The default action for one element vectors is to scalarize - if (VT.getVectorNumElements() == 1) + if (VT.getVectorElementCount().isOne()) return TypeScalarizeVector; // The default action for an odd-width vector is to widen. if (!VT.isPow2VectorType()) Index: llvm/include/llvm/Support/TypeSize.h =================================================================== --- llvm/include/llvm/Support/TypeSize.h +++ llvm/include/llvm/Support/TypeSize.h @@ -15,6 +15,7 @@ #ifndef LLVM_SUPPORT_TYPESIZE_H #define LLVM_SUPPORT_TYPESIZE_H +#include "llvm/Support/MathExtras.h" #include "llvm/Support/WithColor.h" #include @@ -49,6 +50,12 @@ bool operator!=(const ElementCount& RHS) const { return !(*this == RHS); } + + bool isOne() const { return Min == 1 && !Scalable; } + + ElementCount NextPowerOf2() const { + return ElementCount(llvm::NextPowerOf2(Min), Scalable); + } }; // This class is used to represent the size of types. If the type is of fixed Index: llvm/lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- llvm/lib/CodeGen/TargetLoweringBase.cpp +++ llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -825,7 +825,7 @@ if (LA == TypeSplitVector) return LegalizeKind(LA, EVT::getVectorVT(Context, SVT.getVectorElementType(), - SVT.getVectorNumElements() / 2)); + SVT.getVectorElementCount() / 2)); if (LA == TypeScalarizeVector) return LegalizeKind(LA, SVT.getVectorElementType()); return LegalizeKind(LA, NVT); @@ -852,11 +852,11 @@ } // Handle vector types. - unsigned NumElts = VT.getVectorNumElements(); + ElementCount NumElts = VT.getVectorElementCount(); EVT EltVT = VT.getVectorElementType(); // Vectors with only one element are always scalarized. - if (NumElts == 1) + if (NumElts.isOne()) return LegalizeKind(TypeScalarizeVector, EltVT); // Try to widen vector elements until the element type is a power of two and @@ -866,7 +866,7 @@ // Vectors with a number of elements that is not a power of two are always // widened, for example <3 x i8> -> <4 x i8>. if (!VT.isPow2VectorType()) { - NumElts = (unsigned)NextPowerOf2(NumElts); + NumElts = NumElts.NextPowerOf2(); EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts); return LegalizeKind(TypeWidenVector, NVT); } @@ -915,7 +915,7 @@ // If there is no wider legal type, split the vector. while (true) { // Round up to the next power of 2. - NumElts = (unsigned)NextPowerOf2(NumElts); + NumElts = NumElts.NextPowerOf2(); // If there is no simple vector type with this many elements then there // cannot be a larger legal vector type. Note that this assumes that @@ -937,8 +937,11 @@ return LegalizeKind(TypeWidenVector, NVT); } + if (VT.getVectorElementCount() == ElementCount(1, true)) + llvm_unreachable("Cannot legalize this vector"); + // Vectors with illegal element types are expanded. - EVT NVT = EVT::getVectorVT(Context, EltVT, VT.getVectorNumElements() / 2); + EVT NVT = EVT::getVectorVT(Context, EltVT, VT.getVectorElementCount() / 2); return LegalizeKind(TypeSplitVector, NVT); } @@ -1261,7 +1264,7 @@ continue; MVT EltVT = VT.getVectorElementType(); - unsigned NElts = VT.getVectorNumElements(); + ElementCount EC = VT.getVectorElementCount(); bool IsLegalWiderType = false; bool IsScalable = VT.isScalableVector(); LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT); @@ -1278,8 +1281,7 @@ // Promote vectors of integers to vectors with the same number // of elements, with a wider element type. if (SVT.getScalarSizeInBits() > EltVT.getSizeInBits() && - SVT.getVectorNumElements() == NElts && - SVT.isScalableVector() == IsScalable && isTypeLegal(SVT)) { + SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) { TransformToType[i] = SVT; RegisterTypeForVT[i] = SVT; NumRegistersForVT[i] = 1; @@ -1294,13 +1296,13 @@ } case TypeWidenVector: - if (isPowerOf2_32(NElts)) { + if (isPowerOf2_32(EC.Min)) { // Try to widen the vector. for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { MVT SVT = (MVT::SimpleValueType) nVT; - if (SVT.getVectorElementType() == EltVT - && SVT.getVectorNumElements() > NElts - && SVT.isScalableVector() == IsScalable && isTypeLegal(SVT)) { + if (SVT.getVectorElementType() == EltVT && + SVT.isScalableVector() == IsScalable && + SVT.getVectorElementCount().Min > EC.Min && isTypeLegal(SVT)) { TransformToType[i] = SVT; RegisterTypeForVT[i] = SVT; NumRegistersForVT[i] = 1; @@ -1344,10 +1346,16 @@ ValueTypeActions.setTypeAction(VT, TypeScalarizeVector); else if (PreferredAction == TypeSplitVector) ValueTypeActions.setTypeAction(VT, TypeSplitVector); + else if (EC.Min > 1) + ValueTypeActions.setTypeAction(VT, TypeSplitVector); + else if (EC.isOne()) + ValueTypeActions.setTypeAction(VT, TypeScalarizeVector); else - // Set type action according to the number of elements. - ValueTypeActions.setTypeAction(VT, NElts == 1 ? TypeScalarizeVector - : TypeSplitVector); + ; // Can't split or scalarize `vscale x 1 x ``, but cannot use + // llvm_unreachable here because targets that support scalable + // vectors can address this with widening/promotion, whereas targets + // that don't support scalable vectors can't lower them at all and + // end up here. } else { TransformToType[i] = NVT; ValueTypeActions.setTypeAction(VT, TypeWidenVector); Index: llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp =================================================================== --- llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp +++ llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp @@ -17,9 +17,7 @@ #include "llvm/Target/TargetMachine.h" #include "gtest/gtest.h" -using namespace llvm; - -namespace { +namespace llvm { class AArch64SelectionDAGTest : public testing::Test { protected: @@ -42,7 +40,7 @@ TargetOptions Options; TM = std::unique_ptr(static_cast( - T->createTargetMachine("AArch64", "", "", Options, None, None, + T->createTargetMachine("AArch64", "", "+sve", Options, None, None, CodeGenOpt::Aggressive))); if (!TM) return; @@ -69,6 +67,10 @@ DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr); } + TargetLoweringBase::LegalizeKind getTypeConversion(EVT VT) { + return DAG->getTargetLoweringInfo().getTypeConversion(Context, VT); + } + LLVMContext Context; std::unique_ptr TM; std::unique_ptr M; @@ -377,4 +379,62 @@ EXPECT_EQ(SplatIdx, 0); } -} // end anonymous namespace +TEST_F(AArch64SelectionDAGTest, getTypeConversion_SplitScalableMVT) { + if (!TM) + return; + + TargetLoweringBase::LegalizeKind Legality = getTypeConversion(MVT::nxv4i64); + EXPECT_EQ(Legality.first, TargetLoweringBase::TypeSplitVector); + ASSERT_TRUE(Legality.second.isScalableVector()); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_PromoteScalableMVT) { + if (!TM) + return; + + TargetLoweringBase::LegalizeKind Legality = getTypeConversion(MVT::nxv2i32); + EXPECT_EQ(Legality.first, TargetLoweringBase::TypePromoteInteger); + ASSERT_TRUE(Legality.second.isScalableVector()); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_NoScalarizeMVT_nxv1f32) { + if (!TM) + return; + + TargetLoweringBase::LegalizeKind Legality = getTypeConversion(MVT::nxv1f32); + EXPECT_NE(Legality.first, TargetLoweringBase::TypeScalarizeVector); + ASSERT_TRUE(Legality.second.isScalableVector()); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_SplitScalableEVT) { + if (!TM) + return; + + EVT VT = EVT::getVectorVT(Context, MVT::i64, 256, true); + TargetLoweringBase::LegalizeKind Legality = getTypeConversion(VT); + EXPECT_EQ(Legality.first, TargetLoweringBase::TypeSplitVector); + EXPECT_EQ(Legality.second, VT.getHalfNumVectorElementsVT(Context)); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_WidenScalableEVT) { + if (!TM) + return; + + EVT FromVT = EVT::getVectorVT(Context, MVT::i64, 6, true); + TargetLoweringBase::LegalizeKind Legality = getTypeConversion(FromVT); + + EXPECT_EQ(Legality.first, TargetLoweringBase::TypeWidenVector); + EVT ToVT = EVT::getVectorVT(Context, MVT::i64, 8, true); + EXPECT_EQ(Legality.second, ToVT); +} + +TEST_F(AArch64SelectionDAGTest, getTypeConversion_NoScalarizeEVT_nxv1f128) { + if (!TM) + return; + + EVT FromVT = EVT::getVectorVT(Context, MVT::f128, 1, true); + EXPECT_DEATH(getTypeConversion(FromVT), "Cannot legalize this vector"); +} + + +} // end namespace llvm