Index: include/llvm/CodeGen/GlobalISel/LegalizerInfo.h =================================================================== --- include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -215,12 +215,27 @@ /// True iff the specified type index is a scalar that's narrower than the given /// size. LegalityPredicate narrowerThan(unsigned TypeIdx, unsigned Size); + /// True iff the specified type index is a scalar that's wider than the given /// size. LegalityPredicate widerThan(unsigned TypeIdx, unsigned Size); + +/// True iff the specified type index is a scalar or vector with an element type +/// that's narrower than the given size. +LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); + +/// True iff the specified type index is a scalar or a vector with an element +/// type that's wider than the given size. +LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); + /// True iff the specified type index is a scalar whose size is not a power of /// 2. LegalityPredicate sizeNotPow2(unsigned TypeIdx); + +/// True iff the specified type index is a scalar or vector whose element size +/// is not a power of 2. +LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); + /// True iff the specified type indices are both the same bit size. LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); /// True iff the specified MMO index has a size that is not a power of 2 @@ -237,10 +252,20 @@ namespace LegalizeMutations { /// Select this specific type for the given type index. LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); + /// Keep the same type as the given type index. LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); -/// Widen the type for the given type index to the next power of 2. -LegalizeMutation widenScalarToNextPow2(unsigned TypeIdx, unsigned Min = 0); + +/// Keep the same scalar or element type as the given type index. +LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); + +/// Keep the same scalar or element type as the given type. +LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); + +/// Widen the scalar type or vector element type for the given type index to the +/// next power of 2. +LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); + /// Add more elements to the type for the given type index to the next power of /// 2. LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); @@ -618,8 +643,19 @@ LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize = 0) { using namespace LegalityPredicates; - return actionIf(LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), - LegalizeMutations::widenScalarToNextPow2(TypeIdx, MinSize)); + return actionIf( + LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); + } + + /// Widen the scalar or vector element type to the next power of two that is + /// at least MinSize. No effect if the scalar size is a power of two. + LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, + unsigned MinSize = 0) { + using namespace LegalityPredicates; + return actionIf( + LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); } LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { @@ -634,6 +670,15 @@ LegalizeMutations::scalarize(TypeIdx)); } + /// Ensure the scalar is at least as wide as Ty. + LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::WidenScalar, + scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + /// Ensure the scalar is at least as wide as Ty. LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT &Ty) { using namespace LegalityPredicates; @@ -643,6 +688,15 @@ changeTo(typeIdx(TypeIdx), Ty)); } + /// Ensure the scalar is at most as wide as Ty. + LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::NarrowScalar, + scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + /// Ensure the scalar is at most as wide as Ty. LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT &Ty) { using namespace LegalityPredicates; @@ -659,12 +713,12 @@ const LLT &Ty) { using namespace LegalityPredicates; using namespace LegalizeMutations; - return actionIf(LegalizeAction::NarrowScalar, - [=](const LegalityQuery &Query) { - return widerThan(TypeIdx, Ty.getSizeInBits()) && - Predicate(Query); - }, - changeTo(typeIdx(TypeIdx), Ty)); + return actionIf( + LegalizeAction::NarrowScalar, + [=](const LegalityQuery &Query) { + return widerThan(TypeIdx, Ty.getSizeInBits()) && Predicate(Query); + }, + changeElementTo(typeIdx(TypeIdx), Ty)); } /// Limit the range of scalar sizes to MinTy and MaxTy. @@ -674,6 +728,12 @@ return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); } + /// Limit the range of scalar sizes to MinTy and MaxTy. + LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT &MinTy, + const LLT &MaxTy) { + return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy); + } + /// Widen the scalar to match the size of another. LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { typeIdx(TypeIdx); Index: include/llvm/Support/LowLevelTypeImpl.h =================================================================== --- include/llvm/Support/LowLevelTypeImpl.h +++ include/llvm/Support/LowLevelTypeImpl.h @@ -115,6 +115,22 @@ return isVector() ? getElementType() : *this; } + /// If this type is a vector, return a vector with the same number of elements + /// but the new element type. Otherwise, return the new element type. + LLT changeElementType(LLT NewEltTy) const { + return isVector() ? LLT::vector(getNumElements(), NewEltTy) : NewEltTy; + } + + /// If this type is a vector, return a vector with the same number of elements + /// but the new element size. Otherwise, return the new element type. Invalid + /// for pointer types. For pointer types, use changeElementType. + LLT changeElementSize(unsigned NewEltSize) const { + assert(!getScalarType().isPointer() && + "invalid to directly change element size for pointers"); + return isVector() ? LLT::vector(getNumElements(), NewEltSize) + : LLT::scalar(NewEltSize); + } + unsigned getScalarSizeInBits() const { assert(RawData != 0 && "Invalid Type"); if (!IsVector) { Index: lib/CodeGen/GlobalISel/LegalityPredicates.cpp =================================================================== --- lib/CodeGen/GlobalISel/LegalityPredicates.cpp +++ lib/CodeGen/GlobalISel/LegalityPredicates.cpp @@ -79,7 +79,7 @@ LegalityPredicate LegalityPredicates::narrowerThan(unsigned TypeIdx, unsigned Size) { return [=](const LegalityQuery &Query) { - const LLT &QueryTy = Query.Types[TypeIdx]; + const LLT QueryTy = Query.Types[TypeIdx]; return QueryTy.isScalar() && QueryTy.getSizeInBits() < Size; }; } @@ -87,14 +87,37 @@ LegalityPredicate LegalityPredicates::widerThan(unsigned TypeIdx, unsigned Size) { return [=](const LegalityQuery &Query) { - const LLT &QueryTy = Query.Types[TypeIdx]; + const LLT QueryTy = Query.Types[TypeIdx]; return QueryTy.isScalar() && QueryTy.getSizeInBits() > Size; }; } +LegalityPredicate LegalityPredicates::scalarOrEltNarrowerThan(unsigned TypeIdx, + unsigned Size) { + return [=](const LegalityQuery &Query) { + const LLT QueryTy = Query.Types[TypeIdx]; + return QueryTy.getScalarSizeInBits() < Size; + }; +} + +LegalityPredicate LegalityPredicates::scalarOrEltWiderThan(unsigned TypeIdx, + unsigned Size) { + return [=](const LegalityQuery &Query) { + const LLT QueryTy = Query.Types[TypeIdx]; + return QueryTy.getScalarSizeInBits() > Size; + }; +} + +LegalityPredicate LegalityPredicates::scalarOrEltSizeNotPow2(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + const LLT QueryTy = Query.Types[TypeIdx]; + return !isPowerOf2_32(QueryTy.getScalarSizeInBits()); + }; +} + LegalityPredicate LegalityPredicates::sizeNotPow2(unsigned TypeIdx) { return [=](const LegalityQuery &Query) { - const LLT &QueryTy = Query.Types[TypeIdx]; + const LLT QueryTy = Query.Types[TypeIdx]; return QueryTy.isScalar() && !isPowerOf2_32(QueryTy.getSizeInBits()); }; } Index: lib/CodeGen/GlobalISel/LegalizeMutations.cpp =================================================================== --- lib/CodeGen/GlobalISel/LegalizeMutations.cpp +++ lib/CodeGen/GlobalISel/LegalizeMutations.cpp @@ -26,14 +26,30 @@ }; } -LegalizeMutation LegalizeMutations::widenScalarToNextPow2(unsigned TypeIdx, - unsigned Min) { +LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx, + unsigned FromTypeIdx) { return [=](const LegalityQuery &Query) { - unsigned NewSizeInBits = - 1 << Log2_32_Ceil(Query.Types[TypeIdx].getSizeInBits()); - if (NewSizeInBits < Min) - NewSizeInBits = Min; - return std::make_pair(TypeIdx, LLT::scalar(NewSizeInBits)); + const LLT OldTy = Query.Types[TypeIdx]; + const LLT NewTy = Query.Types[FromTypeIdx]; + return std::make_pair(TypeIdx, OldTy.changeElementType(NewTy)); + }; +} + +LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx, + LLT NewEltTy) { + return [=](const LegalityQuery &Query) { + const LLT OldTy = Query.Types[TypeIdx]; + return std::make_pair(TypeIdx, OldTy.changeElementType(NewEltTy)); + }; +} + +LegalizeMutation LegalizeMutations::widenScalarOrEltToNextPow2(unsigned TypeIdx, + unsigned Min) { + return [=](const LegalityQuery &Query) { + const LLT Ty = Query.Types[TypeIdx]; + unsigned NewEltSizeInBits = + std::max(1u << Log2_32_Ceil(Ty.getScalarSizeInBits()), Min); + return std::make_pair(TypeIdx, Ty.changeElementSize(NewEltSizeInBits)); }; } Index: lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -471,19 +471,19 @@ // FIXME: Doesn't handle extract of illegal sizes. getActionDefinitionsBuilder({G_EXTRACT, G_INSERT}) - .legalIf([=](const LegalityQuery &Query) { + .legalIf([=](const LegalityQuery &Query) { const LLT &Ty0 = Query.Types[0]; const LLT &Ty1 = Query.Types[1]; return (Ty0.getSizeInBits() % 16 == 0) && (Ty1.getSizeInBits() % 16 == 0); }) - .widenScalarIf( - [=](const LegalityQuery &Query) { - const LLT &Ty0 = Query.Types[0]; - const LLT &Ty1 = Query.Types[1]; - return (Ty1.getScalarSizeInBits() < 16); - }, - LegalizeMutations::widenScalarToNextPow2(1, 16)); + .widenScalarIf( + [=](const LegalityQuery &Query) { + const LLT &Ty0 = Query.Types[0]; + const LLT &Ty1 = Query.Types[1]; + return (Ty1.getScalarSizeInBits() < 16); + }, + LegalizeMutations::widenScalarOrEltToNextPow2(1, 16)); // TODO: vectors of pointers getActionDefinitionsBuilder(G_BUILD_VECTOR) Index: unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp =================================================================== --- unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp +++ unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp @@ -41,6 +41,12 @@ OS << SS.str(); return OS; } + +std::ostream &operator<<(std::ostream &OS, const llvm::LegalizeActionStep Ty) { + OS << "LegalizeActionStep(" << Ty.Action << ", " << Ty.TypeIdx << ", " + << Ty.NewType << ')'; + return OS; +} } namespace { @@ -198,3 +204,143 @@ LegalizeActionStep(Unsupported, 0, LLT::scalar(33))); } } + +#define EXPECT_ACTION(Action, Index, Type, Query) \ + do { \ + auto A = LI.getAction(Query); \ + EXPECT_EQ(LegalizeActionStep(Action, Index, Type), A) << A; \ + } while (0) + +TEST(LegalizerInfoTest, RuleSets) { + using namespace TargetOpcode; + + const LLT s5 = LLT::scalar(5); + const LLT s8 = LLT::scalar(8); + const LLT s16 = LLT::scalar(16); + const LLT s32 = LLT::scalar(32); + const LLT s33 = LLT::scalar(33); + const LLT s64 = LLT::scalar(64); + + const LLT v2s5 = LLT::vector(2, 5); + const LLT v2s8 = LLT::vector(2, 8); + const LLT v2s16 = LLT::vector(2, 16); + const LLT v2s32 = LLT::vector(2, 32); + const LLT v2s33 = LLT::vector(2, 33); + const LLT v2s64 = LLT::vector(2, 64); + + // Test minScalarOrElt + { + LegalizerInfo LI; + LI.getActionDefinitionsBuilder(G_OR) + .legalFor({s32}) + .minScalarOrElt(0, s32); + LI.computeTables(); + + EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_OR, {s16})); + EXPECT_ACTION(WidenScalar, 0, v2s32, LegalityQuery(G_OR, {v2s16})); + } + + // Test maxScalarOrELt + { + LegalizerInfo LI; + LI.getActionDefinitionsBuilder(G_AND) + .legalFor({s16}) + .maxScalarOrElt(0, s16); + LI.computeTables(); + + EXPECT_ACTION(NarrowScalar, 0, s16, LegalityQuery(G_AND, {s32})); + EXPECT_ACTION(NarrowScalar, 0, v2s16, LegalityQuery(G_AND, {v2s32})); + } + + // Test clampScalarOrElt + { + LegalizerInfo LI; + LI.getActionDefinitionsBuilder(G_XOR) + .legalFor({s16}) + .clampScalarOrElt(0, s16, s32); + LI.computeTables(); + + EXPECT_ACTION(NarrowScalar, 0, s32, LegalityQuery(G_XOR, {s64})); + EXPECT_ACTION(WidenScalar, 0, s16, LegalityQuery(G_XOR, {s8})); + + // Make sure the number of elements is preserved. + EXPECT_ACTION(NarrowScalar, 0, v2s32, LegalityQuery(G_XOR, {v2s64})); + EXPECT_ACTION(WidenScalar, 0, v2s16, LegalityQuery(G_XOR, {v2s8})); + } + + // Test minScalar + { + LegalizerInfo LI; + LI.getActionDefinitionsBuilder(G_OR) + .legalFor({s32}) + .minScalar(0, s32); + LI.computeTables(); + + // Only handle scalars, ignore vectors. + EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_OR, {s16})); + EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_OR, {v2s16})); + } + + // Test maxScalar + { + LegalizerInfo LI; + LI.getActionDefinitionsBuilder(G_AND) + .legalFor({s16}) + .maxScalar(0, s16); + LI.computeTables(); + + // Only handle scalars, ignore vectors. + EXPECT_ACTION(NarrowScalar, 0, s16, LegalityQuery(G_AND, {s32})); + EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_AND, {v2s32})); + } + + // Test clampScalar + { + LegalizerInfo LI; + + LI.getActionDefinitionsBuilder(G_XOR) + .legalFor({s16}) + .clampScalar(0, s16, s32); + LI.computeTables(); + + EXPECT_ACTION(NarrowScalar, 0, s32, LegalityQuery(G_XOR, {s64})); + EXPECT_ACTION(WidenScalar, 0, s16, LegalityQuery(G_XOR, {s8})); + + // Only handle scalars, ignore vectors. + EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_XOR, {v2s64})); + EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_XOR, {v2s8})); + } + + // Test widenScalarOrEltToNextPow2 + { + LegalizerInfo LI; + + LI.getActionDefinitionsBuilder(G_AND) + .legalFor({s32}) + .widenScalarOrEltToNextPow2(0, 32); + LI.computeTables(); + + // Handle scalars and vectors + EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_AND, {s5})); + EXPECT_ACTION(WidenScalar, 0, v2s32, LegalityQuery(G_AND, {v2s5})); + EXPECT_ACTION(WidenScalar, 0, s64, LegalityQuery(G_AND, {s33})); + EXPECT_ACTION(WidenScalar, 0, v2s64, LegalityQuery(G_AND, {v2s33})); + } + + // Test widenScalarToNextPow2 + { + LegalizerInfo LI; + + LI.getActionDefinitionsBuilder(G_AND) + .legalFor({s32}) + .widenScalarToNextPow2(0, 32); + LI.computeTables(); + + EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_AND, {s5})); + EXPECT_ACTION(WidenScalar, 0, s64, LegalityQuery(G_AND, {s33})); + + // Do nothing for vectors. + EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_AND, {v2s5})); + EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_AND, {v2s33})); + } +} Index: unittests/CodeGen/LowLevelTypeTest.cpp =================================================================== --- unittests/CodeGen/LowLevelTypeTest.cpp +++ unittests/CodeGen/LowLevelTypeTest.cpp @@ -118,6 +118,61 @@ LLT::scalarOrVector(2, LLT::pointer(1, 32))); } +TEST(LowLevelTypeTest, ChangeElementType) { + const LLT P0 = LLT::pointer(0, 32); + const LLT P1 = LLT::pointer(1, 64); + + const LLT S32 = LLT::scalar(32); + const LLT S64 = LLT::scalar(64); + + const LLT V2S32 = LLT::vector(2, 32); + const LLT V2S64 = LLT::vector(2, 64); + + const LLT V2P0 = LLT::vector(2, P0); + const LLT V2P1 = LLT::vector(2, P1); + + EXPECT_EQ(S64, S32.changeElementType(S64)); + EXPECT_EQ(S32, S32.changeElementType(S32)); + + EXPECT_EQ(S32, S64.changeElementSize(32)); + EXPECT_EQ(S32, S32.changeElementSize(32)); + + EXPECT_EQ(V2S64, V2S32.changeElementType(S64)); + EXPECT_EQ(V2S32, V2S64.changeElementType(S32)); + + EXPECT_EQ(V2S64, V2S32.changeElementSize(64)); + EXPECT_EQ(V2S32, V2S64.changeElementSize(32)); + + EXPECT_EQ(P0, S32.changeElementType(P0)); + EXPECT_EQ(S32, P0.changeElementType(S32)); + + EXPECT_EQ(V2P1, V2P0.changeElementType(P1)); + EXPECT_EQ(V2S32, V2P0.changeElementType(S32)); +} + +#ifdef GTEST_HAS_DEATH_TEST +#ifndef NDEBUG + +// Invalid to directly change the element size for pointers. +TEST(LowLevelTypeTest, ChangeElementTypeDeath) { + const LLT P0 = LLT::pointer(0, 32); + const LLT V2P0 = LLT::vector(2, P0); + + EXPECT_DEATH(P0.changeElementSize(64), + "invalid to directly change element size for pointers"); + EXPECT_DEATH(V2P0.changeElementSize(64), + "invalid to directly change element size for pointers"); + + // Make sure this still fails even without a change in size. + EXPECT_DEATH(P0.changeElementSize(32), + "invalid to directly change element size for pointers"); + EXPECT_DEATH(V2P0.changeElementSize(32), + "invalid to directly change element size for pointers"); +} + +#endif +#endif + TEST(LowLevelTypeTest, Pointer) { LLVMContext C; DataLayout DL("p64:64:64-p127:512:512:512-p16777215:65528:8");