Index: include/llvm/CodeGen/GlobalISel/LegalizerInfo.h =================================================================== --- include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -21,6 +21,7 @@ #include #include +#include namespace llvm { class LLVMContext; @@ -99,19 +100,126 @@ LegalizerInfo(); virtual ~LegalizerInfo() = default; - /// Compute any ancillary tables needed to quickly decide how an operation - /// should be handled. This must be called after all "set*Action"methods but - /// before any query is made or incorrect results may be returned. - void computeTables(); + static bool needsDifferentType(const LegalizeAction Action) { + switch (Action) { + case NarrowScalar: + case WidenScalar: + case FewerElements: + case MoreElements: + case Unsupported: + return true; + default: + return false; + } + } - /// More friendly way to set an action for common types that have an LLT - /// representation. - void setAction(const InstrAspect &Aspect, LegalizeAction Action) { - TablesInitialized = false; + typedef std::pair SizeAndAction; + typedef std::vector SizeAndActionsVec; + typedef std::map ElemSize2LegalizeNumElementsMap; + + /// The SizeAndActionsVec is a representation mapping between all natural + /// numbers and an Action. The natural number represents the bit size of + /// the InstrAspect. For example, for a target with native support for 32-bit + /// and 64-bit additions, you'd express that as: + /// setAction({G_ADD, LLT:scalar(1)}, + /// {{1, WidenScalar}, // bit sizes [ 1, 31[ + /// {32, Legal}, // bit sizes [32, 33[ + /// {33, WidenScalar}, // bit sizes [33, 64[ + /// {64, Legal}, // bit sizes [64, 65[ + /// {65, NarrowScalar} // bit sizes [65, +inf[ + /// }); + /// It may be that only 64-bit pointers are supported on your target: + /// setAction({G_GEP, LLT:pointer(1)}, + /// {{1, Unsupported}, // bit sizes [ 1, 63[ + /// {64, Legal}, // bit sizes [64, 65[ + /// {65, Unsupported}, // bit sizes [65, +inf[ + /// }); + /// FIXME: should we create a new ADT for mapping an integer range to an + /// Action, so the above could be written in a way that's easier to + /// comprehend without documentation? + void setAction(const InstrAspect &Aspect, + const SizeAndActionsVec &SizeAndActions) { + // sizeInBits is meaningless on the Aspect, just make sure it's 1 for + // consistency here. + assert(Aspect.Type.isPointer() || + (Aspect.Type.isScalar() && Aspect.Type.getSizeInBits() == 1)); unsigned Opcode = Aspect.Opcode - FirstOp; - if (Actions[Opcode].size() <= Aspect.Idx) - Actions[Opcode].resize(Aspect.Idx + 1); - Actions[Aspect.Opcode - FirstOp][Aspect.Idx][Aspect.Type] = Action; + SmallVector &Actions = Aspect.Type.isPointer() + ? PointerActions[Opcode] + : ScalarActions[Opcode]; + if (Actions.size() <= Aspect.Idx) + Actions.resize(Aspect.Idx + 1); + Actions[Aspect.Idx] = SizeAndActions; + // FIXME: run a bunch of verifications on assumptions that should be + // met by SizeAndActions, like: + // - for every Widen action, there should be a larger bitsize that + // can be legalized towards (e.g. Legal, Lower, Libcall or Custom + // action). + // - for every Narrow action, there should be a smaller bitsize that + // can be legalized towards. + // - the first bit size is size 1. + // - ... + } + + /// Helper function for the common case where legalization for a particular + /// operation consists of widening the type to a large legal type, unless + /// there is no such type and then instead it should be narrowed to the + /// largest legal type. E.g. + /// setAction({G_ADD, LLT:scalar(1)}, + /// {{1, WidenScalar}, // bit sizes [ 1, 31[ + /// {32, Legal}, // bit sizes [32, 33[ + /// {33, WidenScalar}, // bit sizes [33, 64[ + /// {64, Legal}, // bit sizes [64, 65[ + /// {65, NarrowScalar} // bit sizes [65, +inf[ + /// }); + /// can be shortened to: + /// setAction({G_ADD, LLT:scalar(1)}, + /// getWidenToLargerTypesAndNarrowToLargest( + /// {32, Legal}, {64, Legal})); + static SizeAndActionsVec + getWidenToLargerTypesAndNarrowToLargest(const SizeAndActionsVec &v) { + return getIncreaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, + NarrowScalar); + } + + /// Helper function for the common case where legalization for a particular + /// vector operation consists of having more elements in the vector, to a + /// type that is legal. Unless / there is no such type and then instead it + /// should be legalized towards the widest vector that's still legal. E.g. + /// {8,{ + /// {1, MoreElements}, // [ <1x8>, <8x8> [ :increase to <8x8> + /// {8, Legal}, // [ <8x8>, <9x8> [ :is Legal + /// {9, MoreElements}, // [ <9x8>, <16x8> [ :increase to <16x8> + /// {16, Legal}, // [ <16x8>, <16x8> [ :is Legal + /// {17, FewerElements}// [ <17x8>, [ :decrease to <16x8> + /// }} + /// can be shortened to: + /// {8, + /// getMoreToWiderTypesAndLessToWidest({8, Legal}, {16, Legal}) + /// } + static SizeAndActionsVec + getMoreToWiderTypesAndLessToWidest(const SizeAndActionsVec &v) { + return getIncreaseToLargerTypesAndDecreaseToLargest(v, MoreElements, + FewerElements); + } + + static SizeAndActionsVec + getIncreaseToLargerTypesAndDecreaseToLargest(const SizeAndActionsVec &v, + LegalizeAction IncreaseAction, + LegalizeAction DecreaseAction) { + SizeAndActionsVec result; + assert(v.size() >= 1); + if (v[0].first != 1) + result.push_back({1, IncreaseAction}); + for (unsigned i = 0; i < v.size(); ++i) { + assert(!needsDifferentType(v[i].second)); + result.push_back(v[i]); + if (i + 1 == v.size()) + result.push_back({v[i].first + 1, DecreaseAction}); + else + result.push_back({v[i].first + 1, IncreaseAction}); + } + return result; } /// If an operation on a given vector type (say ) isn't explicitly @@ -119,20 +227,50 @@ /// (so that there's at least one legal vector with that scalar), then we /// adjust the number of elements in the vector so that it is legal. The /// desired action in the first step is controlled by this function. - void setScalarInVectorAction(unsigned Opcode, LLT ScalarTy, - LegalizeAction Action) { - assert(!ScalarTy.isVector()); - ScalarInVectorActions[std::make_pair(Opcode, ScalarTy)] = Action; + void setScalarInVectorAction(unsigned Opcode, + const SizeAndActionsVec &SizeAndActions) { + unsigned Index = Opcode - FirstOp; + ScalarInVectorActions[Index] = SizeAndActions; } + /// See also setScalarInVectorAction. + /// This function let's you specify the number of elements in a vector that + /// are legal for various element sizes. + /// E.g. + /// setScalarInVectorAction( + /// G_ADD, getWidenToLargerTypesAndNarrowToLargest( + /// {{8, Legal}, {16, Legal}, {32, Legal}, {64, Legal}})); + /// setLegalNrVectorLanes(BinOp, + /// { + /// {8, {4, 8, 16}}, // 4x8, 8x8 and 16x8 are legal + /// {16, {2, 4, 8}}, // 2x16, 4x16 and 8x16 are legal + /// {32, {1, 2, 4}}, // 1x32, 2x32 and 4x32 are legal + /// {64, {1, 2}}, // 1x64 and 2x642 are legal + /// }); + void setLegalNrVectorLanes( + unsigned Opcode, + const std::vector>> + &ElemSize2LegalNumElements) { + ElemSize2LegalizeNumElementsMap M; + for (auto ElemSizeAndLegalVectorWidths : ElemSize2LegalNumElements) { + const uint16_t ElemSize = ElemSizeAndLegalVectorWidths.first; + const auto &LegalVectorWidths = ElemSizeAndLegalVectorWidths.second; + SizeAndActionsVec V; + for (uint16_t Size : LegalVectorWidths) + V.push_back({Size, Legal}); + M[ElemSize] = getMoreToWiderTypesAndLessToWidest(V); + } + unsigned Index = Opcode - FirstOp; + VectorNumElementsActions[Index] = M; + } /// Determine what action should be taken to legalize the given generic - /// instruction opcode, type-index and type. Requires computeTables to have - /// been called. + /// instruction opcode, type-index and type. /// /// \returns a pair consisting of the kind of legalization that should be /// performed and the destination type. - std::pair getAction(const InstrAspect &Aspect) const; + SmallVector, 4> + getAction(const InstrAspect &Aspect) const; /// Determine what action should be taken to legalize the given generic /// instruction. @@ -140,74 +278,162 @@ /// \returns a tuple consisting of the LegalizeAction that should be /// performed, the type-index it should be performed on and the destination /// type. - std::tuple + SmallVector, 4> getAction(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; - /// Iterate the given function (typically something like doubling the width) - /// on Ty until we find a legal type for this operation. - LLT findLegalType(const InstrAspect &Aspect, - function_ref NextType) const { - LegalizeAction Action; - const TypeMap &Map = Actions[Aspect.Opcode - FirstOp][Aspect.Idx]; - LLT Ty = Aspect.Type; - do { - Ty = NextType(Ty); - auto ActionIt = Map.find(Ty); - if (ActionIt == Map.end()) - Action = DefaultActions.find(Aspect.Opcode)->second; - else - Action = ActionIt->second; - } while(Action != Legal); - return Ty; + static SmallVector, 4> + findActionInSAAV(const SizeAndActionsVec &Vec, const uint32_t Size) { + assert(Size >= 1); + // Find the first element in Vec that has a bitsize equal to or smaller + // than the requested bit size. We can do that using reverse iterators + int VecIdx = -1; + for (int i = Vec.size() - 1; i >= 0; --i) { + const uint32_t BitSize = Vec[i].first; + if (BitSize <= Size) { + VecIdx = i; + break; + } + } + assert(VecIdx != -1); + LegalizeAction Action = Vec[VecIdx].second; + switch (Action) { + case Legal: + case Lower: + case Libcall: + case Custom: + return {{Action, Size}}; + case NarrowScalar: + case FewerElements: + for (int i = VecIdx - 1; i >= 0; --i) + if (!needsDifferentType(Vec[i].second)) { + if (Vec[i].second == Legal) + return {{Action, Vec[i].first}}; + else + return {{Action, Vec[i].first}, {Vec[i].second, Vec[i].first}}; + } + llvm_unreachable(""); + case WidenScalar: + case MoreElements: + for (std::size_t i = VecIdx + 1; i < Vec.size(); ++i) + if (!needsDifferentType(Vec[i].second)) { + if (Vec[i].second == Legal) + return {{Action, Vec[i].first}}; + else + return {{Action, Vec[i].first}, {Vec[i].second, Vec[i].first}}; + } + llvm_unreachable(""); + case Unsupported: + return {{Unsupported, Size}}; + case NotFound: + llvm_unreachable("NotFound"); + } } - /// Find what type it's actually OK to perform the given operation on, given - /// the general approach we've decided to take. - LLT findLegalType(const InstrAspect &Aspect, LegalizeAction Action) const; - - std::pair findLegalAction(const InstrAspect &Aspect, - LegalizeAction Action) const { - return std::make_pair(Action, findLegalType(Aspect, Action)); + /// Returns the sequence of actions the target requested to legalize + /// the scalar or pointer type. + /// E.g. findLegalAction({G_REM, 13}) should return + /// [(WidenScalar, 32), (Lower, 32)], indicating to first widen the s13 + /// scalar to 32 bits, and to then lower it, assuming the setAction on SREM + /// was something like: + /// setAction({G_REM, LLT:scalar(1)}, + /// {{1, WidenScalar}, // bit sizes [ 1, 31[ + /// {32, Lower}, // bit sizes [32, 33[ + /// {33, NarrowScalar} // bit sizes [65, +inf[ + /// }); + SmallVector, 4> + findScalarLegalAction(const InstrAspect &Aspect) const { + assert(Aspect.Type.isScalar() || Aspect.Type.isPointer()); + if (Aspect.Opcode < FirstOp || Aspect.Opcode > LastOp) + return {{NotFound, LLT()}}; + unsigned Opcode = Aspect.Opcode - FirstOp; + const SmallVector &Actions = + Aspect.Type.isPointer() ? PointerActions[Opcode] + : ScalarActions[Opcode]; + if (Aspect.Idx >= Actions.size()) + return {{NotFound, LLT()}}; + const SizeAndActionsVec &Vec = Actions[Aspect.Idx]; + // FIXME: speed up this search, e.g. by using a results cache for repeated + // queries? + + SmallVector, 4> result; + for (auto ActionAndSize : + findActionInSAAV(Vec, Aspect.Type.getSizeInBits())) { + result.push_back({ActionAndSize.first, + Aspect.Type.isScalar() + ? LLT::scalar(ActionAndSize.second) + : LLT::pointer(Aspect.Type.getAddressSpace(), + ActionAndSize.second)}); + } + return result; } - /// Find the specified \p Aspect in the primary (explicitly set) Actions - /// table. Returns either the action the target requested or NotFound if there - /// was no setAction call. - LegalizeAction findInActions(const InstrAspect &Aspect) const { + /// Returns the sequence of actions the target requested to legalize the + /// vector type. + SmallVector, 4> + findVectorLegalAction(const InstrAspect &Aspect) const { + assert(Aspect.Type.isVector()); + // First legalize the vector element size, then legalize the number of + // lanes in the vector. + assert(Aspect.Type.isVector()); if (Aspect.Opcode < FirstOp || Aspect.Opcode > LastOp) - return NotFound; - if (Aspect.Idx >= Actions[Aspect.Opcode - FirstOp].size()) - return NotFound; - const TypeMap &Map = Actions[Aspect.Opcode - FirstOp][Aspect.Idx]; - auto ActionIt = Map.find(Aspect.Type); - if (ActionIt == Map.end()) - return NotFound; - - return ActionIt->second; + return {{NotFound, Aspect.Type}}; + if (Aspect.Idx >= ScalarInVectorActions[Aspect.Opcode - FirstOp].size()) + return {{NotFound, Aspect.Type}}; + const SizeAndActionsVec &ElemSizeVec = + ScalarInVectorActions[Aspect.Opcode - FirstOp]; + + SmallVector, 4> result; + LLT IntermediateType; + for (auto &ActionAndSize : + findActionInSAAV(ElemSizeVec, Aspect.Type.getScalarSizeInBits())) { + IntermediateType = + LLT::vector(Aspect.Type.getNumElements(), ActionAndSize.second); + // No need to push in an "already-legal" action here, as that will be + // pushed in by the number-of-elements legalization below. + if (ActionAndSize.first == Legal) + continue; + result.push_back({ActionAndSize.first, IntermediateType}); + if (ActionAndSize.first == NotFound || ActionAndSize.first == Unsupported) + return result; + } + + ElemSize2LegalizeNumElementsMap::const_iterator i = + VectorNumElementsActions[Aspect.Opcode - FirstOp].find( + IntermediateType.getScalarSizeInBits()); + if (i == VectorNumElementsActions[Aspect.Opcode - FirstOp].end()) { + return {{NotFound, IntermediateType}}; + } + const SizeAndActionsVec &NumElementsVec = (*i).second; + for (auto &ActionAndSize : + findActionInSAAV(NumElementsVec, IntermediateType.getNumElements())) { + // No need to push in an "already-legal" action here, if there was + // already another action recorded. + if (ActionAndSize.first == Legal && result.size() != 0) + continue; + result.push_back({ActionAndSize.first, + LLT::vector(ActionAndSize.second, + IntermediateType.getScalarSizeInBits())}); + } + + return result; } bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; - virtual bool legalizeCustom(MachineInstr &MI, - MachineRegisterInfo &MRI, + virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const; private: static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; - typedef DenseMap TypeMap; - typedef DenseMap, LegalizeAction> SIVActionMap; - - SmallVector Actions[LastOp - FirstOp + 1]; - SIVActionMap ScalarInVectorActions; - DenseMap, uint16_t> MaxLegalVectorElts; - DenseMap DefaultActions; - - bool TablesInitialized; + SmallVector ScalarActions[LastOp - FirstOp + 1]; + SmallVector PointerActions[LastOp - FirstOp + 1]; + SizeAndActionsVec ScalarInVectorActions[LastOp - FirstOp + 1]; + ElemSize2LegalizeNumElementsMap + VectorNumElementsActions[LastOp - FirstOp + 1]; }; - } // End namespace llvm. #endif Index: include/llvm/CodeGen/LowLevelType.h =================================================================== --- include/llvm/CodeGen/LowLevelType.h +++ include/llvm/CodeGen/LowLevelType.h @@ -122,47 +122,6 @@ return scalar(SizeInBits); } - /// Get a low-level type with half the size of the original, by halving the - /// size of the scalar type involved. For example `s32` will become `s16`, - /// `<2 x s32>` will become `<2 x s16>`. - LLT halfScalarSize() const { - assert(!isPointer() && getScalarSizeInBits() > 1 && - getScalarSizeInBits() % 2 == 0 && "cannot half size of this type"); - return LLT{Kind, ElementsOrAddrSpace, SizeInBits / 2}; - } - - /// Get a low-level type with twice the size of the original, by doubling the - /// size of the scalar type involved. For example `s32` will become `s64`, - /// `<2 x s32>` will become `<2 x s64>`. - LLT doubleScalarSize() const { - assert(!isPointer() && "cannot change size of this type"); - return LLT{Kind, ElementsOrAddrSpace, SizeInBits * 2}; - } - - /// Get a low-level type with half the size of the original, by halving the - /// number of vector elements of the scalar type involved. The source must be - /// a vector type with an even number of elements. For example `<4 x s32>` - /// will become `<2 x s32>`, `<2 x s32>` will become `s32`. - LLT halfElements() const { - assert(isVector() && ElementsOrAddrSpace % 2 == 0 && - "cannot half odd vector"); - if (ElementsOrAddrSpace == 2) - return scalar(SizeInBits); - - return LLT{Vector, static_cast(ElementsOrAddrSpace / 2), - SizeInBits}; - } - - /// Get a low-level type with twice the size of the original, by doubling the - /// number of vector elements of the scalar type involved. The source must be - /// a vector type. For example `<2 x s32>` will become `<4 x s32>`. Doubling - /// the number of elements in sN produces <2 x sN>. - LLT doubleElements() const { - assert(!isPointer() && "cannot double elements in pointer"); - return LLT{Vector, static_cast(ElementsOrAddrSpace * 2), - SizeInBits}; - } - void print(raw_ostream &OS) const; bool operator==(const LLT &RHS) const { Index: lib/CodeGen/GlobalISel/LegalizerHelper.cpp =================================================================== --- lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -36,26 +36,46 @@ LegalizerHelper::LegalizeResult LegalizerHelper::legalizeInstrStep(MachineInstr &MI, const LegalizerInfo &LegalizerInfo) { - auto Action = LegalizerInfo.getAction(MI, MRI); - switch (std::get<0>(Action)) { - case LegalizerInfo::Legal: - return AlreadyLegal; - case LegalizerInfo::Libcall: - return libcall(MI); - case LegalizerInfo::NarrowScalar: - return narrowScalar(MI, std::get<1>(Action), std::get<2>(Action)); - case LegalizerInfo::WidenScalar: - return widenScalar(MI, std::get<1>(Action), std::get<2>(Action)); - case LegalizerInfo::Lower: - return lower(MI, std::get<1>(Action), std::get<2>(Action)); - case LegalizerInfo::FewerElements: - return fewerElementsVector(MI, std::get<1>(Action), std::get<2>(Action)); - case LegalizerInfo::Custom: - return LegalizerInfo.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized - : UnableToLegalize; - default: - return UnableToLegalize; + const auto Actions = LegalizerInfo.getAction(MI, MRI); + LegalizeResult result; + bool AllLegal = true; + for (unsigned i = 0; i < Actions.size(); ++i) { + const auto Action = Actions[i]; + if (std::get<0>(Action) != LegalizerInfo::Legal) + AllLegal = false; + switch (std::get<0>(Action)) { + case LegalizerInfo::Legal: + result = AlreadyLegal; + break; + case LegalizerInfo::Libcall: + result = libcall(MI); + break; + case LegalizerInfo::NarrowScalar: + result = narrowScalar(MI, std::get<1>(Action), std::get<2>(Action)); + break; + case LegalizerInfo::WidenScalar: + result = widenScalar(MI, std::get<1>(Action), std::get<2>(Action)); + break; + case LegalizerInfo::Lower: + result = lower(MI, std::get<1>(Action), std::get<2>(Action)); + break; + case LegalizerInfo::FewerElements: + result = + fewerElementsVector(MI, std::get<1>(Action), std::get<2>(Action)); + break; + case LegalizerInfo::Custom: + result = LegalizerInfo.legalizeCustom(MI, MRI, MIRBuilder) + ? Legalized + : UnableToLegalize; + break; + default: + result = UnableToLegalize; + break; + } + if (result == UnableToLegalize) + return UnableToLegalize; } + return AllLegal ? AlreadyLegal : Legalized; } LegalizerHelper::LegalizeResult Index: lib/CodeGen/GlobalISel/LegalizerInfo.cpp =================================================================== --- lib/CodeGen/GlobalISel/LegalizerInfo.cpp +++ lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -27,113 +27,44 @@ #include "llvm/Target/TargetOpcodes.h" using namespace llvm; -LegalizerInfo::LegalizerInfo() : TablesInitialized(false) { - // FIXME: these two can be legalized to the fundamental load/store Jakob - // proposed. Once loads & stores are supported. - DefaultActions[TargetOpcode::G_ANYEXT] = Legal; - DefaultActions[TargetOpcode::G_TRUNC] = Legal; - - DefaultActions[TargetOpcode::G_INTRINSIC] = Legal; - DefaultActions[TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS] = Legal; - - DefaultActions[TargetOpcode::G_ADD] = NarrowScalar; - DefaultActions[TargetOpcode::G_LOAD] = NarrowScalar; - DefaultActions[TargetOpcode::G_STORE] = NarrowScalar; - - DefaultActions[TargetOpcode::G_BRCOND] = WidenScalar; - DefaultActions[TargetOpcode::G_INSERT] = NarrowScalar; -} - -void LegalizerInfo::computeTables() { - for (unsigned Opcode = 0; Opcode <= LastOp - FirstOp; ++Opcode) { - for (unsigned Idx = 0; Idx != Actions[Opcode].size(); ++Idx) { - for (auto &Action : Actions[Opcode][Idx]) { - LLT Ty = Action.first; - if (!Ty.isVector()) - continue; - - auto &Entry = MaxLegalVectorElts[std::make_pair(Opcode + FirstOp, - Ty.getElementType())]; - Entry = std::max(Entry, Ty.getNumElements()); - } - } - } - - TablesInitialized = true; -} +LegalizerInfo::LegalizerInfo() {} // FIXME: inefficient implementation for now. Without ComputeValueVTs we're // probably going to need specialized lookup structures for various types before // we have any hope of doing well with something like <13 x i3>. Even the common // cases should do better than what we have now. -std::pair +SmallVector, 4> LegalizerInfo::getAction(const InstrAspect &Aspect) const { - assert(TablesInitialized && "backend forgot to call computeTables"); // These *have* to be implemented for now, they're the fundamental basis of // how everything else is transformed. - // Nothing is going to go well with types that aren't a power of 2 yet, so - // don't even try because we might make things worse. - if (!isPowerOf2_64(Aspect.Type.getSizeInBits())) - return std::make_pair(Unsupported, LLT()); - // FIXME: the long-term plan calls for expansion in terms of load/store (if // they're not legal). if (Aspect.Opcode == TargetOpcode::G_SEQUENCE || Aspect.Opcode == TargetOpcode::G_EXTRACT) - return std::make_pair(Legal, Aspect.Type); - - LegalizeAction Action = findInActions(Aspect); - if (Action != NotFound) - return findLegalAction(Aspect, Action); - - unsigned Opcode = Aspect.Opcode; - LLT Ty = Aspect.Type; - if (!Ty.isVector()) { - auto DefaultAction = DefaultActions.find(Aspect.Opcode); - if (DefaultAction != DefaultActions.end() && DefaultAction->second == Legal) - return std::make_pair(Legal, Ty); - - if (DefaultAction == DefaultActions.end() || - DefaultAction->second != NarrowScalar) - return std::make_pair(Unsupported, LLT()); - return findLegalAction(Aspect, NarrowScalar); - } - - LLT EltTy = Ty.getElementType(); - int NumElts = Ty.getNumElements(); - - auto ScalarAction = ScalarInVectorActions.find(std::make_pair(Opcode, EltTy)); - if (ScalarAction != ScalarInVectorActions.end() && - ScalarAction->second != Legal) - return findLegalAction(Aspect, ScalarAction->second); - - // The element type is legal in principle, but the number of elements is - // wrong. - auto MaxLegalElts = MaxLegalVectorElts.lookup(std::make_pair(Opcode, EltTy)); - if (MaxLegalElts > NumElts) - return findLegalAction(Aspect, MoreElements); + return {std::make_pair(Legal, Aspect.Type)}; - if (MaxLegalElts == 0) { - // Scalarize if there's no legal vector type, which is just a special case - // of FewerElements. - return std::make_pair(FewerElements, EltTy); - } - - return findLegalAction(Aspect, FewerElements); + if (Aspect.Type.isScalar() || Aspect.Type.isPointer()) + return findScalarLegalAction(Aspect); + if (Aspect.Type.isVector()) + return findVectorLegalAction(Aspect); + return {}; } -std::tuple +SmallVector, 4> LegalizerInfo::getAction(const MachineInstr &MI, const MachineRegisterInfo &MRI) const { + SmallVector, 4> + result; SmallBitVector SeenTypes(8); const MCOperandInfo *OpInfo = MI.getDesc().OpInfo; + // FIXME: probably we'll need to cache the results here somehow? for (unsigned i = 0; i < MI.getDesc().getNumOperands(); ++i) { if (!OpInfo[i].isGenericType()) continue; - // We don't want to repeatedly check the same operand index, that - // could get expensive. + // We must only record actions once for each TypeIdx; otherwise we'd + // try to legalize operands multiple times down the line. unsigned TypeIdx = OpInfo[i].getGenericTypeIndex(); if (SeenTypes[TypeIdx]) continue; @@ -141,50 +72,24 @@ SeenTypes.set(TypeIdx); LLT Ty = MRI.getType(MI.getOperand(i).getReg()); - auto Action = getAction({MI.getOpcode(), TypeIdx, Ty}); - if (Action.first != Legal) - return std::make_tuple(Action.first, TypeIdx, Action.second); + auto Actions = getAction({MI.getOpcode(), TypeIdx, Ty}); + for (auto &Action : Actions) + result.push_back(std::make_tuple(Action.first, TypeIdx, Action.second)); } - return std::make_tuple(Legal, 0, LLT{}); + return result; } bool LegalizerInfo::isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const { - return std::get<0>(getAction(MI, MRI)) == Legal; -} - -LLT LegalizerInfo::findLegalType(const InstrAspect &Aspect, - LegalizeAction Action) const { - switch(Action) { - default: - llvm_unreachable("Cannot find legal type"); - case Legal: - case Lower: - case Libcall: - case Custom: - return Aspect.Type; - case NarrowScalar: { - return findLegalType(Aspect, - [&](LLT Ty) -> LLT { return Ty.halfScalarSize(); }); - } - case WidenScalar: { - return findLegalType(Aspect, [&](LLT Ty) -> LLT { - return Ty.getSizeInBits() < 8 ? LLT::scalar(8) : Ty.doubleScalarSize(); - }); - } - case FewerElements: { - return findLegalType(Aspect, - [&](LLT Ty) -> LLT { return Ty.halfElements(); }); - } - case MoreElements: { - return findLegalType(Aspect, - [&](LLT Ty) -> LLT { return Ty.doubleElements(); }); - } - } + SmallVector, 4> + Actions = getAction(MI, MRI); + for (auto Action : Actions) + if (std::get<0>(Action) != Legal) + return false; + return true; } -bool LegalizerInfo::legalizeCustom(MachineInstr &MI, - MachineRegisterInfo &MRI, +bool LegalizerInfo::legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const { return false; } Index: lib/Target/AArch64/AArch64LegalizerInfo.cpp =================================================================== --- lib/Target/AArch64/AArch64LegalizerInfo.cpp +++ lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -31,220 +31,244 @@ using namespace TargetOpcode; const LLT p0 = LLT::pointer(0, 64); const LLT s1 = LLT::scalar(1); - const LLT s8 = LLT::scalar(8); - const LLT s16 = LLT::scalar(16); - const LLT s32 = LLT::scalar(32); - const LLT s64 = LLT::scalar(64); - const LLT v2s32 = LLT::vector(2, 32); - const LLT v4s32 = LLT::vector(4, 32); - const LLT v2s64 = LLT::vector(2, 64); + const SizeAndActionsVec p0Legality = { + {1, Unsupported}, {64, Legal}, {65, Unsupported}}; + const SizeAndActionsVec ResizeTo32Or64 = + getWidenToLargerTypesAndNarrowToLargest({{32, Legal}, {64, Legal}}); + const SizeAndActionsVec ResizeTo8Or16Or32Or64 = + getWidenToLargerTypesAndNarrowToLargest( + {{8, Legal}, {16, Legal}, {32, Legal}, {64, Legal}}); + const SizeAndActionsVec ResizeTo1Or8Or16Or32Or64 = + getWidenToLargerTypesAndNarrowToLargest( + {{1, Legal}, {8, Legal}, {16, Legal}, {32, Legal}, {64, Legal}}); + const SizeAndActionsVec UpsizeTo32Or64_LargerUnsupported = { + {1, WidenScalar}, + {32, Legal}, + {33, WidenScalar}, + {64, Legal}, + {65, Unsupported}}; + const SizeAndActionsVec LoadStoreLegalizationActions = { + {{1, WidenScalar}, + {8, Legal}, + {9, WidenScalar}, + {16, Legal}, + {17, NarrowScalar}, + {32, Legal}, + {33, NarrowScalar}, + {64, Legal}, + {65, NarrowScalar}, + {128, Legal}, + {129, NarrowScalar}}}; + const SizeAndActionsVec LoadStoreOnlyAllowPow2Actions = { + {{1, WidenScalar}, + {8, Legal}, + {9, Unsupported}, + {16, Legal}, + {17, Unsupported}, + {32, Legal}, + {33, Unsupported}, + {64, Legal}, + {65, Unsupported}, + {128, NarrowScalar}, + {129, Unsupported}}}; + + const SizeAndActionsVec AllLegal = {{1, Legal}}; + const SizeAndActionsVec Only1IsLegal = {{1, Legal}, {2, Unsupported}}; + const SizeAndActionsVec Only32IsLegal = { + {1, Unsupported}, {32, Legal}, {33, Unsupported}}; + const SizeAndActionsVec Only64IsLegal = { + {1, Unsupported}, {64, Legal}, {65, Unsupported}}; + const SizeAndActionsVec Only32Or64IsLegal = {{1, Unsupported}, + {32, Legal}, + {33, Unsupported}, + {64, Legal}, + {65, Unsupported}}; + const SizeAndActionsVec Only16Or32IsLegal = {{1, Unsupported}, + {16, Legal}, + {17, Unsupported}, + {32, Legal}, + {33, Unsupported}}; + const SizeAndActionsVec Only1Or8Or16Or32IsLegal = { + {1, Legal}, {2, Unsupported}, {8, Legal}, {9, Unsupported}, + {16, Legal}, {17, Unsupported}, {32, Legal}, {33, Unsupported}}; + + // FIXME: carefully review all the legalization actions defined on + // non-power-of-2 types below. for (unsigned BinOp : {G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR, G_SHL}) { // These operations naturally get the right answer when used on // GPR32, even if the actual type is narrower. - for (auto Ty : {s32, s64, v2s32, v4s32, v2s64}) - setAction({BinOp, Ty}, Legal); - - for (auto Ty : {s1, s8, s16}) - setAction({BinOp, Ty}, WidenScalar); + setAction({BinOp, s1}, ResizeTo32Or64); + setScalarInVectorAction( + BinOp, getWidenToLargerTypesAndNarrowToLargest( + {{8, Legal}, {16, Legal}, {32, Legal}, {64, Legal}})); + setLegalNrVectorLanes(BinOp, + { + {8, {4, 8, 16}}, // 4x8, 8x8 and 16x8 are legal + {16, {2, 4, 8}}, // 2x16, 4x16 and 8x16 are legal + {32, {1, 2, 4}}, // 1x32, 2x32 and 4x32 are legal + {64, {1, 2}}, // 1x64 and 2x642 are legal + }); } - setAction({G_GEP, p0}, Legal); - setAction({G_GEP, 1, s64}, Legal); - - for (auto Ty : {s1, s8, s16, s32}) - setAction({G_GEP, 1, Ty}, WidenScalar); + setAction({G_GEP, p0}, p0Legality); + setAction({G_GEP, 1, s1}, + getWidenToLargerTypesAndNarrowToLargest({{64, Legal}})); - setAction({G_PTR_MASK, p0}, Legal); + setAction({G_PTR_MASK, p0}, p0Legality); - for (unsigned BinOp : {G_LSHR, G_ASHR, G_SDIV, G_UDIV}) { - for (auto Ty : {s32, s64}) - setAction({BinOp, Ty}, Legal); - - for (auto Ty : {s1, s8, s16}) - setAction({BinOp, Ty}, WidenScalar); - } + for (unsigned BinOp : {G_LSHR, G_ASHR, G_SDIV, G_UDIV}) + setAction({BinOp, s1}, ResizeTo32Or64); for (unsigned BinOp : {G_SREM, G_UREM}) - for (auto Ty : { s1, s8, s16, s32, s64 }) - setAction({BinOp, Ty}, Lower); - - for (unsigned Op : {G_SMULO, G_UMULO}) - setAction({Op, s64}, Lower); + setAction({BinOp, s1}, {{1, Lower}, + {2, WidenScalar}, + {8, Lower}, + {9, WidenScalar}, + {16, Lower}, + {17, WidenScalar}, + {32, Lower}, + {33, WidenScalar}, + {64, Lower}, + {65, Unsupported}}); + + for (unsigned Op : {G_SMULO, G_UMULO}) { + setAction({Op, s1}, {{1, WidenScalar}, {64, Lower}, {65, Unsupported}}); + setAction({Op, 1, s1}, {{1, Legal}, {2, Unsupported}}); + } for (unsigned Op : {G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_SMULH, G_UMULH}) { - for (auto Ty : { s32, s64 }) - setAction({Op, Ty}, Legal); - - setAction({Op, 1, s1}, Legal); + setAction({Op, s1}, UpsizeTo32Or64_LargerUnsupported); + setAction({Op, 1, s1}, {{1, Legal}, {2, Unsupported}}); } for (unsigned BinOp : {G_FADD, G_FSUB, G_FMUL, G_FDIV}) - for (auto Ty : {s32, s64}) - setAction({BinOp, Ty}, Legal); + setAction({BinOp, s1}, Only32Or64IsLegal); - for (unsigned BinOp : {G_FREM, G_FPOW}) { - setAction({BinOp, s32}, Libcall); - setAction({BinOp, s64}, Libcall); - } + for (unsigned BinOp : {G_FREM, G_FPOW}) + setAction({BinOp, s1}, {{1, Unsupported}, + {32, Libcall}, + {33, Unsupported}, + {64, Libcall}, + {65, Unsupported}}); // FIXME: what should we do about G_INSERTs with more than one source value? // For now the default of not specifying means we'll fall back. - for (auto Ty : {s32, s64}) { - setAction({G_INSERT, Ty}, Legal); - setAction({G_INSERT, 1, Ty}, Legal); - } - for (auto Ty : {s1, s8, s16}) { - setAction({G_INSERT, Ty}, WidenScalar); - // FIXME: Can't widen the sources because that violates the constraints on - // G_INSERT (It seems entirely reasonable that inputs shouldn't overlap). - } + setAction({G_INSERT, s1}, ResizeTo32Or64); + // FIXME: Can't widen the sources because that violates the constraints on + // G_INSERT (It seems entirely reasonable that inputs shouldn't overlap). + setAction({G_INSERT, 1, s1}, {{1, Legal}}); for (unsigned MemOp : {G_LOAD, G_STORE}) { - for (auto Ty : {s8, s16, s32, s64, p0, v2s32}) - setAction({MemOp, Ty}, Legal); - - setAction({MemOp, s1}, WidenScalar); + // Make sure to not widen the loaded/stored data beyond byte boundaries. + const bool OnlySupportPow2SizesForNow = true; + if (!OnlySupportPow2SizesForNow) + setAction({MemOp, s1}, LoadStoreLegalizationActions); + else + setAction({MemOp, s1}, LoadStoreOnlyAllowPow2Actions); + setAction({MemOp, p0}, p0Legality); + setScalarInVectorAction(MemOp, {{1, WidenScalar}, + {8, Legal}, + {9, WidenScalar}, + {16, Legal}, + {17, NarrowScalar}, + {32, Legal}, + {33, NarrowScalar}, + {64, Legal}, + {65, NarrowScalar}}); + // FIXME: could this increase the number of bytes loaded or stored? + setLegalNrVectorLanes(MemOp, + { + {8, {4, 8, 16}}, // 4x8, 8x8 and 16x8 are legal + {16, {2, 4, 8}}, // 2x16, 4x16 and 8x16 are legal + {32, {1, 2, 4}}, // 1x32, 2x32 and 4x32 are legal + {64, {1, 2}}, // 1x64 and 2x642 are legal + }); // And everything's fine in addrspace 0. - setAction({MemOp, 1, p0}, Legal); + setAction({MemOp, 1, p0}, p0Legality); } // Constants - for (auto Ty : {s32, s64}) { - setAction({TargetOpcode::G_CONSTANT, Ty}, Legal); - setAction({TargetOpcode::G_FCONSTANT, Ty}, Legal); - } - - setAction({G_CONSTANT, p0}, Legal); + setAction({G_CONSTANT, s1}, ResizeTo32Or64); + setAction({G_FCONSTANT, s1}, ResizeTo32Or64); + setAction({G_CONSTANT, p0}, p0Legality); - for (auto Ty : {s1, s8, s16}) - setAction({TargetOpcode::G_CONSTANT, Ty}, WidenScalar); - - setAction({TargetOpcode::G_FCONSTANT, s16}, WidenScalar); - - setAction({G_ICMP, s1}, Legal); - setAction({G_ICMP, 1, s32}, Legal); - setAction({G_ICMP, 1, s64}, Legal); - setAction({G_ICMP, 1, p0}, Legal); - - for (auto Ty : {s1, s8, s16}) { - setAction({G_ICMP, 1, Ty}, WidenScalar); - } + setAction({G_ICMP, s1}, {{1, Legal}, {2, Unsupported}}); + setAction({G_ICMP, 1, s1}, ResizeTo32Or64); + setAction({G_ICMP, 1, p0}, p0Legality); - setAction({G_FCMP, s1}, Legal); - setAction({G_FCMP, 1, s32}, Legal); - setAction({G_FCMP, 1, s64}, Legal); + setAction({G_FCMP, s1}, {{1, Legal}, {2, Unsupported}}); + setAction({G_FCMP, 1, s1}, UpsizeTo32Or64_LargerUnsupported); // Extensions - for (auto Ty : { s1, s8, s16, s32, s64 }) { - setAction({G_ZEXT, Ty}, Legal); - setAction({G_SEXT, Ty}, Legal); - setAction({G_ANYEXT, Ty}, Legal); + for (unsigned ExtOp : {G_ZEXT, G_SEXT, G_ANYEXT}) { + setAction({ExtOp, s1}, ResizeTo8Or16Or32Or64); + setAction({ExtOp, 1, s1}, {{1, Legal}}); } - for (auto Ty : { s1, s8, s16, s32 }) { - setAction({G_ZEXT, 1, Ty}, Legal); - setAction({G_SEXT, 1, Ty}, Legal); - setAction({G_ANYEXT, 1, Ty}, Legal); - } - - setAction({G_FPEXT, s64}, Legal); - setAction({G_FPEXT, 1, s32}, Legal); + setAction({G_FPEXT, s1}, Only64IsLegal); + setAction({G_FPEXT, 1, s1}, Only32IsLegal); // Truncations - for (auto Ty : { s16, s32 }) - setAction({G_FPTRUNC, Ty}, Legal); - - for (auto Ty : { s32, s64 }) - setAction({G_FPTRUNC, 1, Ty}, Legal); - - for (auto Ty : { s1, s8, s16, s32 }) - setAction({G_TRUNC, Ty}, Legal); - - for (auto Ty : { s8, s16, s32, s64 }) - setAction({G_TRUNC, 1, Ty}, Legal); + setAction({G_FPTRUNC, 0, s1}, Only16Or32IsLegal); + setAction({G_FPTRUNC, 1, s1}, Only32Or64IsLegal); + setAction({G_TRUNC, 0, s1}, Only1Or8Or16Or32IsLegal); + setAction({G_TRUNC, 1, s1}, ResizeTo8Or16Or32Or64); // Conversions - for (auto Ty : { s32, s64 }) { - setAction({G_FPTOSI, 0, Ty}, Legal); - setAction({G_FPTOUI, 0, Ty}, Legal); - setAction({G_SITOFP, 1, Ty}, Legal); - setAction({G_UITOFP, 1, Ty}, Legal); - } - for (auto Ty : { s1, s8, s16 }) { - setAction({G_FPTOSI, 0, Ty}, WidenScalar); - setAction({G_FPTOUI, 0, Ty}, WidenScalar); - setAction({G_SITOFP, 1, Ty}, WidenScalar); - setAction({G_UITOFP, 1, Ty}, WidenScalar); - } - - for (auto Ty : { s32, s64 }) { - setAction({G_FPTOSI, 1, Ty}, Legal); - setAction({G_FPTOUI, 1, Ty}, Legal); - setAction({G_SITOFP, 0, Ty}, Legal); - setAction({G_UITOFP, 0, Ty}, Legal); - } + setAction({G_FPTOSI, 0, s1}, ResizeTo32Or64); + setAction({G_FPTOSI, 1, s1}, Only32Or64IsLegal); + setAction({G_FPTOUI, 0, s1}, ResizeTo32Or64); + setAction({G_FPTOUI, 1, s1}, Only32Or64IsLegal); + setAction({G_SITOFP, 0, s1}, Only32Or64IsLegal); + setAction({G_SITOFP, 1, s1}, ResizeTo32Or64); + setAction({G_UITOFP, 0, s1}, Only32Or64IsLegal); + setAction({G_UITOFP, 1, s1}, ResizeTo32Or64); // Control-flow - for (auto Ty : {s1, s8, s16, s32}) - setAction({G_BRCOND, Ty}, Legal); - setAction({G_BRINDIRECT, p0}, Legal); + setAction({G_BRCOND, s1}, Only1Or8Or16Or32IsLegal); + setAction({G_BRINDIRECT, p0}, p0Legality); // Select - for (auto Ty : {s1, s8, s16}) - setAction({G_SELECT, Ty}, WidenScalar); - - for (auto Ty : {s32, s64, p0}) - setAction({G_SELECT, Ty}, Legal); - - setAction({G_SELECT, 1, s1}, Legal); + setAction({G_SELECT, s1}, ResizeTo32Or64); + setAction({G_SELECT, p0}, p0Legality); + setAction({G_SELECT, 1, s1}, Only1IsLegal); // Pointer-handling - setAction({G_FRAME_INDEX, p0}, Legal); - setAction({G_GLOBAL_VALUE, p0}, Legal); + setAction({G_FRAME_INDEX, p0}, p0Legality); + setAction({G_GLOBAL_VALUE, p0}, p0Legality); - for (auto Ty : {s1, s8, s16, s32, s64}) - setAction({G_PTRTOINT, 0, Ty}, Legal); + setAction({G_PTRTOINT, 0, s1}, ResizeTo1Or8Or16Or32Or64); + setAction({G_PTRTOINT, 1, p0}, p0Legality); - setAction({G_PTRTOINT, 1, p0}, Legal); - - setAction({G_INTTOPTR, 0, p0}, Legal); - setAction({G_INTTOPTR, 1, s64}, Legal); + setAction({G_INTTOPTR, 0, p0}, p0Legality); + setAction({G_INTTOPTR, 1, s1}, Only64IsLegal); // Casts for 32 and 64-bit width type are just copies. - for (auto Ty : {s1, s8, s16, s32, s64}) { - setAction({G_BITCAST, 0, Ty}, Legal); - setAction({G_BITCAST, 1, Ty}, Legal); - } + setAction({G_BITCAST, 0, s1}, AllLegal); + setAction({G_BITCAST, 1, s1}, AllLegal); // For the sake of copying bits around, the type does not really // matter as long as it fits a register. - for (int EltSize = 8; EltSize <= 64; EltSize *= 2) { - setAction({G_BITCAST, 0, LLT::vector(128/EltSize, EltSize)}, Legal); - setAction({G_BITCAST, 1, LLT::vector(128/EltSize, EltSize)}, Legal); - if (EltSize >= 64) - continue; - - setAction({G_BITCAST, 0, LLT::vector(64/EltSize, EltSize)}, Legal); - setAction({G_BITCAST, 1, LLT::vector(64/EltSize, EltSize)}, Legal); - if (EltSize >= 32) - continue; - - setAction({G_BITCAST, 0, LLT::vector(32/EltSize, EltSize)}, Legal); - setAction({G_BITCAST, 1, LLT::vector(32/EltSize, EltSize)}, Legal); - } - - setAction({G_VASTART, p0}, Legal); + setScalarInVectorAction( + G_BITCAST, getWidenToLargerTypesAndNarrowToLargest( + {{8, Legal}, {16, Legal}, {32, Legal}, {64, Legal}})); + setLegalNrVectorLanes(G_BITCAST, + { + {8, {4, 8, 16}}, // 4x8, 8x8 and 16x8 are legal + {16, {2, 4, 8}}, // 2x16, 4x16 and 8x16 are legal + {32, {1, 2, 4}}, // 1x32, 2x32 and 4x32 are legal + {64, {1, 2}}, // 1x64 and 2x642 are legal + }); + + setAction({G_VASTART, p0}, p0Legality); // va_list must be a pointer, but most sized types are pretty easy to handle // as the destination. - setAction({G_VAARG, 1, p0}, Legal); - - for (auto Ty : {s8, s16, s32, s64, p0}) - setAction({G_VAARG, Ty}, Custom); - - computeTables(); + setAction({G_VAARG, 1, p0}, p0Legality); + setAction({G_VAARG, s1}, {{1, Custom}}); + setAction({G_VAARG, p0}, {{1, Custom}}); } bool AArch64LegalizerInfo::legalizeCustom(MachineInstr &MI, Index: lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -28,25 +28,33 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo() { using namespace TargetOpcode; - const LLT S32 = LLT::scalar(32); - const LLT S64 = LLT::scalar(64); + const LLT s1 = LLT::scalar(1); const LLT P1 = LLT::pointer(1, 64); const LLT P2 = LLT::pointer(2, 64); - setAction({G_CONSTANT, S64}, Legal); + const SizeAndActionsVec P1Legality = { + {1, Unsupported}, {64, Legal}, {65, Unsupported}}; + const SizeAndActionsVec P2Legality = P1Legality; - setAction({G_GEP, P1}, Legal); - setAction({G_GEP, P2}, Legal); - setAction({G_GEP, 1, S64}, Legal); + const SizeAndActionsVec Only32IsLegal = { + {1, Unsupported}, {32, Legal}, {33, Unsupported}}; + const SizeAndActionsVec Only64IsLegal = { + {1, Unsupported}, {64, Legal}, {65, Unsupported}}; - setAction({G_LOAD, P1}, Legal); - setAction({G_LOAD, P2}, Legal); - setAction({G_LOAD, S32}, Legal); - setAction({G_LOAD, 1, P1}, Legal); - setAction({G_LOAD, 1, P2}, Legal); + setAction({G_CONSTANT, s1}, Only64IsLegal); - setAction({G_STORE, S32}, Legal); - setAction({G_STORE, 1, P1}, Legal); + setAction({G_GEP, P1}, P1Legality); + setAction({G_GEP, P2}, P2Legality); + setAction({G_GEP, 1, s1}, Only64IsLegal); + + setAction({G_LOAD, P1}, P1Legality); + setAction({G_LOAD, P2}, P2Legality); + setAction({G_LOAD, s1}, Only32IsLegal); + setAction({G_LOAD, 1, P1}, P1Legality); + setAction({G_LOAD, 1, P2}, P2Legality); + + setAction({G_STORE, s1}, Only32IsLegal); + setAction({G_STORE, 1, P1}, P1Legality); // FIXME: When RegBankSelect inserts copies, it will only create new // registers with scalar types. This means we can end up with @@ -54,9 +62,7 @@ // operands. In assert builds, the instruction selector will assert // if it sees a generic instruction which isn't legal, so we need to // tell it that scalar types are legal for pointer operands - setAction({G_GEP, S64}, Legal); - setAction({G_LOAD, 1, S64}, Legal); - setAction({G_STORE, 1, S64}, Legal); - - computeTables(); + setAction({G_GEP, s1}, Only64IsLegal); + setAction({G_LOAD, 1, s1}, Only64IsLegal); + setAction({G_STORE, 1, s1}, Only64IsLegal); } Index: lib/Target/ARM/ARMLegalizerInfo.cpp =================================================================== --- lib/Target/ARM/ARMLegalizerInfo.cpp +++ lib/Target/ARM/ARMLegalizerInfo.cpp @@ -28,34 +28,52 @@ using namespace TargetOpcode; const LLT p0 = LLT::pointer(0, 32); - const LLT s1 = LLT::scalar(1); - const LLT s8 = LLT::scalar(8); - const LLT s16 = LLT::scalar(16); - const LLT s32 = LLT::scalar(32); - const LLT s64 = LLT::scalar(64); - setAction({G_FRAME_INDEX, p0}, Legal); + const SizeAndActionsVec p0Legality = { + {1, Unsupported}, {32, Legal}, {33, Unsupported}}; + const SizeAndActionsVec ResizeTo1Or8Or16Or32 = + getWidenToLargerTypesAndNarrowToLargest( + {{1, Legal}, {8, Legal}, {16, Legal}, {32, Legal}}); + const SizeAndActionsVec ResizeTo1Or8Or16Or32Or64 = + getWidenToLargerTypesAndNarrowToLargest( + {{1, Legal}, {8, Legal}, {16, Legal}, {32, Legal}, {64, Legal}}); + const SizeAndActionsVec Only32IsLegal = { + {1, Unsupported}, {32, Legal}, {33, Unsupported}}; + const SizeAndActionsVec Only64IsLegal = { + {1, Unsupported}, {64, Legal}, {65, Unsupported}}; + const SizeAndActionsVec Only32Or64IsLegal = {{1, Unsupported}, + {32, Legal}, + {33, Unsupported}, + {64, Legal}, + {65, Unsupported}}; + const SizeAndActionsVec Only16Or32IsLegal = {{1, Unsupported}, + {16, Legal}, + {17, Unsupported}, + {32, Legal}, + {33, Unsupported}}; + const SizeAndActionsVec Only1Or8Or16IsLegal = { + {1, Legal}, {2, Unsupported}, {8, Legal}, + {9, Unsupported}, {16, Legal}, {17, Unsupported}}; + + setAction({G_FRAME_INDEX, p0}, p0Legality); + + if (ST.hasVFP2()) + setAction({G_LOAD, s1}, ResizeTo1Or8Or16Or32Or64); + else + setAction({G_LOAD, s1}, ResizeTo1Or8Or16Or32); - for (auto Ty : {s1, s8, s16, s32, p0}) - setAction({G_LOAD, Ty}, Legal); - setAction({G_LOAD, 1, p0}, Legal); + setAction({G_LOAD, p0}, p0Legality); + setAction({G_LOAD, 1, p0}, p0Legality); - for (auto Ty : {s1, s8, s16, s32}) - setAction({G_ADD, Ty}, Legal); + setAction({G_ADD, s1}, ResizeTo1Or8Or16Or32); for (unsigned Op : {G_SEXT, G_ZEXT}) { - setAction({Op, s32}, Legal); - for (auto Ty : {s1, s8, s16}) - setAction({Op, 1, Ty}, Legal); + setAction({Op, s1}, Only32IsLegal); + setAction({Op, 1, s1}, Only1Or8Or16IsLegal); } if (ST.hasVFP2()) { - setAction({G_FADD, s32}, Legal); - setAction({G_FADD, s64}, Legal); - - setAction({G_LOAD, s64}, Legal); + setAction({G_FADD, s1}, Only32Or64IsLegal); } - - computeTables(); } Index: lib/Target/X86/X86LegalizerInfo.cpp =================================================================== --- lib/Target/X86/X86LegalizerInfo.cpp +++ lib/Target/X86/X86LegalizerInfo.cpp @@ -25,32 +25,33 @@ #endif X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI) : Subtarget(STI) { - setLegalizerInfo32bit(); setLegalizerInfo64bit(); - - computeTables(); } void X86LegalizerInfo::setLegalizerInfo32bit() { + if (Subtarget.is64Bit()) + return; - const LLT s8 = LLT::scalar(8); - const LLT s16 = LLT::scalar(16); - const LLT s32 = LLT::scalar(32); + const LLT s1 = LLT::scalar(1); + const SizeAndActionsVec Only8Or16Or32IsLegal = { + {1, Unsupported}, {8, Legal}, {9, Unsupported}, {16, Legal}, + {17, Unsupported}, {32, Legal}, {33, Unsupported}}; - for (auto Ty : {s8, s16, s32}) { - setAction({TargetOpcode::G_ADD, Ty}, Legal); - setAction({TargetOpcode::G_SUB, Ty}, Legal); - } + setAction({TargetOpcode::G_ADD, s1}, Only8Or16Or32IsLegal); + setAction({TargetOpcode::G_SUB, s1}, Only8Or16Or32IsLegal); } void X86LegalizerInfo::setLegalizerInfo64bit() { - if (!Subtarget.is64Bit()) return; - const LLT s64 = LLT::scalar(64); + const LLT s1 = LLT::scalar(1); + const SizeAndActionsVec Only8Or16Or32Or64IsLegal = { + {1, Unsupported}, {8, Legal}, {9, Unsupported}, + {16, Legal}, {17, Unsupported}, {32, Legal}, + {33, Unsupported}, {64, Legal}, {65, Unsupported}}; - setAction({TargetOpcode::G_ADD, s64}, Legal); - setAction({TargetOpcode::G_SUB, s64}, Legal); + setAction({TargetOpcode::G_ADD, s1}, Only8Or16Or32Or64IsLegal); + setAction({TargetOpcode::G_SUB, s1}, Only8Or16Or32Or64IsLegal); } Index: unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp =================================================================== --- unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp +++ unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp @@ -44,77 +44,134 @@ namespace { +typedef SmallVector, 4> + ActionsType; TEST(LegalizerInfoTest, ScalarRISC) { using namespace TargetOpcode; LegalizerInfo L; // Typical RISCy set of operations based on AArch64. - L.setAction({G_ADD, LLT::scalar(8)}, LegalizerInfo::WidenScalar); - L.setAction({G_ADD, LLT::scalar(16)}, LegalizerInfo::WidenScalar); - L.setAction({G_ADD, LLT::scalar(32)}, LegalizerInfo::Legal); - L.setAction({G_ADD, LLT::scalar(64)}, LegalizerInfo::Legal); - L.computeTables(); - - // Check we infer the correct types and actually do what we're told. - ASSERT_EQ(L.getAction({G_ADD, LLT::scalar(8)}), - std::make_pair(LegalizerInfo::WidenScalar, LLT::scalar(32))); - ASSERT_EQ(L.getAction({G_ADD, LLT::scalar(16)}), - std::make_pair(LegalizerInfo::WidenScalar, LLT::scalar(32))); - ASSERT_EQ(L.getAction({G_ADD, LLT::scalar(32)}), - std::make_pair(LegalizerInfo::Legal, LLT::scalar(32))); - ASSERT_EQ(L.getAction({G_ADD, LLT::scalar(64)}), - std::make_pair(LegalizerInfo::Legal, LLT::scalar(64))); - - // Make sure the default for over-sized types applies. - ASSERT_EQ(L.getAction({G_ADD, LLT::scalar(128)}), - std::make_pair(LegalizerInfo::NarrowScalar, LLT::scalar(64))); + L.setAction({G_ADD, LLT::scalar(1)}, {{1, LegalizerInfo::WidenScalar}, + {32, LegalizerInfo::Legal}, + {33, LegalizerInfo::WidenScalar}, + {64, LegalizerInfo::Legal}, + {65, LegalizerInfo::NarrowScalar}}); + L.setAction({G_SUB, LLT::scalar(1)}, + LegalizerInfo::getWidenToLargerTypesAndNarrowToLargest( + {{32, LegalizerInfo::Legal}, {64, LegalizerInfo::Legal}})); + + for (auto &opcode : {G_ADD, G_SUB}) { + // Check we infer the correct types and actually do what we're told. + ASSERT_EQ(L.getAction({opcode, LLT::scalar(8)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::scalar(32)}})); + ASSERT_EQ(L.getAction({opcode, LLT::scalar(16)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::scalar(32)}})); + ASSERT_EQ(L.getAction({opcode, LLT::scalar(32)}), + ActionsType({{LegalizerInfo::Legal, LLT::scalar(32)}})); + ASSERT_EQ(L.getAction({opcode, LLT::scalar(64)}), + ActionsType({{LegalizerInfo::Legal, LLT::scalar(64)}})); + + // Make sure the default for over-sized types applies. + ASSERT_EQ(L.getAction({opcode, LLT::scalar(128)}), + ActionsType({{LegalizerInfo::NarrowScalar, LLT::scalar(64)}})); + // Make sure we also handle unusual sizes + ASSERT_EQ(L.getAction({opcode, LLT::scalar(1)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::scalar(32)}})); + ASSERT_EQ(L.getAction({opcode, LLT::scalar(31)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::scalar(32)}})); + ASSERT_EQ(L.getAction({opcode, LLT::scalar(33)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::scalar(64)}})); + ASSERT_EQ(L.getAction({opcode, LLT::scalar(63)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::scalar(64)}})); + ASSERT_EQ(L.getAction({opcode, LLT::scalar(65)}), + ActionsType({{LegalizerInfo::NarrowScalar, LLT::scalar(64)}})); + } } TEST(LegalizerInfoTest, VectorRISC) { using namespace TargetOpcode; LegalizerInfo L; // Typical RISCy set of operations based on ARM. - L.setScalarInVectorAction(G_ADD, LLT::scalar(8), LegalizerInfo::Legal); - L.setScalarInVectorAction(G_ADD, LLT::scalar(16), LegalizerInfo::Legal); - L.setScalarInVectorAction(G_ADD, LLT::scalar(32), LegalizerInfo::Legal); - - L.setAction({G_ADD, LLT::vector(8, 8)}, LegalizerInfo::Legal); - L.setAction({G_ADD, LLT::vector(16, 8)}, LegalizerInfo::Legal); - L.setAction({G_ADD, LLT::vector(4, 16)}, LegalizerInfo::Legal); - L.setAction({G_ADD, LLT::vector(8, 16)}, LegalizerInfo::Legal); - L.setAction({G_ADD, LLT::vector(2, 32)}, LegalizerInfo::Legal); - L.setAction({G_ADD, LLT::vector(4, 32)}, LegalizerInfo::Legal); - L.computeTables(); + L.setScalarInVectorAction( + G_ADD, LegalizerInfo::getWidenToLargerTypesAndNarrowToLargest( + {{8, LegalizerInfo::Legal}, + {16, LegalizerInfo::Legal}, + {32, LegalizerInfo::Legal}})); + L.setLegalNrVectorLanes(G_ADD, {{8, {8, 16}}, // 8x8 and 16x8 are legal + {16, {4, 8}}, // 16x4 and 16x8 are legal + {32, {2, 4}}}); // 32x2 and 32x4 are legal // Check we infer the correct types and actually do what we're told for some // simple cases. - ASSERT_EQ(L.getAction({G_ADD, LLT::vector(2, 8)}), - std::make_pair(LegalizerInfo::MoreElements, LLT::vector(8, 8))); ASSERT_EQ(L.getAction({G_ADD, LLT::vector(8, 8)}), - std::make_pair(LegalizerInfo::Legal, LLT::vector(8, 8))); - ASSERT_EQ( - L.getAction({G_ADD, LLT::vector(8, 32)}), - std::make_pair(LegalizerInfo::FewerElements, LLT::vector(4, 32))); + ActionsType({{LegalizerInfo::Legal, LLT::vector(8, 8)}})); + ASSERT_EQ(L.getAction({G_ADD, LLT::vector(8, 7)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::vector(8, 8)}})); + ASSERT_EQ(L.getAction({G_ADD, LLT::vector(2, 8)}), + ActionsType({{LegalizerInfo::MoreElements, LLT::vector(8, 8)}})); + ASSERT_EQ(L.getAction({G_ADD, LLT::vector(8, 32)}), + ActionsType({{LegalizerInfo::FewerElements, LLT::vector(4, 32)}})); + // Check a few non-power-of-2 sizes: + ASSERT_EQ(L.getAction({G_ADD, LLT::vector(3, 8)}), + ActionsType({{LegalizerInfo::MoreElements, LLT::vector(8, 8)}})); + ASSERT_EQ(L.getAction({G_ADD, LLT::vector(3, 3)}), + ActionsType({{LegalizerInfo::WidenScalar, LLT::vector(3, 8)}, + {LegalizerInfo::MoreElements, LLT::vector(8, 8)}})); } TEST(LegalizerInfoTest, MultipleTypes) { using namespace TargetOpcode; LegalizerInfo L; LLT p0 = LLT::pointer(0, 64); - LLT s32 = LLT::scalar(32); LLT s64 = LLT::scalar(64); // Typical RISCy set of operations based on AArch64. - L.setAction({G_PTRTOINT, 0, s64}, LegalizerInfo::Legal); - L.setAction({G_PTRTOINT, 1, p0}, LegalizerInfo::Legal); - - L.setAction({G_PTRTOINT, 0, s32}, LegalizerInfo::WidenScalar); - L.computeTables(); + L.setAction({G_PTRTOINT, 0, LLT::scalar(1)}, + {{1, LegalizerInfo::WidenScalar}, + {64, LegalizerInfo::Legal}, + {65, LegalizerInfo::NarrowScalar}}); + L.setAction({G_PTRTOINT, 1, LLT::pointer(0, 1)}, + {{1, LegalizerInfo::Unsupported}, + {64, LegalizerInfo::Legal}, + {65, LegalizerInfo::Unsupported}}); // Check we infer the correct types and actually do what we're told. ASSERT_EQ(L.getAction({G_PTRTOINT, 0, s64}), - std::make_pair(LegalizerInfo::Legal, s64)); + ActionsType({{LegalizerInfo::Legal, s64}})); ASSERT_EQ(L.getAction({G_PTRTOINT, 1, p0}), - std::make_pair(LegalizerInfo::Legal, p0)); + ActionsType({{LegalizerInfo::Legal, p0}})); + // Make sure we also handle unusual sizes + ASSERT_EQ(L.getAction({G_PTRTOINT, 0, LLT::scalar(65)}), + ActionsType({{LegalizerInfo::NarrowScalar, s64}})); + ASSERT_EQ(L.getAction({G_PTRTOINT, 1, LLT::pointer(0, 32)}), + ActionsType({{LegalizerInfo::Unsupported, LLT::pointer(0, 32)}})); +} + +TEST(LegalizerInfoTest, MultipleSteps) { + using namespace TargetOpcode; + LegalizerInfo L; + L.setAction({G_UREM, LLT::scalar(1)}, {{1, LegalizerInfo::WidenScalar}, + {32, LegalizerInfo::Lower}, + {33, LegalizerInfo::WidenScalar}, + {64, LegalizerInfo::Lower}, + {65, LegalizerInfo::NarrowScalar}}); + L.setAction({G_LOAD, LLT::scalar(1)}, {{1, LegalizerInfo::WidenScalar}, + {8, LegalizerInfo::Legal}, + {9, LegalizerInfo::Unsupported}, + {16, LegalizerInfo::NarrowScalar}, + {17, LegalizerInfo::Unsupported}}); + + // Check we infer the correct types and actually do what we're told. + ASSERT_EQ(L.getAction({G_UREM, LLT::scalar(8)}), + ActionsType({ + std::make_pair(LegalizerInfo::WidenScalar, LLT::scalar(32)), + std::make_pair(LegalizerInfo::Lower, LLT::scalar(32)), + })); + // Check that we correctly jump over Unsupported sizes when looking for + // a larger/smaller type size to legalize towards. + ASSERT_EQ(L.getAction({G_LOAD, LLT::scalar(16)}), + ActionsType({ + std::make_pair(LegalizerInfo::NarrowScalar, LLT::scalar(8)), + })); } } Index: unittests/CodeGen/LowLevelTypeTest.cpp =================================================================== --- unittests/CodeGen/LowLevelTypeTest.cpp +++ unittests/CodeGen/LowLevelTypeTest.cpp @@ -36,36 +36,22 @@ for (unsigned S : {1U, 17U, 32U, 64U, 0xfffffU}) { const LLT Ty = LLT::scalar(S); - const LLT HalfTy = (S % 2) == 0 ? Ty.halfScalarSize() : Ty; - const LLT DoubleTy = Ty.doubleScalarSize(); // Test kind. - for (const LLT TestTy : {Ty, HalfTy, DoubleTy}) { - ASSERT_TRUE(TestTy.isValid()); - ASSERT_TRUE(TestTy.isScalar()); + ASSERT_TRUE(Ty.isValid()); + ASSERT_TRUE(Ty.isScalar()); - ASSERT_FALSE(TestTy.isPointer()); - ASSERT_FALSE(TestTy.isVector()); - } + ASSERT_FALSE(Ty.isPointer()); + ASSERT_FALSE(Ty.isVector()); // Test sizes. EXPECT_EQ(S, Ty.getSizeInBits()); EXPECT_EQ(S, Ty.getScalarSizeInBits()); - EXPECT_EQ(S*2, DoubleTy.getSizeInBits()); - EXPECT_EQ(S*2, DoubleTy.getScalarSizeInBits()); - - if ((S % 2) == 0) { - EXPECT_EQ(S/2, HalfTy.getSizeInBits()); - EXPECT_EQ(S/2, HalfTy.getScalarSizeInBits()); - } - // Test equality operators. EXPECT_TRUE(Ty == Ty); EXPECT_FALSE(Ty != Ty); - EXPECT_NE(Ty, DoubleTy); - // Test Type->LLT conversion. Type *IRTy = IntegerType::get(C, S); EXPECT_EQ(Ty, LLT(*IRTy, DL)); @@ -90,61 +76,18 @@ // Test getElementType(). EXPECT_EQ(STy, VTy.getElementType()); - const LLT HalfSzTy = ((S % 2) == 0) ? VTy.halfScalarSize() : VTy; - const LLT DoubleSzTy = VTy.doubleScalarSize(); - - // halfElements requires an even number of elements. - const LLT HalfEltIfEvenTy = ((Elts % 2) == 0) ? VTy.halfElements() : VTy; - const LLT DoubleEltTy = VTy.doubleElements(); - // Test kind. - for (const LLT TestTy : {VTy, HalfSzTy, DoubleSzTy, DoubleEltTy}) { - ASSERT_TRUE(TestTy.isValid()); - ASSERT_TRUE(TestTy.isVector()); - - ASSERT_FALSE(TestTy.isScalar()); - ASSERT_FALSE(TestTy.isPointer()); - } - - // Test halving elements to a scalar. - { - ASSERT_TRUE(HalfEltIfEvenTy.isValid()); - ASSERT_FALSE(HalfEltIfEvenTy.isPointer()); - if (Elts > 2) { - ASSERT_TRUE(HalfEltIfEvenTy.isVector()); - } else { - ASSERT_FALSE(HalfEltIfEvenTy.isVector()); - EXPECT_EQ(STy, HalfEltIfEvenTy); - } - } + ASSERT_TRUE(VTy.isValid()); + ASSERT_TRUE(VTy.isVector()); + ASSERT_FALSE(VTy.isScalar()); + ASSERT_FALSE(VTy.isPointer()); // Test sizes. EXPECT_EQ(S * Elts, VTy.getSizeInBits()); EXPECT_EQ(S, VTy.getScalarSizeInBits()); EXPECT_EQ(Elts, VTy.getNumElements()); - if ((S % 2) == 0) { - EXPECT_EQ((S / 2) * Elts, HalfSzTy.getSizeInBits()); - EXPECT_EQ(S / 2, HalfSzTy.getScalarSizeInBits()); - EXPECT_EQ(Elts, HalfSzTy.getNumElements()); - } - - EXPECT_EQ((S * 2) * Elts, DoubleSzTy.getSizeInBits()); - EXPECT_EQ(S * 2, DoubleSzTy.getScalarSizeInBits()); - EXPECT_EQ(Elts, DoubleSzTy.getNumElements()); - - if ((Elts % 2) == 0) { - EXPECT_EQ(S * (Elts / 2), HalfEltIfEvenTy.getSizeInBits()); - EXPECT_EQ(S, HalfEltIfEvenTy.getScalarSizeInBits()); - if (Elts > 2) - EXPECT_EQ(Elts / 2, HalfEltIfEvenTy.getNumElements()); - } - - EXPECT_EQ(S * (Elts * 2), DoubleEltTy.getSizeInBits()); - EXPECT_EQ(S, DoubleEltTy.getScalarSizeInBits()); - EXPECT_EQ(Elts * 2, DoubleEltTy.getNumElements()); - // Test equality operators. EXPECT_TRUE(VTy == VTy); EXPECT_FALSE(VTy != VTy); @@ -152,10 +95,6 @@ // Test inequality operators on.. // ..different kind. EXPECT_NE(VTy, STy); - // ..different #elts. - EXPECT_NE(VTy, DoubleEltTy); - // ..different scalar size. - EXPECT_NE(VTy, DoubleSzTy); // Test Type->LLT conversion. Type *IRSTy = IntegerType::get(C, S);