Index: llvm/lib/Target/AArch64/AArch64Combine.td =================================================================== --- llvm/lib/Target/AArch64/AArch64Combine.td +++ llvm/lib/Target/AArch64/AArch64Combine.td @@ -33,43 +33,43 @@ def rev : GICombineRule< (defs root:$root, shuffle_matchdata:$matchinfo), (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return matchREV(*${root}, MRI, ${matchinfo}); }]), - (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) + [{ return Helper.matchREV(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) >; def zip : GICombineRule< (defs root:$root, shuffle_matchdata:$matchinfo), (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return matchZip(*${root}, MRI, ${matchinfo}); }]), - (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) + [{ return Helper.matchZip(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) >; def uzp : GICombineRule< (defs root:$root, shuffle_matchdata:$matchinfo), (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return matchUZP(*${root}, MRI, ${matchinfo}); }]), - (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) + [{ return Helper.matchUZP(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) >; def dup: GICombineRule < (defs root:$root, shuffle_matchdata:$matchinfo), (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return matchDup(*${root}, MRI, ${matchinfo}); }]), - (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) + [{ return Helper.matchDup(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) >; def trn : GICombineRule< (defs root:$root, shuffle_matchdata:$matchinfo), (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return matchTRN(*${root}, MRI, ${matchinfo}); }]), - (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) + [{ return Helper.matchTRN(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyShuffleVectorPseudo(*${root}, ${matchinfo}); }]) >; def ext: GICombineRule < (defs root:$root, shuffle_matchdata:$matchinfo), (match (wip_match_opcode G_SHUFFLE_VECTOR):$root, - [{ return matchEXT(*${root}, MRI, ${matchinfo}); }]), - (apply [{ applyEXT(*${root}, ${matchinfo}); }]) + [{ return Helper.matchEXT(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyEXT(*${root}, ${matchinfo}); }]) >; // Combines which replace a G_SHUFFLE_VECTOR with a target-specific pseudo @@ -81,4 +81,9 @@ [erase_undef_store, combines_for_extload, sext_trunc_sextload, shuffle_vector_pseudos]> { let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule"; + let StateClass = "AArch64PostLegalizerCombinerHelperState"; + let AdditionalArguments = [ + GICombinerHelperArg<"AArch64PostLegalizerCombinerHelper &", + "Helper"> + ]; } Index: llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp =================================================================== --- llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp +++ llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp @@ -1,4 +1,4 @@ - //=== lib/CodeGen/GlobalISel/AArch64PostLegalizerCombiner.cpp -------------===// +//=== lib/CodeGen/GlobalISel/AArch64PostLegalizerCombiner.cpp -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -44,10 +44,110 @@ ShuffleVectorPseudo() {} }; -/// Check if a vector shuffle corresponds to a REV instruction with the -/// specified blocksize. -static bool isREVMask(ArrayRef M, unsigned EltSize, unsigned NumElts, - unsigned BlockSize) { +class AArch64PostLegalizerCombinerHelper : public CombinerHelper { +private: + /// Check if a vector shuffle corresponds to a REV instruction with the + /// specified blocksize. + bool isREVMask(ArrayRef M, unsigned EltSize, unsigned NumElts, + unsigned BlockSize); + + /// Determines if \p M is a shuffle vector mask for a TRN of \p NumElts. + /// Whether or not G_TRN1 or G_TRN2 should be used is stored in \p + /// WhichResult. + bool isTRNMask(ArrayRef M, unsigned NumElts, unsigned &WhichResult); + + /// Check if a G_EXT instruction can handle a shuffle mask \p M when the + /// vector + /// sources of the shuffle are different. + Optional> getExtMask(ArrayRef M, + unsigned NumElts); + /// Determines if \p M is a shuffle vector mask for a UZP of \p NumElts. + /// Whether or not G_UZP1 or G_UZP2 should be used is stored in \p + /// WhichResult. + bool isUZPMask(ArrayRef M, unsigned NumElts, unsigned &WhichResult); + + /// \return true if \p M is a zip mask for a shuffle vector of \p NumElts. + /// Whether or not G_ZIP1 or G_ZIP2 should be used is stored in \p + /// WhichResult. + bool isZipMask(ArrayRef M, unsigned NumElts, unsigned &WhichResult); + + /// Helper function for matchDup. + bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI, + ShuffleVectorPseudo &MatchInfo); + + /// Helper function for matchDup. + bool matchDupFromBuildVector(int Lane, MachineInstr &MI, + ShuffleVectorPseudo &MatchInfo); + +public: + AArch64PostLegalizerCombinerHelper(GISelChangeObserver &Observer, + MachineIRBuilder &B, + GISelKnownBits *KB = nullptr, + MachineDominatorTree *MDT = nullptr, + const LegalizerInfo *LI = nullptr); + + /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with + /// a G_REV instruction. + /// + /// \param [in] MI - The shuffle vector instruction. + /// \param [out] MatchInfo - One of G_REV16, G_REV2, or G_REV64 on success. + bool matchREV(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo); + + /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with + /// a G_TRN1 or G_TRN2 instruction. + /// + /// \param [in] MI - The shuffle vector instruction. + /// \param [out] MatchInfo - Either G_TRN1 or G_TRN2 on success. + bool matchTRN(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo); + + /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with + /// a G_UZP1 or G_UZP2 instruction. + /// + /// \param [in] MI - The shuffle vector instruction. + /// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success. + bool matchUZP(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo); + + /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with + /// a G_ZIP1 or G_ZIP2 instruction. + /// + /// \param [in] MI - The shuffle vector instruction. + /// \param [out] MatchInfo - Either G_ZIP1 or G_ZIP2 on success. + bool matchZip(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo); + + /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with + /// a G_DUP instruction. + /// + /// \param [in] MI - The shuffle vector instruction. + /// \param [out] MatchInfo - G_DUP on success. + bool matchDup(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo); + + /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with + /// a G_EXT instruction. + /// + /// \param [in] MI - The shuffle vector instruction. + /// \param [out] MatchInfo - G_EXT on success. + bool matchEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo); + + /// Replace a G_SHUFFLE_VECTOR instruction with a pseudo. + /// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR. + bool applyShuffleVectorPseudo(MachineInstr &MI, + ShuffleVectorPseudo &MatchInfo); + + /// Replace a G_SHUFFLE_VECTOR instruction with G_EXT. + /// Special-cased because the constant operand must be emitted as a G_CONSTANT + /// for the imported tablegen patterns to work. + bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo); +}; + +AArch64PostLegalizerCombinerHelper::AArch64PostLegalizerCombinerHelper( + GISelChangeObserver &Observer, MachineIRBuilder &B, GISelKnownBits *KB, + MachineDominatorTree *MDT, const LegalizerInfo *LI) + : CombinerHelper(Observer, B, KB, MDT, LI) {} + +bool AArch64PostLegalizerCombinerHelper::isREVMask(ArrayRef M, + unsigned EltSize, + unsigned NumElts, + unsigned BlockSize) { assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) && "Only possible block sizes for REV are: 16, 32, 64"); assert(EltSize != 64 && "EltSize cannot be 64 for REV mask."); @@ -73,10 +173,9 @@ return true; } -/// Determines if \p M is a shuffle vector mask for a TRN of \p NumElts. -/// Whether or not G_TRN1 or G_TRN2 should be used is stored in \p WhichResult. -static bool isTRNMask(ArrayRef M, unsigned NumElts, - unsigned &WhichResult) { +bool AArch64PostLegalizerCombinerHelper::isTRNMask(ArrayRef M, + unsigned NumElts, + unsigned &WhichResult) { if (NumElts % 2 != 0) return false; WhichResult = (M[0] == 0 ? 0 : 1); @@ -89,10 +188,9 @@ return true; } -/// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector -/// sources of the shuffle are different. -static Optional> getExtMask(ArrayRef M, - unsigned NumElts) { +Optional> +AArch64PostLegalizerCombinerHelper::getExtMask(ArrayRef M, + unsigned NumElts) { // Look for the first non-undef element. auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; }); if (FirstRealElt == M.end()) @@ -131,10 +229,9 @@ return std::make_pair(ReverseExt, Imm); } -/// Determines if \p M is a shuffle vector mask for a UZP of \p NumElts. -/// Whether or not G_UZP1 or G_UZP2 should be used is stored in \p WhichResult. -static bool isUZPMask(ArrayRef M, unsigned NumElts, - unsigned &WhichResult) { +bool AArch64PostLegalizerCombinerHelper::isUZPMask(ArrayRef M, + unsigned NumElts, + unsigned &WhichResult) { WhichResult = (M[0] == 0 ? 0 : 1); for (unsigned i = 0; i != NumElts; ++i) { // Skip undef indices. @@ -146,10 +243,9 @@ return true; } -/// \return true if \p M is a zip mask for a shuffle vector of \p NumElts. -/// Whether or not G_ZIP1 or G_ZIP2 should be used is stored in \p WhichResult. -static bool isZipMask(ArrayRef M, unsigned NumElts, - unsigned &WhichResult) { +bool AArch64PostLegalizerCombinerHelper::isZipMask(ArrayRef M, + unsigned NumElts, + unsigned &WhichResult) { if (NumElts % 2 != 0) return false; @@ -157,18 +253,16 @@ WhichResult = (M[0] == 0 ? 0 : 1); unsigned Idx = WhichResult * NumElts / 2; for (unsigned i = 0; i != NumElts; i += 2) { - if ((M[i] >= 0 && static_cast(M[i]) != Idx) || - (M[i + 1] >= 0 && static_cast(M[i + 1]) != Idx + NumElts)) - return false; + if ((M[i] >= 0 && static_cast(M[i]) != Idx) || + (M[i + 1] >= 0 && static_cast(M[i + 1]) != Idx + NumElts)) + return false; Idx += 1; } return true; } -/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a -/// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc. -static bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchREV( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); Register Dst = MI.getOperand(0).getReg(); @@ -194,10 +288,8 @@ return false; } -/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with -/// a G_TRN1 or G_TRN2 instruction. -static bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchTRN( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); unsigned WhichResult; ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); @@ -212,13 +304,8 @@ return true; } -/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with -/// a G_UZP1 or G_UZP2 instruction. -/// -/// \param [in] MI - The shuffle vector instruction. -/// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success. -static bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchUZP( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); unsigned WhichResult; ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); @@ -233,8 +320,8 @@ return true; } -static bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchZip( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); unsigned WhichResult; ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); @@ -249,10 +336,8 @@ return true; } -/// Helper function for matchDup. -static bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI, - MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchDupFromInsertVectorElt( + int Lane, MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { if (Lane != 0) return false; @@ -290,9 +375,8 @@ } /// Helper function for matchDup. -static bool matchDupFromBuildVector(int Lane, MachineInstr &MI, - MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchDupFromBuildVector( + int Lane, MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { assert(Lane >= 0 && "Expected positive lane?"); // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the // lane's definition directly. @@ -306,8 +390,8 @@ return true; } -static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchDup( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); auto MaybeLane = getSplatIndex(MI); if (!MaybeLane) @@ -316,15 +400,15 @@ // If this is undef splat, generate it via "just" vdup, if possible. if (Lane < 0) Lane = 0; - if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo)) + if (matchDupFromInsertVectorElt(Lane, MI, MatchInfo)) return true; - if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo)) + if (matchDupFromBuildVector(Lane, MI, MatchInfo)) return true; return false; } -static bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { +bool AArch64PostLegalizerCombinerHelper::matchEXT( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); Register Dst = MI.getOperand(0).getReg(); auto ExtInfo = getExtMask(MI.getOperand(3).getShuffleMask(), @@ -344,30 +428,36 @@ return true; } -/// Replace a G_SHUFFLE_VECTOR instruction with a pseudo. -/// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR. -static bool applyShuffleVectorPseudo(MachineInstr &MI, - ShuffleVectorPseudo &MatchInfo) { - MachineIRBuilder MIRBuilder(MI); - MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps); +bool AArch64PostLegalizerCombinerHelper::applyShuffleVectorPseudo( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { + Builder.setInstr(MI); + Builder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps); MI.eraseFromParent(); return true; } -/// Replace a G_SHUFFLE_VECTOR instruction with G_EXT. -/// Special-cased because the constant operand must be emitted as a G_CONSTANT -/// for the imported tablegen patterns to work. -static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { - MachineIRBuilder MIRBuilder(MI); +bool AArch64PostLegalizerCombinerHelper::applyEXT( + MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { + Builder.setInstr(MI); // Tablegen patterns expect an i32 G_CONSTANT as the final op. auto Cst = - MIRBuilder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm()); - MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, - {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst}); + Builder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm()); + Builder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, + {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst}); MI.eraseFromParent(); return true; } +class AArch64PostLegalizerCombinerHelperState { +protected: + AArch64PostLegalizerCombinerHelper &Helper; + +public: + AArch64PostLegalizerCombinerHelperState( + AArch64PostLegalizerCombinerHelper &Helper) + : Helper(Helper) {} +}; + #define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS #include "AArch64GenPostLegalizeGICombiner.inc" #undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS @@ -403,8 +493,8 @@ MachineIRBuilder &B) const { const auto *LI = MI.getParent()->getParent()->getSubtarget().getLegalizerInfo(); - CombinerHelper Helper(Observer, B, KB, MDT, LI); - AArch64GenPostLegalizerCombinerHelper Generated(GeneratedRuleCfg); + AArch64PostLegalizerCombinerHelper Helper(Observer, B, KB, MDT, LI); + AArch64GenPostLegalizerCombinerHelper Generated(GeneratedRuleCfg, Helper); return Generated.tryCombineAll(Observer, MI, B, Helper); }