Index: llvm/lib/Target/AArch64/AArch64.h =================================================================== --- llvm/lib/Target/AArch64/AArch64.h +++ llvm/lib/Target/AArch64/AArch64.h @@ -60,6 +60,7 @@ AArch64Subtarget &, AArch64RegisterBankInfo &); FunctionPass *createAArch64PreLegalizeCombiner(bool IsOptNone); FunctionPass *createAArch64PostLegalizeCombiner(bool IsOptNone); +FunctionPass *createAArch64PostLegalizeLowering(bool IsOptNone); FunctionPass *createAArch64StackTaggingPass(bool IsOptNone); FunctionPass *createAArch64StackTaggingPreRAPass(); @@ -80,6 +81,7 @@ void initializeAArch64SIMDInstrOptPass(PassRegistry&); void initializeAArch64PreLegalizerCombinerPass(PassRegistry&); void initializeAArch64PostLegalizerCombinerPass(PassRegistry &); +void initializeAArch64PostLegalizerLoweringPass(PassRegistry &); void initializeAArch64PromoteConstantPass(PassRegistry&); void initializeAArch64RedundantCopyEliminationPass(PassRegistry&); void initializeAArch64StorePairSuppressPass(PassRegistry&); Index: llvm/lib/Target/AArch64/AArch64Combine.td =================================================================== --- llvm/lib/Target/AArch64/AArch64Combine.td +++ llvm/lib/Target/AArch64/AArch64Combine.td @@ -83,12 +83,21 @@ (apply [{ applyVAshrLshrImm(*${root}, MRI, ${matchinfo}); }]) >; +// Post-legalization combines which should happen at all optimization levels. +// (E.g. ones that facilitate matching for the selector) For example, matching +// pseudos. +def AArch64PostLegalizerLoweringHelper + : GICombinerHelper<"AArch64GenPostLegalizerLoweringHelper", + [shuffle_vector_pseudos, vashr_vlshr_imm]> { + let DisableRuleOption = "aarch64postlegalizerlowering-disable-rule"; +} +// Post-legalization combines which are primarily optimizations. def AArch64PostLegalizerCombinerHelper : GICombinerHelper<"AArch64GenPostLegalizerCombinerHelper", [copy_prop, erase_undef_store, combines_for_extload, - sext_trunc_sextload, shuffle_vector_pseudos, + sext_trunc_sextload, hoist_logic_op_with_same_opcode_hands, - and_trivial_mask, vashr_vlshr_imm, xor_of_and_with_same_reg]> { + and_trivial_mask, xor_of_and_with_same_reg]> { let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule"; } Index: llvm/lib/Target/AArch64/AArch64TargetMachine.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -184,6 +184,7 @@ initializeAArch64SIMDInstrOptPass(*PR); initializeAArch64PreLegalizerCombinerPass(*PR); initializeAArch64PostLegalizerCombinerPass(*PR); + initializeAArch64PostLegalizerLoweringPass(*PR); initializeAArch64PromoteConstantPass(*PR); initializeAArch64RedundantCopyEliminationPass(*PR); initializeAArch64StorePairSuppressPass(*PR); @@ -564,6 +565,7 @@ bool IsOptNone = getOptLevel() == CodeGenOpt::None; if (!IsOptNone) addPass(createAArch64PostLegalizeCombiner(IsOptNone)); + addPass(createAArch64PostLegalizeLowering(IsOptNone)); } bool AArch64PassConfig::addRegBankSelect() { Index: llvm/lib/Target/AArch64/CMakeLists.txt =================================================================== --- llvm/lib/Target/AArch64/CMakeLists.txt +++ llvm/lib/Target/AArch64/CMakeLists.txt @@ -12,6 +12,8 @@ -combiners="AArch64PreLegalizerCombinerHelper") tablegen(LLVM AArch64GenPostLegalizeGICombiner.inc -gen-global-isel-combiner -combiners="AArch64PostLegalizerCombinerHelper") +tablegen(LLVM AArch64GenPostLegalizeGILowering.inc -gen-global-isel-combiner + -combiners="AArch64PostLegalizerLoweringHelper") tablegen(LLVM AArch64GenInstrInfo.inc -gen-instr-info) tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter) tablegen(LLVM AArch64GenMCPseudoLowering.inc -gen-pseudo-lowering) @@ -29,6 +31,7 @@ GISel/AArch64LegalizerInfo.cpp GISel/AArch64PreLegalizerCombiner.cpp GISel/AArch64PostLegalizerCombiner.cpp + GISel/AArch64PostLegalizerLowering.cpp GISel/AArch64RegisterBankInfo.cpp AArch64A57FPLoadBalancing.cpp AArch64AdvSIMDScalarPass.cpp Index: llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp =================================================================== --- llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp +++ llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp @@ -34,383 +34,6 @@ #define DEBUG_TYPE "aarch64-postlegalizer-combiner" using namespace llvm; -using namespace MIPatternMatch; - -/// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR. -/// -/// Used for matching target-supported shuffles before codegen. -struct ShuffleVectorPseudo { - unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1) - Register Dst; ///< Destination register. - SmallVector SrcOps; ///< Source registers. - ShuffleVectorPseudo(unsigned Opc, Register Dst, - std::initializer_list SrcOps) - : Opc(Opc), Dst(Dst), SrcOps(SrcOps){}; - ShuffleVectorPseudo() {} -}; - -/// Check if a vector shuffle corresponds to a REV instruction with the -/// specified blocksize. -static bool isREVMask(ArrayRef M, unsigned EltSize, unsigned NumElts, - unsigned BlockSize) { - assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) && - "Only possible block sizes for REV are: 16, 32, 64"); - assert(EltSize != 64 && "EltSize cannot be 64 for REV mask."); - - unsigned BlockElts = M[0] + 1; - - // If the first shuffle index is UNDEF, be optimistic. - if (M[0] < 0) - BlockElts = BlockSize / EltSize; - - if (BlockSize <= EltSize || BlockSize != BlockElts * EltSize) - return false; - - for (unsigned i = 0; i < NumElts; ++i) { - // Ignore undef indices. - if (M[i] < 0) - continue; - if (static_cast(M[i]) != - (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts)) - return false; - } - - return true; -} - -/// Determines if \p M is a shuffle vector mask for a TRN of \p NumElts. -/// Whether or not G_TRN1 or G_TRN2 should be used is stored in \p WhichResult. -static bool isTRNMask(ArrayRef M, unsigned NumElts, - unsigned &WhichResult) { - if (NumElts % 2 != 0) - return false; - WhichResult = (M[0] == 0 ? 0 : 1); - for (unsigned i = 0; i < NumElts; i += 2) { - if ((M[i] >= 0 && static_cast(M[i]) != i + WhichResult) || - (M[i + 1] >= 0 && - static_cast(M[i + 1]) != i + NumElts + WhichResult)) - return false; - } - return true; -} - -/// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector -/// sources of the shuffle are different. -static Optional> getExtMask(ArrayRef M, - unsigned NumElts) { - // Look for the first non-undef element. - auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; }); - if (FirstRealElt == M.end()) - return None; - - // Use APInt to handle overflow when calculating expected element. - unsigned MaskBits = APInt(32, NumElts * 2).logBase2(); - APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1); - - // The following shuffle indices must be the successive elements after the - // first real element. - if (any_of( - make_range(std::next(FirstRealElt), M.end()), - [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; })) - return None; - - // The index of an EXT is the first element if it is not UNDEF. - // Watch out for the beginning UNDEFs. The EXT index should be the expected - // value of the first element. E.g. - // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>. - // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>. - // ExpectedElt is the last mask index plus 1. - uint64_t Imm = ExpectedElt.getZExtValue(); - bool ReverseExt = false; - - // There are two difference cases requiring to reverse input vectors. - // For example, for vector <4 x i32> we have the following cases, - // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>) - // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>) - // For both cases, we finally use mask <5, 6, 7, 0>, which requires - // to reverse two input vectors. - if (Imm < NumElts) - ReverseExt = true; - else - Imm -= NumElts; - return std::make_pair(ReverseExt, Imm); -} - -/// Determines if \p M is a shuffle vector mask for a UZP of \p NumElts. -/// Whether or not G_UZP1 or G_UZP2 should be used is stored in \p WhichResult. -static bool isUZPMask(ArrayRef M, unsigned NumElts, - unsigned &WhichResult) { - WhichResult = (M[0] == 0 ? 0 : 1); - for (unsigned i = 0; i != NumElts; ++i) { - // Skip undef indices. - if (M[i] < 0) - continue; - if (static_cast(M[i]) != 2 * i + WhichResult) - return false; - } - return true; -} - -/// \return true if \p M is a zip mask for a shuffle vector of \p NumElts. -/// Whether or not G_ZIP1 or G_ZIP2 should be used is stored in \p WhichResult. -static bool isZipMask(ArrayRef M, unsigned NumElts, - unsigned &WhichResult) { - if (NumElts % 2 != 0) - return false; - - // 0 means use ZIP1, 1 means use ZIP2. - WhichResult = (M[0] == 0 ? 0 : 1); - unsigned Idx = WhichResult * NumElts / 2; - for (unsigned i = 0; i != NumElts; i += 2) { - if ((M[i] >= 0 && static_cast(M[i]) != Idx) || - (M[i + 1] >= 0 && static_cast(M[i + 1]) != Idx + NumElts)) - return false; - Idx += 1; - } - return true; -} - -/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a -/// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc. -static bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); - ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); - Register Dst = MI.getOperand(0).getReg(); - Register Src = MI.getOperand(1).getReg(); - LLT Ty = MRI.getType(Dst); - unsigned EltSize = Ty.getScalarSizeInBits(); - - // Element size for a rev cannot be 64. - if (EltSize == 64) - return false; - - unsigned NumElts = Ty.getNumElements(); - - // Try to produce G_REV64 - if (isREVMask(ShuffleMask, EltSize, NumElts, 64)) { - MatchInfo = ShuffleVectorPseudo(AArch64::G_REV64, Dst, {Src}); - return true; - } - - // TODO: Produce G_REV32 and G_REV16 once we have proper legalization support. - // This should be identical to above, but with a constant 32 and constant - // 16. - return false; -} - -/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with -/// a G_TRN1 or G_TRN2 instruction. -static bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); - unsigned WhichResult; - ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); - Register Dst = MI.getOperand(0).getReg(); - unsigned NumElts = MRI.getType(Dst).getNumElements(); - if (!isTRNMask(ShuffleMask, NumElts, WhichResult)) - return false; - unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2; - Register V1 = MI.getOperand(1).getReg(); - Register V2 = MI.getOperand(2).getReg(); - MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2}); - return true; -} - -/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with -/// a G_UZP1 or G_UZP2 instruction. -/// -/// \param [in] MI - The shuffle vector instruction. -/// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success. -static bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); - unsigned WhichResult; - ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); - Register Dst = MI.getOperand(0).getReg(); - unsigned NumElts = MRI.getType(Dst).getNumElements(); - if (!isUZPMask(ShuffleMask, NumElts, WhichResult)) - return false; - unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2; - Register V1 = MI.getOperand(1).getReg(); - Register V2 = MI.getOperand(2).getReg(); - MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2}); - return true; -} - -static bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); - unsigned WhichResult; - ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); - Register Dst = MI.getOperand(0).getReg(); - unsigned NumElts = MRI.getType(Dst).getNumElements(); - if (!isZipMask(ShuffleMask, NumElts, WhichResult)) - return false; - unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2; - Register V1 = MI.getOperand(1).getReg(); - Register V2 = MI.getOperand(2).getReg(); - MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2}); - return true; -} - -/// Helper function for matchDup. -static bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI, - MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - if (Lane != 0) - return false; - - // Try to match a vector splat operation into a dup instruction. - // We're looking for this pattern: - // - // %scalar:gpr(s64) = COPY $x0 - // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF - // %cst0:gpr(s32) = G_CONSTANT i32 0 - // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32) - // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32) - // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef, %zerovec(<2 x s32>) - // - // ...into: - // %splat = G_DUP %scalar - - // Begin matching the insert. - auto *InsMI = getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT, - MI.getOperand(1).getReg(), MRI); - if (!InsMI) - return false; - // Match the undef vector operand. - if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(), - MRI)) - return false; - - // Match the index constant 0. - int64_t Index = 0; - if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ICst(Index)) || Index) - return false; - - MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), - {InsMI->getOperand(2).getReg()}); - return true; -} - -/// Helper function for matchDup. -static bool matchDupFromBuildVector(int Lane, MachineInstr &MI, - MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - assert(Lane >= 0 && "Expected positive lane?"); - // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the - // lane's definition directly. - auto *BuildVecMI = getOpcodeDef(TargetOpcode::G_BUILD_VECTOR, - MI.getOperand(1).getReg(), MRI); - if (!BuildVecMI) - return false; - Register Reg = BuildVecMI->getOperand(Lane + 1).getReg(); - MatchInfo = - ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), {Reg}); - return true; -} - -static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); - auto MaybeLane = getSplatIndex(MI); - if (!MaybeLane) - return false; - int Lane = *MaybeLane; - // If this is undef splat, generate it via "just" vdup, if possible. - if (Lane < 0) - Lane = 0; - if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo)) - return true; - if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo)) - return true; - return false; -} - -static bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI, - ShuffleVectorPseudo &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); - Register Dst = MI.getOperand(0).getReg(); - auto ExtInfo = getExtMask(MI.getOperand(3).getShuffleMask(), - MRI.getType(Dst).getNumElements()); - if (!ExtInfo) - return false; - bool ReverseExt; - uint64_t Imm; - std::tie(ReverseExt, Imm) = *ExtInfo; - Register V1 = MI.getOperand(1).getReg(); - Register V2 = MI.getOperand(2).getReg(); - if (ReverseExt) - std::swap(V1, V2); - uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8; - Imm *= ExtFactor; - MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm}); - return true; -} - -/// Replace a G_SHUFFLE_VECTOR instruction with a pseudo. -/// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR. -static bool applyShuffleVectorPseudo(MachineInstr &MI, - ShuffleVectorPseudo &MatchInfo) { - MachineIRBuilder MIRBuilder(MI); - MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps); - MI.eraseFromParent(); - return true; -} - -/// Replace a G_SHUFFLE_VECTOR instruction with G_EXT. -/// Special-cased because the constant operand must be emitted as a G_CONSTANT -/// for the imported tablegen patterns to work. -static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { - MachineIRBuilder MIRBuilder(MI); - // Tablegen patterns expect an i32 G_CONSTANT as the final op. - auto Cst = - MIRBuilder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm()); - MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, - {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst}); - MI.eraseFromParent(); - return true; -} - -/// isVShiftRImm - Check if this is a valid vector for the immediate -/// operand of a vector shift right operation. The value must be in the range: -/// 1 <= Value <= ElementBits for a right shift. -static bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty, - int64_t &Cnt) { - assert(Ty.isVector() && "vector shift count is not a vector type"); - MachineInstr *MI = MRI.getVRegDef(Reg); - auto Cst = getBuildVectorConstantSplat(*MI, MRI); - if (!Cst) - return false; - Cnt = *Cst; - int64_t ElementBits = Ty.getScalarSizeInBits(); - return Cnt >= 1 && Cnt <= ElementBits; -} - -/// Match a vector G_ASHR or G_LSHR with a valid immediate shift. -static bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, - int64_t &Imm) { - assert(MI.getOpcode() == TargetOpcode::G_ASHR || - MI.getOpcode() == TargetOpcode::G_LSHR); - LLT Ty = MRI.getType(MI.getOperand(1).getReg()); - if (!Ty.isVector()) - return false; - return isVShiftRImm(MI.getOperand(2).getReg(), MRI, Ty, Imm); -} - -static bool applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, - int64_t &Imm) { - unsigned Opc = MI.getOpcode(); - assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR); - unsigned NewOpc = - Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR; - MachineIRBuilder MIB(MI); - auto ImmDef = MIB.buildConstant(LLT::scalar(32), Imm); - MIB.buildInstr(NewOpc, {MI.getOperand(0)}, {MI.getOperand(1), ImmDef}); - MI.eraseFromParent(); - return true; -} #define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS #include "AArch64GenPostLegalizeGICombiner.inc" Index: llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp =================================================================== --- /dev/null +++ llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp @@ -0,0 +1,525 @@ + //=== lib/CodeGen/GlobalISel/AArch64PostLegalizerLowering.cpp ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Post-legalization lowering for instructions. (E.g. pseudos). +// +//===----------------------------------------------------------------------===// + +#include "AArch64TargetMachine.h" +#include "MCTargetDesc/AArch64MCTargetDesc.h" +#include "llvm/CodeGen/GlobalISel/Combiner.h" +#include "llvm/CodeGen/GlobalISel/CombinerHelper.h" +#include "llvm/CodeGen/GlobalISel/CombinerInfo.h" +#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "aarch64-postlegalizer-lowering" + +using namespace llvm; +using namespace MIPatternMatch; + +/// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR. +/// +/// Used for matching target-supported shuffles before codegen. +struct ShuffleVectorPseudo { + unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1) + Register Dst; ///< Destination register. + SmallVector SrcOps; ///< Source registers. + ShuffleVectorPseudo(unsigned Opc, Register Dst, + std::initializer_list SrcOps) + : Opc(Opc), Dst(Dst), SrcOps(SrcOps){}; + ShuffleVectorPseudo() {} +}; + +/// Check if a vector shuffle corresponds to a REV instruction with the +/// specified blocksize. +static bool isREVMask(ArrayRef M, unsigned EltSize, unsigned NumElts, + unsigned BlockSize) { + assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) && + "Only possible block sizes for REV are: 16, 32, 64"); + assert(EltSize != 64 && "EltSize cannot be 64 for REV mask."); + + unsigned BlockElts = M[0] + 1; + + // If the first shuffle index is UNDEF, be optimistic. + if (M[0] < 0) + BlockElts = BlockSize / EltSize; + + if (BlockSize <= EltSize || BlockSize != BlockElts * EltSize) + return false; + + for (unsigned i = 0; i < NumElts; ++i) { + // Ignore undef indices. + if (M[i] < 0) + continue; + if (static_cast(M[i]) != + (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts)) + return false; + } + + return true; +} + +/// Determines if \p M is a shuffle vector mask for a TRN of \p NumElts. +/// Whether or not G_TRN1 or G_TRN2 should be used is stored in \p WhichResult. +static bool isTRNMask(ArrayRef M, unsigned NumElts, + unsigned &WhichResult) { + if (NumElts % 2 != 0) + return false; + WhichResult = (M[0] == 0 ? 0 : 1); + for (unsigned i = 0; i < NumElts; i += 2) { + if ((M[i] >= 0 && static_cast(M[i]) != i + WhichResult) || + (M[i + 1] >= 0 && + static_cast(M[i + 1]) != i + NumElts + WhichResult)) + return false; + } + return true; +} + +/// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector +/// sources of the shuffle are different. +static Optional> getExtMask(ArrayRef M, + unsigned NumElts) { + // Look for the first non-undef element. + auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; }); + if (FirstRealElt == M.end()) + return None; + + // Use APInt to handle overflow when calculating expected element. + unsigned MaskBits = APInt(32, NumElts * 2).logBase2(); + APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1); + + // The following shuffle indices must be the successive elements after the + // first real element. + if (any_of( + make_range(std::next(FirstRealElt), M.end()), + [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; })) + return None; + + // The index of an EXT is the first element if it is not UNDEF. + // Watch out for the beginning UNDEFs. The EXT index should be the expected + // value of the first element. E.g. + // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>. + // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>. + // ExpectedElt is the last mask index plus 1. + uint64_t Imm = ExpectedElt.getZExtValue(); + bool ReverseExt = false; + + // There are two difference cases requiring to reverse input vectors. + // For example, for vector <4 x i32> we have the following cases, + // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>) + // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>) + // For both cases, we finally use mask <5, 6, 7, 0>, which requires + // to reverse two input vectors. + if (Imm < NumElts) + ReverseExt = true; + else + Imm -= NumElts; + return std::make_pair(ReverseExt, Imm); +} + +/// Determines if \p M is a shuffle vector mask for a UZP of \p NumElts. +/// Whether or not G_UZP1 or G_UZP2 should be used is stored in \p WhichResult. +static bool isUZPMask(ArrayRef M, unsigned NumElts, + unsigned &WhichResult) { + WhichResult = (M[0] == 0 ? 0 : 1); + for (unsigned i = 0; i != NumElts; ++i) { + // Skip undef indices. + if (M[i] < 0) + continue; + if (static_cast(M[i]) != 2 * i + WhichResult) + return false; + } + return true; +} + +/// \return true if \p M is a zip mask for a shuffle vector of \p NumElts. +/// Whether or not G_ZIP1 or G_ZIP2 should be used is stored in \p WhichResult. +static bool isZipMask(ArrayRef M, unsigned NumElts, + unsigned &WhichResult) { + if (NumElts % 2 != 0) + return false; + + // 0 means use ZIP1, 1 means use ZIP2. + WhichResult = (M[0] == 0 ? 0 : 1); + unsigned Idx = WhichResult * NumElts / 2; + for (unsigned i = 0; i != NumElts; i += 2) { + if ((M[i] >= 0 && static_cast(M[i]) != Idx) || + (M[i + 1] >= 0 && static_cast(M[i + 1]) != Idx + NumElts)) + return false; + Idx += 1; + } + return true; +} + +/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a +/// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc. +static bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); + ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + LLT Ty = MRI.getType(Dst); + unsigned EltSize = Ty.getScalarSizeInBits(); + + // Element size for a rev cannot be 64. + if (EltSize == 64) + return false; + + unsigned NumElts = Ty.getNumElements(); + + // Try to produce G_REV64 + if (isREVMask(ShuffleMask, EltSize, NumElts, 64)) { + MatchInfo = ShuffleVectorPseudo(AArch64::G_REV64, Dst, {Src}); + return true; + } + + // TODO: Produce G_REV32 and G_REV16 once we have proper legalization support. + // This should be identical to above, but with a constant 32 and constant + // 16. + return false; +} + +/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with +/// a G_TRN1 or G_TRN2 instruction. +static bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); + unsigned WhichResult; + ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); + Register Dst = MI.getOperand(0).getReg(); + unsigned NumElts = MRI.getType(Dst).getNumElements(); + if (!isTRNMask(ShuffleMask, NumElts, WhichResult)) + return false; + unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2; + Register V1 = MI.getOperand(1).getReg(); + Register V2 = MI.getOperand(2).getReg(); + MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2}); + return true; +} + +/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with +/// a G_UZP1 or G_UZP2 instruction. +/// +/// \param [in] MI - The shuffle vector instruction. +/// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success. +static bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); + unsigned WhichResult; + ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); + Register Dst = MI.getOperand(0).getReg(); + unsigned NumElts = MRI.getType(Dst).getNumElements(); + if (!isUZPMask(ShuffleMask, NumElts, WhichResult)) + return false; + unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2; + Register V1 = MI.getOperand(1).getReg(); + Register V2 = MI.getOperand(2).getReg(); + MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2}); + return true; +} + +static bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); + unsigned WhichResult; + ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); + Register Dst = MI.getOperand(0).getReg(); + unsigned NumElts = MRI.getType(Dst).getNumElements(); + if (!isZipMask(ShuffleMask, NumElts, WhichResult)) + return false; + unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2; + Register V1 = MI.getOperand(1).getReg(); + Register V2 = MI.getOperand(2).getReg(); + MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2}); + return true; +} + +/// Helper function for matchDup. +static bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI, + MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + if (Lane != 0) + return false; + + // Try to match a vector splat operation into a dup instruction. + // We're looking for this pattern: + // + // %scalar:gpr(s64) = COPY $x0 + // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF + // %cst0:gpr(s32) = G_CONSTANT i32 0 + // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32) + // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32) + // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef, %zerovec(<2 x s32>) + // + // ...into: + // %splat = G_DUP %scalar + + // Begin matching the insert. + auto *InsMI = getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT, + MI.getOperand(1).getReg(), MRI); + if (!InsMI) + return false; + // Match the undef vector operand. + if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(), + MRI)) + return false; + + // Match the index constant 0. + int64_t Index = 0; + if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ICst(Index)) || Index) + return false; + + MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), + {InsMI->getOperand(2).getReg()}); + return true; +} + +/// Helper function for matchDup. +static bool matchDupFromBuildVector(int Lane, MachineInstr &MI, + MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + assert(Lane >= 0 && "Expected positive lane?"); + // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the + // lane's definition directly. + auto *BuildVecMI = getOpcodeDef(TargetOpcode::G_BUILD_VECTOR, + MI.getOperand(1).getReg(), MRI); + if (!BuildVecMI) + return false; + Register Reg = BuildVecMI->getOperand(Lane + 1).getReg(); + MatchInfo = + ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), {Reg}); + return true; +} + +static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); + auto MaybeLane = getSplatIndex(MI); + if (!MaybeLane) + return false; + int Lane = *MaybeLane; + // If this is undef splat, generate it via "just" vdup, if possible. + if (Lane < 0) + Lane = 0; + if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo)) + return true; + if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo)) + return true; + return false; +} + +static bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI, + ShuffleVectorPseudo &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); + Register Dst = MI.getOperand(0).getReg(); + auto ExtInfo = getExtMask(MI.getOperand(3).getShuffleMask(), + MRI.getType(Dst).getNumElements()); + if (!ExtInfo) + return false; + bool ReverseExt; + uint64_t Imm; + std::tie(ReverseExt, Imm) = *ExtInfo; + Register V1 = MI.getOperand(1).getReg(); + Register V2 = MI.getOperand(2).getReg(); + if (ReverseExt) + std::swap(V1, V2); + uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8; + Imm *= ExtFactor; + MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm}); + return true; +} + +/// Replace a G_SHUFFLE_VECTOR instruction with a pseudo. +/// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR. +static bool applyShuffleVectorPseudo(MachineInstr &MI, + ShuffleVectorPseudo &MatchInfo) { + MachineIRBuilder MIRBuilder(MI); + MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps); + MI.eraseFromParent(); + return true; +} + +/// Replace a G_SHUFFLE_VECTOR instruction with G_EXT. +/// Special-cased because the constant operand must be emitted as a G_CONSTANT +/// for the imported tablegen patterns to work. +static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) { + MachineIRBuilder MIRBuilder(MI); + // Tablegen patterns expect an i32 G_CONSTANT as the final op. + auto Cst = + MIRBuilder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm()); + MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, + {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst}); + MI.eraseFromParent(); + return true; +} + +/// isVShiftRImm - Check if this is a valid vector for the immediate +/// operand of a vector shift right operation. The value must be in the range: +/// 1 <= Value <= ElementBits for a right shift. +static bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty, + int64_t &Cnt) { + assert(Ty.isVector() && "vector shift count is not a vector type"); + MachineInstr *MI = MRI.getVRegDef(Reg); + auto Cst = getBuildVectorConstantSplat(*MI, MRI); + if (!Cst) + return false; + Cnt = *Cst; + int64_t ElementBits = Ty.getScalarSizeInBits(); + return Cnt >= 1 && Cnt <= ElementBits; +} + +/// Match a vector G_ASHR or G_LSHR with a valid immediate shift. +static bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, + int64_t &Imm) { + assert(MI.getOpcode() == TargetOpcode::G_ASHR || + MI.getOpcode() == TargetOpcode::G_LSHR); + LLT Ty = MRI.getType(MI.getOperand(1).getReg()); + if (!Ty.isVector()) + return false; + return isVShiftRImm(MI.getOperand(2).getReg(), MRI, Ty, Imm); +} + +static bool applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, + int64_t &Imm) { + unsigned Opc = MI.getOpcode(); + assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR); + unsigned NewOpc = + Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR; + MachineIRBuilder MIB(MI); + auto ImmDef = MIB.buildConstant(LLT::scalar(32), Imm); + MIB.buildInstr(NewOpc, {MI.getOperand(0)}, {MI.getOperand(1), ImmDef}); + MI.eraseFromParent(); + return true; +} + +#define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS +#include "AArch64GenPostLegalizeGILowering.inc" +#undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS + +namespace { +#define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H +#include "AArch64GenPostLegalizeGILowering.inc" +#undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H + +class AArch64PostLegalizerLoweringInfo : public CombinerInfo { + GISelKnownBits *KB; + MachineDominatorTree *MDT; + +public: + AArch64GenPostLegalizerLoweringHelperRuleConfig GeneratedRuleCfg; + + AArch64PostLegalizerLoweringInfo(bool EnableOpt, bool OptSize, bool MinSize, + GISelKnownBits *KB, + MachineDominatorTree *MDT) + : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false, + /*LegalizerInfo*/ nullptr, EnableOpt, OptSize, MinSize), + KB(KB), MDT(MDT) { + if (!GeneratedRuleCfg.parseCommandLineOption()) + report_fatal_error("Invalid rule identifier"); + } + + virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI, + MachineIRBuilder &B) const override; +}; + +bool AArch64PostLegalizerLoweringInfo::combine(GISelChangeObserver &Observer, + MachineInstr &MI, + MachineIRBuilder &B) const { + const auto *LI = + MI.getParent()->getParent()->getSubtarget().getLegalizerInfo(); + CombinerHelper Helper(Observer, B, KB, MDT, LI); + AArch64GenPostLegalizerLoweringHelper Generated(GeneratedRuleCfg); + return Generated.tryCombineAll(Observer, MI, B, Helper); +} + +#define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP +#include "AArch64GenPostLegalizeGILowering.inc" +#undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP + +class AArch64PostLegalizerLowering : public MachineFunctionPass { +public: + static char ID; + + AArch64PostLegalizerLowering(bool IsOptNone = false); + + StringRef getPassName() const override { + return "AArch64PostLegalizerLowering"; + } + + bool runOnMachineFunction(MachineFunction &MF) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + +private: + bool IsOptNone; +}; +} // end anonymous namespace + +void AArch64PostLegalizerLowering::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); + AU.setPreservesCFG(); + getSelectionDAGFallbackAnalysisUsage(AU); + AU.addRequired(); + AU.addPreserved(); + if (!IsOptNone) { + AU.addRequired(); + AU.addPreserved(); + } + MachineFunctionPass::getAnalysisUsage(AU); +} + +AArch64PostLegalizerLowering::AArch64PostLegalizerLowering(bool IsOptNone) + : MachineFunctionPass(ID), IsOptNone(IsOptNone) { + initializeAArch64PostLegalizerLoweringPass(*PassRegistry::getPassRegistry()); +} + +bool AArch64PostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) { + if (MF.getProperties().hasProperty( + MachineFunctionProperties::Property::FailedISel)) + return false; + assert(MF.getProperties().hasProperty( + MachineFunctionProperties::Property::Legalized) && + "Expected a legalized function?"); + auto *TPC = &getAnalysis(); + const Function &F = MF.getFunction(); + bool EnableOpt = + MF.getTarget().getOptLevel() != CodeGenOpt::None && !skipFunction(F); + GISelKnownBits *KB = &getAnalysis().get(MF); + MachineDominatorTree *MDT = + IsOptNone ? nullptr : &getAnalysis(); + AArch64PostLegalizerLoweringInfo PCInfo(EnableOpt, F.hasOptSize(), + F.hasMinSize(), KB, MDT); + Combiner C(PCInfo, TPC); + return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr); +} + +char AArch64PostLegalizerLowering::ID = 0; +INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLowering, DEBUG_TYPE, + "Lower AArch64 MachineInstrs after legalization", false, + false) +INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_DEPENDENCY(GISelKnownBitsAnalysis) +INITIALIZE_PASS_END(AArch64PostLegalizerLowering, DEBUG_TYPE, + "Lower AArch64 MachineInstrs after legalization", false, + false) + +namespace llvm { +FunctionPass *createAArch64PostLegalizeLowering(bool IsOptNone) { + return new AArch64PostLegalizerLowering(IsOptNone); +} +} // end namespace llvm Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-ext.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-ext.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-ext.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s # # Check that we can combine a G_SHUFFLE_VECTOR into a G_EXT. Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-rev.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-rev.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-rev.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s # # Test producing a G_REV from an appropriate G_SHUFFLE_VECTOR. Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-shuffle-splat.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-shuffle-splat.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-shuffle-splat.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple=aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s --- name: splat_4xi32 Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-trn.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-trn.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-trn.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s # # Check that we produce G_TRN1 or G_TRN2 when we have an appropriate shuffle # mask. Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-uzp.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-uzp.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-uzp.mir @@ -3,7 +3,7 @@ # Check that we can recognize a shuffle mask for a uzp instruction and produce # a G_UZP1 or G_UZP2 where appropriate. # -# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s ... --- Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-vashr-vlshr.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-vashr-vlshr.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-vashr-vlshr.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s ... --- Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-zip.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-zip.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-zip.mir @@ -3,7 +3,7 @@ # Check that we can recognize a shuffle mask for a zip instruction, and produce # G_ZIP1 or G_ZIP2 where appropriate. # -# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s ... --- Index: llvm/test/CodeGen/AArch64/O0-pipeline.ll =================================================================== --- llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -37,6 +37,8 @@ ; CHECK-NEXT: AArch64PreLegalizerCombiner ; CHECK-NEXT: Analysis containing CSE Info ; CHECK-NEXT: Legalizer +; CHECK-NEXT: Analysis for ComputingKnownBits +; CHECK-NEXT: AArch64PostLegalizerLowering ; CHECK-NEXT: RegBankSelect ; CHECK-NEXT: Localizer ; CHECK-NEXT: Analysis for ComputingKnownBits