Index: llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h =================================================================== --- llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -205,6 +205,9 @@ /// Return true if a G_SHUFFLE_VECTOR instruction \p MI has an undef mask. bool matchUndefShuffleVectorMask(MachineInstr &MI); + /// Return true if a G_STORE instruction \p MI is storing an undef value. + bool matchUndefStore(MachineInstr &MI); + /// Replace an instruction with a G_FCONSTANT with value \p C. bool replaceInstWithFConstant(MachineInstr &MI, double C); @@ -231,6 +234,9 @@ /// Optimize (x op x) -> x bool matchBinOpSameVal(MachineInstr &MI); + /// Erase \p MI + bool eraseInst(MachineInstr &MI); + /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); Index: llvm/include/llvm/Target/GlobalISel/Combine.td =================================================================== --- llvm/include/llvm/Target/GlobalISel/Combine.td +++ llvm/include/llvm/Target/GlobalISel/Combine.td @@ -208,12 +208,21 @@ (apply [{ return Helper.replaceSingleDefInstWithOperand(*${root}, 1); }]) >; +// Erase stores of undef values. +def erase_undef_store : GICombineRule< + (defs root:$root), + (match (wip_match_opcode G_STORE):$root, + [{ return Helper.matchUndefStore(*${root}); }]), + (apply [{ return Helper.eraseInst(*${root}); }]) +>; + // FIXME: These should use the custom predicate feature once it lands. def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero, undef_to_negative_one, propagate_undef_any_op, propagate_undef_all_ops, - propagate_undef_shuffle_mask]>; + propagate_undef_shuffle_mask, + erase_undef_store]>; def identity_combines : GICombineGroup<[select_same_val, right_identity_zero, binop_same_val]>; Index: llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp =================================================================== --- llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -1512,6 +1512,17 @@ return all_of(Mask, [](int Elt) { return Elt < 0; }); } +bool CombinerHelper::matchUndefStore(MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::G_STORE); + return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(0).getReg(), + MRI); +} + +bool CombinerHelper::eraseInst(MachineInstr &MI) { + MI.eraseFromParent(); + return true; +} + bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1, const MachineOperand &MOP2) { if (!MOP1.isReg() || !MOP2.isReg()) Index: llvm/lib/Target/AArch64/AArch64.h =================================================================== --- llvm/lib/Target/AArch64/AArch64.h +++ llvm/lib/Target/AArch64/AArch64.h @@ -57,6 +57,7 @@ createAArch64InstructionSelector(const AArch64TargetMachine &, AArch64Subtarget &, AArch64RegisterBankInfo &); FunctionPass *createAArch64PreLegalizeCombiner(bool IsOptNone); +FunctionPass *createAArch64PostLegalizeCombiner(bool IsOptNone); FunctionPass *createAArch64StackTaggingPass(bool MergeInit); FunctionPass *createAArch64StackTaggingPreRAPass(); @@ -75,6 +76,7 @@ void initializeAArch64LoadStoreOptPass(PassRegistry&); void initializeAArch64SIMDInstrOptPass(PassRegistry&); void initializeAArch64PreLegalizerCombinerPass(PassRegistry&); +void initializeAArch64PostLegalizerCombinerPass(PassRegistry &); void initializeAArch64PromoteConstantPass(PassRegistry&); void initializeAArch64RedundantCopyEliminationPass(PassRegistry&); void initializeAArch64StorePairSuppressPass(PassRegistry&); Index: llvm/lib/Target/AArch64/AArch64Combine.td =================================================================== --- llvm/lib/Target/AArch64/AArch64Combine.td +++ llvm/lib/Target/AArch64/AArch64Combine.td @@ -23,3 +23,8 @@ fconstant_to_constant]> { let DisableRuleOption = "aarch64prelegalizercombiner-disable-rule"; } + +def AArch64PostLegalizerCombinerHelper: GICombinerHelper< + "AArch64GenPostLegalizerCombinerHelper", [erase_undef_store]> { + let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule"; +} Index: llvm/lib/Target/AArch64/AArch64PostLegalizerCombiner.cpp =================================================================== --- /dev/null +++ llvm/lib/Target/AArch64/AArch64PostLegalizerCombiner.cpp @@ -0,0 +1,142 @@ +//=== lib/CodeGen/GlobalISel/AArch64PostLegalizerCombiner.cpp -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This performs post-legalization combines on generic MachineInstrs. +// +// Any combine that this pass performs must preserve instruction legality. +// Combines unconcerned with legality should be handled by the +// PreLegalizerCombiner instead. +// +//===----------------------------------------------------------------------===// + +#include "AArch64TargetMachine.h" +#include "llvm/CodeGen/GlobalISel/Combiner.h" +#include "llvm/CodeGen/GlobalISel/CombinerHelper.h" +#include "llvm/CodeGen/GlobalISel/CombinerInfo.h" +#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "aarch64-postlegalizer-combiner" + +using namespace llvm; + +#define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS +#include "AArch64GenPostLegalizeGICombiner.inc" +#undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS + +namespace { +#define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H +#include "AArch64GenPostLegalizeGICombiner.inc" +#undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H + +class AArch64PostLegalizerCombinerInfo : public CombinerInfo { + GISelKnownBits *KB; + MachineDominatorTree *MDT; + +public: + AArch64GenPostLegalizerCombinerHelper Generated; + + AArch64PostLegalizerCombinerInfo(bool EnableOpt, bool OptSize, bool MinSize, + GISelKnownBits *KB, + MachineDominatorTree *MDT) + : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false, + /*LegalizerInfo*/ nullptr, EnableOpt, OptSize, MinSize), + KB(KB), MDT(MDT) { + if (!Generated.parseCommandLineOption()) + report_fatal_error("Invalid rule identifier"); + } + + virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI, + MachineIRBuilder &B) const override; +}; + +bool AArch64PostLegalizerCombinerInfo::combine(GISelChangeObserver &Observer, + MachineInstr &MI, + MachineIRBuilder &B) const { + CombinerHelper Helper(Observer, B, KB, MDT); + return Generated.tryCombineAll(Observer, MI, B, Helper); +} + +#define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP +#include "AArch64GenPostLegalizeGICombiner.inc" +#undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP + +class AArch64PostLegalizerCombiner : public MachineFunctionPass { +public: + static char ID; + + AArch64PostLegalizerCombiner(bool IsOptNone = false); + + StringRef getPassName() const override { + return "AArch64PostLegalizerCombiner"; + } + + bool runOnMachineFunction(MachineFunction &MF) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + +private: + bool IsOptNone; +}; +} // end anonymous namespace + +void AArch64PostLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); + AU.setPreservesCFG(); + getSelectionDAGFallbackAnalysisUsage(AU); + AU.addRequired(); + AU.addPreserved(); + if (!IsOptNone) { + AU.addRequired(); + AU.addPreserved(); + } + MachineFunctionPass::getAnalysisUsage(AU); +} + +AArch64PostLegalizerCombiner::AArch64PostLegalizerCombiner(bool IsOptNone) + : MachineFunctionPass(ID), IsOptNone(IsOptNone) { + initializeAArch64PostLegalizerCombinerPass(*PassRegistry::getPassRegistry()); +} + +bool AArch64PostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) { + if (MF.getProperties().hasProperty( + MachineFunctionProperties::Property::FailedISel)) + return false; + assert(MF.getProperties().hasProperty( + MachineFunctionProperties::Property::Legalized) && + "Expected a legalized function?"); + auto *TPC = &getAnalysis(); + const Function &F = MF.getFunction(); + bool EnableOpt = + MF.getTarget().getOptLevel() != CodeGenOpt::None && !skipFunction(F); + GISelKnownBits *KB = &getAnalysis().get(MF); + MachineDominatorTree *MDT = + IsOptNone ? nullptr : &getAnalysis(); + AArch64PostLegalizerCombinerInfo PCInfo(EnableOpt, F.hasOptSize(), + F.hasMinSize(), KB, MDT); + Combiner C(PCInfo, TPC); + return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr); +} + +char AArch64PostLegalizerCombiner::ID = 0; +INITIALIZE_PASS_BEGIN(AArch64PostLegalizerCombiner, DEBUG_TYPE, + "Combine AArch64 MachineInstrs after legalization", false, + false) +INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_DEPENDENCY(GISelKnownBitsAnalysis) +INITIALIZE_PASS_END(AArch64PostLegalizerCombiner, DEBUG_TYPE, + "Combine AArch64 MachineInstrs after legalization", false, + false) + +namespace llvm { +FunctionPass *createAArch64PostLegalizeCombiner(bool IsOptNone) { + return new AArch64PostLegalizerCombiner(IsOptNone); +} +} // end namespace llvm Index: llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp +++ llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp @@ -54,12 +54,12 @@ } #define AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS -#include "AArch64GenGICombiner.inc" +#include "AArch64GenPreLegalizeGICombiner.inc" #undef AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS namespace { #define AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H -#include "AArch64GenGICombiner.inc" +#include "AArch64GenPreLegalizeGICombiner.inc" #undef AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H class AArch64PreLegalizerCombinerInfo : public CombinerInfo { @@ -119,7 +119,7 @@ } #define AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP -#include "AArch64GenGICombiner.inc" +#include "AArch64GenPreLegalizeGICombiner.inc" #undef AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP // Pass boilerplate Index: llvm/lib/Target/AArch64/AArch64TargetMachine.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -183,6 +183,7 @@ initializeAArch64LoadStoreOptPass(*PR); initializeAArch64SIMDInstrOptPass(*PR); initializeAArch64PreLegalizerCombinerPass(*PR); + initializeAArch64PostLegalizerCombinerPass(*PR); initializeAArch64PromoteConstantPass(*PR); initializeAArch64RedundantCopyEliminationPass(*PR); initializeAArch64StorePairSuppressPass(*PR); @@ -410,6 +411,7 @@ bool addIRTranslator() override; void addPreLegalizeMachineIR() override; bool addLegalizeMachineIR() override; + void addPreRegBankSelect() override; bool addRegBankSelect() override; void addPreGlobalInstructionSelect() override; bool addGlobalInstructionSelect() override; @@ -552,6 +554,11 @@ return false; } +void AArch64PassConfig::addPreRegBankSelect() { + bool IsOptNone = getOptLevel() == CodeGenOpt::None; + addPass(createAArch64PostLegalizeCombiner(IsOptNone)); +} + bool AArch64PassConfig::addRegBankSelect() { addPass(new RegBankSelect()); return false; Index: llvm/lib/Target/AArch64/CMakeLists.txt =================================================================== --- llvm/lib/Target/AArch64/CMakeLists.txt +++ llvm/lib/Target/AArch64/CMakeLists.txt @@ -8,8 +8,10 @@ tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler) tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel) tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel) -tablegen(LLVM AArch64GenGICombiner.inc -gen-global-isel-combiner +tablegen(LLVM AArch64GenPreLegalizeGICombiner.inc -gen-global-isel-combiner -combiners="AArch64PreLegalizerCombinerHelper") +tablegen(LLVM AArch64GenPostLegalizeGICombiner.inc -gen-global-isel-combiner + -combiners="AArch64PostLegalizerCombinerHelper") tablegen(LLVM AArch64GenInstrInfo.inc -gen-instr-info) tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter) tablegen(LLVM AArch64GenMCPseudoLowering.inc -gen-pseudo-lowering) @@ -52,6 +54,7 @@ AArch64MacroFusion.cpp AArch64MCInstLower.cpp AArch64PreLegalizerCombiner.cpp + AArch64PostLegalizerCombiner.cpp AArch64PromoteConstant.cpp AArch64PBQPRegAlloc.cpp AArch64RegisterBankInfo.cpp Index: llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll +++ llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll @@ -61,6 +61,10 @@ ; ENABLED-NEXT: Analysis containing CSE Info ; ENABLED-NEXT: Legalizer ; VERIFY-NEXT: Verify generated machine code +; ENABLED-NEXT: Analysis for ComputingKnownBits +; ENABLED-O1-NEXT: MachineDominator Tree Construction +; ENABLED-NEXT: AArch64PostLegalizerCombiner +; VERIFY-NEXT: Verify generated machine code ; ENABLED-NEXT: RegBankSelect ; VERIFY-NEXT: Verify generated machine code ; ENABLED-NEXT: Localizer Index: llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-store-undef.mir =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-store-undef.mir @@ -0,0 +1,25 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s + +... +--- +name: delete_store_undef +legalized: true +tracksRegLiveness: true +body: | + bb.1.entry: + liveins: $x0 + + ; A store of an implicit def can safely be deleted before or after + ; legalization. + + ; CHECK-LABEL: name: delete_store_undef + ; CHECK: liveins: $x0 + ; CHECK-NOT: G_STORE + ; CHECK: RET_ReallyLR + %0:_(p0) = COPY $x0 + %1:_(s32) = G_IMPLICIT_DEF + G_STORE %1(s32), %0(p0) :: (store 4) + RET_ReallyLR + +... Index: llvm/test/CodeGen/AArch64/O0-pipeline.ll =================================================================== --- llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -42,6 +42,8 @@ ; CHECK-NEXT: AArch64PreLegalizerCombiner ; CHECK-NEXT: Analysis containing CSE Info ; CHECK-NEXT: Legalizer +; CHECK-NEXT: Analysis for ComputingKnownBits +; CHECK-NEXT: AArch64PostLegalizerCombiner ; CHECK-NEXT: RegBankSelect ; CHECK-NEXT: Localizer ; CHECK-NEXT: Analysis for ComputingKnownBits