diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -331,6 +331,8 @@ /// machine instructions. extern char &MachineCopyPropagationID; + MachineFunctionPass *createMachineCopyPropagationPass(bool UseCopyInstr); + /// PeepholeOptimizer - This pass performs peephole optimizations - /// like extension and comparison eliminations. extern char &PeepholeOptimizerID; diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp --- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp +++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp @@ -61,6 +61,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/InitializePasses.h" @@ -82,8 +83,24 @@ DEBUG_COUNTER(FwdCounter, "machine-cp-fwd", "Controls which register COPYs are forwarded"); +static cl::opt MCPUseCopyInstr("mcp-use-is-copy-instr", cl::init(false), + cl::Hidden); + namespace { +static Optional isCopyInstr(const MachineInstr &MI, + const TargetInstrInfo &TII, + bool UseCopyInstr) { + if (UseCopyInstr) + return TII.isCopyInstr(MI); + + if (MI.isCopy()) + return Optional( + DestSourcePair{MI.getOperand(0), MI.getOperand(1)}); + + return None; +} + class CopyTracker { struct CopyInfo { MachineInstr *MI; @@ -109,7 +126,8 @@ } /// Remove register from copy maps. - void invalidateRegister(MCRegister Reg, const TargetRegisterInfo &TRI) { + void invalidateRegister(MCRegister Reg, const TargetRegisterInfo &TRI, + const TargetInstrInfo &TII, bool UseCopyInstr) { // Since Reg might be a subreg of some registers, only invalidate Reg is not // enough. We have to find the COPY defines Reg or registers defined by Reg // and invalidate all of them. @@ -119,8 +137,13 @@ auto I = Copies.find(*RUI); if (I != Copies.end()) { if (MachineInstr *MI = I->second.MI) { - RegsToInvalidate.insert(MI->getOperand(0).getReg().asMCReg()); - RegsToInvalidate.insert(MI->getOperand(1).getReg().asMCReg()); + Optional CopyOperands = + isCopyInstr(*MI, TII, UseCopyInstr); + assert(CopyOperands && "Expect copy"); + + RegsToInvalidate.insert( + CopyOperands->Destination->getReg().asMCReg()); + RegsToInvalidate.insert(CopyOperands->Source->getReg().asMCReg()); } RegsToInvalidate.insert(I->second.DefRegs.begin(), I->second.DefRegs.end()); @@ -132,7 +155,8 @@ } /// Clobber a single register, removing it from the tracker's copy maps. - void clobberRegister(MCRegister Reg, const TargetRegisterInfo &TRI) { + void clobberRegister(MCRegister Reg, const TargetRegisterInfo &TRI, + const TargetInstrInfo &TII, bool UseCopyInstr) { for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI) { auto I = Copies.find(*RUI); if (I != Copies.end()) { @@ -141,8 +165,12 @@ markRegsUnavailable(I->second.DefRegs, TRI); // When we clobber the destination of a copy, we need to clobber the // whole register it defined. - if (MachineInstr *MI = I->second.MI) - markRegsUnavailable({MI->getOperand(0).getReg().asMCReg()}, TRI); + if (MachineInstr *MI = I->second.MI) { + Optional CopyOperands = + isCopyInstr(*MI, TII, UseCopyInstr); + markRegsUnavailable({CopyOperands->Destination->getReg().asMCReg()}, + TRI); + } // Now we can erase the copy. Copies.erase(I); } @@ -150,11 +178,13 @@ } /// Add this copy's registers into the tracker's copy maps. - void trackCopy(MachineInstr *MI, const TargetRegisterInfo &TRI) { - assert(MI->isCopy() && "Tracking non-copy?"); + void trackCopy(MachineInstr *MI, const TargetRegisterInfo &TRI, + const TargetInstrInfo &TII, bool UseCopyInstr) { + Optional CopyOperands = isCopyInstr(*MI, TII, UseCopyInstr); + assert(CopyOperands && "Tracking non-copy?"); - MCRegister Def = MI->getOperand(0).getReg().asMCReg(); - MCRegister Src = MI->getOperand(1).getReg().asMCReg(); + MCRegister Src = CopyOperands->Source->getReg().asMCReg(); + MCRegister Def = CopyOperands->Destination->getReg().asMCReg(); // Remember Def is defined by the copy. for (MCRegUnitIterator RUI(Def, &TRI); RUI.isValid(); ++RUI) @@ -197,15 +227,22 @@ } MachineInstr *findAvailBackwardCopy(MachineInstr &I, MCRegister Reg, - const TargetRegisterInfo &TRI) { + const TargetRegisterInfo &TRI, + const TargetInstrInfo &TII, + bool UseCopyInstr) { MCRegUnitIterator RUI(Reg, &TRI); MachineInstr *AvailCopy = findCopyDefViaUnit(*RUI, TRI); - if (!AvailCopy || - !TRI.isSubRegisterEq(AvailCopy->getOperand(1).getReg(), Reg)) + + if (!AvailCopy) + return nullptr; + + Optional CopyOperands = + isCopyInstr(*AvailCopy, TII, UseCopyInstr); + Register AvailSrc = CopyOperands->Source->getReg(); + Register AvailDef = CopyOperands->Destination->getReg(); + if (!TRI.isSubRegisterEq(AvailSrc, Reg)) return nullptr; - Register AvailSrc = AvailCopy->getOperand(1).getReg(); - Register AvailDef = AvailCopy->getOperand(0).getReg(); for (const MachineInstr &MI : make_range(AvailCopy->getReverseIterator(), I.getReverseIterator())) for (const MachineOperand &MO : MI.operands()) @@ -218,20 +255,26 @@ } MachineInstr *findAvailCopy(MachineInstr &DestCopy, MCRegister Reg, - const TargetRegisterInfo &TRI) { + const TargetRegisterInfo &TRI, + const TargetInstrInfo &TII, bool UseCopyInstr) { // We check the first RegUnit here, since we'll only be interested in the // copy if it copies the entire register anyway. MCRegUnitIterator RUI(Reg, &TRI); MachineInstr *AvailCopy = findCopyForUnit(*RUI, TRI, /*MustBeAvailable=*/true); - if (!AvailCopy || - !TRI.isSubRegisterEq(AvailCopy->getOperand(0).getReg(), Reg)) + + if (!AvailCopy) + return nullptr; + + Optional CopyOperands = + isCopyInstr(*AvailCopy, TII, UseCopyInstr); + Register AvailSrc = CopyOperands->Source->getReg(); + Register AvailDef = CopyOperands->Destination->getReg(); + if (!TRI.isSubRegisterEq(AvailDef, Reg)) return nullptr; // Check that the available copy isn't clobbered by any regmasks between // itself and the destination. - Register AvailSrc = AvailCopy->getOperand(1).getReg(); - Register AvailDef = AvailCopy->getOperand(0).getReg(); for (const MachineInstr &MI : make_range(AvailCopy->getIterator(), DestCopy.getIterator())) for (const MachineOperand &MO : MI.operands()) @@ -252,10 +295,14 @@ const TargetInstrInfo *TII; const MachineRegisterInfo *MRI; + // Return true if this is a copy instruction and false otherwise. + bool UseCopyInstr; + public: static char ID; // Pass identification, replacement for typeid - MachineCopyPropagation() : MachineFunctionPass(ID) { + MachineCopyPropagation(bool CopyInstr = false) + : MachineFunctionPass(ID), UseCopyInstr(CopyInstr || MCPUseCopyInstr) { initializeMachineCopyPropagationPass(*PassRegistry::getPassRegistry()); } @@ -333,9 +380,13 @@ /// isNopCopy("ecx = COPY eax", AX, CX) == true /// isNopCopy("ecx = COPY eax", AH, CL) == false static bool isNopCopy(const MachineInstr &PreviousCopy, MCRegister Src, - MCRegister Def, const TargetRegisterInfo *TRI) { - MCRegister PreviousSrc = PreviousCopy.getOperand(1).getReg().asMCReg(); - MCRegister PreviousDef = PreviousCopy.getOperand(0).getReg().asMCReg(); + MCRegister Def, const TargetRegisterInfo *TRI, + const TargetInstrInfo *TII, bool UseCopyInstr) { + + Optional CopyOperands = + isCopyInstr(PreviousCopy, *TII, UseCopyInstr); + MCRegister PreviousSrc = CopyOperands->Source->getReg().asMCReg(); + MCRegister PreviousDef = CopyOperands->Destination->getReg().asMCReg(); if (Src == PreviousSrc && Def == PreviousDef) return true; if (!TRI->isSubRegister(PreviousSrc, Src)) @@ -355,22 +406,26 @@ return false; // Search for an existing copy. - MachineInstr *PrevCopy = Tracker.findAvailCopy(Copy, Def, *TRI); + MachineInstr *PrevCopy = + Tracker.findAvailCopy(Copy, Def, *TRI, *TII, UseCopyInstr); if (!PrevCopy) return false; + auto PrevCopyOperands = isCopyInstr(*PrevCopy, *TII, UseCopyInstr); // Check that the existing copy uses the correct sub registers. - if (PrevCopy->getOperand(0).isDead()) + if (PrevCopyOperands->Destination->isDead()) return false; - if (!isNopCopy(*PrevCopy, Src, Def, TRI)) + if (!isNopCopy(*PrevCopy, Src, Def, TRI, TII, UseCopyInstr)) return false; LLVM_DEBUG(dbgs() << "MCP: copy is a NOP, removing: "; Copy.dump()); // Copy was redundantly redefining either Src or Def. Remove earlier kill // flags between Copy and PrevCopy because the value will be reused now. - assert(Copy.isCopy()); - Register CopyDef = Copy.getOperand(0).getReg(); + Optional CopyOperands = isCopyInstr(Copy, *TII, UseCopyInstr); + assert(CopyOperands); + + Register CopyDef = CopyOperands->Destination->getReg(); assert(CopyDef == Src || CopyDef == Def); for (MachineInstr &MI : make_range(PrevCopy->getIterator(), Copy.getIterator())) @@ -384,7 +439,9 @@ bool MachineCopyPropagation::isBackwardPropagatableRegClassCopy( const MachineInstr &Copy, const MachineInstr &UseI, unsigned UseIdx) { - Register Def = Copy.getOperand(0).getReg(); + + Optional CopyOperands = isCopyInstr(Copy, *TII, UseCopyInstr); + Register Def = CopyOperands->Destination->getReg(); if (const TargetRegisterClass *URC = UseI.getRegClassConstraint(UseIdx, TII, TRI)) @@ -402,7 +459,8 @@ const MachineInstr &UseI, unsigned UseIdx) { - Register CopySrcReg = Copy.getOperand(1).getReg(); + Optional CopyOperands = isCopyInstr(Copy, *TII, UseCopyInstr); + Register CopySrcReg = CopyOperands->Source->getReg(); // If the new register meets the opcode register constraints, then allow // forwarding. @@ -410,7 +468,8 @@ UseI.getRegClassConstraint(UseIdx, TII, TRI)) return URC->contains(CopySrcReg); - if (!UseI.isCopy()) + auto UseICopyOperands = isCopyInstr(UseI, *TII, UseCopyInstr); + if (!UseICopyOperands) return false; /// COPYs don't have register class constraints, so if the user instruction @@ -433,7 +492,7 @@ // Allow forwarding if src and dst belong to any common class, so long as they // don't belong to any (possibly smaller) common class that requires copies to // go via a different class. - Register UseDstReg = UseI.getOperand(0).getReg(); + Register UseDstReg = UseICopyOperands->Destination->getReg(); bool Found = false; bool IsCrossClass = false; for (const TargetRegisterClass *RC : TRI->regclasses()) { @@ -451,7 +510,7 @@ return true; // The forwarded copy would be cross-class. Only do this if the original copy // was also cross-class. - Register CopyDstReg = Copy.getOperand(0).getReg(); + Register CopyDstReg = CopyOperands->Destination->getReg(); for (const TargetRegisterClass *RC : TRI->regclasses()) { if (RC->contains(CopySrcReg) && RC->contains(CopyDstReg) && TRI->getCrossCopyRegClass(RC) != RC) @@ -523,13 +582,15 @@ if (!MOUse.isRenamable()) continue; - MachineInstr *Copy = - Tracker.findAvailCopy(MI, MOUse.getReg().asMCReg(), *TRI); + MachineInstr *Copy = Tracker.findAvailCopy(MI, MOUse.getReg().asMCReg(), + *TRI, *TII, UseCopyInstr); if (!Copy) continue; - Register CopyDstReg = Copy->getOperand(0).getReg(); - const MachineOperand &CopySrc = Copy->getOperand(1); + Optional CopyOperands = + isCopyInstr(*Copy, *TII, UseCopyInstr); + Register CopyDstReg = CopyOperands->Destination->getReg(); + const MachineOperand &CopySrc = *CopyOperands->Source; Register CopySrcReg = CopySrc.getReg(); // FIXME: Don't handle partial uses of wider COPYs yet. @@ -553,7 +614,8 @@ // Check that the instruction is not a copy that partially overwrites the // original copy source that we are about to use. The tracker mechanism // cannot cope with that. - if (MI.isCopy() && MI.modifiesRegister(CopySrcReg, TRI) && + if (isCopyInstr(MI, *TII, UseCopyInstr) && + MI.modifiesRegister(CopySrcReg, TRI) && !MI.definesRegister(CopySrcReg)) { LLVM_DEBUG(dbgs() << "MCP: Copy source overlap with dest in " << MI); continue; @@ -592,14 +654,20 @@ for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) { // Analyze copies (which don't overlap themselves). - if (MI.isCopy() && !TRI->regsOverlap(MI.getOperand(0).getReg(), - MI.getOperand(1).getReg())) { - assert(MI.getOperand(0).getReg().isPhysical() && - MI.getOperand(1).getReg().isPhysical() && + Optional CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr); + if (CopyOperands) { + + Register RegSrc = CopyOperands->Source->getReg(); + Register RegDef = CopyOperands->Destination->getReg(); + + if (TRI->regsOverlap(RegDef, RegSrc)) + continue; + + assert(RegDef.isPhysical() && RegSrc.isPhysical() && "MachineCopyPropagation should be run after register allocation!"); - MCRegister Def = MI.getOperand(0).getReg().asMCReg(); - MCRegister Src = MI.getOperand(1).getReg().asMCReg(); + MCRegister Def = RegDef.asMCReg(); + MCRegister Src = RegSrc.asMCReg(); // The two copies cancel out and the source of the first copy // hasn't been overridden, eliminate the second one. e.g. @@ -622,7 +690,8 @@ forwardUses(MI); // Src may have been changed by forwardUses() - Src = MI.getOperand(1).getReg().asMCReg(); + CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr); + Src = CopyOperands->Source->getReg().asMCReg(); // If Src is defined by a previous copy, the previous copy cannot be // eliminated. @@ -649,17 +718,17 @@ // %xmm2 = copy %xmm0 // ... // %xmm2 = copy %xmm9 - Tracker.clobberRegister(Def, *TRI); + Tracker.clobberRegister(Def, *TRI, *TII, UseCopyInstr); for (const MachineOperand &MO : MI.implicit_operands()) { if (!MO.isReg() || !MO.isDef()) continue; MCRegister Reg = MO.getReg().asMCReg(); if (!Reg) continue; - Tracker.clobberRegister(Reg, *TRI); + Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr); } - Tracker.trackCopy(&MI, *TRI); + Tracker.trackCopy(&MI, *TRI, *TII, UseCopyInstr); continue; } @@ -673,7 +742,7 @@ // later. if (MO.isTied()) ReadRegister(Reg, MI, RegularUse); - Tracker.clobberRegister(Reg, *TRI); + Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr); } forwardUses(MI); @@ -709,7 +778,9 @@ MaybeDeadCopies.begin(); DI != MaybeDeadCopies.end();) { MachineInstr *MaybeDead = *DI; - MCRegister Reg = MaybeDead->getOperand(0).getReg().asMCReg(); + Optional CopyOperands = + isCopyInstr(*MaybeDead, *TII, UseCopyInstr); + MCRegister Reg = CopyOperands->Destination->getReg().asMCReg(); assert(!MRI->isReserved(Reg)); if (!RegMask->clobbersPhysReg(Reg)) { @@ -722,7 +793,7 @@ // Make sure we invalidate any entries in the copy maps before erasing // the instruction. - Tracker.clobberRegister(Reg, *TRI); + Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr); // erase() will return the next valid iterator pointing to the next // element after the erased one. @@ -735,7 +806,7 @@ // Any previous copy definition or reading the Defs is no longer available. for (MCRegister Reg : Defs) - Tracker.clobberRegister(Reg, *TRI); + Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr); } // If MBB doesn't have successors, delete the copies whose defs are not used. @@ -745,12 +816,16 @@ for (MachineInstr *MaybeDead : MaybeDeadCopies) { LLVM_DEBUG(dbgs() << "MCP: Removing copy due to no live-out succ: "; MaybeDead->dump()); - assert(!MRI->isReserved(MaybeDead->getOperand(0).getReg())); + + Optional CopyOperands = + isCopyInstr(*MaybeDead, *TII, UseCopyInstr); + assert(CopyOperands); + + Register SrcReg = CopyOperands->Source->getReg(); + Register DestReg = CopyOperands->Destination->getReg(); + assert(!MRI->isReserved(DestReg)); // Update matching debug values, if any. - assert(MaybeDead->isCopy()); - Register SrcReg = MaybeDead->getOperand(1).getReg(); - Register DestReg = MaybeDead->getOperand(0).getReg(); SmallVector MaybeDeadDbgUsers( CopyDbgUsers[MaybeDead].begin(), CopyDbgUsers[MaybeDead].end()); MRI->updateDbgUsersToReg(DestReg.asMCReg(), SrcReg.asMCReg(), @@ -768,10 +843,14 @@ } static bool isBackwardPropagatableCopy(MachineInstr &MI, - const MachineRegisterInfo &MRI) { - assert(MI.isCopy() && "MI is expected to be a COPY"); - Register Def = MI.getOperand(0).getReg(); - Register Src = MI.getOperand(1).getReg(); + const MachineRegisterInfo &MRI, + const TargetInstrInfo &TII, + bool UseCopyInstr) { + Optional CopyOperands = isCopyInstr(MI, TII, UseCopyInstr); + assert(CopyOperands && "MI is expected to be a COPY"); + + Register Def = CopyOperands->Destination->getReg(); + Register Src = CopyOperands->Source->getReg(); if (!Def || !Src) return false; @@ -779,7 +858,7 @@ if (MRI.isReserved(Def) || MRI.isReserved(Src)) return false; - return MI.getOperand(1).isRenamable() && MI.getOperand(1).isKill(); + return CopyOperands->Source->isRenamable() && CopyOperands->Source->isKill(); } void MachineCopyPropagation::propagateDefs(MachineInstr &MI) { @@ -804,13 +883,15 @@ if (!MODef.isRenamable()) continue; - MachineInstr *Copy = - Tracker.findAvailBackwardCopy(MI, MODef.getReg().asMCReg(), *TRI); + MachineInstr *Copy = Tracker.findAvailBackwardCopy( + MI, MODef.getReg().asMCReg(), *TRI, *TII, UseCopyInstr); if (!Copy) continue; - Register Def = Copy->getOperand(0).getReg(); - Register Src = Copy->getOperand(1).getReg(); + Optional CopyOperands = + isCopyInstr(*Copy, *TII, UseCopyInstr); + Register Def = CopyOperands->Destination->getReg(); + Register Src = CopyOperands->Source->getReg(); if (MODef.getReg() != Src) continue; @@ -829,7 +910,7 @@ << MI << " from " << *Copy); MODef.setReg(Def); - MODef.setIsRenamable(Copy->getOperand(0).isRenamable()); + MODef.setIsRenamable(CopyOperands->Destination->isRenamable()); LLVM_DEBUG(dbgs() << "MCP: After replacement: " << MI << "\n"); MaybeDeadCopies.insert(Copy); @@ -845,19 +926,23 @@ for (MachineInstr &MI : llvm::make_early_inc_range(llvm::reverse(MBB))) { // Ignore non-trivial COPYs. - if (MI.isCopy() && MI.getNumOperands() == 2 && - !TRI->regsOverlap(MI.getOperand(0).getReg(), - MI.getOperand(1).getReg())) { + Optional CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr); + if (CopyOperands) { + Register DefReg = CopyOperands->Destination->getReg(); + Register SrcReg = CopyOperands->Source->getReg(); - MCRegister Def = MI.getOperand(0).getReg().asMCReg(); - MCRegister Src = MI.getOperand(1).getReg().asMCReg(); + if (TRI->regsOverlap(DefReg, SrcReg)) + continue; + + MCRegister Def = DefReg.asMCReg(); + MCRegister Src = SrcReg.asMCReg(); // Unlike forward cp, we don't invoke propagateDefs here, // just let forward cp do COPY-to-COPY propagation. - if (isBackwardPropagatableCopy(MI, *MRI)) { - Tracker.invalidateRegister(Src, *TRI); - Tracker.invalidateRegister(Def, *TRI); - Tracker.trackCopy(&MI, *TRI); + if (isBackwardPropagatableCopy(MI, *MRI, *TII, UseCopyInstr)) { + Tracker.invalidateRegister(Src, *TRI, *TII, UseCopyInstr); + Tracker.invalidateRegister(Def, *TRI, *TII, UseCopyInstr); + Tracker.trackCopy(&MI, *TRI, *TII, UseCopyInstr); continue; } } @@ -868,7 +953,7 @@ MCRegister Reg = MO.getReg().asMCReg(); if (!Reg) continue; - Tracker.invalidateRegister(Reg, *TRI); + Tracker.invalidateRegister(Reg, *TRI, *TII, UseCopyInstr); } propagateDefs(MI); @@ -880,7 +965,8 @@ continue; if (MO.isDef()) - Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI); + Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI, *TII, + UseCopyInstr); if (MO.readsReg()) { if (MO.isDebug()) { @@ -894,7 +980,8 @@ } } } else { - Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI); + Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI, *TII, + UseCopyInstr); } } } @@ -902,8 +989,10 @@ for (auto *Copy : MaybeDeadCopies) { - Register Src = Copy->getOperand(1).getReg(); - Register Def = Copy->getOperand(0).getReg(); + Optional CopyOperands = + isCopyInstr(*Copy, *TII, UseCopyInstr); + Register Src = CopyOperands->Source->getReg(); + Register Def = CopyOperands->Destination->getReg(); SmallVector MaybeDeadDbgUsers(CopyDbgUsers[Copy].begin(), CopyDbgUsers[Copy].end()); @@ -934,3 +1023,8 @@ return Changed; } + +MachineFunctionPass * +llvm::createMachineCopyPropagationPass(bool UseCopyInstr = false) { + return new MachineCopyPropagation(UseCopyInstr); +} diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -34,6 +34,7 @@ #include "llvm/CodeGen/MIRParser/MIParser.h" #include "llvm/CodeGen/MachineScheduler.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Function.h" @@ -62,6 +63,11 @@ cl::desc("Enable the conditional branch tuning pass"), cl::init(true), cl::Hidden); +static cl::opt EnableAArch64CopyPropagation( + "aarch64-enable-copy-propagation", + cl::desc("Enable the copy propagation with AArch64 copy instr"), + cl::init(true), cl::Hidden); + static cl::opt EnableMCR("aarch64-enable-mcr", cl::desc("Enable the machine combiner pass"), cl::init(true), cl::Hidden); @@ -772,6 +778,10 @@ if (TM->getOptLevel() >= CodeGenOpt::Aggressive && EnableLoadStoreOpt) addPass(createAArch64LoadStoreOptimizationPass()); + if (TM->getOptLevel() >= CodeGenOpt::Aggressive && + EnableAArch64CopyPropagation) + addPass(createMachineCopyPropagationPass(true)); + addPass(createAArch64A53Fix835769()); if (EnableBranchTargets) diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -200,6 +200,7 @@ ; CHECK-NEXT: Insert XRay ops ; CHECK-NEXT: Implement the 'patchable-function' attribute ; CHECK-NEXT: AArch64 load / store optimization pass +; CHECK-NEXT: Machine Copy Propagation Pass ; CHECK-NEXT: Workaround A53 erratum 835769 pass ; CHECK-NEXT: AArch64 Branch Targets ; CHECK-NEXT: Branch relaxation pass diff --git a/llvm/test/CodeGen/AArch64/copyprop.ll b/llvm/test/CodeGen/AArch64/copyprop.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/copyprop.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -O3 -mtriple=aarch64-- | FileCheck %s + +define void @copyprop_after_mbp(i32 %v, i32* %a, i32* %b, i32* %c, i32* %d) { +; CHECK-LABEL: copyprop_after_mbp: +; CHECK: // %bb.0: +; CHECK-NEXT: cmp w0, #10 +; CHECK-NEXT: b.ne .LBB0_2 +; CHECK-NEXT: // %bb.1: // %bb.0 +; CHECK-NEXT: mov w9, #15 +; CHECK-NEXT: mov w8, #1 +; CHECK-NEXT: str w9, [x2] +; CHECK-NEXT: mov w9, #12 +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: str w9, [x4] +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB0_2: // %bb.1 +; CHECK-NEXT: mov w9, #25 +; CHECK-NEXT: str w9, [x3] +; CHECK-NEXT: mov w9, #12 +; CHECK-NEXT: str wzr, [x1] +; CHECK-NEXT: str w9, [x4] +; CHECK-NEXT: ret + %1 = icmp eq i32 %v, 10 + br i1 %1, label %bb.0, label %bb.1 + +bb.0: + store i32 15, i32* %b, align 4 + br label %bb.2 + +bb.1: + store i32 25, i32* %c, align 4 + br label %bb.2 + +bb.2: + %2 = phi i32 [ 1, %bb.0 ], [ 0, %bb.1 ] + store i32 %2, i32* %a, align 4 + store i32 12, i32* %d, align 4 + ret void +} diff --git a/llvm/test/CodeGen/AArch64/copyprop.mir b/llvm/test/CodeGen/AArch64/copyprop.mir --- a/llvm/test/CodeGen/AArch64/copyprop.mir +++ b/llvm/test/CodeGen/AArch64/copyprop.mir @@ -1,4 +1,4 @@ -# RUN: llc -mtriple=aarch64-linux-gnu -run-pass machine-cp -o - %s | FileCheck %s +# RUN: llc -mtriple=aarch64-linux-gnu -run-pass machine-cp -mcp-use-is-copy-instr -o - %s | FileCheck %s # Tests for MachineCopyPropagation copy forwarding. --- # Simple forwarding. @@ -102,3 +102,14 @@ $x0 = ADDXri $x0, 1, 0 $x2 = SUBXri renamable $x1, 1, 0 ... +--- +# CHECK-LABEL: name: test1_orr_as_copy +# CHECK: STRBBui $wzr, killed renamable $x8, 36 +name: test1_orr_as_copy +tracksRegLiveness: true +body: | + bb.0: + liveins: $x8 + $w9 = ORRWrs $wzr, $wzr, 0 + STRBBui killed renamable $w9, killed renamable $x8, 36 +...