Index: include/llvm/CodeGen/TargetInstrInfo.h =================================================================== --- include/llvm/CodeGen/TargetInstrInfo.h +++ include/llvm/CodeGen/TargetInstrInfo.h @@ -18,6 +18,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/None.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineCombinerPattern.h" #include "llvm/CodeGen/MachineFunction.h" @@ -958,8 +959,9 @@ virtual bool canCopyGluedNodeDuringSchedule(SDNode *N) const { return false; } /// Remember what registers the specified instruction uses and modifies. - virtual void trackRegDefsUses(const MachineInstr &MI, BitVector &ModifiedRegs, - BitVector &UsedRegs, + virtual void trackRegDefsUses(const MachineInstr &MI, + LiveRegUnits &ModifiedRegUnits, + LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI) const; protected: Index: lib/CodeGen/MachineSink.cpp =================================================================== --- lib/CodeGen/MachineSink.cpp +++ lib/CodeGen/MachineSink.cpp @@ -959,8 +959,8 @@ } private: - /// Track which registers have been modified and used. - BitVector ModifiedRegs, UsedRegs; + /// Track which register units have been modified and used. + LiveRegUnits ModifiedRegUnits, UsedRegUnits; /// Sink Copy instructions unused in the same block close to their uses in /// successors. @@ -975,22 +975,17 @@ INITIALIZE_PASS(PostRAMachineSinking, "postra-machine-sink", "PostRA Machine Sink", false, false) -static bool aliasWithRegsInLiveIn(MachineBasicBlock *SI, - SmallSet &AliasedRegs) { - for (const auto LI : SI->liveins()) - if (AliasedRegs.count(LI.PhysReg)) - return true; - return false; +static bool aliasWithRegsInLiveIn(MachineBasicBlock &MBB, unsigned Reg, + const TargetRegisterInfo *TRI) { + LiveRegUnits LiveInRegUnits(*TRI); + LiveInRegUnits.addLiveIns(MBB); + return !LiveInRegUnits.available(Reg); } static MachineBasicBlock * getSingleLiveInSuccBB(MachineBasicBlock &CurBB, ArrayRef SinkableBBs, unsigned Reg, const TargetRegisterInfo *TRI) { - SmallSet AliasedRegs; - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - AliasedRegs.insert(*AI); - // Try to find a single sinkable successor in which Reg is live-in. MachineBasicBlock *BB = nullptr; for (auto *SI : SinkableBBs) { @@ -1010,7 +1005,7 @@ for (auto *SI : CurBB.successors()) { if (SI == BB) continue; - if (aliasWithRegsInLiveIn(SI, AliasedRegs)) + if (aliasWithRegsInLiveIn(*SI, Reg, TRI)) return nullptr; } return BB; @@ -1032,11 +1027,11 @@ static void clearKillFlags(MachineInstr *MI, MachineBasicBlock &CurBB, SmallVectorImpl &UsedOpsInCopy, - BitVector &UsedRegs, const TargetRegisterInfo *TRI) { + LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI) { for (auto U : UsedOpsInCopy) { MachineOperand &MO = MI->getOperand(U); unsigned SrcReg = MO.getReg(); - if (UsedRegs[SrcReg]) { + if (!UsedRegUnits.available(SrcReg)) { MachineBasicBlock::iterator NI = std::next(MI->getIterator()); for (MachineInstr &UI : make_range(NI, CurBB.end())) { if (UI.killsRegister(SrcReg, TRI)) { @@ -1064,8 +1059,8 @@ static bool hasRegisterDependency(MachineInstr *MI, SmallVectorImpl &UsedOpsInCopy, SmallVectorImpl &DefedRegsInCopy, - BitVector &ModifiedRegs, - BitVector &UsedRegs) { + LiveRegUnits &ModifiedRegUnits, + LiveRegUnits &UsedRegUnits) { bool HasRegDependency = false; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); @@ -1075,7 +1070,7 @@ if (!Reg) continue; if (MO.isDef()) { - if (ModifiedRegs[Reg] || UsedRegs[Reg]) { + if (!ModifiedRegUnits.available(Reg) || !UsedRegUnits.available(Reg)) { HasRegDependency = true; break; } @@ -1086,7 +1081,7 @@ // it's not perfectly clear if skipping the internal read is safe in all // other targets. } else if (MO.isUse()) { - if (ModifiedRegs[Reg]) { + if (!ModifiedRegUnits.available(Reg)) { HasRegDependency = true; break; } @@ -1115,8 +1110,8 @@ // Track which registers have been modified and used between the end of the // block and the current instruction. - ModifiedRegs.reset(); - UsedRegs.reset(); + ModifiedRegUnits.clear(); + UsedRegUnits.clear(); for (auto I = CurBB.rbegin(), E = CurBB.rend(); I != E;) { MachineInstr *MI = &*I; @@ -1127,7 +1122,7 @@ return false; if (!MI->isCopy() || !MI->getOperand(0).isRenamable()) { - TII->trackRegDefsUses(*MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(*MI, ModifiedRegUnits, UsedRegUnits, TRI); continue; } @@ -1137,9 +1132,9 @@ SmallVector DefedRegsInCopy; // Don't sink the COPY if it would violate a register dependency. - if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy, ModifiedRegs, - UsedRegs)) { - TII->trackRegDefsUses(*MI, ModifiedRegs, UsedRegs, TRI); + if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy, ModifiedRegUnits, + UsedRegUnits)) { + TII->trackRegDefsUses(*MI, ModifiedRegUnits, UsedRegUnits, TRI); continue; } assert((!UsedOpsInCopy.empty() && !DefedRegsInCopy.empty()) && @@ -1149,7 +1144,7 @@ // Don't sink if we cannot find a single sinkable successor in which Reg // is live-in. if (!SuccBB) { - TII->trackRegDefsUses(*MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(*MI, ModifiedRegUnits, UsedRegUnits, TRI); continue; } assert((SuccBB->pred_size() == 1 && *SuccBB->pred_begin() == &CurBB) && @@ -1157,7 +1152,7 @@ // Clear the kill flag if SrcReg is killed between MI and the end of the // block. - clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegs, TRI); + clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegUnits, TRI); MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI(); SuccBB->splice(InsertPos, &CurBB, MI); updateLiveIn(MI, SuccBB, UsedOpsInCopy, DefedRegsInCopy); @@ -1172,9 +1167,9 @@ bool Changed = false; const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); - ModifiedRegs.resize(TRI->getNumRegs()); - UsedRegs.resize(TRI->getNumRegs()); + ModifiedRegUnits.init(*TRI); + UsedRegUnits.init(*TRI); for (auto &BB : MF) Changed |= tryToSinkCopy(BB, MF, TRI, TII); Index: lib/CodeGen/TargetInstrInfo.cpp =================================================================== --- lib/CodeGen/TargetInstrInfo.cpp +++ lib/CodeGen/TargetInstrInfo.cpp @@ -883,28 +883,26 @@ } void TargetInstrInfo::trackRegDefsUses(const MachineInstr &MI, - BitVector &ModifiedRegs, - BitVector &UsedRegs, + LiveRegUnits &ModifiedRegUnits, + LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI) const { - for (const MachineOperand &MO : MI.operands()) { - if (MO.isRegMask()) - ModifiedRegs.setBitsNotInMask(MO.getRegMask()); - if (!MO.isReg()) + for (ConstMIBundleOperands O(MI); O.isValid(); ++O) { + if (O->isRegMask()) + ModifiedRegUnits.addRegsInMask(O->getRegMask()); + if (!O->isReg()) continue; - unsigned Reg = MO.getReg(); + unsigned Reg = O->getReg(); if (!Reg) continue; - if (MO.isDef()) { + if (O->isDef()) { // Some architectures (e.g. AArch64 XZR/WZR) have registers that are // constant and may be used as destinations to indicate the generated // value is discarded. No need to track such case as a def. if (!TRI->isConstantPhysReg(Reg)) - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - ModifiedRegs.set(*AI); + ModifiedRegUnits.addReg(Reg); } else { - assert(MO.isUse() && "Reg operand not a def and not a use"); - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - UsedRegs.set(*AI); + assert(O->isUse() && "Reg operand not a def and not a use"); + UsedRegUnits.addReg(Reg); } } } Index: lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp =================================================================== --- lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -98,8 +98,8 @@ const TargetRegisterInfo *TRI; const AArch64Subtarget *Subtarget; - // Track which registers have been modified and used. - BitVector ModifiedRegs, UsedRegs; + // Track which register units have been modified and used. + LiveRegUnits ModifiedRegUnits, UsedRegUnits; void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); @@ -1051,10 +1051,10 @@ if (MBBI == B) return false; - // Track which registers have been modified and used between the first insn - // and the second insn. - ModifiedRegs.reset(); - UsedRegs.reset(); + // Track which register units have been modified and used between the first + // insn and the second insn. + ModifiedRegUnits.clear(); + UsedRegUnits.clear(); unsigned Count = 0; do { @@ -1073,7 +1073,7 @@ if (MI.mayStore() && isMatchingStore(LoadMI, MI) && BaseReg == getLdStBaseOp(MI).getReg() && isLdOffsetInRangeOfSt(LoadMI, MI, TII) && - !ModifiedRegs[getLdStRegOp(MI).getReg()]) { + ModifiedRegUnits.available(getLdStRegOp(MI).getReg())) { StoreI = MBBI; return true; } @@ -1081,12 +1081,12 @@ if (MI.isCall()) return false; - // Update modified / uses register lists. - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + // Update modified / uses register units. + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); // Otherwise, if the base register is modified, we have no match, so // return early. - if (ModifiedRegs[BaseReg]) + if (!ModifiedRegUnits.available(BaseReg)) return false; // If we encounter a store aliased with the load, return early. @@ -1164,10 +1164,10 @@ int OffsetStride = IsUnscaled ? getMemScale(FirstMI) : 1; bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI); - // Track which registers have been modified and used between the first insn - // (inclusive) and the second insn. - ModifiedRegs.reset(); - UsedRegs.reset(); + // Track which register units have been modified and used between the first + // insn (inclusive) and the second insn. + ModifiedRegUnits.clear(); + UsedRegUnits.clear(); // Remember any instructions that read/write memory between FirstMI and MI. SmallVector MemInsns; @@ -1202,7 +1202,7 @@ // If the unscaled offset isn't a multiple of the MemSize, we can't // pair the operations together: bail and keep looking. if (MIOffset % MemSize) { - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); MemInsns.push_back(&MI); continue; } @@ -1222,7 +1222,7 @@ // the stored value is the same (i.e., WZR). if ((!IsUnscaled && alignTo(MinOffset, 2) != MinOffset) || (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) { - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); MemInsns.push_back(&MI); continue; } @@ -1232,7 +1232,7 @@ // immediate offset of merging these instructions is out of range for // a pairwise instruction, bail and keep looking. if (!inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) { - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); MemInsns.push_back(&MI); continue; } @@ -1240,7 +1240,7 @@ // can't express the offset of the unscaled input, bail and keep // looking. if (IsUnscaled && (alignTo(MinOffset, OffsetStride) != MinOffset)) { - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); MemInsns.push_back(&MI); continue; } @@ -1249,7 +1249,7 @@ // and keep looking. A load-pair instruction with both destination // registers the same is UNPREDICTABLE and will result in an exception. if (MayLoad && Reg == getLdStRegOp(MI).getReg()) { - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); MemInsns.push_back(&MI); continue; } @@ -1258,8 +1258,9 @@ // the two instructions and none of the instructions between the second // and first alias with the second, we can combine the second into the // first. - if (!ModifiedRegs[getLdStRegOp(MI).getReg()] && - !(MI.mayLoad() && UsedRegs[getLdStRegOp(MI).getReg()]) && + if (ModifiedRegUnits.available(getLdStRegOp(MI).getReg()) && + !(MI.mayLoad() && + !UsedRegUnits.available(getLdStRegOp(MI).getReg())) && !mayAlias(MI, MemInsns, AA)) { Flags.setMergeForward(false); return MBBI; @@ -1269,8 +1270,9 @@ // between the two instructions and none of the instructions between the // first and the second alias with the first, we can combine the first // into the second. - if (!ModifiedRegs[getLdStRegOp(FirstMI).getReg()] && - !(MayLoad && UsedRegs[getLdStRegOp(FirstMI).getReg()]) && + if (ModifiedRegUnits.available(getLdStRegOp(FirstMI).getReg()) && + !(MayLoad && + !UsedRegUnits.available(getLdStRegOp(FirstMI).getReg())) && !mayAlias(FirstMI, MemInsns, AA)) { Flags.setMergeForward(true); return MBBI; @@ -1285,12 +1287,12 @@ if (MI.isCall()) return E; - // Update modified / uses register lists. - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + // Update modified / uses register units. + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); // Otherwise, if the base register is modified, we have no match, so // return early. - if (ModifiedRegs[BaseReg]) + if (!ModifiedRegUnits.available(BaseReg)) return E; // Update list of instructions that read/write memory. @@ -1446,10 +1448,10 @@ return E; } - // Track which registers have been modified and used between the first insn - // (inclusive) and the second insn. - ModifiedRegs.reset(); - UsedRegs.reset(); + // Track which register units have been modified and used between the first + // insn (inclusive) and the second insn. + ModifiedRegUnits.clear(); + UsedRegUnits.clear(); ++MBBI; for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) { MachineInstr &MI = *MBBI; @@ -1464,11 +1466,12 @@ return MBBI; // Update the status of what the instruction clobbered and used. - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); // Otherwise, if the base register is used or modified, we have no match, so // return early. - if (ModifiedRegs[BaseReg] || UsedRegs[BaseReg]) + if (!ModifiedRegUnits.available(BaseReg) || + !UsedRegUnits.available(BaseReg)) return E; } return E; @@ -1497,10 +1500,10 @@ return E; } - // Track which registers have been modified and used between the first insn - // (inclusive) and the second insn. - ModifiedRegs.reset(); - UsedRegs.reset(); + // Track which register units have been modified and used between the first + // insn (inclusive) and the second insn. + ModifiedRegUnits.clear(); + UsedRegUnits.clear(); unsigned Count = 0; do { --MBBI; @@ -1516,11 +1519,12 @@ return MBBI; // Update the status of what the instruction clobbered and used. - TII->trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); + TII->trackRegDefsUses(MI, ModifiedRegUnits, UsedRegUnits, TRI); // Otherwise, if the base register is used or modified, we have no match, so // return early. - if (ModifiedRegs[BaseReg] || UsedRegs[BaseReg]) + if (!ModifiedRegUnits.available(BaseReg) || + !UsedRegUnits.available(BaseReg)) return E; } while (MBBI != B && Count < Limit); return E; @@ -1750,8 +1754,8 @@ // Resize the modified and used register bitfield trackers. We do this once // per function and then clear the bitfield each time we optimize a load or // store. - ModifiedRegs.resize(TRI->getNumRegs()); - UsedRegs.resize(TRI->getNumRegs()); + ModifiedRegUnits.init(*TRI); + UsedRegUnits.init(*TRI); bool Modified = false; bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();