Index: lib/CodeGen/MachineSink.cpp =================================================================== --- lib/CodeGen/MachineSink.cpp +++ lib/CodeGen/MachineSink.cpp @@ -975,6 +975,14 @@ INITIALIZE_PASS(PostRAMachineSinking, "postra-machine-sink", "PostRA Machine Sink", false, false) +static bool aliasWithRegsInLiveIn(MachineBasicBlock *SI, + SmallSet &AliasedRegs) { + for (const auto LI : SI->liveins()) + if (AliasedRegs.count(LI.PhysReg)) + return true; + return false; +} + static MachineBasicBlock * getSingleLiveInSuccBB(MachineBasicBlock &CurBB, ArrayRef SinkableBBs, unsigned Reg, @@ -1002,13 +1010,95 @@ for (auto *SI : CurBB.successors()) { if (SI == BB) continue; - for (const auto LI : SI->liveins()) - if (AliasedRegs.count(LI.PhysReg)) - return nullptr; + if (aliasWithRegsInLiveIn(SI, AliasedRegs)) + return nullptr; } return BB; } +static MachineBasicBlock *getSingleLiveInSuccBB( + MachineBasicBlock &CurBB, ArrayRef SinkableBBs, + ArrayRef DefedRegsInCopy, const TargetRegisterInfo *TRI) { + MachineBasicBlock *SingleBB = nullptr; + for (auto DefReg : DefedRegsInCopy) { + MachineBasicBlock *BB = + getSingleLiveInSuccBB(CurBB, SinkableBBs, DefReg, TRI); + if (!BB || (SingleBB && SingleBB != BB)) + return nullptr; + SingleBB = BB; + } + return SingleBB; +} + +static void +clearKillFlags(MachineInstr *MI, MachineBasicBlock &CurBB, + SmallVectorImpl> &UsedRegsInCopy, + BitVector &UsedRegs, const TargetRegisterInfo *TRI) { + for (auto U : UsedRegsInCopy) { + unsigned OpIdx = U.first; + unsigned SrcReg = U.second; + if (UsedRegs[SrcReg]) { + MachineBasicBlock::iterator NI = std::next(MI->getIterator()); + for (MachineInstr &UI : make_range(NI, CurBB.end())) { + if (UI.killsRegister(SrcReg, TRI)) { + UI.clearRegisterKills(SrcReg, TRI); + MI->getOperand(OpIdx).setIsKill(true); + break; + } + } + } + } +} + +static void +updateLiveIn(MachineBasicBlock *SuccBB, + SmallVectorImpl> &UsedRegsInCopy, + SmallVectorImpl &DefedRegsInCopy) { + for (auto DefReg : DefedRegsInCopy) + SuccBB->removeLiveIn(DefReg); + for (auto U : UsedRegsInCopy) { + unsigned Reg = U.second; + if (!SuccBB->isLiveIn(Reg)) + SuccBB->addLiveIn(Reg); + } +} + +static bool hasRegisterDependency( + MachineInstr *MI, + SmallVectorImpl> &UsedRegsInCopy, + SmallVectorImpl &DefedRegsInCopy, BitVector &ModifiedRegs, + BitVector &UsedRegs) { + + bool HasRegDependency = false; + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + MachineOperand &MO = MI->getOperand(i); + if (!MO.isReg()) + continue; + unsigned Reg = MO.getReg(); + if (!Reg) + continue; + if (MO.isDef()) { + if (ModifiedRegs[Reg] || UsedRegs[Reg]) { + HasRegDependency = true; + break; + } + DefedRegsInCopy.push_back(Reg); + + // FIXME: instead of isUse(), readsReg() would be a better fix here, + // For example, we can ignore modifications in reg with undef. However, + // it's not perfectly clear if skipping the internal read is safe in all + // other targets. + } else if (MO.isUse()) { + if (ModifiedRegs[Reg]) { + HasRegDependency = true; + break; + } + UsedRegsInCopy.push_back(std::make_pair(i, Reg)); + } + } + return HasRegDependency; +} + bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB, MachineFunction &MF, const TargetRegisterInfo *TRI, @@ -1044,16 +1134,21 @@ continue; } - unsigned DefReg = MI->getOperand(0).getReg(); - unsigned SrcReg = MI->getOperand(1).getReg(); + // Track the register number used and the operand index in Copy. + SmallVector, 2> UsedRegsInCopy; + // Track the register number defed in Copy. + SmallVector DefedRegsInCopy; + // Don't sink the COPY if it would violate a register dependency. - if (ModifiedRegs[DefReg] || ModifiedRegs[SrcReg] || UsedRegs[DefReg]) { + if (hasRegisterDependency(MI, UsedRegsInCopy, DefedRegsInCopy, ModifiedRegs, + UsedRegs)) { TII->trackRegDefsUses(*MI, ModifiedRegs, UsedRegs, TRI); continue; } - + assert((!UsedRegsInCopy.empty() && !DefedRegsInCopy.empty()) && + "Unexpect SrcReg or DefReg"); MachineBasicBlock *SuccBB = - getSingleLiveInSuccBB(CurBB, SinkableBBs, DefReg, TRI); + getSingleLiveInSuccBB(CurBB, SinkableBBs, DefedRegsInCopy, TRI); // Don't sink if we cannot find a single sinkable successor in which Reg // is live-in. if (!SuccBB) { @@ -1065,22 +1160,10 @@ // Clear the kill flag if SrcReg is killed between MI and the end of the // block. - if (UsedRegs[SrcReg]) { - MachineBasicBlock::iterator NI = std::next(MI->getIterator()); - for (MachineInstr &UI : make_range(NI, CurBB.end())) { - if (UI.killsRegister(SrcReg, TRI)) { - UI.clearRegisterKills(SrcReg, TRI); - MI->getOperand(1).setIsKill(true); - break; - } - } - } - + clearKillFlags(MI, CurBB, UsedRegsInCopy, UsedRegs, TRI); MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI(); SuccBB->splice(InsertPos, &CurBB, MI); - SuccBB->removeLiveIn(DefReg); - if (!SuccBB->isLiveIn(SrcReg)) - SuccBB->addLiveIn(SrcReg); + updateLiveIn(SuccBB, UsedRegsInCopy, DefedRegsInCopy); Changed = true; ++NumPostRACopySink; Index: test/CodeGen/SystemZ/no-postra-sink.mir =================================================================== --- /dev/null +++ test/CodeGen/SystemZ/no-postra-sink.mir @@ -0,0 +1,63 @@ +# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z13 -run-pass=postra-machine-sink -verify-machineinstrs -o - %s | FileCheck %s + +--- +# Don't sink COPY to bb.2 since SLLK define r13l that is aliased with r12q. +# CHECK-LABEL: name: donot_sink_copy +# CHECK-LABEL: bb.0: +# CHECK: renamable $r0l = COPY renamable $r12l, implicit killed $r12q +# CHECK-LABEL: bb.2: +# CHECK-NOT: COPY +name: donot_sink_copy +tracksRegLiveness: true +body: | + bb.0 : + successors: %bb.1, %bb.2 + liveins: $r2d, $r3d, $r4d, $r5d, $r12q + + renamable $r0l = COPY renamable $r12l, implicit killed $r12q + renamable $r13l = SLLK renamable $r4l, $noreg, 1 + CHIMux renamable $r3l, 0, implicit-def $cc, implicit killed $r3d + BRC 14, 6, %bb.2, implicit killed $cc + J %bb.1 + + bb.1: + successors: + + bb.2: + successors: + liveins: $r2d, $r4d, $r5d, $r0l, $r13l + + renamable $r0d = LGFR killed renamable $r0l + renamable $r11d = LGFR killed renamable $r13l +... + +# Don't sink COPY to bb.2 since SLLK use r1l that is aliased with r0q. +# CHECK-LABEL: name: donot_sink_copy2 +# CHECK-LABEL: bb.0: +# CHECK: renamable $r0l = COPY renamable $r12l, implicit-def $r0q +# CHECK-LABEL: bb.2: +# CHECK-NOT: COPY +name: donot_sink_copy2 +tracksRegLiveness: true +body: | + bb.0 : + successors: %bb.1, %bb.2 + liveins: $r2d, $r3d, $r4d, $r5d, $r12q + + renamable $r0l = COPY renamable $r12l, implicit def $r0q + renamable $r13l = SLLK renamable $r1l, $noreg, 1 + CHIMux renamable $r3l, 0, implicit-def $cc, implicit killed $r3d + BRC 14, 6, %bb.2, implicit killed $cc + J %bb.1 + + bb.1: + successors: + + bb.2: + successors: + liveins: $r2d, $r4d, $r5d, $r0l, $r13l + + renamable $r0d = LGFR killed renamable $r0l + renamable $r11d = LGFR killed renamable $r13l +... +