Index: llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -1140,7 +1140,7 @@ // then remove the early-clobber flag. for (unsigned Reg : ECRegs) { if (MIB->readsRegister(Reg, TRI)) { - MachineOperand *MO = + MachineOperand *MO = MIB->findRegisterDefOperand(Reg, false, false, TRI); assert(MO && "No def operand for clobbered register?"); MO->setIsEarlyClobber(false); Index: llvm/lib/Target/PowerPC/PPCInstrInfo.h =================================================================== --- llvm/lib/Target/PowerPC/PPCInstrInfo.h +++ llvm/lib/Target/PowerPC/PPCInstrInfo.h @@ -128,12 +128,12 @@ // If the inst has imm-form and one of its operand is produced by a LI, // put the imm into the inst directly and remove the LI if possible. bool transformToImmFormFedByLI(MachineInstr &MI, const ImmInstrInfo &III, - unsigned ConstantOpNo, int64_t Imm) const; + unsigned ConstantOpNo, MachineInstr &DefMI, + int64_t Imm) const; // If the inst has imm-form and one of its operand is produced by an // add-immediate, try to transform it when possible. bool transformToImmFormFedByAdd(MachineInstr &MI, const ImmInstrInfo &III, - unsigned ConstantOpNo, - MachineInstr &DefMI, + unsigned ConstantOpNo, MachineInstr &DefMI, bool KillDefMI) const; // Try to find that, if the instruction 'MI' contains any operand that // could be forwarded from some inst that feeds it. If yes, return the @@ -158,8 +158,8 @@ int64_t &Imm) const; bool isRegElgibleForForwarding(const MachineOperand &RegMO, const MachineInstr &DefMI, - const MachineInstr &MI, - bool KillDefMI) const; + const MachineInstr &MI, bool KillDefMI, + bool &IsFwdFeederRegKilled) const; const unsigned *getStoreOpcodesForSpillArray() const; const unsigned *getLoadOpcodesForSpillArray() const; virtual void anchor(); @@ -411,6 +411,7 @@ bool convertToImmediateForm(MachineInstr &MI, MachineInstr **KilledDef = nullptr) const; + void fixUpKillFlags(MachineInstr &MI, MachineInstr &DefMI, unsigned RegNo) const; void replaceInstrWithLI(MachineInstr &MI, const LoadImmediateInfo &LII) const; void replaceInstrOperandWithImm(MachineInstr &MI, unsigned OpNo, int64_t Imm) const; Index: llvm/lib/Target/PowerPC/PPCInstrInfo.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -2416,6 +2416,56 @@ return OpcodesForSpill[(Subtarget.hasP9Vector()) ? 1 : 0]; } +// Fixup kill flag after transformation from reg+reg to reg+imm for dead +// register RegNo. +void PPCInstrInfo::fixUpKillFlags(MachineInstr &MI, MachineInstr &DefMI, + unsigned RegNo) const { + const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); + if (MRI.isSSA()) + return; + + bool IsKillSet = false; + + // Set kill flag for MI. + if (MachineOperand *MO = + MI.findRegisterUseOperand(RegNo, false, &getRegisterInfo())) { + MO->setIsKill(true); + IsKillSet = true; + } + + // Walking the inst in reverse order (MI<->DefMI). + MachineBasicBlock::reverse_iterator It = MI; + MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(); + It++; + MachineOperand *MO = nullptr; + for (; It != E; ++It) { + // If killed is not set, set killed for its use or set dead for its def if + // no use found. + if (!IsKillSet) { + if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) { + // Use found, set it killed. + IsKillSet = true; + MO->setIsKill(true); + continue; + } else if ((MO = It->findRegisterDefOperand(RegNo, false, true, + &getRegisterInfo()))) { + // No use found, set dead for its def. + assert(&*It == &DefMI && "No new def between DefMI and MI."); + MO->setIsDead(true); + break; + } + } else { + // If killed is already set, clear old killed flag. + if ((MO = It->findRegisterUseOperand(RegNo, true, &getRegisterInfo()))) { + MO->setIsKill(false); + break; + } + } + if ((&*It) == &DefMI) + break; + } +} + // If this instruction has an immediate form and one of its operands is a // result of a load-immediate or an add-immediate, convert it to // the immediate form if the constant is in range. @@ -2432,8 +2482,9 @@ return false; assert(ForwardingOperand < MI.getNumOperands() && "The forwarding operand needs to be valid at this point"); - bool KillFwdDefMI = !SeenIntermediateUse && - MI.getOperand(ForwardingOperand).isKill(); + bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill(); + bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled; + unsigned ForwardOperandReg = MI.getOperand(ForwardingOperand).getReg(); if (KilledDef && KillFwdDefMI) *KilledDef = DefMI; @@ -2442,8 +2493,9 @@ // If this is a reg+reg instruction that has a reg+imm form, // and one of the operands is produced by an add-immediate, // try to convert it. - if (HasImmForm && transformToImmFormFedByAdd(MI, III, ForwardingOperand, - *DefMI, KillFwdDefMI)) + if (HasImmForm && + transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI, + KillFwdDefMI)) return true; if ((DefMI->getOpcode() != PPC::LI && DefMI->getOpcode() != PPC::LI8) || @@ -2458,7 +2510,7 @@ // If this is a reg+reg instruction that has a reg+imm form, // and one of the operands is produced by LI, convert it now. if (HasImmForm) - return transformToImmFormFedByLI(MI, III, ForwardingOperand, SExtImm); + return transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI, SExtImm); bool ReplaceWithLI = false; bool Is64BitLI = false; @@ -2478,6 +2530,8 @@ case PPC::CMPLDI: { // Doing this post-RA would require dataflow analysis to reliably find uses // of the CR register set by the compare. + // No need to fixup killed/dead flag since this transformation is only valid + // before RA. if (PostRA) return false; // If a compare-immediate is fed by an immediate and is itself an input of @@ -2654,6 +2708,12 @@ if (KilledDef && SetCR) *KilledDef = nullptr; replaceInstrWithLI(MI, LII); + + // Fixup kill/dead flag after transformation. + // ForwardOperandReg = LI imm1 + // y = op2 imm2, ForwardOperandReg(killed) + if (IsForwardingOperandKilled) + fixUpKillFlags(MI, *DefMI, ForwardOperandReg); LLVM_DEBUG(dbgs() << "With:\n"); LLVM_DEBUG(MI.dump()); return true; @@ -3161,11 +3221,10 @@ return isAnImmediateOperand(*ImmMO); } -bool PPCInstrInfo::isRegElgibleForForwarding(const MachineOperand &RegMO, - const MachineInstr &DefMI, - const MachineInstr &MI, - bool KillDefMI - ) const { +bool PPCInstrInfo::isRegElgibleForForwarding( + const MachineOperand &RegMO, const MachineInstr &DefMI, + const MachineInstr &MI, bool KillDefMI, + bool &IsFwdFeederRegKilled) const { // x = addi y, imm // ... // z = lfdx 0, x -> z = lfd imm(y) @@ -3185,6 +3244,8 @@ for (; It != E; ++It) { if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI) return false; + else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI) + IsFwdFeederRegKilled = true; // Made it to DefMI without encountering a clobber. if ((&*It) == &DefMI) break; @@ -3256,11 +3317,9 @@ // is the literal zero, attempt to forward the source of the add-immediate to // the corresponding D-Form instruction with the displacement coming from // the immediate being added. -bool PPCInstrInfo::transformToImmFormFedByAdd(MachineInstr &MI, - const ImmInstrInfo &III, - unsigned OpNoForForwarding, - MachineInstr &DefMI, - bool KillDefMI) const { +bool PPCInstrInfo::transformToImmFormFedByAdd( + MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding, + MachineInstr &DefMI, bool KillDefMI) const { // RegMO ImmMO // | | // x = addi reg, imm <----- DefMI @@ -3285,10 +3344,19 @@ if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm)) return false; + bool IsFwdFeederRegKilled = false; // Check if the RegMO can be forwarded to MI. - if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI)) + if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI, + IsFwdFeederRegKilled)) return false; + // Get Killed info in case fixup needed after transformation. + unsigned ForwardKilledOperandReg = ~0U; + MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); + bool PostRA = !MRI.isSSA(); + if (PostRA && MI.getOperand(OpNoForForwarding).isKill()) + ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg(); + // We know that, the MI and DefMI both meet the pattern, and // the Imm also meet the requirement with the new Imm-form. // It is safe to do the transformation now. @@ -3342,12 +3410,31 @@ LLVM_DEBUG(dbgs() << "With:\n"); LLVM_DEBUG(MI.dump()); + // Fixup kill/dead flag after transformation. + // Pattern 1: + // x = op1 KilledFwdFeederReg, imm + // n = opn KilledFwdFeederReg(killed), regn + // y = op2 0, x + if (IsFwdFeederRegKilled) + fixUpKillFlags(MI, DefMI, RegMO->getReg()); + // Pattern 2: + // ForwardKilledOperandReg = op1 reg, imm + // y = op2 0, ForwardKilledOperandReg(killed) + if (ForwardKilledOperandReg != ~0U) + fixUpKillFlags(MI, DefMI, ForwardKilledOperandReg); + // Pattern 3: + // x = op1 reg(killed), imm + // y = op2 0, x + if (RegMO->isKill()) + fixUpKillFlags(MI, DefMI, RegMO->getReg()); + return true; } bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI, const ImmInstrInfo &III, unsigned ConstantOpNo, + MachineInstr &DefMI, int64_t Imm) const { MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); bool PostRA = !MRI.isSSA(); @@ -3386,6 +3473,11 @@ return false; } + // Get Killed info in case fixup needed after transformation. + unsigned ForwardKilledOperandReg = ~0U; + if (PostRA && MI.getOperand(ConstantOpNo).isKill()) + ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg(); + unsigned Opc = MI.getOpcode(); bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLWo || Opc == PPC::SRW || Opc == PPC::SRWo; @@ -3468,6 +3560,12 @@ } } } + + // Fixup kill/dead flag after transformation. + // ForwardKilledOperandReg = op1 reg, imm + // y = op2 0, ForwardKilledOperandReg(killed) + if (ForwardKilledOperandReg != ~0U) + fixUpKillFlags(MI, DefMI, ForwardKilledOperandReg); return true; } Index: llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instr-add.mir =================================================================== --- llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instr-add.mir +++ llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instr-add.mir @@ -27,7 +27,7 @@ ; Following instruction $r3 also reads $x3, ADDI8 can not be erased ; CHECK: $x3 = ADDI8 $x5, 100, implicit-def $r3 STW $r3, $x5, 100 - ; CHECK: STW $r3, $x5, 100 + ; CHECK: STW killed $r3, $x5, 100 STFSX killed $f1, $zero8, $x3 ; CHECK: STFS killed $f1, 100, $x5 STD $x5, $x5, 100 Index: llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-kill-flag.mir =================================================================== --- /dev/null +++ llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-kill-flag.mir @@ -0,0 +1,137 @@ +# RUN: llc -mtriple=powerpc64le--linux-gnu -stop-after ppc-pre-emit-peephole %s -o - -verify-machineinstrs | FileCheck %s + +--- +name: testKillPassUpLI1 +# CHECK: name: testKillPassUpLI1 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = LI8 100 + ; CHECK-NOT: LI8 + STFSX killed $f1, killed $x3, killed $x5 + ; CHECK: STFS killed $f1, 100, killed $x5 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassUpLI2 +# CHECK: name: testKillPassUpLI2 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = LI8 100 + STFSX killed $f1, $x3, $x5 + ; CHECK: STFS killed $f1, 100, $x5 + STD killed $x3, killed $x5, 100 + ; CHECK: STD killed $x3, killed $x5, 100 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassUpLI3 +# CHECK: name: testKillPassUpLI3 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = LI8 100 + STD $x3, $x5, 100 + ; CHECK: STD killed $x3, $x5, 100 + STFSX killed $f1, killed $x3, $x5 + ; CHECK: STFS killed $f1, 100, $x5 + STD killed $x5, $x5, 100 + ; CHECK: STD killed $x5, $x5, 100 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassUpLI4 +# CHECK: name: testKillPassUpLI4 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $x5 + $x3 = LI8 100 + STD $x3, killed $x5, 100 + ; CHECK: STD killed $x3, killed $x5, 100 + $x5 = ADDI8 killed $x3, 200 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassUpADD1 +# CHECK: name: testKillPassUpADD1 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = ADDI8 $x5, 100 + ; CHECK-NOT: ADDI8 + STFSX killed $f1, $zero8, killed $x3 + ; CHECK: STFS killed $f1, 100, killed $x5 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassUpADD2 +# CHECK: name: testKillPassUpADD2 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = ADDI8 $x5, 100 + STFSX killed $f1, $zero8, $x3 + ; CHECK: STFS killed $f1, 100, $x5 + STD killed $x3, killed $x5, 100 + ; CHECK: STD killed $x3, killed $x5, 100 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassUpADD3 +# CHECK: name: testKillPassUpADD3 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = ADDI8 $x5, 100 + STD $x3, $x5, 100 + ; CHECK: STD killed $x3, $x5, 100 + STFSX killed $f1, $zero8, killed $x3 + ; CHECK: STFS killed $f1, 100, $x5 + STD killed $x5, $x5, 100 + ; CHECK: STD killed $x5, $x5, 100 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassDownADD1 +# CHECK: name: testKillPassDownADD1 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = ADDI8 killed $x5, 100 + ; CHECK-NOT: ADDI8 $x5, 100 + STFSX killed $f1, $zero8, killed $x3 + ; CHECK: STFS killed $f1, 100, killed $x5 + BLR8 implicit $lr8, implicit $rm + +... +--- +name: testKillPassDownADD2 +# CHECK: name: testKillPassDownADD2 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x3, $f1, $x5 + $x3 = ADDI8 $x5, 100 + STD killed $x5, $x5, 100 + ; CHECK: STD $x5, $x5, 100 + STFSX killed $f1, $zero8, killed $x3 + ; CHECK: STFS killed $f1, 100, killed $x5 + BLR8 implicit $lr8, implicit $rm + +...