Index: lib/Target/SystemZ/SystemZInstrInfo.h =================================================================== --- lib/Target/SystemZ/SystemZInstrInfo.h +++ lib/Target/SystemZ/SystemZInstrInfo.h @@ -316,6 +316,8 @@ SystemZII::FusedCompareType Type, const MachineInstr *MI = nullptr) const; + bool trySwapCompareOperands(MachineBasicBlock::iterator MBBI) const; + // If Opcode is a LOAD opcode for with an associated LOAD AND TRAP // operation exists, returh the opcode for the latter, otherwise return 0. unsigned getLoadAndTrap(unsigned Opcode) const; Index: lib/Target/SystemZ/SystemZInstrInfo.cpp =================================================================== --- lib/Target/SystemZ/SystemZInstrInfo.cpp +++ lib/Target/SystemZ/SystemZInstrInfo.cpp @@ -1246,14 +1246,21 @@ // commutable, try to change R into . unsigned NumOps = MI.getNumExplicitOperands(); int MemOpcode = SystemZ::getMemOpcode(Opcode); + if (MemOpcode == -1) + return nullptr; + + // Try to swap compare operands if possible. + if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR) && + OpNum == 0 && trySwapCompareOperands(MI)) + OpNum = 1; // See if this is a 3-address instruction that is convertible to 2-address // and suitable for folding below. Only try this with virtual registers // and a provided VRM (during regalloc). bool NeedsCommute = false; - if (SystemZ::getTwoOperandOpcode(Opcode) != -1 && MemOpcode != -1) { + if (SystemZ::getTwoOperandOpcode(Opcode) != -1) { if (VRM == nullptr) - MemOpcode = -1; + return nullptr; else { assert(NumOps == 3 && "Expected two source registers."); Register DstReg = MI.getOperand(0).getReg(); @@ -1268,31 +1275,29 @@ DstPhys == VRM->getPhys(SrcReg)) NeedsCommute = (OpNum == 1); else - MemOpcode = -1; + return nullptr; } } - if (MemOpcode >= 0) { - if ((OpNum == NumOps - 1) || NeedsCommute) { - const MCInstrDesc &MemDesc = get(MemOpcode); - uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags); - assert(AccessBytes != 0 && "Size of access should be known"); - assert(AccessBytes <= Size && "Access outside the frame index"); - uint64_t Offset = Size - AccessBytes; - MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, - MI.getDebugLoc(), get(MemOpcode)); - MIB.add(MI.getOperand(0)); - if (NeedsCommute) - MIB.add(MI.getOperand(2)); - else - for (unsigned I = 1; I < OpNum; ++I) - MIB.add(MI.getOperand(I)); - MIB.addFrameIndex(FrameIndex).addImm(Offset); - if (MemDesc.TSFlags & SystemZII::HasIndex) - MIB.addReg(0); - transferDeadCC(&MI, MIB); - return MIB; - } + if ((OpNum == NumOps - 1) || NeedsCommute) { + const MCInstrDesc &MemDesc = get(MemOpcode); + uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags); + assert(AccessBytes != 0 && "Size of access should be known"); + assert(AccessBytes <= Size && "Access outside the frame index"); + uint64_t Offset = Size - AccessBytes; + MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, + MI.getDebugLoc(), get(MemOpcode)); + MIB.add(MI.getOperand(0)); + if (NeedsCommute) + MIB.add(MI.getOperand(2)); + else + for (unsigned I = 1; I < OpNum; ++I) + MIB.add(MI.getOperand(I)); + MIB.addFrameIndex(FrameIndex).addImm(Offset); + if (MemDesc.TSFlags & SystemZII::HasIndex) + MIB.addReg(0); + transferDeadCC(&MI, MIB); + return MIB; } return nullptr; @@ -1810,6 +1815,75 @@ return 0; } +bool SystemZInstrInfo:: +trySwapCompareOperands(MachineBasicBlock::iterator const MBBI) const { + assert(MBBI->isCompare() && MBBI->getOperand(0).isReg() && + MBBI->getOperand(1).isReg() && "Not a compare reg/reg."); + + MachineBasicBlock *MBB = MBBI->getParent(); + if (SystemZRegisterInfo::isCCLiveOut(*MBB)) + return false; + + SmallVector CCUsers; + for (MachineBasicBlock::iterator Itr = std::next(MBBI); + Itr != MBB->end(); ++Itr) { + if (Itr->readsRegister(SystemZ::CC)) { + unsigned Flags = Itr->getDesc().TSFlags; + if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast)) + CCUsers.push_back(&*Itr); + else + return false; + } + if (Itr->definesRegister(SystemZ::CC)) + break; + } + assert(CCUsers.size() && "No CC users found?"); + + // Update all CC users. + for (unsigned Idx = 0; Idx < CCUsers.size(); ++Idx) { + unsigned Flags = CCUsers[Idx]->getDesc().TSFlags; + unsigned FirstOpNum = ((Flags & SystemZII::CCMaskFirst) ? + 0 : CCUsers[Idx]->getNumExplicitOperands() - 2); + MachineOperand &CCMaskMO = CCUsers[Idx]->getOperand(FirstOpNum + 1); + switch(CCMaskMO.getImm()) { + case SystemZ::CCMASK_CMP_LT: + CCMaskMO.setImm(SystemZ::CCMASK_CMP_GT); + break; + case SystemZ::CCMASK_CMP_GT: + CCMaskMO.setImm(SystemZ::CCMASK_CMP_LT); + break; + case SystemZ::CCMASK_CMP_LE: + CCMaskMO.setImm(SystemZ::CCMASK_CMP_GE); + break; + case SystemZ::CCMASK_CMP_GE: + CCMaskMO.setImm(SystemZ::CCMASK_CMP_LE); + break; + case SystemZ::CCMASK_CMP_EQ: + case SystemZ::CCMASK_CMP_NE: + break; + default: + llvm_unreachable("Unexpected CCMask value."); + break; + } + } + + // Swap the registers and flags of the compare operands. MBBI is expected + // to remain without constructing a new one. TODO: Is there a better way? + MachineOperand &LHS = MBBI->getOperand(0); + MachineOperand &RHS = MBBI->getOperand(1); + MachineOperand Tmp = MachineOperand(LHS); + LHS.setReg(RHS.getReg()); + LHS.setSubReg(RHS.getSubReg()); + LHS.setIsKill(RHS.isKill()); + LHS.setIsUndef(RHS.isUndef()); + RHS.setReg(Tmp.getReg()); + RHS.setSubReg(Tmp.getSubReg()); + RHS.setIsKill(Tmp.isKill()); + RHS.setIsUndef(Tmp.isUndef()); + + return true; +} + unsigned SystemZInstrInfo::getLoadAndTrap(unsigned Opcode) const { if (!STI.hasLoadAndTrap()) return 0; Index: lib/Target/SystemZ/SystemZPostRewrite.cpp =================================================================== --- lib/Target/SystemZ/SystemZPostRewrite.cpp +++ lib/Target/SystemZ/SystemZPostRewrite.cpp @@ -54,6 +54,7 @@ bool selectMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NextMBBI); bool selectMBB(MachineBasicBlock &MBB); + bool verifyLiveInLists_CC(MachineBasicBlock &MBB); }; char SystemZPostRewrite::ID = 0; @@ -112,9 +113,38 @@ return Modified; } +// EXPERIMENTAL +bool SystemZPostRewrite::verifyLiveInLists_CC(MachineBasicBlock &MBB) { + if (MBB.isLiveIn(SystemZ::CC)) + return false; + bool Missing = false; + MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); + for (; MBBI != E; ++MBBI) { + if (MBBI->isDebugInstr()) + continue; + if (MBBI->readsRegister(SystemZ::CC)) { + Missing = true; + break; + } + if (MBBI->definesRegister(SystemZ::CC)) + break; + } + if (Missing || (MBBI == E && SystemZRegisterInfo::isCCLiveOut(MBB))) { + assert(&MBB != &*MBB.getParent()->begin() && + "CC can not be live-in in the entry block of the function."); + llvm_unreachable("missing CC in live-in list!"); + } + return false; +} + + bool SystemZPostRewrite::runOnMachineFunction(MachineFunction &MF) { TII = static_cast(MF.getSubtarget().getInstrInfo()); + // EXPERIMENTAL + for (auto &MBB : MF) + verifyLiveInLists_CC(MBB); + bool Modified = false; for (auto &MBB : MF) Modified |= selectMBB(MBB); Index: lib/Target/SystemZ/SystemZRegisterInfo.h =================================================================== --- lib/Target/SystemZ/SystemZRegisterInfo.h +++ lib/Target/SystemZ/SystemZRegisterInfo.h @@ -49,6 +49,8 @@ const TargetRegisterClass * getCrossCopyRegClass(const TargetRegisterClass *RC) const override; + static bool isCCLiveOut(MachineBasicBlock &MBB); + bool getRegAllocationHints(unsigned VirtReg, ArrayRef Order, SmallVectorImpl &Hints, Index: lib/Target/SystemZ/SystemZRegisterInfo.cpp =================================================================== --- lib/Target/SystemZ/SystemZRegisterInfo.cpp +++ lib/Target/SystemZ/SystemZRegisterInfo.cpp @@ -73,6 +73,13 @@ Hints.push_back(Reg); } +bool SystemZRegisterInfo::isCCLiveOut(MachineBasicBlock &MBB) { + for (auto SI = MBB.succ_begin(), SE = MBB.succ_end(); SI != SE; ++SI) + if ((*SI)->isLiveIn(SystemZ::CC)) + return true; + return false; +} + bool SystemZRegisterInfo::getRegAllocationHints(unsigned VirtReg, ArrayRef Order,