Index: lib/Target/SystemZ/CMakeLists.txt =================================================================== --- lib/Target/SystemZ/CMakeLists.txt +++ lib/Target/SystemZ/CMakeLists.txt @@ -17,7 +17,6 @@ SystemZCallingConv.cpp SystemZConstantPoolValue.cpp SystemZElimCompare.cpp - SystemZExpandPseudo.cpp SystemZFrameLowering.cpp SystemZHazardRecognizer.cpp SystemZISelDAGToDAG.cpp Index: lib/Target/SystemZ/SystemZ.h =================================================================== --- lib/Target/SystemZ/SystemZ.h +++ lib/Target/SystemZ/SystemZ.h @@ -190,7 +190,6 @@ FunctionPass *createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel); FunctionPass *createSystemZElimComparePass(SystemZTargetMachine &TM); -FunctionPass *createSystemZExpandPseudoPass(SystemZTargetMachine &TM); FunctionPass *createSystemZShortenInstPass(SystemZTargetMachine &TM); FunctionPass *createSystemZLongBranchPass(SystemZTargetMachine &TM); FunctionPass *createSystemZLDCleanupPass(SystemZTargetMachine &TM); Index: lib/Target/SystemZ/SystemZExpandPseudo.cpp =================================================================== --- lib/Target/SystemZ/SystemZExpandPseudo.cpp +++ /dev/null @@ -1,152 +0,0 @@ -//==-- SystemZExpandPseudo.cpp - Expand pseudo instructions -------*- C++ -*-=// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file contains a pass that expands pseudo instructions into target -// instructions to allow proper scheduling and other late optimizations. This -// pass should be run after register allocation but before the post-regalloc -// scheduling pass. -// -//===----------------------------------------------------------------------===// - -#include "SystemZ.h" -#include "SystemZInstrInfo.h" -#include "SystemZSubtarget.h" -#include "llvm/CodeGen/LivePhysRegs.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -using namespace llvm; - -#define SYSTEMZ_EXPAND_PSEUDO_NAME "SystemZ pseudo instruction expansion pass" - -namespace llvm { - void initializeSystemZExpandPseudoPass(PassRegistry&); -} - -namespace { -class SystemZExpandPseudo : public MachineFunctionPass { -public: - static char ID; - SystemZExpandPseudo() : MachineFunctionPass(ID) { - initializeSystemZExpandPseudoPass(*PassRegistry::getPassRegistry()); - } - - const SystemZInstrInfo *TII; - - bool runOnMachineFunction(MachineFunction &Fn) override; - - StringRef getPassName() const override { return SYSTEMZ_EXPAND_PSEUDO_NAME; } - -private: - bool expandMBB(MachineBasicBlock &MBB); - bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI); - bool expandLOCRMux(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI); -}; -char SystemZExpandPseudo::ID = 0; -} - -INITIALIZE_PASS(SystemZExpandPseudo, "systemz-expand-pseudo", - SYSTEMZ_EXPAND_PSEUDO_NAME, false, false) - -/// Returns an instance of the pseudo instruction expansion pass. -FunctionPass *llvm::createSystemZExpandPseudoPass(SystemZTargetMachine &TM) { - return new SystemZExpandPseudo(); -} - -// MI is a load-register-on-condition pseudo instruction that could not be -// handled as a single hardware instruction. Replace it by a branch sequence. -bool SystemZExpandPseudo::expandLOCRMux(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI) { - MachineFunction &MF = *MBB.getParent(); - const BasicBlock *BB = MBB.getBasicBlock(); - MachineInstr &MI = *MBBI; - DebugLoc DL = MI.getDebugLoc(); - Register DestReg = MI.getOperand(0).getReg(); - Register SrcReg = MI.getOperand(2).getReg(); - unsigned CCValid = MI.getOperand(3).getImm(); - unsigned CCMask = MI.getOperand(4).getImm(); - - LivePhysRegs LiveRegs(TII->getRegisterInfo()); - LiveRegs.addLiveOuts(MBB); - for (auto I = std::prev(MBB.end()); I != MBBI; --I) - LiveRegs.stepBackward(*I); - - // Splice MBB at MI, moving the rest of the block into RestMBB. - MachineBasicBlock *RestMBB = MF.CreateMachineBasicBlock(BB); - MF.insert(std::next(MachineFunction::iterator(MBB)), RestMBB); - RestMBB->splice(RestMBB->begin(), &MBB, MI, MBB.end()); - RestMBB->transferSuccessors(&MBB); - for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I) - RestMBB->addLiveIn(*I); - - // Create a new block MoveMBB to hold the move instruction. - MachineBasicBlock *MoveMBB = MF.CreateMachineBasicBlock(BB); - MF.insert(std::next(MachineFunction::iterator(MBB)), MoveMBB); - MoveMBB->addLiveIn(SrcReg); - for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I) - MoveMBB->addLiveIn(*I); - - // At the end of MBB, create a conditional branch to RestMBB if the - // condition is false, otherwise fall through to MoveMBB. - BuildMI(&MBB, DL, TII->get(SystemZ::BRC)) - .addImm(CCValid).addImm(CCMask ^ CCValid).addMBB(RestMBB); - MBB.addSuccessor(RestMBB); - MBB.addSuccessor(MoveMBB); - - // In MoveMBB, emit an instruction to move SrcReg into DestReg, - // then fall through to RestMBB. - TII->copyPhysReg(*MoveMBB, MoveMBB->end(), DL, DestReg, SrcReg, - MI.getOperand(2).isKill()); - MoveMBB->addSuccessor(RestMBB); - - NextMBBI = MBB.end(); - MI.eraseFromParent(); - return true; -} - -/// If MBBI references a pseudo instruction that should be expanded here, -/// do the expansion and return true. Otherwise return false. -bool SystemZExpandPseudo::expandMI(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI) { - MachineInstr &MI = *MBBI; - switch (MI.getOpcode()) { - case SystemZ::LOCRMux: - return expandLOCRMux(MBB, MBBI, NextMBBI); - default: - break; - } - return false; -} - -/// Iterate over the instructions in basic block MBB and expand any -/// pseudo instructions. Return true if anything was modified. -bool SystemZExpandPseudo::expandMBB(MachineBasicBlock &MBB) { - bool Modified = false; - - MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); - while (MBBI != E) { - MachineBasicBlock::iterator NMBBI = std::next(MBBI); - Modified |= expandMI(MBB, MBBI, NMBBI); - MBBI = NMBBI; - } - - return Modified; -} - -bool SystemZExpandPseudo::runOnMachineFunction(MachineFunction &MF) { - TII = static_cast(MF.getSubtarget().getInstrInfo()); - - bool Modified = false; - for (auto &MBB : MF) - Modified |= expandMBB(MBB); - return Modified; -} - Index: lib/Target/SystemZ/SystemZInstrInfo.h =================================================================== --- lib/Target/SystemZ/SystemZInstrInfo.h +++ lib/Target/SystemZ/SystemZInstrInfo.h @@ -170,20 +170,10 @@ unsigned HighOpcode) const; void expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned HighOpcode) const; - void expandLOCRPseudo(MachineInstr &MI, unsigned LowOpcode, - unsigned HighOpcode) const; - void expandSELRPseudo(MachineInstr &MI, unsigned LowOpcode, - unsigned HighOpcode, unsigned MixedOpcode) const; void expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned Size) const; void expandLoadStackGuard(MachineInstr *MI) const; - MachineInstrBuilder - emitGRX32Move(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, - unsigned LowLowOpcode, unsigned Size, bool KillSrc, - bool UndefSrc) const; - virtual void anchor(); protected: @@ -326,6 +316,12 @@ MachineBasicBlock::iterator MBBI, unsigned Reg, uint64_t Value) const; + MachineInstrBuilder + emitGRX32Move(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, + unsigned LowLowOpcode, unsigned Size, bool KillSrc, + bool UndefSrc) const; + // Sometimes, it is possible for the target to tell, even without // aliasing information, that two MIs access different memory // addresses. This function returns true if two MIs access different Index: lib/Target/SystemZ/SystemZInstrInfo.cpp =================================================================== --- lib/Target/SystemZ/SystemZInstrInfo.cpp +++ lib/Target/SystemZ/SystemZInstrInfo.cpp @@ -46,22 +46,12 @@ #include "SystemZGenInstrInfo.inc" #define DEBUG_TYPE "systemz-II" -STATISTIC(LOCRMuxJumps, "Number of LOCRMux jump-sequences (lower is better)"); // Return a mask with Count low bits set. static uint64_t allOnes(unsigned int Count) { return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1; } -// Reg should be a 32-bit GPR. Return true if it is a high register rather -// than a low register. -static bool isHighReg(unsigned int Reg) { - if (SystemZ::GRH32BitRegClass.contains(Reg)) - return true; - assert(SystemZ::GR32BitRegClass.contains(Reg) && "Invalid GRX32"); - return false; -} - // Pin the vtable to this file. void SystemZInstrInfo::anchor() {} @@ -148,7 +138,7 @@ unsigned HighOpcode, bool ConvertHigh) const { Register Reg = MI.getOperand(0).getReg(); - bool IsHigh = isHighReg(Reg); + bool IsHigh = SystemZRegisterInfo::isHighReg(Reg); MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode)); if (IsHigh && ConvertHigh) MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm())); @@ -163,8 +153,8 @@ unsigned HighOpcode) const { Register DestReg = MI.getOperand(0).getReg(); Register SrcReg = MI.getOperand(1).getReg(); - bool DestIsHigh = isHighReg(DestReg); - bool SrcIsHigh = isHighReg(SrcReg); + bool DestIsHigh = SystemZRegisterInfo::isHighReg(DestReg); + bool SrcIsHigh = SystemZRegisterInfo::isHighReg(SrcReg); if (!DestIsHigh && !SrcIsHigh) MI.setDesc(get(LowOpcodeK)); else { @@ -185,8 +175,9 @@ void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned HighOpcode) const { Register Reg = MI.getOperand(0).getReg(); - unsigned Opcode = getOpcodeForOffset(isHighReg(Reg) ? HighOpcode : LowOpcode, - MI.getOperand(2).getImm()); + unsigned Opcode = getOpcodeForOffset( + SystemZRegisterInfo::isHighReg(Reg) ? HighOpcode : LowOpcode, + MI.getOperand(2).getImm()); MI.setDesc(get(Opcode)); } @@ -196,92 +187,11 @@ void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned HighOpcode) const { Register Reg = MI.getOperand(0).getReg(); - unsigned Opcode = isHighReg(Reg) ? HighOpcode : LowOpcode; + unsigned Opcode = + SystemZRegisterInfo::isHighReg(Reg) ? HighOpcode : LowOpcode; MI.setDesc(get(Opcode)); } -// MI is a load-register-on-condition pseudo instruction. Replace it with -// LowOpcode if source and destination are both low GR32s and HighOpcode if -// source and destination are both high GR32s. -void SystemZInstrInfo::expandLOCRPseudo(MachineInstr &MI, unsigned LowOpcode, - unsigned HighOpcode) const { - Register DestReg = MI.getOperand(0).getReg(); - Register SrcReg = MI.getOperand(2).getReg(); - bool DestIsHigh = isHighReg(DestReg); - bool SrcIsHigh = isHighReg(SrcReg); - - if (!DestIsHigh && !SrcIsHigh) - MI.setDesc(get(LowOpcode)); - else if (DestIsHigh && SrcIsHigh) - MI.setDesc(get(HighOpcode)); - else - LOCRMuxJumps++; - - // If we were unable to implement the pseudo with a single instruction, we - // need to convert it back into a branch sequence. This cannot be done here - // since the caller of expandPostRAPseudo does not handle changes to the CFG - // correctly. This change is defered to the SystemZExpandPseudo pass. -} - -// MI is a select pseudo instruction. Replace it with LowOpcode if source -// and destination are all low GR32s and HighOpcode if source and destination -// are all high GR32s. Otherwise, use the two-operand MixedOpcode. -void SystemZInstrInfo::expandSELRPseudo(MachineInstr &MI, unsigned LowOpcode, - unsigned HighOpcode, - unsigned MixedOpcode) const { - Register DestReg = MI.getOperand(0).getReg(); - Register Src1Reg = MI.getOperand(1).getReg(); - Register Src2Reg = MI.getOperand(2).getReg(); - bool DestIsHigh = isHighReg(DestReg); - bool Src1IsHigh = isHighReg(Src1Reg); - bool Src2IsHigh = isHighReg(Src2Reg); - - // If sources and destination aren't all high or all low, we may be able to - // simplify the operation by moving one of the sources to the destination - // first. But only if this doesn't clobber the other source. - if (DestReg != Src1Reg && DestReg != Src2Reg) { - if (DestIsHigh != Src1IsHigh) { - emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, Src1Reg, - SystemZ::LR, 32, MI.getOperand(1).isKill(), - MI.getOperand(1).isUndef()); - MI.getOperand(1).setReg(DestReg); - Src1Reg = DestReg; - Src1IsHigh = DestIsHigh; - } else if (DestIsHigh != Src2IsHigh) { - emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, Src2Reg, - SystemZ::LR, 32, MI.getOperand(2).isKill(), - MI.getOperand(2).isUndef()); - MI.getOperand(2).setReg(DestReg); - Src2Reg = DestReg; - Src2IsHigh = DestIsHigh; - } - } - - // If the destination (now) matches one source, prefer this to be first. - if (DestReg != Src1Reg && DestReg == Src2Reg) { - commuteInstruction(MI, false, 1, 2); - std::swap(Src1Reg, Src2Reg); - std::swap(Src1IsHigh, Src2IsHigh); - } - - if (!DestIsHigh && !Src1IsHigh && !Src2IsHigh) - MI.setDesc(get(LowOpcode)); - else if (DestIsHigh && Src1IsHigh && Src2IsHigh) - MI.setDesc(get(HighOpcode)); - else { - // Given the simplifcation above, we must already have a two-operand case. - assert (DestReg == Src1Reg); - MI.setDesc(get(MixedOpcode)); - MI.tieOperands(0, 1); - LOCRMuxJumps++; - } - - // If we were unable to implement the pseudo with a single instruction, we - // need to convert it back into a branch sequence. This cannot be done here - // since the caller of expandPostRAPseudo does not handle changes to the CFG - // correctly. This change is defered to the SystemZExpandPseudo pass. -} - // MI is an RR-style pseudo instruction that zero-extends the low Size bits // of one GRX32 into another. Replace it with LowOpcode if both operands // are low registers, otherwise use RISB[LH]G. @@ -328,38 +238,6 @@ MachineInstrBuilder(MF, MI).addReg(Reg64).addImm(40).addReg(0); } -// Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR -// DestReg before MBBI in MBB. Use LowLowOpcode when both DestReg and SrcReg -// are low registers, otherwise use RISB[LH]G. Size is the number of bits -// taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR). -// KillSrc is true if this move is the last use of SrcReg. -MachineInstrBuilder -SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - const DebugLoc &DL, unsigned DestReg, - unsigned SrcReg, unsigned LowLowOpcode, - unsigned Size, bool KillSrc, - bool UndefSrc) const { - unsigned Opcode; - bool DestIsHigh = isHighReg(DestReg); - bool SrcIsHigh = isHighReg(SrcReg); - if (DestIsHigh && SrcIsHigh) - Opcode = SystemZ::RISBHH; - else if (DestIsHigh && !SrcIsHigh) - Opcode = SystemZ::RISBHL; - else if (!DestIsHigh && SrcIsHigh) - Opcode = SystemZ::RISBLH; - else { - return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg) - .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)); - } - unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0); - return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) - .addReg(DestReg, RegState::Undef) - .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)) - .addImm(32 - Size).addImm(128 + 31).addImm(Rotate); -} - MachineInstr *SystemZInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, @@ -1359,15 +1237,6 @@ expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI); return true; - case SystemZ::LOCRMux: - expandLOCRPseudo(MI, SystemZ::LOCR, SystemZ::LOCFHR); - return true; - - case SystemZ::SELRMux: - expandSELRPseudo(MI, SystemZ::SELR, SystemZ::SELFHR, - SystemZ::LOCRMux); - return true; - case SystemZ::STCMux: expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH); return true; @@ -1469,8 +1338,8 @@ return true; case SystemZ::RISBMux: { - bool DestIsHigh = isHighReg(MI.getOperand(0).getReg()); - bool SrcIsHigh = isHighReg(MI.getOperand(2).getReg()); + bool DestIsHigh = SystemZRegisterInfo::isHighReg(MI.getOperand(0).getReg()); + bool SrcIsHigh = SystemZRegisterInfo::isHighReg(MI.getOperand(2).getReg()); if (SrcIsHigh == DestIsHigh) MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL)); else { @@ -1848,6 +1717,38 @@ BuildMI(MBB, MBBI, DL, get(Opcode), Reg).addImm(Value); } +// Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR +// DestReg before MBBI in MBB. Use LowLowOpcode when both DestReg and SrcReg +// are low registers, otherwise use RISB[LH]G. Size is the number of bits +// taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR). +// KillSrc is true if this move is the last use of SrcReg. +MachineInstrBuilder +SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, unsigned DestReg, + unsigned SrcReg, unsigned LowLowOpcode, + unsigned Size, bool KillSrc, + bool UndefSrc) const { + unsigned Opcode; + bool DestIsHigh = SystemZRegisterInfo::isHighReg(DestReg); + bool SrcIsHigh = SystemZRegisterInfo::isHighReg(SrcReg); + if (DestIsHigh && SrcIsHigh) + Opcode = SystemZ::RISBHH; + else if (DestIsHigh && !SrcIsHigh) + Opcode = SystemZ::RISBHL; + else if (!DestIsHigh && SrcIsHigh) + Opcode = SystemZ::RISBLH; + else { + return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)); + } + unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0); + return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) + .addReg(DestReg, RegState::Undef) + .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)) + .addImm(32 - Size).addImm(128 + 31).addImm(Rotate); +} + bool SystemZInstrInfo:: areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb, Index: lib/Target/SystemZ/SystemZPostRewrite.cpp =================================================================== --- lib/Target/SystemZ/SystemZPostRewrite.cpp +++ lib/Target/SystemZ/SystemZPostRewrite.cpp @@ -25,6 +25,7 @@ #define DEBUG_TYPE "systemz-postrewrite" STATISTIC(MemFoldCopies, "Number of copies inserted before folded mem ops."); +STATISTIC(LOCRMuxJumps, "Number of LOCRMux jump-sequences (lower is better)"); namespace llvm { void initializeSystemZPostRewritePass(PassRegistry&); @@ -45,12 +46,17 @@ StringRef getPassName() const override { return SYSTEMZ_POSTREWRITE_NAME; } - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesAll(); - MachineFunctionPass::getAnalysisUsage(AU); - } - private: + void trySelectLOCRMux(MachineInstr &MI, + unsigned LowOpcode, + unsigned HighOpcode); + void trySelectSELRMux(MachineInstr &MI, + unsigned LowOpcode, + unsigned HighOpcode, + unsigned MixedOpcode); + bool expandLOCRMux(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); bool selectMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NextMBBI); bool selectMBB(MachineBasicBlock &MBB); @@ -68,11 +74,136 @@ return new SystemZPostRewrite(); } +// MI is a load-register-on-condition pseudo instruction. Replace it with +// LowOpcode if source and destination are both low GR32s and HighOpcode if +// source and destination are both high GR32s. +void SystemZPostRewrite::trySelectLOCRMux(MachineInstr &MI, + unsigned LowOpcode, + unsigned HighOpcode) { + Register DestReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(2).getReg(); + bool DestIsHigh = SystemZRegisterInfo::isHighReg(DestReg); + bool SrcIsHigh = SystemZRegisterInfo::isHighReg(SrcReg); + + if (!DestIsHigh && !SrcIsHigh) + MI.setDesc(TII->get(LowOpcode)); + else if (DestIsHigh && SrcIsHigh) + MI.setDesc(TII->get(HighOpcode)); +} + +// MI is a select pseudo instruction. Replace it with LowOpcode if source +// and destination are all low GR32s and HighOpcode if source and destination +// are all high GR32s. Otherwise, use the two-operand MixedOpcode. +void SystemZPostRewrite::trySelectSELRMux(MachineInstr &MI, + unsigned LowOpcode, + unsigned HighOpcode, + unsigned MixedOpcode) { + Register DestReg = MI.getOperand(0).getReg(); + Register Src1Reg = MI.getOperand(1).getReg(); + Register Src2Reg = MI.getOperand(2).getReg(); + bool DestIsHigh = SystemZRegisterInfo::isHighReg(DestReg); + bool Src1IsHigh = SystemZRegisterInfo::isHighReg(Src1Reg); + bool Src2IsHigh = SystemZRegisterInfo::isHighReg(Src2Reg); + + // If sources and destination aren't all high or all low, we may be able to + // simplify the operation by moving one of the sources to the destination + // first. But only if this doesn't clobber the other source. + if (DestReg != Src1Reg && DestReg != Src2Reg) { + if (DestIsHigh != Src1IsHigh) { + TII->emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, Src1Reg, + SystemZ::LR, 32, MI.getOperand(1).isKill(), + MI.getOperand(1).isUndef()); + MI.getOperand(1).setReg(DestReg); + Src1Reg = DestReg; + Src1IsHigh = DestIsHigh; + } else if (DestIsHigh != Src2IsHigh) { + TII->emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, Src2Reg, + SystemZ::LR, 32, MI.getOperand(2).isKill(), + MI.getOperand(2).isUndef()); + MI.getOperand(2).setReg(DestReg); + Src2Reg = DestReg; + Src2IsHigh = DestIsHigh; + } + } + + // If the destination (now) matches one source, prefer this to be first. + if (DestReg != Src1Reg && DestReg == Src2Reg) { + TII->commuteInstruction(MI, false, 1, 2); + std::swap(Src1Reg, Src2Reg); + std::swap(Src1IsHigh, Src2IsHigh); + } + + if (!DestIsHigh && !Src1IsHigh && !Src2IsHigh) + MI.setDesc(TII->get(LowOpcode)); + else if (DestIsHigh && Src1IsHigh && Src2IsHigh) + MI.setDesc(TII->get(HighOpcode)); + else { + // Given the simplification above, we must already have a two-operand case. + assert (DestReg == Src1Reg); + MI.setDesc(TII->get(MixedOpcode)); + MI.tieOperands(0, 1); + } +} + +// MI is a load-register-on-condition pseudo instruction that could not be +// handled as a single hardware instruction. Replace it by a branch sequence. +bool SystemZPostRewrite::expandLOCRMux(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + MachineFunction &MF = *MBB.getParent(); + const BasicBlock *BB = MBB.getBasicBlock(); + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + Register DestReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(2).getReg(); + unsigned CCValid = MI.getOperand(3).getImm(); + unsigned CCMask = MI.getOperand(4).getImm(); + + LivePhysRegs LiveRegs(TII->getRegisterInfo()); + LiveRegs.addLiveOuts(MBB); + for (auto I = std::prev(MBB.end()); I != MBBI; --I) + LiveRegs.stepBackward(*I); + + // Splice MBB at MI, moving the rest of the block into RestMBB. + MachineBasicBlock *RestMBB = MF.CreateMachineBasicBlock(BB); + MF.insert(std::next(MachineFunction::iterator(MBB)), RestMBB); + RestMBB->splice(RestMBB->begin(), &MBB, MI, MBB.end()); + RestMBB->transferSuccessors(&MBB); + for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I) + RestMBB->addLiveIn(*I); + + // Create a new block MoveMBB to hold the move instruction. + MachineBasicBlock *MoveMBB = MF.CreateMachineBasicBlock(BB); + MF.insert(std::next(MachineFunction::iterator(MBB)), MoveMBB); + MoveMBB->addLiveIn(SrcReg); + for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I) + MoveMBB->addLiveIn(*I); + + // At the end of MBB, create a conditional branch to RestMBB if the + // condition is false, otherwise fall through to MoveMBB. + BuildMI(&MBB, DL, TII->get(SystemZ::BRC)) + .addImm(CCValid).addImm(CCMask ^ CCValid).addMBB(RestMBB); + MBB.addSuccessor(RestMBB); + MBB.addSuccessor(MoveMBB); + + // In MoveMBB, emit an instruction to move SrcReg into DestReg, + // then fall through to RestMBB. + TII->copyPhysReg(*MoveMBB, MoveMBB->end(), DL, DestReg, SrcReg, + MI.getOperand(2).isKill()); + MoveMBB->addSuccessor(RestMBB); + + NextMBBI = MBB.end(); + MI.eraseFromParent(); + LOCRMuxJumps++; + return true; +} + /// If MBBI references a pseudo instruction that should be selected here, /// do it and return true. Otherwise return false. bool SystemZPostRewrite::selectMI(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI) { + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + bool Modified = false; MachineInstr &MI = *MBBI; unsigned Opcode = MI.getOpcode(); @@ -94,7 +225,20 @@ return true; } - return false; + switch (Opcode) { + case SystemZ::LOCRMux: + trySelectLOCRMux(MI, SystemZ::LOCR, SystemZ::LOCFHR); + Modified = true; + break; + case SystemZ::SELRMux: + trySelectSELRMux(MI, SystemZ::SELR, SystemZ::SELFHR, SystemZ::LOCRMux); + Modified = true; + break; + } + if (MI.getOpcode() == SystemZ::LOCRMux) + Modified |= expandLOCRMux(MBB, MBBI, NextMBBI); + + return Modified; } /// Iterate over the instructions in basic block MBB and select any Index: lib/Target/SystemZ/SystemZRegisterInfo.h =================================================================== --- lib/Target/SystemZ/SystemZRegisterInfo.h +++ lib/Target/SystemZ/SystemZRegisterInfo.h @@ -49,6 +49,10 @@ const TargetRegisterClass * getCrossCopyRegClass(const TargetRegisterClass *RC) const override; + // Reg should be a 32-bit GPR. Return true if it is a high register rather + // than a low register. + static bool isHighReg(unsigned int Reg); + bool getRegAllocationHints(unsigned VirtReg, ArrayRef Order, SmallVectorImpl &Hints, Index: lib/Target/SystemZ/SystemZRegisterInfo.cpp =================================================================== --- lib/Target/SystemZ/SystemZRegisterInfo.cpp +++ lib/Target/SystemZ/SystemZRegisterInfo.cpp @@ -73,6 +73,13 @@ Hints.push_back(Reg); } +bool SystemZRegisterInfo::isHighReg(unsigned int Reg) { + if (SystemZ::GRH32BitRegClass.contains(Reg)) + return true; + assert(SystemZ::GR32BitRegClass.contains(Reg) && "Invalid GRX32"); + return false; +} + bool SystemZRegisterInfo::getRegAllocationHints(unsigned VirtReg, ArrayRef Order, Index: lib/Target/SystemZ/SystemZTargetMachine.cpp =================================================================== --- lib/Target/SystemZ/SystemZTargetMachine.cpp +++ lib/Target/SystemZ/SystemZTargetMachine.cpp @@ -223,8 +223,6 @@ if (getOptLevel() == CodeGenOpt::None) addPass(createSystemZPostRewritePass(getSystemZTargetMachine())); - addPass(createSystemZExpandPseudoPass(getSystemZTargetMachine())); - if (getOptLevel() != CodeGenOpt::None) addPass(&IfConverterID); }