Index: include/llvm/CodeGen/TargetRegisterInfo.h =================================================================== --- include/llvm/CodeGen/TargetRegisterInfo.h +++ include/llvm/CodeGen/TargetRegisterInfo.h @@ -813,6 +813,21 @@ const LiveRegMatrix *Matrix = nullptr) const; + /// Let target disallow a replacement of a physical register. This would + /// typically be useful during copy propagation for the cases where + /// getRegAllocationHints() returned hard hints. This could either be + /// called with for a virtual register (with non-null VReg and VRM + /// arguments), or for a physical register at OpIdx to be replaced. + virtual bool isRegSuitableReplacement(const MachineInstr *MI, + unsigned VReg, + unsigned NewPhysReg, + const VirtRegMap *VRM = nullptr, + unsigned OpIdx = UINT_MAX) const { + assert(((OpIdx == UINT_MAX) != (VReg == 0)) && + ((VReg == 0) == (VRM == nullptr)) && "Bad arguments."); + return true; + } + /// A callback to allow target a chance to update register allocation hints /// when a register is "changed" (e.g. coalesced) to another register. /// e.g. On ARM, some virtual registers should target register pairs, Index: lib/CodeGen/MachineCopyPropagation.cpp =================================================================== --- lib/CodeGen/MachineCopyPropagation.cpp +++ lib/CodeGen/MachineCopyPropagation.cpp @@ -309,6 +309,11 @@ unsigned CopySrcReg = Copy.getOperand(1).getReg(); + // Let target disallow a replacement of a physical register + if (!TRI->isRegSuitableReplacement(&UseI, 0/*NoVReg*/, CopySrcReg, + nullptr/*VRM*/, UseIdx)) + return false; + // If the new register meets the opcode register constraints, then allow // forwarding. if (const TargetRegisterClass *URC = Index: lib/CodeGen/RegAllocGreedy.cpp =================================================================== --- lib/CodeGen/RegAllocGreedy.cpp +++ lib/CodeGen/RegAllocGreedy.cpp @@ -2966,6 +2966,18 @@ LLVM_DEBUG(dbgs() << "=> Not profitable.\n"); continue; } + + // Check with target for any special replacements to avoid. + bool SuitableReplacement = true; + for (const MachineInstr &Instr : MRI->reg_nodbg_instructions(Reg)) + if (!TRI->isRegSuitableReplacement(&Instr, Reg, PhysReg, VRM)) { + LLVM_DEBUG(dbgs() << "=> Not suitable.\n"); + SuitableReplacement = false; + break; + } + if (!SuitableReplacement) + continue; + // At this point, the cost is either cheaper or equal. If it is // equal, we consider this is profitable because it may expose // more recoloring opportunities. Index: lib/Target/SystemZ/SystemZInstrFormats.td =================================================================== --- lib/Target/SystemZ/SystemZInstrFormats.td +++ lib/Target/SystemZ/SystemZInstrFormats.td @@ -4574,6 +4574,19 @@ let OpType = "reg"; } +// Like BinaryRRAndK, but expanded after RA depending on the choice of register. +multiclass BinaryRRAndKPseudo { + let NumOpsKey = key, Predicates = [FeatureHighWord] in { + let NumOpsValue = "3" in + def K : Pseudo<(outs cls1:$R1), (ins cls2:$R2, cls3:$R3), []>; + let NumOpsValue = "2", isConvertibleToThreeAddress = 1, + Constraints = "$R1 = $R1src" in + def "" : Pseudo<(outs cls1:$R1), (ins cls2:$R1src, cls3:$R3), + [(set cls1:$R1, (operator cls2:$R1src, cls3:$R3))]>; + } +} + // Like BinaryRI, but expanded after RA depending on the choice of register. class BinaryRIPseudo @@ -4621,6 +4634,15 @@ let AccessBytes = bytes; } +// Like CompareRRE, but expanded after RA depending on the choice of +// register. +class CompareRREPseudo + : Pseudo<(outs), (ins cls1:$R1, cls2:$R2), + [(set CC, (operator cls1:$R1, cls2:$R2))]> { + let isCompare = 1; +} + // Like TestBinarySIL, but expanded later. class TestBinarySILPseudo : Pseudo<(outs), (ins bdaddr12only:$BD1, imm:$I2), Index: lib/Target/SystemZ/SystemZInstrInfo.h =================================================================== --- lib/Target/SystemZ/SystemZInstrInfo.h +++ lib/Target/SystemZ/SystemZInstrInfo.h @@ -153,10 +153,14 @@ unsigned LowOpcodeK, unsigned HighOpcode) const; void expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned HighOpcode) const; + void expandRRAndKPseudo(MachineInstr &MI, unsigned LowOpcode, + unsigned HiOpcode, unsigned HiHiLoOpcode) const; void expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned HighOpcode) const; void expandLOCRPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned HighOpcode) const; + void expandCmpRMux(MachineInstr &MI, unsigned LowLowOpcode, + unsigned HiHiOpcode, unsigned HiLowOpcode) const; void expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode, unsigned Size) const; void expandLoadStackGuard(MachineInstr *MI) const; Index: lib/Target/SystemZ/SystemZInstrInfo.cpp =================================================================== --- lib/Target/SystemZ/SystemZInstrInfo.cpp +++ lib/Target/SystemZ/SystemZInstrInfo.cpp @@ -188,6 +188,50 @@ MI.setDesc(get(Opcode)); } +// TODO: comment +void SystemZInstrInfo::expandRRAndKPseudo(MachineInstr &MI, unsigned LowOpcode, + unsigned HiOpcode, + unsigned HiHiLoOpcode) const { + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned LHSReg = MI.getOperand(1).getReg(); + unsigned RHSReg = MI.getOperand(2).getReg(); + bool LHSisKill = MI.getOperand(1).isKill(); + bool RHSisKill = MI.getOperand(2).isKill(); + bool DstIsHigh = isHighReg(DstReg); + bool LHSIsHigh = isHighReg(LHSReg); + bool RHSIsHigh = isHighReg(RHSReg); + bool IsAdd = MI.isCommutable(); + bool Commute = false; + unsigned Opcode; + if (!DstIsHigh && !LHSIsHigh && !RHSIsHigh) + Opcode = (DstReg == LHSReg ? + LowOpcode : SystemZ::getThreeOperandOpcode(LowOpcode)); + else if (DstIsHigh && LHSIsHigh && RHSIsHigh) + Opcode = HiOpcode; + else if (DstIsHigh && (LHSIsHigh || RHSIsHigh)) { + Opcode = HiHiLoOpcode; + if (!LHSIsHigh && RHSIsHigh) { // HLH is unsupported. + if (IsAdd) + Commute = true; + else + Opcode = SystemZ::SUBCOMMUX; // EXPERIMENTAL + } + } + else + Opcode = SystemZ::ADDSUBMUX; // EXPERIMENTAL + + MachineBasicBlock *MBB = MI.getParent(); + if (!Commute) + BuildMI(*MBB, MI, MI.getDebugLoc(), get(Opcode), DstReg) + .addReg(LHSReg, getKillRegState(LHSisKill)) + .addReg(RHSReg, getKillRegState(RHSisKill)); + else + BuildMI(*MBB, MI, MI.getDebugLoc(), get(Opcode), DstReg) + .addReg(RHSReg, getKillRegState(RHSisKill)) + .addReg(LHSReg, getKillRegState(LHSisKill)); + MI.eraseFromParent(); +} + // MI is a load-on-condition pseudo instruction with a single register // (source or destination) operand. Replace it with LowOpcode if the // register is a low GR32 and HighOpcode if the register is a high GR32. @@ -221,6 +265,27 @@ // correctly. This change is defered to the SystemZExpandPseudo pass. } +// MI is a 32 bit compare with register pseudo instruction. Replace it with +// one of the three possible opcodes. "LowHi" is not supported and must +// therefore be inverted. +void SystemZInstrInfo::expandCmpRMux(MachineInstr &MI, unsigned LowLowOpcode, + unsigned HiHiOpcode, + unsigned HiLowOpcode) const { + unsigned Src0Reg = MI.getOperand(0).getReg(); + unsigned Src1Reg = MI.getOperand(1).getReg(); + bool Src0IsHigh = isHighReg(Src0Reg); + bool Src1IsHigh = isHighReg(Src1Reg); + + if (!Src0IsHigh && !Src1IsHigh) + MI.setDesc(get(LowLowOpcode)); + else if (Src0IsHigh && Src1IsHigh) + MI.setDesc(get(HiHiOpcode)); + else if (Src0IsHigh && !Src1IsHigh) + MI.setDesc(get(HiLowOpcode)); + else // LowHi + MI.setDesc(get(SystemZ::CMPMUX)); // EXPERIMENTAL +} + // MI is an RR-style pseudo instruction that zero-extends the low Size bits // of one GRX32 into another. Replace it with LowOpcode if both operands // are low registers, otherwise use RISB[LH]G. @@ -1373,6 +1438,26 @@ expandRIPseudo(MI, SystemZ::TMLH, SystemZ::TMHH, false); return true; + case SystemZ::ARMux: + case SystemZ::ARMuxK: + expandRRAndKPseudo(MI, SystemZ::AR, SystemZ::AHHHR, SystemZ::AHHLR); + return true; + + case SystemZ::SRMux: + case SystemZ::SRMuxK: + expandRRAndKPseudo(MI, SystemZ::SR, SystemZ::SHHHR, SystemZ::SHHLR); + return true; + + case SystemZ::ALRMux: + case SystemZ::ALRMuxK: + expandRRAndKPseudo(MI, SystemZ::ALR, SystemZ::ALHHHR, SystemZ::ALHHLR); + return true; + + case SystemZ::SLRMux: + case SystemZ::SLRMuxK: + expandRRAndKPseudo(MI, SystemZ::SLR, SystemZ::SLHHHR, SystemZ::SLHHLR); + return true; + case SystemZ::AHIMux: expandRIPseudo(MI, SystemZ::AHI, SystemZ::AIH, false); return true; @@ -1405,6 +1490,14 @@ expandRXYPseudo(MI, SystemZ::CL, SystemZ::CLHF); return true; + case SystemZ::CRMux: + expandCmpRMux(MI, SystemZ::CR, SystemZ::CHHR, SystemZ::CHLR); + return true; + + case SystemZ::CLRMux: + expandCmpRMux(MI, SystemZ::CLR, SystemZ::CLHHR, SystemZ::CLHLR); + return true; + case SystemZ::RISBMux: { bool DestIsHigh = isHighReg(MI.getOperand(0).getReg()); bool SrcIsHigh = isHighReg(MI.getOperand(2).getReg()); Index: lib/Target/SystemZ/SystemZInstrInfo.td =================================================================== --- lib/Target/SystemZ/SystemZInstrInfo.td +++ lib/Target/SystemZ/SystemZInstrInfo.td @@ -893,11 +893,19 @@ let Defs = [CC], CCValues = 0xF, CompareZeroCCMask = 0x8 in { // Addition of a register. let isCommutable = 1 in { + defm ARMux : BinaryRRAndKPseudo<"armux", z_sadd, GRX32, GRX32, GRX32>; defm AR : BinaryRRAndK<"ar", 0x1A, 0xB9F8, z_sadd, GR32, GR32>; defm AGR : BinaryRREAndK<"agr", 0xB908, 0xB9E8, z_sadd, GR64, GR64>; } def AGFR : BinaryRRE<"agfr", 0xB918, null_frag, GR64, GR32>; + // EXPERIMENTAL! -- just to be able to compile and see the occurences. + let hasNoSchedulingInfo = 1, isCodeGenOnly = 1 in { + def ADDSUBMUX : BinaryRRFa<"# addsubmux", 0x0, null_frag, GRX32, GRX32, GRX32>; + def SUBCOMMUX : BinaryRRFa<"# commsub", 0x0, null_frag, GRX32, GRX32, GRX32>; + def CMPMUX : CompareRRE<"# cmpmux", 0x0, null_frag, GRX32, GRX32>; + } + // Addition to a high register. def AHHHR : BinaryRRFa<"ahhhr", 0xB9C8, null_frag, GRH32, GRH32, GRH32>, Requires<[FeatureHighWord]>; @@ -935,6 +943,7 @@ let Defs = [CC] in { // Addition of a register. let isCommutable = 1 in { + defm ALRMux : BinaryRRAndKPseudo<"alrmux", z_uadd, GRX32, GRX32, GRX32>; defm ALR : BinaryRRAndK<"alr", 0x1E, 0xB9FA, z_uadd, GR32, GR32>; defm ALGR : BinaryRREAndK<"algr", 0xB90A, 0xB9EA, z_uadd, GR64, GR64>; } @@ -994,6 +1003,7 @@ // Subtraction producing a signed overflow flag. let Defs = [CC], CCValues = 0xF, CompareZeroCCMask = 0x8 in { // Subtraction of a register. + defm SRMux : BinaryRRAndKPseudo<"srmux", z_ssub, GRX32, GRX32, GRX32>; defm SR : BinaryRRAndK<"sr", 0x1B, 0xB9F9, z_ssub, GR32, GR32>; def SGFR : BinaryRRE<"sgfr", 0xB919, null_frag, GR64, GR32>; defm SGR : BinaryRREAndK<"sgr", 0xB909, 0xB9E9, z_ssub, GR64, GR64>; @@ -1035,6 +1045,7 @@ // Subtraction producing a carry. let Defs = [CC] in { // Subtraction of a register. + defm SLRMux : BinaryRRAndKPseudo<"slrmux", z_usub, GRX32, GRX32, GRX32>; defm SLR : BinaryRRAndK<"slr", 0x1F, 0xB9FB, z_usub, GR32, GR32>; def SLGFR : BinaryRRE<"slgfr", 0xB91B, null_frag, GR64, GR32>; defm SLGR : BinaryRREAndK<"slgr", 0xB90B, 0xB9EB, z_usub, GR64, GR64>; @@ -1420,7 +1431,10 @@ // some of the signed forms have COMPARE AND BRANCH equivalents whereas none // of the unsigned forms do. let Defs = [CC], CCValues = 0xE in { - // Comparison with a register. + // Comparison with a register. CRMux expands to CR, CHHR or CHLR, + // depending on the choice of register. + def CRMux : CompareRREPseudo, + Requires<[FeatureHighWord]>; def CR : CompareRR <"cr", 0x19, z_scmp, GR32, GR32>; def CGFR : CompareRRE<"cgfr", 0xB930, null_frag, GR64, GR32>; def CGR : CompareRRE<"cgr", 0xB920, z_scmp, GR64, GR64>; @@ -1472,7 +1486,10 @@ // Unsigned comparisons. let Defs = [CC], CCValues = 0xE, IsLogical = 1 in { - // Comparison with a register. + // Comparison with a register. CLRMux expands to CLR, CLHHR or CLHLR, + // depending on the choice of register. + def CLRMux : CompareRREPseudo, + Requires<[FeatureHighWord]>; def CLR : CompareRR <"clr", 0x15, z_ucmp, GR32, GR32>; def CLGFR : CompareRRE<"clgfr", 0xB931, null_frag, GR64, GR32>; def CLGR : CompareRRE<"clgr", 0xB921, z_ucmp, GR64, GR64>; Index: lib/Target/SystemZ/SystemZRegisterInfo.h =================================================================== --- lib/Target/SystemZ/SystemZRegisterInfo.h +++ lib/Target/SystemZ/SystemZRegisterInfo.h @@ -56,6 +56,12 @@ const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const override; + bool isRegSuitableReplacement(const MachineInstr *MI, + unsigned VReg, + unsigned NewPhysReg, + const VirtRegMap *VRM = nullptr, + unsigned OpIdx = UINT_MAX) const override; + // Override TargetRegisterInfo.h. bool requiresRegisterScavenging(const MachineFunction &MF) const override { return true; Index: lib/Target/SystemZ/SystemZRegisterInfo.cpp =================================================================== --- lib/Target/SystemZ/SystemZRegisterInfo.cpp +++ lib/Target/SystemZ/SystemZRegisterInfo.cpp @@ -21,9 +21,19 @@ #define GET_REGINFO_TARGET_DESC #include "SystemZGenRegisterInfo.inc" +#define DEBUG_TYPE "regalloc" + SystemZRegisterInfo::SystemZRegisterInfo() : SystemZGenRegisterInfo(SystemZ::R14D) {} +static const TargetRegisterClass *getRCFromPhysReg(unsigned PhysReg) { + if (SystemZ::GR32BitRegClass.contains(PhysReg)) + return &SystemZ::GR32BitRegClass; + assert (SystemZ::GRH32BitRegClass.contains(PhysReg) && + "Phys reg not in GR32 or GRH32?"); + return &SystemZ::GRH32BitRegClass; +} + // Given that MO is a GRX32 operand, return either GR32 or GRH32 if MO // somehow belongs in it. Otherwise, return GRX32. static const TargetRegisterClass *getRC32(MachineOperand &MO, @@ -40,14 +50,8 @@ MO.getSubReg() == SystemZ::subreg_hh32) return &SystemZ::GRH32BitRegClass; - if (VRM && VRM->hasPhys(MO.getReg())) { - unsigned PhysReg = VRM->getPhys(MO.getReg()); - if (SystemZ::GR32BitRegClass.contains(PhysReg)) - return &SystemZ::GR32BitRegClass; - assert (SystemZ::GRH32BitRegClass.contains(PhysReg) && - "Phys reg not in GR32 or GRH32?"); - return &SystemZ::GRH32BitRegClass; - } + if (VRM && VRM->hasPhys(MO.getReg())) + return getRCFromPhysReg(VRM->getPhys(MO.getReg())); assert (RC == &SystemZ::GRX32BitRegClass); return RC; @@ -87,14 +91,161 @@ VirtReg, Order, Hints, MF, VRM, Matrix); if (MRI->getRegClass(VirtReg) == &SystemZ::GRX32BitRegClass) { + // Compare/Add/Sub "High" Muxes. Not all combinations are supported by + // the target. + struct Reg2RCMap : std::map { + unsigned VirtReg_; // XXX + bool Change; + unsigned RequiredHigh; // USEFUL? + unsigned RequiredLow; + void reset(unsigned Reg) { + VirtReg_ = Reg; + Change = false; + RequiredHigh = 0; + RequiredLow = 0; + } + void findRC(MachineOperand &MO, + const VirtRegMap *VRM, + const MachineRegisterInfo *MRI) { + if (find(MO.getReg()) != end()) + return; + const TargetRegisterClass *RC = getRC32(MO, VRM, MRI); + (*this)[MO.getReg()] = RC; + } + void constrainReg(unsigned Reg, const TargetRegisterClass *RC) { + if ((*this)[Reg] != &SystemZ::GRX32BitRegClass) + return; + if (Reg == VirtReg_) { + if (RC == &SystemZ::GR32BitRegClass) + RequiredLow++; + else + RequiredHigh++; + } else { + (*this)[Reg] = RC; + Change = true; + } + } + } Reg2RC; + // Search all use/def connected instructions iteratively to propagate the + // requirements of registers being in either GR32 or GR32H. At the end, + // pass hints for VirtReg if either GR32 or GR32H seems preferred. + // XXX: Make sure not to revisit same instruction multiple times? SmallVector Worklist; SmallSet DoneRegs; + Reg2RC.Change = true; + while (Reg2RC.Change) { + Reg2RC.reset(VirtReg); + Worklist.clear(); + Worklist.push_back(VirtReg); + DoneRegs.clear(); + while (Worklist.size()) { + unsigned Reg = Worklist.pop_back_val(); + if (!DoneRegs.insert(Reg).second) + continue; + for (MachineRegisterInfo::reg_instr_nodbg_iterator + RI = MRI->reg_instr_nodbg_begin(Reg), + E = MRI->reg_instr_nodbg_end(); RI != E; ) { + MachineInstr &MI = *RI++; + unsigned Regs[3]; + unsigned OpI = 0; + auto processOperands = [&]() -> void { + for (; OpI < MI.getNumOperands(); ++OpI) { + MachineOperand &MO = MI.getOperand(OpI); + if (MO.isImplicit()) + break; + Regs[OpI] = MO.getReg(); + Reg2RC.findRC(MO, VRM, MRI); + // Add GRX32 registers to worklist + if (Reg2RC[MO.getReg()] == &SystemZ::GRX32BitRegClass) + Worklist.push_back(MO.getReg()); + } + }; + + const TargetRegisterClass *LOW = &SystemZ::GR32BitRegClass; + const TargetRegisterClass *HIGH = &SystemZ::GRH32BitRegClass; + switch (MI.getOpcode()) { + case SystemZ::CRMux: + case SystemZ::CLRMux: + processOperands(); + // Avoid LH + if (Reg2RC[Regs[0]] == LOW) + Reg2RC.constrainReg(Regs[1], LOW); + else if (Reg2RC[Regs[1]] == HIGH) + Reg2RC.constrainReg(Regs[0], HIGH); + break; + + case SystemZ::SRMux: + case SystemZ::SRMuxK: + case SystemZ::SLRMux: + case SystemZ::SLRMuxK: + processOperands(); + // H__ -> HH_ + if (Reg2RC[Regs[0]] == HIGH) { + Reg2RC.constrainReg(Regs[1], HIGH); + break; + } + // _L_ -> LLL + if (Reg2RC[Regs[1]] == LOW) { + Reg2RC.constrainReg(Regs[0], LOW); + Reg2RC.constrainReg(Regs[2], LOW); + break; + } + LLVM_FALLTHROUGH; + case SystemZ::ARMux: + case SystemZ::ARMuxK: + case SystemZ::ALRMux: + case SystemZ::ALRMuxK: + processOperands(); + // L__ -> LLL + if (Reg2RC[Regs[0]] == LOW) { + Reg2RC.constrainReg(Regs[1], LOW); + Reg2RC.constrainReg(Regs[2], LOW); + } + // _LL -> LLL + else if (Reg2RC[Regs[1]] == LOW && Reg2RC[Regs[2]] == LOW) + Reg2RC.constrainReg(Regs[0], LOW); + else if (Reg2RC[Regs[0]] == HIGH) { + // HL_ -> HLH + if (Reg2RC[Regs[1]] == LOW) + Reg2RC.constrainReg(Regs[2], HIGH); + // H_L -> HHL + else if (Reg2RC[Regs[2]] == LOW) + Reg2RC.constrainReg(Regs[1], HIGH); + // Hrr -> HHH + else if (Regs[1] == Regs[2]) + Reg2RC.constrainReg(Regs[1], HIGH); + } + // _H? -> HH? + else if (Reg2RC[Regs[1]] == HIGH || Reg2RC[Regs[2]] == HIGH) + Reg2RC.constrainReg(Regs[0], HIGH); + break; + case SystemZ::COPY: + if (MI.getOperand(1).getReg() == Reg) + Worklist.push_back(MI.getOperand(0).getReg()); // USEFUL? + break; + } + } // Reg + } // Worklist + } // Change + if (Reg2RC.RequiredHigh > Reg2RC.RequiredLow) { + addHints(Order, Hints, &SystemZ::GRH32BitRegClass, MRI); + LLVM_DEBUG(dbgs() << "SystemZ: Cmp/Add/Sub Mux hints\n"); + return true; + } else if (Reg2RC.RequiredLow > Reg2RC.RequiredHigh) { + addHints(Order, Hints, &SystemZ::GR32BitRegClass, MRI); + LLVM_DEBUG(dbgs() << "SystemZ: Cmp/Add/Sub Mux hints\n"); + return true; + } + + // XXX: Incorporate rest of hints above? Or do LOCRMux first? + // LOCRMux + Worklist.clear(); + DoneRegs.clear(); Worklist.push_back(VirtReg); while (Worklist.size()) { unsigned Reg = Worklist.pop_back_val(); if (!DoneRegs.insert(Reg).second) continue; - for (auto &Use : MRI->use_instructions(Reg)) { // For LOCRMux, see if the other operand is already a high or low // register, and in that case give the correpsonding hints for @@ -108,6 +259,7 @@ getRC32(TrueMO, VRM, MRI)); if (RC && RC != &SystemZ::GRX32BitRegClass) { addHints(Order, Hints, RC, MRI); + LLVM_DEBUG(dbgs() << "SystemZ: LOCRMux hints\n"); // Return true to make these hints the only regs available to // RA. This may mean extra spilling but since the alternative is // a jump sequence expansion of the LOCRMux, it is preferred. @@ -119,28 +271,116 @@ (TrueMO.getReg() == Reg ? FalseMO.getReg() : TrueMO.getReg()); if (MRI->getRegClass(OtherReg) == &SystemZ::GRX32BitRegClass) Worklist.push_back(OtherReg); - } // end LOCRMux - else if (Use.getOpcode() == SystemZ::CHIMux || - Use.getOpcode() == SystemZ::CFIMux) { - if (Use.getOperand(1).getImm() == 0) { - bool OnlyLMuxes = true; - for (MachineInstr &DefMI : MRI->def_instructions(VirtReg)) - if (DefMI.getOpcode() != SystemZ::LMux) - OnlyLMuxes = false; - if (OnlyLMuxes) { - addHints(Order, Hints, &SystemZ::GR32BitRegClass, MRI); - // Return false to make these hints preferred but not obligatory. - return false; - } - } - } // end CHIMux / CFIMux + } } - } + } // end LOCRMux + + // CHIMux / CFIMux + for (auto &Use : MRI->use_instructions(VirtReg)) + if ((Use.getOpcode() == SystemZ::CHIMux || + Use.getOpcode() == SystemZ::CFIMux) && + Use.getOperand(1).getImm() == 0) { + bool OnlyLMuxes = true; + for (MachineInstr &DefMI : MRI->def_instructions(VirtReg)) + if (DefMI.getOpcode() != SystemZ::LMux) + OnlyLMuxes = false; + if (OnlyLMuxes) { + LLVM_DEBUG(dbgs() << "SystemZ: CHIMux/CFIMux hints\n"); + addHints(Order, Hints, &SystemZ::GR32BitRegClass, MRI); + // Return false to make these hints preferred but not obligatory. + return false; + } + } // end CHIMux / CFIMux } return BaseImplRetVal; } +bool SystemZRegisterInfo::isRegSuitableReplacement(const MachineInstr *MI, + unsigned VReg, + unsigned NewPhysReg, + const VirtRegMap *VRM, + unsigned OpIdx) const { + if (!SystemZ::GRX32BitRegClass.contains(NewPhysReg)) + return true; + const TargetRegisterClass *NewRC = getRCFromPhysReg(NewPhysReg); + unsigned PhysRegs[3]; + const TargetRegisterClass *RCs[3]; + unsigned PRegToReplace = 0; + unsigned OpI = 0; + auto processOperands = [&]() -> bool { + for (; OpI < MI->getNumOperands(); ++OpI) { + const MachineOperand &MO = MI->getOperand(OpI); + if (MO.isImplicit()) + break; + unsigned Reg = MO.getReg(); + unsigned PhysReg = (isPhysicalRegister(Reg) ? Reg : VRM->getPhys(Reg)); + if ((VReg && VReg == Reg) || (!VReg && OpIdx == OpI)) { + PRegToReplace = PhysReg; + if (NewRC == getRCFromPhysReg(PRegToReplace)) + return false; + } + assert(PhysReg && "Expected phys reg for operand"); + if (MO.getSubReg() != SystemZ::NoSubRegister) + PhysReg = getSubReg(PhysReg, MO.getSubReg()); + PhysRegs[OpI] = PhysReg; + RCs[OpI] = getRCFromPhysReg(PhysReg); + } + return true; + }; + + bool Unsupported = false; + switch (MI->getOpcode()) { + case SystemZ::CRMux: + case SystemZ::CLRMux: { + if (!processOperands()) + break; + bool LHS = (PRegToReplace == PhysRegs[0]); + if ((LHS && NewRC == &SystemZ::GR32BitRegClass && // Avoid LH + RCs[1] == &SystemZ::GRH32BitRegClass) || + (!LHS && NewRC == &SystemZ::GRH32BitRegClass && + RCs[0] == &SystemZ::GR32BitRegClass)) + Unsupported = true; + break; + } + case SystemZ::SRMuxK: + case SystemZ::SLRMuxK: + if (!processOperands()) + break; + if (RCs[0] == &SystemZ::GRH32BitRegClass && // Avoid HLH + PRegToReplace == PhysRegs[1] && NewRC == &SystemZ::GR32BitRegClass) { + Unsupported = true; + break; + } + LLVM_FALLTHROUGH; + case SystemZ::SRMux: + case SystemZ::SLRMux: + case SystemZ::ARMux: + case SystemZ::ARMuxK: + case SystemZ::ALRMux: + case SystemZ::ALRMuxK: + if (!processOperands()) + break; + if ((RCs[0] == &SystemZ::GR32BitRegClass && // Avoid LLH / LHL / HLL + RCs[1] == &SystemZ::GR32BitRegClass && + RCs[2] == &SystemZ::GR32BitRegClass) || + ((PRegToReplace == PhysRegs[0]) && // Avoid LHH + RCs[0] == &SystemZ::GRH32BitRegClass && + RCs[1] == &SystemZ::GRH32BitRegClass && + RCs[2] == &SystemZ::GRH32BitRegClass)) + Unsupported = true; + break; + default: break; + } + if (Unsupported) { + LLVM_DEBUG(dbgs() << "SystemZ: Avoiding replacing " + << getName(PRegToReplace) << " with " + << getName(NewPhysReg) << " in "; MI->dump()); + return false; + } + return true; +} + const MCPhysReg * SystemZRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { const SystemZSubtarget &Subtarget = MF->getSubtarget(); Index: lib/Target/SystemZ/SystemZScheduleZ13.td =================================================================== --- lib/Target/SystemZ/SystemZScheduleZ13.td +++ lib/Target/SystemZ/SystemZScheduleZ13.td @@ -366,6 +366,7 @@ def : InstRW<[WLat1, FXa, NormalGr], (instregex "AR(K)?$")>; def : InstRW<[WLat1, FXa, NormalGr], (instregex "A(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXa, NormalGr], (instregex "A(L)?HHLR$")>; +def : InstRW<[WLat1, FXa, NormalGr], (instregex "A(L)?RMux(K)?$")>; def : InstRW<[WLat1, FXa, NormalGr], (instregex "ALSIH(N)?$")>; def : InstRW<[WLat2LSU, FXb, LSU, NormalGr], (instregex "A(L)?(G)?SI$")>; @@ -397,6 +398,7 @@ def : InstRW<[WLat1, FXa, NormalGr], (instregex "SR(K)?$")>; def : InstRW<[WLat1, FXa, NormalGr], (instregex "S(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXa, NormalGr], (instregex "S(L)?HHLR$")>; +def : InstRW<[WLat1, FXa, NormalGr], (instregex "S(L)?RMux(K)?$")>; // Subtraction with borrow def : InstRW<[WLat2LSU, WLat2LSU, RegReadAdv, FXa, LSU, GroupAlone], @@ -547,6 +549,7 @@ def : InstRW<[WLat1LSU, FXb, LSU, NormalGr], (instregex "CLRL$")>; def : InstRW<[WLat1, FXb, NormalGr], (instregex "C(L)?HHR$")>; def : InstRW<[WLat2, FXb, NormalGr], (instregex "C(L)?HLR$")>; +def : InstRW<[WLat1, FXb, NormalGr], (instregex "C(L)?RMux")>; // Compare halfword def : InstRW<[WLat2LSU, RegReadAdv, FXb, LSU, NormalGr], (instregex "CH(Y)?$")>; Index: lib/Target/SystemZ/SystemZScheduleZ14.td =================================================================== --- lib/Target/SystemZ/SystemZScheduleZ14.td +++ lib/Target/SystemZ/SystemZScheduleZ14.td @@ -367,6 +367,7 @@ def : InstRW<[WLat1, FXa, NormalGr], (instregex "AR(K)?$")>; def : InstRW<[WLat1, FXa, NormalGr], (instregex "A(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXa, NormalGr], (instregex "A(L)?HHLR$")>; +def : InstRW<[WLat1, FXa, NormalGr], (instregex "A(L)?RMux(K)?$")>; def : InstRW<[WLat1, FXa, NormalGr], (instregex "ALSIH(N)?$")>; def : InstRW<[WLat2LSU, FXb, LSU, NormalGr], (instregex "A(L)?(G)?SI$")>; @@ -398,6 +399,7 @@ def : InstRW<[WLat1, FXa, NormalGr], (instregex "SR(K)?$")>; def : InstRW<[WLat1, FXa, NormalGr], (instregex "S(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXa, NormalGr], (instregex "S(L)?HHLR$")>; +def : InstRW<[WLat1, FXa, NormalGr], (instregex "S(L)?RMux(K)?$")>; // Subtraction with borrow def : InstRW<[WLat2LSU, WLat2LSU, RegReadAdv, FXa, LSU, GroupAlone], @@ -557,6 +559,7 @@ def : InstRW<[WLat1LSU, FXb, LSU, NormalGr], (instregex "CLRL$")>; def : InstRW<[WLat1, FXb, NormalGr], (instregex "C(L)?HHR$")>; def : InstRW<[WLat2, FXb, NormalGr], (instregex "C(L)?HLR$")>; +def : InstRW<[WLat1, FXb, NormalGr], (instregex "C(L)?RMux")>; // Compare halfword def : InstRW<[WLat2LSU, RegReadAdv, FXb, LSU, NormalGr], (instregex "CH(Y)?$")>; Index: lib/Target/SystemZ/SystemZScheduleZ196.td =================================================================== --- lib/Target/SystemZ/SystemZScheduleZ196.td +++ lib/Target/SystemZ/SystemZScheduleZ196.td @@ -326,6 +326,7 @@ def : InstRW<[WLat1, FXU, NormalGr], (instregex "AR(K)?$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "A(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXU2, GroupAlone], (instregex "A(L)?HHLR$")>; +def : InstRW<[WLat1, FXU, NormalGr], (instregex "A(L)?RMux(K)?$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "ALSIH(N)?$")>; def : InstRW<[WLat1LSU, WLat1LSU, RegReadAdv, FXU, LSU, NormalGr], (instregex "A(L)?G$")>; @@ -359,6 +360,7 @@ def : InstRW<[WLat1, FXU, NormalGr], (instregex "SR(K)?$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "S(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXU2, GroupAlone], (instregex "S(L)?HHLR$")>; +def : InstRW<[WLat1, FXU, NormalGr], (instregex "S(L)?RMux(K)?$")>; // Subtraction with borrow def : InstRW<[WLat2LSU, WLat2LSU, RegReadAdv, FXU, LSU, GroupAlone], @@ -508,6 +510,7 @@ def : InstRW<[WLat1LSU, FXU, LSU, NormalGr], (instregex "CLRL$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "C(L)?HHR$")>; def : InstRW<[WLat2, FXU2, GroupAlone], (instregex "C(L)?HLR$")>; +def : InstRW<[WLat1, FXU, NormalGr], (instregex "C(L)?RMux")>; // Compare halfword def : InstRW<[WLat2LSU, RegReadAdv, FXU2, LSU, GroupAlone], Index: lib/Target/SystemZ/SystemZScheduleZEC12.td =================================================================== --- lib/Target/SystemZ/SystemZScheduleZEC12.td +++ lib/Target/SystemZ/SystemZScheduleZEC12.td @@ -337,6 +337,7 @@ def : InstRW<[WLat1, FXU, NormalGr], (instregex "AR(K)?$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "A(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXU, NormalGr], (instregex "A(L)?HHLR$")>; +def : InstRW<[WLat1, FXU, NormalGr], (instregex "A(L)?RMux(K)?$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "ALSIH(N)?$")>; def : InstRW<[WLat1LSU, WLat1LSU, RegReadAdv, FXU, LSU, NormalGr], (instregex "A(L)?G$")>; @@ -370,6 +371,7 @@ def : InstRW<[WLat1, FXU, NormalGr], (instregex "SR(K)?$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "S(L)?HHHR$")>; def : InstRW<[WLat2, WLat2, FXU, NormalGr], (instregex "S(L)?HHLR$")>; +def : InstRW<[WLat1, FXU, NormalGr], (instregex "S(L)?RMux(K)?$")>; // Subtraction with borrow def : InstRW<[WLat2LSU, WLat2LSU, RegReadAdv, FXU, LSU, GroupAlone], @@ -519,6 +521,7 @@ def : InstRW<[WLat1LSU, FXU, LSU, NormalGr], (instregex "CLRL$")>; def : InstRW<[WLat1, FXU, NormalGr], (instregex "C(L)?HHR$")>; def : InstRW<[WLat2, FXU, NormalGr], (instregex "C(L)?HLR$")>; +def : InstRW<[WLat1, FXU, NormalGr], (instregex "C(L)?RMux")>; // Compare halfword def : InstRW<[WLat2LSU, RegReadAdv, FXU, LSU, NormalGr], (instregex "CH(Y)?$")>;