diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index 693a41372a68..e8735b31628c 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -1,5036 +1,5000 @@ //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains the PowerPC implementation of the TargetInstrInfo class. // //===----------------------------------------------------------------------===// #include "PPCInstrInfo.h" #include "MCTargetDesc/PPCPredicates.h" #include "PPC.h" #include "PPCHazardRecognizers.h" #include "PPCInstrBuilder.h" #include "PPCMachineFunctionInfo.h" #include "PPCTargetMachine.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/SlotIndexes.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCInst.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; #define DEBUG_TYPE "ppc-instr-info" #define GET_INSTRMAP_INFO #define GET_INSTRINFO_CTOR_DTOR #include "PPCGenInstrInfo.inc" STATISTIC(NumStoreSPILLVSRRCAsVec, "Number of spillvsrrc spilled to stack as vec"); STATISTIC(NumStoreSPILLVSRRCAsGpr, "Number of spillvsrrc spilled to stack as gpr"); STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc"); STATISTIC(CmpIselsConverted, "Number of ISELs that depend on comparison of constants converted"); STATISTIC(MissedConvertibleImmediateInstrs, "Number of compare-immediate instructions fed by constants"); STATISTIC(NumRcRotatesConvertedToRcAnd, "Number of record-form rotates converted to record-form andi"); static cl:: opt DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden, cl::desc("Disable analysis for CTR loops")); static cl::opt DisableCmpOpt("disable-ppc-cmp-opt", cl::desc("Disable compare instruction optimization"), cl::Hidden); static cl::opt VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy", cl::desc("Causes the backend to crash instead of generating a nop VSX copy"), cl::Hidden); static cl::opt UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden, cl::desc("Use the old (incorrect) instruction latency calculation")); // Pin the vtable to this file. void PPCInstrInfo::anchor() {} PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI) : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP, /* CatchRetOpcode */ -1, STI.isPPC64() ? PPC::BLR8 : PPC::BLR), Subtarget(STI), RI(STI.getTargetMachine()) {} /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for /// this target when scheduling the DAG. ScheduleHazardRecognizer * PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const { unsigned Directive = static_cast(STI)->getCPUDirective(); if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 || Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) { const InstrItineraryData *II = static_cast(STI)->getInstrItineraryData(); return new ScoreboardHazardRecognizer(II, DAG); } return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG); } /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer /// to use for this target when scheduling the DAG. ScheduleHazardRecognizer * PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const { unsigned Directive = DAG->MF.getSubtarget().getCPUDirective(); // FIXME: Leaving this as-is until we have POWER9 scheduling info if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8) return new PPCDispatchGroupSBHazardRecognizer(II, DAG); // Most subtargets use a PPC970 recognizer. if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 && Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) { assert(DAG->TII && "No InstrInfo?"); return new PPCHazardRecognizer970(*DAG); } return new ScoreboardHazardRecognizer(II, DAG); } unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr &MI, unsigned *PredCost) const { if (!ItinData || UseOldLatencyCalc) return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost); // The default implementation of getInstrLatency calls getStageLatency, but // getStageLatency does not do the right thing for us. While we have // itinerary, most cores are fully pipelined, and so the itineraries only // express the first part of the pipeline, not every stage. Instead, we need // to use the listed output operand cycle number (using operand 0 here, which // is an output). unsigned Latency = 1; unsigned DefClass = MI.getDesc().getSchedClass(); for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI.getOperand(i); if (!MO.isReg() || !MO.isDef() || MO.isImplicit()) continue; int Cycle = ItinData->getOperandCycle(DefClass, i); if (Cycle < 0) continue; Latency = std::max(Latency, (unsigned) Cycle); } return Latency; } int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, const MachineInstr &DefMI, unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const { int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx, UseMI, UseIdx); if (!DefMI.getParent()) return Latency; const MachineOperand &DefMO = DefMI.getOperand(DefIdx); Register Reg = DefMO.getReg(); bool IsRegCR; if (Register::isVirtualRegister(Reg)) { const MachineRegisterInfo *MRI = &DefMI.getParent()->getParent()->getRegInfo(); IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) || MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass); } else { IsRegCR = PPC::CRRCRegClass.contains(Reg) || PPC::CRBITRCRegClass.contains(Reg); } if (UseMI.isBranch() && IsRegCR) { if (Latency < 0) Latency = getInstrLatency(ItinData, DefMI); // On some cores, there is an additional delay between writing to a condition // register, and using it from a branch. unsigned Directive = Subtarget.getCPUDirective(); switch (Directive) { default: break; case PPC::DIR_7400: case PPC::DIR_750: case PPC::DIR_970: case PPC::DIR_E5500: case PPC::DIR_PWR4: case PPC::DIR_PWR5: case PPC::DIR_PWR5X: case PPC::DIR_PWR6: case PPC::DIR_PWR6X: case PPC::DIR_PWR7: case PPC::DIR_PWR8: // FIXME: Is this needed for POWER9? Latency += 2; break; } } return Latency; } /// This is an architecture-specific helper function of reassociateOps. /// Set special operand attributes for new instructions after reassociation. void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, MachineInstr &NewMI1, MachineInstr &NewMI2) const { // Propagate FP flags from the original instructions. // But clear poison-generating flags because those may not be valid now. uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags(); NewMI1.setFlags(IntersectedFlags); NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap); NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap); NewMI1.clearFlag(MachineInstr::MIFlag::IsExact); NewMI2.setFlags(IntersectedFlags); NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap); NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap); NewMI2.clearFlag(MachineInstr::MIFlag::IsExact); } void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const { MI.setFlags(Flags); MI.clearFlag(MachineInstr::MIFlag::NoSWrap); MI.clearFlag(MachineInstr::MIFlag::NoUWrap); MI.clearFlag(MachineInstr::MIFlag::IsExact); } // This function does not list all associative and commutative operations, but // only those worth feeding through the machine combiner in an attempt to // reduce the critical path. Mostly, this means floating-point operations, // because they have high latencies(>=5) (compared to other operations, such as // and/or, which are also associative and commutative, but have low latencies). bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const { switch (Inst.getOpcode()) { // Floating point: // FP Add: case PPC::FADD: case PPC::FADDS: // FP Multiply: case PPC::FMUL: case PPC::FMULS: // Altivec Add: case PPC::VADDFP: // VSX Add: case PPC::XSADDDP: case PPC::XVADDDP: case PPC::XVADDSP: case PPC::XSADDSP: // VSX Multiply: case PPC::XSMULDP: case PPC::XVMULDP: case PPC::XVMULSP: case PPC::XSMULSP: return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) && Inst.getFlag(MachineInstr::MIFlag::FmNsz); // Fixed point: // Multiply: case PPC::MULHD: case PPC::MULLD: case PPC::MULHW: case PPC::MULLW: return true; default: return false; } } #define InfoArrayIdxFMAInst 0 #define InfoArrayIdxFAddInst 1 #define InfoArrayIdxFMULInst 2 #define InfoArrayIdxAddOpIdx 3 #define InfoArrayIdxMULOpIdx 4 // Array keeps info for FMA instructions: // Index 0(InfoArrayIdxFMAInst): FMA instruction; // Index 1(InfoArrayIdxFAddInst): ADD instruction assoaicted with FMA; // Index 2(InfoArrayIdxFMULInst): MUL instruction assoaicted with FMA; // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands; // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands; // second MUL operand index is plus 1. static const uint16_t FMAOpIdxInfo[][5] = { // FIXME: Add more FMA instructions like XSNMADDADP and so on. {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2}, {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2}, {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2}, {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2}, {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1}, {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1}}; // Check if an opcode is a FMA instruction. If it is, return the index in array // FMAOpIdxInfo. Otherwise, return -1. int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const { for (unsigned I = 0; I < array_lengthof(FMAOpIdxInfo); I++) if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode) return I; return -1; } // Try to reassociate FMA chains like below: // // Pattern 1: // A = FADD X, Y (Leaf) // B = FMA A, M21, M22 (Prev) // C = FMA B, M31, M32 (Root) // --> // A = FMA X, M21, M22 // B = FMA Y, M31, M32 // C = FADD A, B // // Pattern 2: // A = FMA X, M11, M12 (Leaf) // B = FMA A, M21, M22 (Prev) // C = FMA B, M31, M32 (Root) // --> // A = FMUL M11, M12 // B = FMA X, M21, M22 // D = FMA A, M31, M32 // C = FADD B, D // // breaking the dependency between A and B, allowing FMA to be executed in // parallel (or back-to-back in a pipeline) instead of depending on each other. bool PPCInstrInfo::getFMAPatterns( MachineInstr &Root, SmallVectorImpl &Patterns) const { MachineBasicBlock *MBB = Root.getParent(); const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) { for (const auto &MO : Instr.explicit_operands()) if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg()))) return false; return true; }; auto IsReassociable = [&](const MachineInstr &Instr, int16_t &AddOpIdx, bool IsLeaf, bool IsAdd) { int16_t Idx = -1; if (!IsAdd) { Idx = getFMAOpIdxInfo(Instr.getOpcode()); if (Idx < 0) return false; } else if (Instr.getOpcode() != FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())] [InfoArrayIdxFAddInst]) return false; // Instruction can be reassociated. // fast math flags may prohibit reassociation. if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) && Instr.getFlag(MachineInstr::MIFlag::FmNsz))) return false; // Instruction operands are virtual registers for reassociation. if (!IsAllOpsVirtualReg(Instr)) return false; if (IsAdd && IsLeaf) return true; AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx]; const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx); MachineInstr *MIAdd = MRI.getUniqueVRegDef(OpAdd.getReg()); // If 'add' operand's def is not in current block, don't do ILP related opt. if (!MIAdd || MIAdd->getParent() != MBB) return false; // If this is not Leaf FMA Instr, its 'add' operand should only have one use // as this fma will be changed later. return IsLeaf ? true : MRI.hasOneNonDBGUse(OpAdd.getReg()); }; int16_t AddOpIdx = -1; // Root must be a valid FMA like instruction. if (!IsReassociable(Root, AddOpIdx, false, false)) return false; assert((AddOpIdx >= 0) && "add operand index not right!"); Register RegB = Root.getOperand(AddOpIdx).getReg(); MachineInstr *Prev = MRI.getUniqueVRegDef(RegB); // Prev must be a valid FMA like instruction. AddOpIdx = -1; if (!IsReassociable(*Prev, AddOpIdx, false, false)) return false; assert((AddOpIdx >= 0) && "add operand index not right!"); Register RegA = Prev->getOperand(AddOpIdx).getReg(); MachineInstr *Leaf = MRI.getUniqueVRegDef(RegA); AddOpIdx = -1; if (IsReassociable(*Leaf, AddOpIdx, true, false)) { Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM); return true; } if (IsReassociable(*Leaf, AddOpIdx, true, true)) { Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM); return true; } return false; } bool PPCInstrInfo::getMachineCombinerPatterns( MachineInstr &Root, SmallVectorImpl &Patterns) const { // Using the machine combiner in this way is potentially expensive, so // restrict to when aggressive optimizations are desired. if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive) return false; if (getFMAPatterns(Root, Patterns)) return true; return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns); } void PPCInstrInfo::genAlternativeCodeSequence( MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl &InsInstrs, SmallVectorImpl &DelInstrs, DenseMap &InstrIdxForVirtReg) const { switch (Pattern) { case MachineCombinerPattern::REASSOC_XY_AMM_BMM: case MachineCombinerPattern::REASSOC_XMM_AMM_BMM: reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg); break; default: // Reassociate default patterns. TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg); break; } } // Currently, only handle two patterns REASSOC_XY_AMM_BMM and // REASSOC_XMM_AMM_BMM. See comments for getFMAPatterns. void PPCInstrInfo::reassociateFMA( MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl &InsInstrs, SmallVectorImpl &DelInstrs, DenseMap &InstrIdxForVirtReg) const { MachineFunction *MF = Root.getMF(); MachineRegisterInfo &MRI = MF->getRegInfo(); MachineOperand &OpC = Root.getOperand(0); Register RegC = OpC.getReg(); const TargetRegisterClass *RC = MRI.getRegClass(RegC); MRI.constrainRegClass(RegC, RC); unsigned FmaOp = Root.getOpcode(); int16_t Idx = getFMAOpIdxInfo(FmaOp); assert(Idx >= 0 && "Root must be a FMA instruction"); uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx]; uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx]; MachineInstr *Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg()); MachineInstr *Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg()); uint16_t IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags(); auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg, bool &KillFlag) { Reg = Operand.getReg(); MRI.constrainRegClass(Reg, RC); KillFlag = Operand.isKill(); }; auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1, Register &MulOp2, bool &MulOp1KillFlag, bool &MulOp2KillFlag) { GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag); GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag); }; Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32; bool KillX = false, KillY = false, KillM11 = false, KillM12 = false, KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false; GetFMAInstrInfo(Root, RegM31, RegM32, KillM31, KillM32); GetFMAInstrInfo(*Prev, RegM21, RegM22, KillM21, KillM22); if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) { GetFMAInstrInfo(*Leaf, RegM11, RegM12, KillM11, KillM12); GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX); } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) { GetOperandInfo(Leaf->getOperand(1), RegX, KillX); GetOperandInfo(Leaf->getOperand(2), RegY, KillY); } // Create new virtual registers for the new results instead of // recycling legacy ones because the MachineCombiner's computation of the // critical path requires a new register definition rather than an existing // one. Register NewVRA = MRI.createVirtualRegister(RC); InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0)); Register NewVRB = MRI.createVirtualRegister(RC); InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1)); Register NewVRD = 0; if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) { NewVRD = MRI.createVirtualRegister(RC); InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2)); } auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd, Register RegMul1, bool KillRegMul1, Register RegMul2, bool KillRegMul2) { MI->getOperand(AddOpIdx).setReg(RegAdd); MI->getOperand(AddOpIdx).setIsKill(KillAdd); MI->getOperand(FirstMulOpIdx).setReg(RegMul1); MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1); MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2); MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2); }; if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) { // Create new instructions for insertion. MachineInstrBuilder MINewB = BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB) .addReg(RegX, getKillRegState(KillX)) .addReg(RegM21, getKillRegState(KillM21)) .addReg(RegM22, getKillRegState(KillM22)); MachineInstrBuilder MINewA = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA) .addReg(RegY, getKillRegState(KillY)) .addReg(RegM31, getKillRegState(KillM31)) .addReg(RegM32, getKillRegState(KillM32)); // If AddOpIdx is not 1, adjust the order. if (AddOpIdx != 1) { AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22); AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32); } MachineInstrBuilder MINewC = BuildMI(*MF, Root.getDebugLoc(), get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC) .addReg(NewVRB, getKillRegState(true)) .addReg(NewVRA, getKillRegState(true)); // Update flags for newly created instructions. setSpecialOperandAttr(*MINewA, IntersectedFlags); setSpecialOperandAttr(*MINewB, IntersectedFlags); setSpecialOperandAttr(*MINewC, IntersectedFlags); // Record new instructions for insertion. InsInstrs.push_back(MINewA); InsInstrs.push_back(MINewB); InsInstrs.push_back(MINewC); } else if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) { assert(NewVRD && "new FMA register not created!"); // Create new instructions for insertion. MachineInstrBuilder MINewA = BuildMI(*MF, Leaf->getDebugLoc(), get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA) .addReg(RegM11, getKillRegState(KillM11)) .addReg(RegM12, getKillRegState(KillM12)); MachineInstrBuilder MINewB = BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB) .addReg(RegX, getKillRegState(KillX)) .addReg(RegM21, getKillRegState(KillM21)) .addReg(RegM22, getKillRegState(KillM22)); MachineInstrBuilder MINewD = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD) .addReg(NewVRA, getKillRegState(true)) .addReg(RegM31, getKillRegState(KillM31)) .addReg(RegM32, getKillRegState(KillM32)); // If AddOpIdx is not 1, adjust the order. if (AddOpIdx != 1) { AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22); AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32, KillM32); } MachineInstrBuilder MINewC = BuildMI(*MF, Root.getDebugLoc(), get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC) .addReg(NewVRB, getKillRegState(true)) .addReg(NewVRD, getKillRegState(true)); // Update flags for newly created instructions. setSpecialOperandAttr(*MINewA, IntersectedFlags); setSpecialOperandAttr(*MINewB, IntersectedFlags); setSpecialOperandAttr(*MINewD, IntersectedFlags); setSpecialOperandAttr(*MINewC, IntersectedFlags); // Record new instructions for insertion. InsInstrs.push_back(MINewA); InsInstrs.push_back(MINewB); InsInstrs.push_back(MINewD); InsInstrs.push_back(MINewC); } assert(!InsInstrs.empty() && "Insertion instructions set should not be empty!"); // Record old instructions for deletion. DelInstrs.push_back(Leaf); DelInstrs.push_back(Prev); DelInstrs.push_back(&Root); } // Detect 32 -> 64-bit extensions where we may reuse the low sub-register. bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const { switch (MI.getOpcode()) { default: return false; case PPC::EXTSW: case PPC::EXTSW_32: case PPC::EXTSW_32_64: SrcReg = MI.getOperand(1).getReg(); DstReg = MI.getOperand(0).getReg(); SubIdx = PPC::sub_32; return true; } } unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const { unsigned Opcode = MI.getOpcode(); const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray(); const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill; if (End != std::find(OpcodesForSpill, End, Opcode)) { // Check for the operands added by addFrameReference (the immediate is the // offset which defaults to 0). if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() && MI.getOperand(2).isFI()) { FrameIndex = MI.getOperand(2).getIndex(); return MI.getOperand(0).getReg(); } } return 0; } // For opcodes with the ReMaterializable flag set, this function is called to // verify the instruction is really rematable. bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, AliasAnalysis *AA) const { switch (MI.getOpcode()) { default: // This function should only be called for opcodes with the ReMaterializable // flag set. llvm_unreachable("Unknown rematerializable operation!"); break; case PPC::LI: case PPC::LI8: case PPC::LIS: case PPC::LIS8: case PPC::ADDIStocHA: case PPC::ADDIStocHA8: case PPC::ADDItocL: case PPC::LOAD_STACK_GUARD: case PPC::XXLXORz: case PPC::XXLXORspz: case PPC::XXLXORdpz: case PPC::XXLEQVOnes: case PPC::V_SET0B: case PPC::V_SET0H: case PPC::V_SET0: case PPC::V_SETALLONESB: case PPC::V_SETALLONESH: case PPC::V_SETALLONES: case PPC::CRSET: case PPC::CRUNSET: case PPC::XXSETACCZ: return true; } return false; } unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const { unsigned Opcode = MI.getOpcode(); const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray(); const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill; if (End != std::find(OpcodesForSpill, End, Opcode)) { if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() && MI.getOperand(2).isFI()) { FrameIndex = MI.getOperand(2).getIndex(); return MI.getOperand(0).getReg(); } } return 0; } MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const { MachineFunction &MF = *MI.getParent()->getParent(); // Normal instructions can be commuted the obvious way. if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec) return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because // changing the relative order of the mask operands might change what happens // to the high-bits of the mask (and, thus, the result). // Cannot commute if it has a non-zero rotate count. if (MI.getOperand(3).getImm() != 0) return nullptr; // If we have a zero rotate count, we have: // M = mask(MB,ME) // Op0 = (Op1 & ~M) | (Op2 & M) // Change this to: // M = mask((ME+1)&31, (MB-1)&31) // Op0 = (Op2 & ~M) | (Op1 & M) // Swap op1/op2 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) && "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec."); Register Reg0 = MI.getOperand(0).getReg(); Register Reg1 = MI.getOperand(1).getReg(); Register Reg2 = MI.getOperand(2).getReg(); unsigned SubReg1 = MI.getOperand(1).getSubReg(); unsigned SubReg2 = MI.getOperand(2).getSubReg(); bool Reg1IsKill = MI.getOperand(1).isKill(); bool Reg2IsKill = MI.getOperand(2).isKill(); bool ChangeReg0 = false; // If machine instrs are no longer in two-address forms, update // destination register as well. if (Reg0 == Reg1) { // Must be two address instruction! assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) && "Expecting a two-address instruction!"); assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch"); Reg2IsKill = false; ChangeReg0 = true; } // Masks. unsigned MB = MI.getOperand(4).getImm(); unsigned ME = MI.getOperand(5).getImm(); // We can't commute a trivial mask (there is no way to represent an all-zero // mask). if (MB == 0 && ME == 31) return nullptr; if (NewMI) { // Create a new instruction. Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg(); bool Reg0IsDead = MI.getOperand(0).isDead(); return BuildMI(MF, MI.getDebugLoc(), MI.getDesc()) .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead)) .addReg(Reg2, getKillRegState(Reg2IsKill)) .addReg(Reg1, getKillRegState(Reg1IsKill)) .addImm((ME + 1) & 31) .addImm((MB - 1) & 31); } if (ChangeReg0) { MI.getOperand(0).setReg(Reg2); MI.getOperand(0).setSubReg(SubReg2); } MI.getOperand(2).setReg(Reg1); MI.getOperand(1).setReg(Reg2); MI.getOperand(2).setSubReg(SubReg1); MI.getOperand(1).setSubReg(SubReg2); MI.getOperand(2).setIsKill(Reg1IsKill); MI.getOperand(1).setIsKill(Reg2IsKill); // Swap the mask around. MI.getOperand(4).setImm((ME + 1) & 31); MI.getOperand(5).setImm((MB - 1) & 31); return &MI; } bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const { // For VSX A-Type FMA instructions, it is the first two operands that can be // commuted, however, because the non-encoded tied input operand is listed // first, the operands to swap are actually the second and third. int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode()); if (AltOpc == -1) return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1 // and SrcOpIdx2. return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3); } void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const { // This function is used for scheduling, and the nop wanted here is the type // that terminates dispatch groups on the POWER cores. unsigned Directive = Subtarget.getCPUDirective(); unsigned Opcode; switch (Directive) { default: Opcode = PPC::NOP; break; case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break; case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break; case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */ // FIXME: Update when POWER9 scheduling model is ready. case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break; } DebugLoc DL; BuildMI(MBB, MI, DL, get(Opcode)); } /// Return the noop instruction to use for a noop. void PPCInstrInfo::getNoop(MCInst &NopInst) const { NopInst.setOpcode(PPC::NOP); } // Branch analysis. // Note: If the condition register is set to CTR or CTR8 then this is a // BDNZ (imm == 1) or BDZ (imm == 0) branch. bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl &Cond, bool AllowModify) const { bool isPPC64 = Subtarget.isPPC64(); // If the block has no terminators, it just falls into the block after it. MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); if (I == MBB.end()) return false; if (!isUnpredicatedTerminator(*I)) return false; if (AllowModify) { // If the BB ends with an unconditional branch to the fallthrough BB, // we eliminate the branch instruction. if (I->getOpcode() == PPC::B && MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { I->eraseFromParent(); // We update iterator after deleting the last branch. I = MBB.getLastNonDebugInstr(); if (I == MBB.end() || !isUnpredicatedTerminator(*I)) return false; } } // Get the last instruction in the block. MachineInstr &LastInst = *I; // If there is only one terminator instruction, process it. if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) { if (LastInst.getOpcode() == PPC::B) { if (!LastInst.getOperand(0).isMBB()) return true; TBB = LastInst.getOperand(0).getMBB(); return false; } else if (LastInst.getOpcode() == PPC::BCC) { if (!LastInst.getOperand(2).isMBB()) return true; // Block ends with fall-through condbranch. TBB = LastInst.getOperand(2).getMBB(); Cond.push_back(LastInst.getOperand(0)); Cond.push_back(LastInst.getOperand(1)); return false; } else if (LastInst.getOpcode() == PPC::BC) { if (!LastInst.getOperand(1).isMBB()) return true; // Block ends with fall-through condbranch. TBB = LastInst.getOperand(1).getMBB(); Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); Cond.push_back(LastInst.getOperand(0)); return false; } else if (LastInst.getOpcode() == PPC::BCn) { if (!LastInst.getOperand(1).isMBB()) return true; // Block ends with fall-through condbranch. TBB = LastInst.getOperand(1).getMBB(); Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET)); Cond.push_back(LastInst.getOperand(0)); return false; } else if (LastInst.getOpcode() == PPC::BDNZ8 || LastInst.getOpcode() == PPC::BDNZ) { if (!LastInst.getOperand(0).isMBB()) return true; if (DisableCTRLoopAnal) return true; TBB = LastInst.getOperand(0).getMBB(); Cond.push_back(MachineOperand::CreateImm(1)); Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, true)); return false; } else if (LastInst.getOpcode() == PPC::BDZ8 || LastInst.getOpcode() == PPC::BDZ) { if (!LastInst.getOperand(0).isMBB()) return true; if (DisableCTRLoopAnal) return true; TBB = LastInst.getOperand(0).getMBB(); Cond.push_back(MachineOperand::CreateImm(0)); Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, true)); return false; } // Otherwise, don't know what this is. return true; } // Get the instruction before it if it's a terminator. MachineInstr &SecondLastInst = *I; // If there are three terminators, we don't know what sort of block this is. if (I != MBB.begin() && isUnpredicatedTerminator(*--I)) return true; // If the block ends with PPC::B and PPC:BCC, handle it. if (SecondLastInst.getOpcode() == PPC::BCC && LastInst.getOpcode() == PPC::B) { if (!SecondLastInst.getOperand(2).isMBB() || !LastInst.getOperand(0).isMBB()) return true; TBB = SecondLastInst.getOperand(2).getMBB(); Cond.push_back(SecondLastInst.getOperand(0)); Cond.push_back(SecondLastInst.getOperand(1)); FBB = LastInst.getOperand(0).getMBB(); return false; } else if (SecondLastInst.getOpcode() == PPC::BC && LastInst.getOpcode() == PPC::B) { if (!SecondLastInst.getOperand(1).isMBB() || !LastInst.getOperand(0).isMBB()) return true; TBB = SecondLastInst.getOperand(1).getMBB(); Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); Cond.push_back(SecondLastInst.getOperand(0)); FBB = LastInst.getOperand(0).getMBB(); return false; } else if (SecondLastInst.getOpcode() == PPC::BCn && LastInst.getOpcode() == PPC::B) { if (!SecondLastInst.getOperand(1).isMBB() || !LastInst.getOperand(0).isMBB()) return true; TBB = SecondLastInst.getOperand(1).getMBB(); Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET)); Cond.push_back(SecondLastInst.getOperand(0)); FBB = LastInst.getOperand(0).getMBB(); return false; } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 || SecondLastInst.getOpcode() == PPC::BDNZ) && LastInst.getOpcode() == PPC::B) { if (!SecondLastInst.getOperand(0).isMBB() || !LastInst.getOperand(0).isMBB()) return true; if (DisableCTRLoopAnal) return true; TBB = SecondLastInst.getOperand(0).getMBB(); Cond.push_back(MachineOperand::CreateImm(1)); Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, true)); FBB = LastInst.getOperand(0).getMBB(); return false; } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 || SecondLastInst.getOpcode() == PPC::BDZ) && LastInst.getOpcode() == PPC::B) { if (!SecondLastInst.getOperand(0).isMBB() || !LastInst.getOperand(0).isMBB()) return true; if (DisableCTRLoopAnal) return true; TBB = SecondLastInst.getOperand(0).getMBB(); Cond.push_back(MachineOperand::CreateImm(0)); Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, true)); FBB = LastInst.getOperand(0).getMBB(); return false; } // If the block ends with two PPC:Bs, handle it. The second one is not // executed, so remove it. if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) { if (!SecondLastInst.getOperand(0).isMBB()) return true; TBB = SecondLastInst.getOperand(0).getMBB(); I = LastInst; if (AllowModify) I->eraseFromParent(); return false; } // Otherwise, can't handle this. return true; } unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB, int *BytesRemoved) const { assert(!BytesRemoved && "code size not handled"); MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); if (I == MBB.end()) return 0; if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC && I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn && I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ && I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ) return 0; // Remove the branch. I->eraseFromParent(); I = MBB.end(); if (I == MBB.begin()) return 1; --I; if (I->getOpcode() != PPC::BCC && I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn && I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ && I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ) return 1; // Remove the branch. I->eraseFromParent(); return 2; } unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef Cond, const DebugLoc &DL, int *BytesAdded) const { // Shouldn't be a fall through. assert(TBB && "insertBranch must not be told to insert a fallthrough"); assert((Cond.size() == 2 || Cond.size() == 0) && "PPC branch conditions have two components!"); assert(!BytesAdded && "code size not handled"); bool isPPC64 = Subtarget.isPPC64(); // One-way branch. if (!FBB) { if (Cond.empty()) // Unconditional branch BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB); else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) BuildMI(&MBB, DL, get(Cond[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB); else if (Cond[0].getImm() == PPC::PRED_BIT_SET) BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB); else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET) BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB); else // Conditional branch BuildMI(&MBB, DL, get(PPC::BCC)) .addImm(Cond[0].getImm()) .add(Cond[1]) .addMBB(TBB); return 1; } // Two-way Conditional Branch. if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) BuildMI(&MBB, DL, get(Cond[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB); else if (Cond[0].getImm() == PPC::PRED_BIT_SET) BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB); else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET) BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB); else BuildMI(&MBB, DL, get(PPC::BCC)) .addImm(Cond[0].getImm()) .add(Cond[1]) .addMBB(TBB); BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB); return 2; } // Select analysis. bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, ArrayRef Cond, Register DstReg, Register TrueReg, Register FalseReg, int &CondCycles, int &TrueCycles, int &FalseCycles) const { if (Cond.size() != 2) return false; // If this is really a bdnz-like condition, then it cannot be turned into a // select. if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) return false; // Check register classes. const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); const TargetRegisterClass *RC = RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); if (!RC) return false; // isel is for regular integer GPRs only. if (!PPC::GPRCRegClass.hasSubClassEq(RC) && !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) && !PPC::G8RCRegClass.hasSubClassEq(RC) && !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) return false; // FIXME: These numbers are for the A2, how well they work for other cores is // an open question. On the A2, the isel instruction has a 2-cycle latency // but single-cycle throughput. These numbers are used in combination with // the MispredictPenalty setting from the active SchedMachineModel. CondCycles = 1; TrueCycles = 1; FalseCycles = 1; return true; } void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &dl, Register DestReg, ArrayRef Cond, Register TrueReg, Register FalseReg) const { assert(Cond.size() == 2 && "PPC branch conditions have two components!"); // Get the register classes. MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); const TargetRegisterClass *RC = RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); assert(RC && "TrueReg and FalseReg must have overlapping register classes"); bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) || PPC::G8RC_NOX0RegClass.hasSubClassEq(RC); assert((Is64Bit || PPC::GPRCRegClass.hasSubClassEq(RC) || PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) && "isel is for regular integer GPRs only"); unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL; auto SelectPred = static_cast(Cond[0].getImm()); unsigned SubIdx = 0; bool SwapOps = false; switch (SelectPred) { case PPC::PRED_EQ: case PPC::PRED_EQ_MINUS: case PPC::PRED_EQ_PLUS: SubIdx = PPC::sub_eq; SwapOps = false; break; case PPC::PRED_NE: case PPC::PRED_NE_MINUS: case PPC::PRED_NE_PLUS: SubIdx = PPC::sub_eq; SwapOps = true; break; case PPC::PRED_LT: case PPC::PRED_LT_MINUS: case PPC::PRED_LT_PLUS: SubIdx = PPC::sub_lt; SwapOps = false; break; case PPC::PRED_GE: case PPC::PRED_GE_MINUS: case PPC::PRED_GE_PLUS: SubIdx = PPC::sub_lt; SwapOps = true; break; case PPC::PRED_GT: case PPC::PRED_GT_MINUS: case PPC::PRED_GT_PLUS: SubIdx = PPC::sub_gt; SwapOps = false; break; case PPC::PRED_LE: case PPC::PRED_LE_MINUS: case PPC::PRED_LE_PLUS: SubIdx = PPC::sub_gt; SwapOps = true; break; case PPC::PRED_UN: case PPC::PRED_UN_MINUS: case PPC::PRED_UN_PLUS: SubIdx = PPC::sub_un; SwapOps = false; break; case PPC::PRED_NU: case PPC::PRED_NU_MINUS: case PPC::PRED_NU_PLUS: SubIdx = PPC::sub_un; SwapOps = true; break; case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break; case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break; } Register FirstReg = SwapOps ? FalseReg : TrueReg, SecondReg = SwapOps ? TrueReg : FalseReg; // The first input register of isel cannot be r0. If it is a member // of a register class that can be r0, then copy it first (the // register allocator should eliminate the copy). if (MRI.getRegClass(FirstReg)->contains(PPC::R0) || MRI.getRegClass(FirstReg)->contains(PPC::X0)) { const TargetRegisterClass *FirstRC = MRI.getRegClass(FirstReg)->contains(PPC::X0) ? &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass; Register OldFirstReg = FirstReg; FirstReg = MRI.createVirtualRegister(FirstRC); BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg) .addReg(OldFirstReg); } BuildMI(MBB, MI, dl, get(OpCode), DestReg) .addReg(FirstReg).addReg(SecondReg) .addReg(Cond[1].getReg(), 0, SubIdx); } static unsigned getCRBitValue(unsigned CRBit) { unsigned Ret = 4; if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT || CRBit == PPC::CR2LT || CRBit == PPC::CR3LT || CRBit == PPC::CR4LT || CRBit == PPC::CR5LT || CRBit == PPC::CR6LT || CRBit == PPC::CR7LT) Ret = 3; if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT || CRBit == PPC::CR2GT || CRBit == PPC::CR3GT || CRBit == PPC::CR4GT || CRBit == PPC::CR5GT || CRBit == PPC::CR6GT || CRBit == PPC::CR7GT) Ret = 2; if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ || CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ || CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ || CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ) Ret = 1; if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN || CRBit == PPC::CR2UN || CRBit == PPC::CR3UN || CRBit == PPC::CR4UN || CRBit == PPC::CR5UN || CRBit == PPC::CR6UN || CRBit == PPC::CR7UN) Ret = 0; assert(Ret != 4 && "Invalid CR bit register"); return Ret; } void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const { // We can end up with self copies and similar things as a result of VSX copy // legalization. Promote them here. const TargetRegisterInfo *TRI = &getRegisterInfo(); if (PPC::F8RCRegClass.contains(DestReg) && PPC::VSRCRegClass.contains(SrcReg)) { MCRegister SuperReg = TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass); if (VSXSelfCopyCrash && SrcReg == SuperReg) llvm_unreachable("nop VSX copy"); DestReg = SuperReg; } else if (PPC::F8RCRegClass.contains(SrcReg) && PPC::VSRCRegClass.contains(DestReg)) { MCRegister SuperReg = TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass); if (VSXSelfCopyCrash && DestReg == SuperReg) llvm_unreachable("nop VSX copy"); SrcReg = SuperReg; } // Different class register copy if (PPC::CRBITRCRegClass.contains(SrcReg) && PPC::GPRCRegClass.contains(DestReg)) { MCRegister CRReg = getCRFromCRBit(SrcReg); BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg); getKillRegState(KillSrc); // Rotate the CR bit in the CR fields to be the least significant bit and // then mask with 0x1 (MB = ME = 31). BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg) .addReg(DestReg, RegState::Kill) .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg))) .addImm(31) .addImm(31); return; } else if (PPC::CRRCRegClass.contains(SrcReg) && (PPC::G8RCRegClass.contains(DestReg) || PPC::GPRCRegClass.contains(DestReg))) { bool Is64Bit = PPC::G8RCRegClass.contains(DestReg); unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF; unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM; unsigned CRNum = TRI->getEncodingValue(SrcReg); BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg); getKillRegState(KillSrc); if (CRNum == 7) return; // Shift the CR bits to make the CR field in the lowest 4 bits of GRC. BuildMI(MBB, I, DL, get(ShCode), DestReg) .addReg(DestReg, RegState::Kill) .addImm(CRNum * 4 + 4) .addImm(28) .addImm(31); return; } else if (PPC::G8RCRegClass.contains(SrcReg) && PPC::VSFRCRegClass.contains(DestReg)) { assert(Subtarget.hasDirectMove() && "Subtarget doesn't support directmove, don't know how to copy."); BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg); NumGPRtoVSRSpill++; getKillRegState(KillSrc); return; } else if (PPC::VSFRCRegClass.contains(SrcReg) && PPC::G8RCRegClass.contains(DestReg)) { assert(Subtarget.hasDirectMove() && "Subtarget doesn't support directmove, don't know how to copy."); BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg); getKillRegState(KillSrc); return; } else if (PPC::SPERCRegClass.contains(SrcReg) && PPC::GPRCRegClass.contains(DestReg)) { BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg); getKillRegState(KillSrc); return; } else if (PPC::GPRCRegClass.contains(SrcReg) && PPC::SPERCRegClass.contains(DestReg)) { BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg); getKillRegState(KillSrc); return; } unsigned Opc; if (PPC::GPRCRegClass.contains(DestReg, SrcReg)) Opc = PPC::OR; else if (PPC::G8RCRegClass.contains(DestReg, SrcReg)) Opc = PPC::OR8; else if (PPC::F4RCRegClass.contains(DestReg, SrcReg)) Opc = PPC::FMR; else if (PPC::CRRCRegClass.contains(DestReg, SrcReg)) Opc = PPC::MCRF; else if (PPC::VRRCRegClass.contains(DestReg, SrcReg)) Opc = PPC::VOR; else if (PPC::VSRCRegClass.contains(DestReg, SrcReg)) // There are two different ways this can be done: // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only // issue in VSU pipeline 0. // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but // can go to either pipeline. // We'll always use xxlor here, because in practically all cases where // copies are generated, they are close enough to some use that the // lower-latency form is preferable. Opc = PPC::XXLOR; else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) || PPC::VSSRCRegClass.contains(DestReg, SrcReg)) Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf; else if (Subtarget.pairedVectorMemops() && PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) { if (SrcReg > PPC::VSRp15) SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2; else SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2; if (DestReg > PPC::VSRp15) DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2; else DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2; BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg). addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc)); BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1). addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc)); return; } else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg)) Opc = PPC::CROR; else if (PPC::SPERCRegClass.contains(DestReg, SrcReg)) Opc = PPC::EVOR; else llvm_unreachable("Impossible reg-to-reg copy"); const MCInstrDesc &MCID = get(Opc); if (MCID.getNumOperands() == 3) BuildMI(MBB, I, DL, MCID, DestReg) .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc)); else BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc)); } static unsigned getSpillIndex(const TargetRegisterClass *RC) { int OpcodeIndex = 0; if (PPC::GPRCRegClass.hasSubClassEq(RC) || PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_Int4Spill; } else if (PPC::G8RCRegClass.hasSubClassEq(RC) || PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_Int8Spill; } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_Float8Spill; } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_Float4Spill; } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_SPESpill; } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_CRSpill; } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_CRBitSpill; } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_VRVectorSpill; } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_VSXVectorSpill; } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_VectorFloat8Spill; } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_VectorFloat4Spill; } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) { OpcodeIndex = SOK_SpillToVSR; } else { llvm_unreachable("Unknown regclass!"); } return OpcodeIndex; } unsigned PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const { const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray(); return OpcodesForSpill[getSpillIndex(RC)]; } unsigned PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const { const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray(); return OpcodesForSpill[getSpillIndex(RC)]; } void PPCInstrInfo::StoreRegToStackSlot( MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx, const TargetRegisterClass *RC, SmallVectorImpl &NewMIs) const { unsigned Opcode = getStoreOpcodeForSpill(RC); DebugLoc DL; PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setHasSpills(); NewMIs.push_back(addFrameReference( BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)), FrameIdx)); if (PPC::CRRCRegClass.hasSubClassEq(RC) || PPC::CRBITRCRegClass.hasSubClassEq(RC)) FuncInfo->setSpillsCR(); if (isXFormMemOp(Opcode)) FuncInfo->setHasNonRISpills(); } void PPCInstrInfo::storeRegToStackSlotNoUpd( MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg, bool isKill, int FrameIdx, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const { MachineFunction &MF = *MBB.getParent(); SmallVector NewMIs; StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs); for (unsigned i = 0, e = NewMIs.size(); i != e; ++i) MBB.insert(MI, NewMIs[i]); const MachineFrameInfo &MFI = MF.getFrameInfo(); MachineMemOperand *MMO = MF.getMachineMemOperand( MachinePointerInfo::getFixedStack(MF, FrameIdx), MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx), MFI.getObjectAlign(FrameIdx)); NewMIs.back()->addMemOperand(MF, MMO); } void PPCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIdx, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const { // We need to avoid a situation in which the value from a VRRC register is // spilled using an Altivec instruction and reloaded into a VSRC register // using a VSX instruction. The issue with this is that the VSX // load/store instructions swap the doublewords in the vector and the Altivec // ones don't. The register classes on the spill/reload may be different if // the register is defined using an Altivec instruction and is then used by a // VSX instruction. RC = updatedRC(RC); storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI); } void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL, unsigned DestReg, int FrameIdx, const TargetRegisterClass *RC, SmallVectorImpl &NewMIs) const { unsigned Opcode = getLoadOpcodeForSpill(RC); NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg), FrameIdx)); PPCFunctionInfo *FuncInfo = MF.getInfo(); if (PPC::CRRCRegClass.hasSubClassEq(RC) || PPC::CRBITRCRegClass.hasSubClassEq(RC)) FuncInfo->setSpillsCR(); if (isXFormMemOp(Opcode)) FuncInfo->setHasNonRISpills(); } void PPCInstrInfo::loadRegFromStackSlotNoUpd( MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg, int FrameIdx, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const { MachineFunction &MF = *MBB.getParent(); SmallVector NewMIs; DebugLoc DL; if (MI != MBB.end()) DL = MI->getDebugLoc(); PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setHasSpills(); LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs); for (unsigned i = 0, e = NewMIs.size(); i != e; ++i) MBB.insert(MI, NewMIs[i]); const MachineFrameInfo &MFI = MF.getFrameInfo(); MachineMemOperand *MMO = MF.getMachineMemOperand( MachinePointerInfo::getFixedStack(MF, FrameIdx), MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx), MFI.getObjectAlign(FrameIdx)); NewMIs.back()->addMemOperand(MF, MMO); } void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIdx, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const { // We need to avoid a situation in which the value from a VRRC register is // spilled using an Altivec instruction and reloaded into a VSRC register // using a VSX instruction. The issue with this is that the VSX // load/store instructions swap the doublewords in the vector and the Altivec // ones don't. The register classes on the spill/reload may be different if // the register is defined using an Altivec instruction and is then used by a // VSX instruction. RC = updatedRC(RC); loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI); } bool PPCInstrInfo:: reverseBranchCondition(SmallVectorImpl &Cond) const { assert(Cond.size() == 2 && "Invalid PPC branch opcode!"); if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR) Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0); else // Leave the CR# the same, but invert the condition. Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm())); return false; } // For some instructions, it is legal to fold ZERO into the RA register field. // This function performs that fold by replacing the operand with PPC::ZERO, // it does not consider whether the load immediate zero is no longer in use. bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg) const { // A zero immediate should always be loaded with a single li. unsigned DefOpc = DefMI.getOpcode(); if (DefOpc != PPC::LI && DefOpc != PPC::LI8) return false; if (!DefMI.getOperand(1).isImm()) return false; if (DefMI.getOperand(1).getImm() != 0) return false; // Note that we cannot here invert the arguments of an isel in order to fold // a ZERO into what is presented as the second argument. All we have here // is the condition bit, and that might come from a CR-logical bit operation. const MCInstrDesc &UseMCID = UseMI.getDesc(); // Only fold into real machine instructions. if (UseMCID.isPseudo()) return false; // We need to find which of the User's operands is to be folded, that will be // the operand that matches the given register ID. unsigned UseIdx; for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx) if (UseMI.getOperand(UseIdx).isReg() && UseMI.getOperand(UseIdx).getReg() == Reg) break; assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI"); assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg"); const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx]; // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0 // register (which might also be specified as a pointer class kind). if (UseInfo->isLookupPtrRegClass()) { if (UseInfo->RegClass /* Kind */ != 1) return false; } else { if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID && UseInfo->RegClass != PPC::G8RC_NOX0RegClassID) return false; } // Make sure this is not tied to an output register (or otherwise // constrained). This is true for ST?UX registers, for example, which // are tied to their output registers. if (UseInfo->Constraints != 0) return false; MCRegister ZeroReg; if (UseInfo->isLookupPtrRegClass()) { bool isPPC64 = Subtarget.isPPC64(); ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO; } else { ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ? PPC::ZERO8 : PPC::ZERO; } UseMI.getOperand(UseIdx).setReg(ZeroReg); return true; } // Folds zero into instructions which have a load immediate zero as an operand // but also recognize zero as immediate zero. If the definition of the load // has no more users it is deleted. bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const { bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg); if (MRI->use_nodbg_empty(Reg)) DefMI.eraseFromParent(); return Changed; } static bool MBBDefinesCTR(MachineBasicBlock &MBB) { for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end(); I != IE; ++I) if (I->definesRegister(PPC::CTR) || I->definesRegister(PPC::CTR8)) return true; return false; } // We should make sure that, if we're going to predicate both sides of a // condition (a diamond), that both sides don't define the counter register. We // can predicate counter-decrement-based branches, but while that predicates // the branching, it does not predicate the counter decrement. If we tried to // merge the triangle into one predicated block, we'd decrement the counter // twice. bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB, unsigned NumT, unsigned ExtraT, MachineBasicBlock &FMBB, unsigned NumF, unsigned ExtraF, BranchProbability Probability) const { return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB)); } bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const { // The predicated branches are identified by their type, not really by the // explicit presence of a predicate. Furthermore, some of them can be // predicated more than once. Because if conversion won't try to predicate // any instruction which already claims to be predicated (by returning true // here), always return false. In doing so, we let isPredicable() be the // final word on whether not the instruction can be (further) predicated. return false; } bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const { // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion // across them, since some FP operations may change content of FPSCR. // TODO: Model FPSCR in PPC instruction definitions and remove the workaround if (MI.getOpcode() == PPC::MFFS || MI.getOpcode() == PPC::MTFSF) return true; return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF); } bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI, ArrayRef Pred) const { unsigned OpC = MI.getOpcode(); if (OpC == PPC::BLR || OpC == PPC::BLR8) { if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) { bool isPPC64 = Subtarget.isPPC64(); MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR) : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR))); // Need add Def and Use for CTR implicit operand. MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addReg(Pred[1].getReg(), RegState::Implicit) .addReg(Pred[1].getReg(), RegState::ImplicitDefine); } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) { MI.setDesc(get(PPC::BCLR)); MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) { MI.setDesc(get(PPC::BCLRn)); MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); } else { MI.setDesc(get(PPC::BCCLR)); MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addImm(Pred[0].getImm()) .add(Pred[1]); } return true; } else if (OpC == PPC::B) { if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) { bool isPPC64 = Subtarget.isPPC64(); MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : (isPPC64 ? PPC::BDZ8 : PPC::BDZ))); // Need add Def and Use for CTR implicit operand. MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addReg(Pred[1].getReg(), RegState::Implicit) .addReg(Pred[1].getReg(), RegState::ImplicitDefine); } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) { MachineBasicBlock *MBB = MI.getOperand(0).getMBB(); MI.RemoveOperand(0); MI.setDesc(get(PPC::BC)); MachineInstrBuilder(*MI.getParent()->getParent(), MI) .add(Pred[1]) .addMBB(MBB); } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) { MachineBasicBlock *MBB = MI.getOperand(0).getMBB(); MI.RemoveOperand(0); MI.setDesc(get(PPC::BCn)); MachineInstrBuilder(*MI.getParent()->getParent(), MI) .add(Pred[1]) .addMBB(MBB); } else { MachineBasicBlock *MBB = MI.getOperand(0).getMBB(); MI.RemoveOperand(0); MI.setDesc(get(PPC::BCC)); MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addImm(Pred[0].getImm()) .add(Pred[1]) .addMBB(MBB); } return true; } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL || OpC == PPC::BCTRL8) { if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) llvm_unreachable("Cannot predicate bctr[l] on the ctr register"); bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8; bool isPPC64 = Subtarget.isPPC64(); if (Pred[0].getImm() == PPC::PRED_BIT_SET) { MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8) : (setLR ? PPC::BCCTRL : PPC::BCCTR))); MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) { MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n) : (setLR ? PPC::BCCTRLn : PPC::BCCTRn))); MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); } else { MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8) : (setLR ? PPC::BCCCTRL : PPC::BCCCTR))); MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addImm(Pred[0].getImm()) .add(Pred[1]); } // Need add Def and Use for LR implicit operand. if (setLR) MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit) .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine); return true; } return false; } bool PPCInstrInfo::SubsumesPredicate(ArrayRef Pred1, ArrayRef Pred2) const { assert(Pred1.size() == 2 && "Invalid PPC first predicate"); assert(Pred2.size() == 2 && "Invalid PPC second predicate"); if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR) return false; if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR) return false; // P1 can only subsume P2 if they test the same condition register. if (Pred1[1].getReg() != Pred2[1].getReg()) return false; PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm(); PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm(); if (P1 == P2) return true; // Does P1 subsume P2, e.g. GE subsumes GT. if (P1 == PPC::PRED_LE && (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ)) return true; if (P1 == PPC::PRED_GE && (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ)) return true; return false; } bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI, std::vector &Pred, bool SkipDead) const { // Note: At the present time, the contents of Pred from this function is // unused by IfConversion. This implementation follows ARM by pushing the // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of // predicate, instructions defining CTR or CTR8 are also included as // predicate-defining instructions. const TargetRegisterClass *RCs[] = { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass, &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass }; bool Found = false; for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI.getOperand(i); for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) { const TargetRegisterClass *RC = RCs[c]; if (MO.isReg()) { if (MO.isDef() && RC->contains(MO.getReg())) { Pred.push_back(MO); Found = true; } } else if (MO.isRegMask()) { for (TargetRegisterClass::iterator I = RC->begin(), IE = RC->end(); I != IE; ++I) if (MO.clobbersPhysReg(*I)) { Pred.push_back(MO); Found = true; } } } } return Found; } bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int &Mask, int &Value) const { unsigned Opc = MI.getOpcode(); switch (Opc) { default: return false; case PPC::CMPWI: case PPC::CMPLWI: case PPC::CMPDI: case PPC::CMPLDI: SrcReg = MI.getOperand(1).getReg(); SrcReg2 = 0; Value = MI.getOperand(2).getImm(); Mask = 0xFFFF; return true; case PPC::CMPW: case PPC::CMPLW: case PPC::CMPD: case PPC::CMPLD: case PPC::FCMPUS: case PPC::FCMPUD: SrcReg = MI.getOperand(1).getReg(); SrcReg2 = MI.getOperand(2).getReg(); Value = 0; Mask = 0; return true; } } bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int Mask, int Value, const MachineRegisterInfo *MRI) const { if (DisableCmpOpt) return false; int OpC = CmpInstr.getOpcode(); Register CRReg = CmpInstr.getOperand(0).getReg(); // FP record forms set CR1 based on the exception status bits, not a // comparison with zero. if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD) return false; const TargetRegisterInfo *TRI = &getRegisterInfo(); // The record forms set the condition register based on a signed comparison // with zero (so says the ISA manual). This is not as straightforward as it // seems, however, because this is always a 64-bit comparison on PPC64, even // for instructions that are 32-bit in nature (like slw for example). // So, on PPC32, for unsigned comparisons, we can use the record forms only // for equality checks (as those don't depend on the sign). On PPC64, // we are restricted to equality for unsigned 64-bit comparisons and for // signed 32-bit comparisons the applicability is more restricted. bool isPPC64 = Subtarget.isPPC64(); bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW; bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW; bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD; // Look through copies unless that gets us to a physical register. Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI); if (ActualSrc.isVirtual()) SrcReg = ActualSrc; // Get the unique definition of SrcReg. MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); if (!MI) return false; bool equalityOnly = false; bool noSub = false; if (isPPC64) { if (is32BitSignedCompare) { // We can perform this optimization only if MI is sign-extending. if (isSignExtended(*MI)) noSub = true; else return false; } else if (is32BitUnsignedCompare) { // We can perform this optimization, equality only, if MI is // zero-extending. if (isZeroExtended(*MI)) { noSub = true; equalityOnly = true; } else return false; } else equalityOnly = is64BitUnsignedCompare; } else equalityOnly = is32BitUnsignedCompare; if (equalityOnly) { // We need to check the uses of the condition register in order to reject // non-equality comparisons. for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end(); I != IE; ++I) { MachineInstr *UseMI = &*I; if (UseMI->getOpcode() == PPC::BCC) { PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm(); unsigned PredCond = PPC::getPredicateCondition(Pred); // We ignore hint bits when checking for non-equality comparisons. if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE) return false; } else if (UseMI->getOpcode() == PPC::ISEL || UseMI->getOpcode() == PPC::ISEL8) { unsigned SubIdx = UseMI->getOperand(3).getSubReg(); if (SubIdx != PPC::sub_eq) return false; } else return false; } } MachineBasicBlock::iterator I = CmpInstr; // Scan forward to find the first use of the compare. for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL; ++I) { bool FoundUse = false; for (MachineRegisterInfo::use_instr_iterator J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end(); J != JE; ++J) if (&*J == &*I) { FoundUse = true; break; } if (FoundUse) break; } SmallVector, 4> PredsToUpdate; SmallVector, 4> SubRegsToUpdate; // There are two possible candidates which can be changed to set CR[01]. // One is MI, the other is a SUB instruction. // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1). MachineInstr *Sub = nullptr; if (SrcReg2 != 0) // MI is not a candidate for CMPrr. MI = nullptr; // FIXME: Conservatively refuse to convert an instruction which isn't in the // same BB as the comparison. This is to allow the check below to avoid calls // (and other explicit clobbers); instead we should really check for these // more explicitly (in at least a few predecessors). else if (MI->getParent() != CmpInstr.getParent()) return false; else if (Value != 0) { // The record-form instructions set CR bit based on signed comparison // against 0. We try to convert a compare against 1 or -1 into a compare // against 0 to exploit record-form instructions. For example, we change // the condition "greater than -1" into "greater than or equal to 0" // and "less than 1" into "less than or equal to 0". // Since we optimize comparison based on a specific branch condition, // we don't optimize if condition code is used by more than once. if (equalityOnly || !MRI->hasOneUse(CRReg)) return false; MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg); if (UseMI->getOpcode() != PPC::BCC) return false; PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm(); unsigned PredCond = PPC::getPredicateCondition(Pred); unsigned PredHint = PPC::getPredicateHint(Pred); int16_t Immed = (int16_t)Value; // When modifying the condition in the predicate, we propagate hint bits // from the original predicate to the new one. if (Immed == -1 && PredCond == PPC::PRED_GT) // We convert "greater than -1" into "greater than or equal to 0", // since we are assuming signed comparison by !equalityOnly Pred = PPC::getPredicate(PPC::PRED_GE, PredHint); else if (Immed == -1 && PredCond == PPC::PRED_LE) // We convert "less than or equal to -1" into "less than 0". Pred = PPC::getPredicate(PPC::PRED_LT, PredHint); else if (Immed == 1 && PredCond == PPC::PRED_LT) // We convert "less than 1" into "less than or equal to 0". Pred = PPC::getPredicate(PPC::PRED_LE, PredHint); else if (Immed == 1 && PredCond == PPC::PRED_GE) // We convert "greater than or equal to 1" into "greater than 0". Pred = PPC::getPredicate(PPC::PRED_GT, PredHint); else return false; PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred)); } // Search for Sub. --I; // Get ready to iterate backward from CmpInstr. MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin(); for (; I != E && !noSub; --I) { const MachineInstr &Instr = *I; unsigned IOpC = Instr.getOpcode(); if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) || Instr.readsRegister(PPC::CR0, TRI))) // This instruction modifies or uses the record condition register after // the one we want to change. While we could do this transformation, it // would likely not be profitable. This transformation removes one // instruction, and so even forcing RA to generate one move probably // makes it unprofitable. return false; // Check whether CmpInstr can be made redundant by the current instruction. if ((OpC == PPC::CMPW || OpC == PPC::CMPLW || OpC == PPC::CMPD || OpC == PPC::CMPLD) && (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) && ((Instr.getOperand(1).getReg() == SrcReg && Instr.getOperand(2).getReg() == SrcReg2) || (Instr.getOperand(1).getReg() == SrcReg2 && Instr.getOperand(2).getReg() == SrcReg))) { Sub = &*I; break; } if (I == B) // The 'and' is below the comparison instruction. return false; } // Return false if no candidates exist. if (!MI && !Sub) return false; // The single candidate is called MI. if (!MI) MI = Sub; int NewOpC = -1; int MIOpC = MI->getOpcode(); if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec || MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec) NewOpC = MIOpC; else { NewOpC = PPC::getRecordFormOpcode(MIOpC); if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1) NewOpC = MIOpC; } // FIXME: On the non-embedded POWER architectures, only some of the record // forms are fast, and we should use only the fast ones. // The defining instruction has a record form (or is already a record // form). It is possible, however, that we'll need to reverse the condition // code of the users. if (NewOpC == -1) return false; // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP // needs to be updated to be based on SUB. Push the condition code // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the // condition code of these operands will be modified. // Here, Value == 0 means we haven't converted comparison against 1 or -1 to // comparison against 0, which may modify predicate. bool ShouldSwap = false; if (Sub && Value == 0) { ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 && Sub->getOperand(2).getReg() == SrcReg; // The operands to subf are the opposite of sub, so only in the fixed-point // case, invert the order. ShouldSwap = !ShouldSwap; } if (ShouldSwap) for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end(); I != IE; ++I) { MachineInstr *UseMI = &*I; if (UseMI->getOpcode() == PPC::BCC) { PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm(); unsigned PredCond = PPC::getPredicateCondition(Pred); assert((!equalityOnly || PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) && "Invalid predicate for equality-only optimization"); (void)PredCond; // To suppress warning in release build. PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), PPC::getSwappedPredicate(Pred))); } else if (UseMI->getOpcode() == PPC::ISEL || UseMI->getOpcode() == PPC::ISEL8) { unsigned NewSubReg = UseMI->getOperand(3).getSubReg(); assert((!equalityOnly || NewSubReg == PPC::sub_eq) && "Invalid CR bit for equality-only optimization"); if (NewSubReg == PPC::sub_lt) NewSubReg = PPC::sub_gt; else if (NewSubReg == PPC::sub_gt) NewSubReg = PPC::sub_lt; SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)), NewSubReg)); } else // We need to abort on a user we don't understand. return false; } assert(!(Value != 0 && ShouldSwap) && "Non-zero immediate support and ShouldSwap" "may conflict in updating predicate"); // Create a new virtual register to hold the value of the CR set by the // record-form instruction. If the instruction was not previously in // record form, then set the kill flag on the CR. CmpInstr.eraseFromParent(); MachineBasicBlock::iterator MII = MI; BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(), get(TargetOpcode::COPY), CRReg) .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0); // Even if CR0 register were dead before, it is alive now since the // instruction we just built uses it. MI->clearRegisterDeads(PPC::CR0); if (MIOpC != NewOpC) { // We need to be careful here: we're replacing one instruction with // another, and we need to make sure that we get all of the right // implicit uses and defs. On the other hand, the caller may be holding // an iterator to this instruction, and so we can't delete it (this is // specifically the case if this is the instruction directly after the // compare). // Rotates are expensive instructions. If we're emitting a record-form // rotate that can just be an andi/andis, we should just emit that. if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) { Register GPRRes = MI->getOperand(0).getReg(); int64_t SH = MI->getOperand(2).getImm(); int64_t MB = MI->getOperand(3).getImm(); int64_t ME = MI->getOperand(4).getImm(); // We can only do this if both the start and end of the mask are in the // same halfword. bool MBInLoHWord = MB >= 16; bool MEInLoHWord = ME >= 16; uint64_t Mask = ~0LLU; if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) { Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1); // The mask value needs to shift right 16 if we're emitting andis. Mask >>= MBInLoHWord ? 0 : 16; NewOpC = MIOpC == PPC::RLWINM ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec) : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec); } else if (MRI->use_empty(GPRRes) && (ME == 31) && (ME - MB + 1 == SH) && (MB >= 16)) { // If we are rotating by the exact number of bits as are in the mask // and the mask is in the least significant bits of the register, // that's just an andis. (as long as the GPR result has no uses). Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1); Mask >>= 16; NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec; } // If we've set the mask, we can transform. if (Mask != ~0LLU) { MI->RemoveOperand(4); MI->RemoveOperand(3); MI->getOperand(2).setImm(Mask); NumRcRotatesConvertedToRcAnd++; } } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) { int64_t MB = MI->getOperand(3).getImm(); if (MB >= 48) { uint64_t Mask = (1LLU << (63 - MB + 1)) - 1; NewOpC = PPC::ANDI8_rec; MI->RemoveOperand(3); MI->getOperand(2).setImm(Mask); NumRcRotatesConvertedToRcAnd++; } } const MCInstrDesc &NewDesc = get(NewOpC); MI->setDesc(NewDesc); if (NewDesc.ImplicitDefs) for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs(); *ImpDefs; ++ImpDefs) if (!MI->definesRegister(*ImpDefs)) MI->addOperand(*MI->getParent()->getParent(), MachineOperand::CreateReg(*ImpDefs, true, true)); if (NewDesc.ImplicitUses) for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses(); *ImpUses; ++ImpUses) if (!MI->readsRegister(*ImpUses)) MI->addOperand(*MI->getParent()->getParent(), MachineOperand::CreateReg(*ImpUses, false, true)); } assert(MI->definesRegister(PPC::CR0) && "Record-form instruction does not define cr0?"); // Modify the condition code of operands in OperandsToUpdate. // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++) PredsToUpdate[i].first->setImm(PredsToUpdate[i].second); for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++) SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second); return true; } bool PPCInstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, const TargetRegisterInfo *TRI) const { const MachineOperand *BaseOp; OffsetIsScalable = false; if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI)) return false; BaseOps.push_back(BaseOp); return true; } static bool isLdStSafeToCluster(const MachineInstr &LdSt, const TargetRegisterInfo *TRI) { // If this is a volatile load/store, don't mess with it. if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3) return false; if (LdSt.getOperand(2).isFI()) return true; assert(LdSt.getOperand(2).isReg() && "Expected a reg operand."); // Can't cluster if the instruction modifies the base register // or it is update form. e.g. ld r2,3(r2) if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI)) return false; return true; } // Only cluster instruction pair that have the same opcode, and they are // clusterable according to PowerPC specification. static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc, const PPCSubtarget &Subtarget) { switch (FirstOpc) { default: return false; case PPC::STD: case PPC::STFD: case PPC::STXSD: case PPC::DFSTOREf64: return FirstOpc == SecondOpc; // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with // 32bit and 64bit instruction selection. They are clusterable pair though // they are different opcode. case PPC::STW: case PPC::STW8: return SecondOpc == PPC::STW || SecondOpc == PPC::STW8; } } bool PPCInstrInfo::shouldClusterMemOps( ArrayRef BaseOps1, ArrayRef BaseOps2, unsigned NumLoads, unsigned NumBytes) const { assert(BaseOps1.size() == 1 && BaseOps2.size() == 1); const MachineOperand &BaseOp1 = *BaseOps1.front(); const MachineOperand &BaseOp2 = *BaseOps2.front(); assert((BaseOp1.isReg() || BaseOp1.isFI()) && "Only base registers and frame indices are supported."); // The NumLoads means the number of loads that has been clustered. // Don't cluster memory op if there are already two ops clustered at least. if (NumLoads > 2) return false; // Cluster the load/store only when they have the same base // register or FI. if ((BaseOp1.isReg() != BaseOp2.isReg()) || (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) || (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex())) return false; // Check if the load/store are clusterable according to the PowerPC // specification. const MachineInstr &FirstLdSt = *BaseOp1.getParent(); const MachineInstr &SecondLdSt = *BaseOp2.getParent(); unsigned FirstOpc = FirstLdSt.getOpcode(); unsigned SecondOpc = SecondLdSt.getOpcode(); const TargetRegisterInfo *TRI = &getRegisterInfo(); // Cluster the load/store only when they have the same opcode, and they are // clusterable opcode according to PowerPC specification. if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget)) return false; // Can't cluster load/store that have ordered or volatile memory reference. if (!isLdStSafeToCluster(FirstLdSt, TRI) || !isLdStSafeToCluster(SecondLdSt, TRI)) return false; int64_t Offset1 = 0, Offset2 = 0; unsigned Width1 = 0, Width2 = 0; const MachineOperand *Base1 = nullptr, *Base2 = nullptr; if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) || !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) || Width1 != Width2) return false; assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 && "getMemOperandWithOffsetWidth return incorrect base op"); // The caller should already have ordered FirstMemOp/SecondMemOp by offset. assert(Offset1 <= Offset2 && "Caller should have ordered offsets."); return Offset1 + Width1 == Offset2; } /// GetInstSize - Return the number of bytes of code the specified /// instruction may be. This returns the maximum number of bytes. /// unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { unsigned Opcode = MI.getOpcode(); if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) { const MachineFunction *MF = MI.getParent()->getParent(); const char *AsmStr = MI.getOperand(0).getSymbolName(); return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); } else if (Opcode == TargetOpcode::STACKMAP) { StackMapOpers Opers(&MI); return Opers.getNumPatchBytes(); } else if (Opcode == TargetOpcode::PATCHPOINT) { PatchPointOpers Opers(&MI); return Opers.getNumPatchBytes(); } else { return get(Opcode).getSize(); } } std::pair PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { const unsigned Mask = PPCII::MO_ACCESS_MASK; return std::make_pair(TF & Mask, TF & ~Mask); } ArrayRef> PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { using namespace PPCII; static const std::pair TargetFlags[] = { {MO_LO, "ppc-lo"}, {MO_HA, "ppc-ha"}, {MO_TPREL_LO, "ppc-tprel-lo"}, {MO_TPREL_HA, "ppc-tprel-ha"}, {MO_DTPREL_LO, "ppc-dtprel-lo"}, {MO_TLSLD_LO, "ppc-tlsld-lo"}, {MO_TOC_LO, "ppc-toc-lo"}, {MO_TLS, "ppc-tls"}}; return makeArrayRef(TargetFlags); } ArrayRef> PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { using namespace PPCII; static const std::pair TargetFlags[] = { {MO_PLT, "ppc-plt"}, {MO_PIC_FLAG, "ppc-pic"}, {MO_PCREL_FLAG, "ppc-pcrel"}, {MO_GOT_FLAG, "ppc-got"}, {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"}, {MO_TLSGD_FLAG, "ppc-tlsgd"}, {MO_TLSLD_FLAG, "ppc-tlsld"}, {MO_TPREL_FLAG, "ppc-tprel"}, {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"}, {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"}, {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}}; return makeArrayRef(TargetFlags); } // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction. // The VSX versions have the advantage of a full 64-register target whereas // the FP ones have the advantage of lower latency and higher throughput. So // what we are after is using the faster instructions in low register pressure // situations and using the larger register file in high register pressure // situations. bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const { unsigned UpperOpcode, LowerOpcode; switch (MI.getOpcode()) { case PPC::DFLOADf32: UpperOpcode = PPC::LXSSP; LowerOpcode = PPC::LFS; break; case PPC::DFLOADf64: UpperOpcode = PPC::LXSD; LowerOpcode = PPC::LFD; break; case PPC::DFSTOREf32: UpperOpcode = PPC::STXSSP; LowerOpcode = PPC::STFS; break; case PPC::DFSTOREf64: UpperOpcode = PPC::STXSD; LowerOpcode = PPC::STFD; break; case PPC::XFLOADf32: UpperOpcode = PPC::LXSSPX; LowerOpcode = PPC::LFSX; break; case PPC::XFLOADf64: UpperOpcode = PPC::LXSDX; LowerOpcode = PPC::LFDX; break; case PPC::XFSTOREf32: UpperOpcode = PPC::STXSSPX; LowerOpcode = PPC::STFSX; break; case PPC::XFSTOREf64: UpperOpcode = PPC::STXSDX; LowerOpcode = PPC::STFDX; break; case PPC::LIWAX: UpperOpcode = PPC::LXSIWAX; LowerOpcode = PPC::LFIWAX; break; case PPC::LIWZX: UpperOpcode = PPC::LXSIWZX; LowerOpcode = PPC::LFIWZX; break; case PPC::STIWX: UpperOpcode = PPC::STXSIWX; LowerOpcode = PPC::STFIWX; break; default: llvm_unreachable("Unknown Operation!"); } Register TargetReg = MI.getOperand(0).getReg(); unsigned Opcode; if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) || (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31)) Opcode = LowerOpcode; else Opcode = UpperOpcode; MI.setDesc(get(Opcode)); return true; } static bool isAnImmediateOperand(const MachineOperand &MO) { return MO.isCPI() || MO.isGlobal() || MO.isImm(); } bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { auto &MBB = *MI.getParent(); auto DL = MI.getDebugLoc(); switch (MI.getOpcode()) { case PPC::BUILD_UACC: { MCRegister ACC = MI.getOperand(0).getReg(); MCRegister UACC = MI.getOperand(1).getReg(); if (ACC - PPC::ACC0 != UACC - PPC::UACC0) { MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4; MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4; // FIXME: This can easily be improved to look up to the top of the MBB // to see if the inputs are XXLOR's. If they are and SrcReg is killed, // we can just re-target any such XXLOR's to DstVSR + offset. for (int VecNo = 0; VecNo < 4; VecNo++) BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo) .addReg(SrcVSR + VecNo) .addReg(SrcVSR + VecNo); } // BUILD_UACC is expanded to 4 copies of the underlying vsx regisers. // So after building the 4 copies, we can replace the BUILD_UACC instruction // with a NOP. LLVM_FALLTHROUGH; } case PPC::KILL_PAIR: { MI.setDesc(get(PPC::UNENCODED_NOP)); MI.RemoveOperand(1); MI.RemoveOperand(0); return true; } case TargetOpcode::LOAD_STACK_GUARD: { assert(Subtarget.isTargetLinux() && "Only Linux target is expected to contain LOAD_STACK_GUARD"); const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008; const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2; MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ)); MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addImm(Offset) .addReg(Reg); return true; } case PPC::DFLOADf32: case PPC::DFLOADf64: case PPC::DFSTOREf32: case PPC::DFSTOREf64: { assert(Subtarget.hasP9Vector() && "Invalid D-Form Pseudo-ops on Pre-P9 target."); assert(MI.getOperand(2).isReg() && isAnImmediateOperand(MI.getOperand(1)) && "D-form op must have register and immediate operands"); return expandVSXMemPseudo(MI); } case PPC::XFLOADf32: case PPC::XFSTOREf32: case PPC::LIWAX: case PPC::LIWZX: case PPC::STIWX: { assert(Subtarget.hasP8Vector() && "Invalid X-Form Pseudo-ops on Pre-P8 target."); assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() && "X-form op must have register and register operands"); return expandVSXMemPseudo(MI); } case PPC::XFLOADf64: case PPC::XFSTOREf64: { assert(Subtarget.hasVSX() && "Invalid X-Form Pseudo-ops on target that has no VSX."); assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() && "X-form op must have register and register operands"); return expandVSXMemPseudo(MI); } case PPC::SPILLTOVSR_LD: { Register TargetReg = MI.getOperand(0).getReg(); if (PPC::VSFRCRegClass.contains(TargetReg)) { MI.setDesc(get(PPC::DFLOADf64)); return expandPostRAPseudo(MI); } else MI.setDesc(get(PPC::LD)); return true; } case PPC::SPILLTOVSR_ST: { Register SrcReg = MI.getOperand(0).getReg(); if (PPC::VSFRCRegClass.contains(SrcReg)) { NumStoreSPILLVSRRCAsVec++; MI.setDesc(get(PPC::DFSTOREf64)); return expandPostRAPseudo(MI); } else { NumStoreSPILLVSRRCAsGpr++; MI.setDesc(get(PPC::STD)); } return true; } case PPC::SPILLTOVSR_LDX: { Register TargetReg = MI.getOperand(0).getReg(); if (PPC::VSFRCRegClass.contains(TargetReg)) MI.setDesc(get(PPC::LXSDX)); else MI.setDesc(get(PPC::LDX)); return true; } case PPC::SPILLTOVSR_STX: { Register SrcReg = MI.getOperand(0).getReg(); if (PPC::VSFRCRegClass.contains(SrcReg)) { NumStoreSPILLVSRRCAsVec++; MI.setDesc(get(PPC::STXSDX)); } else { NumStoreSPILLVSRRCAsGpr++; MI.setDesc(get(PPC::STDX)); } return true; } case PPC::CFENCE8: { auto Val = MI.getOperand(0).getReg(); BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val); BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP)) .addImm(PPC::PRED_NE_MINUS) .addReg(PPC::CR7) .addImm(1); MI.setDesc(get(PPC::ISYNC)); MI.RemoveOperand(0); return true; } } return false; } // Essentially a compile-time implementation of a compare->isel sequence. // It takes two constants to compare, along with the true/false registers // and the comparison type (as a subreg to a CR field) and returns one // of the true/false registers, depending on the comparison results. static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc, unsigned TrueReg, unsigned FalseReg, unsigned CRSubReg) { // Signed comparisons. The immediates are assumed to be sign-extended. if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) { switch (CRSubReg) { default: llvm_unreachable("Unknown integer comparison type."); case PPC::sub_lt: return Imm1 < Imm2 ? TrueReg : FalseReg; case PPC::sub_gt: return Imm1 > Imm2 ? TrueReg : FalseReg; case PPC::sub_eq: return Imm1 == Imm2 ? TrueReg : FalseReg; } } // Unsigned comparisons. else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) { switch (CRSubReg) { default: llvm_unreachable("Unknown integer comparison type."); case PPC::sub_lt: return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg; case PPC::sub_gt: return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg; case PPC::sub_eq: return Imm1 == Imm2 ? TrueReg : FalseReg; } } return PPC::NoRegister; } void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI, unsigned OpNo, int64_t Imm) const { assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG"); // Replace the REG with the Immediate. Register InUseReg = MI.getOperand(OpNo).getReg(); MI.getOperand(OpNo).ChangeToImmediate(Imm); if (MI.implicit_operands().empty()) return; // We need to make sure that the MI didn't have any implicit use // of this REG any more. const TargetRegisterInfo *TRI = &getRegisterInfo(); int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI); if (UseOpIdx >= 0) { MachineOperand &MO = MI.getOperand(UseOpIdx); if (MO.isImplicit()) // The operands must always be in the following order: // - explicit reg defs, // - other explicit operands (reg uses, immediates, etc.), // - implicit reg defs // - implicit reg uses // Therefore, removing the implicit operand won't change the explicit // operands layout. MI.RemoveOperand(UseOpIdx); } } // Replace an instruction with one that materializes a constant (and sets // CR0 if the original instruction was a record-form instruction). void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI, const LoadImmediateInfo &LII) const { // Remove existing operands. int OperandToKeep = LII.SetCR ? 1 : 0; for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--) MI.RemoveOperand(i); // Replace the instruction. if (LII.SetCR) { MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec)); // Set the immediate. MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine); return; } else MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI)); // Set the immediate. MachineInstrBuilder(*MI.getParent()->getParent(), MI) .addImm(LII.Imm); } MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI, bool &SeenIntermediateUse) const { assert(!MI.getParent()->getParent()->getRegInfo().isSSA() && "Should be called after register allocation."); const TargetRegisterInfo *TRI = &getRegisterInfo(); MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI; It++; SeenIntermediateUse = false; for (; It != E; ++It) { if (It->modifiesRegister(Reg, TRI)) return &*It; if (It->readsRegister(Reg, TRI)) SeenIntermediateUse = true; } return nullptr; } MachineInstr *PPCInstrInfo::getForwardingDefMI( MachineInstr &MI, unsigned &OpNoForForwarding, bool &SeenIntermediateUse) const { OpNoForForwarding = ~0U; MachineInstr *DefMI = nullptr; MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo(); const TargetRegisterInfo *TRI = &getRegisterInfo(); // If we're in SSA, get the defs through the MRI. Otherwise, only look // within the basic block to see if the register is defined using an // LI/LI8/ADDI/ADDI8. if (MRI->isSSA()) { for (int i = 1, e = MI.getNumOperands(); i < e; i++) { if (!MI.getOperand(i).isReg()) continue; Register Reg = MI.getOperand(i).getReg(); if (!Register::isVirtualRegister(Reg)) continue; unsigned TrueReg = TRI->lookThruCopyLike(Reg, MRI); if (Register::isVirtualRegister(TrueReg)) { DefMI = MRI->getVRegDef(TrueReg); if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 || DefMI->getOpcode() == PPC::ADDI || DefMI->getOpcode() == PPC::ADDI8) { OpNoForForwarding = i; // The ADDI and LI operand maybe exist in one instruction at same // time. we prefer to fold LI operand as LI only has one Imm operand // and is more possible to be converted. So if current DefMI is // ADDI/ADDI8, we continue to find possible LI/LI8. if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8) break; } } } } else { // Looking back through the definition for each operand could be expensive, // so exit early if this isn't an instruction that either has an immediate // form or is already an immediate form that we can handle. ImmInstrInfo III; unsigned Opc = MI.getOpcode(); bool ConvertibleImmForm = Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI || Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 || Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI || Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec || Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 || Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec; bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg()) ? isVFRegister(MI.getOperand(0).getReg()) : false; if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true)) return nullptr; // Don't convert or %X, %Y, %Y since that's just a register move. if ((Opc == PPC::OR || Opc == PPC::OR8) && MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) return nullptr; for (int i = 1, e = MI.getNumOperands(); i < e; i++) { MachineOperand &MO = MI.getOperand(i); SeenIntermediateUse = false; if (MO.isReg() && MO.isUse() && !MO.isImplicit()) { Register Reg = MI.getOperand(i).getReg(); // If we see another use of this reg between the def and the MI, // we want to flat it so the def isn't deleted. MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse); if (DefMI) { // Is this register defined by some form of add-immediate (including // load-immediate) within this basic block? switch (DefMI->getOpcode()) { default: break; case PPC::LI: case PPC::LI8: case PPC::ADDItocL: case PPC::ADDI: case PPC::ADDI8: OpNoForForwarding = i; return DefMI; } } } } } return OpNoForForwarding == ~0U ? nullptr : DefMI; } unsigned PPCInstrInfo::getSpillTarget() const { return Subtarget.hasP9Vector() ? 1 : 0; } const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const { return StoreSpillOpcodesArray[getSpillTarget()]; } const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const { return LoadSpillOpcodesArray[getSpillTarget()]; } void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr *StartMI, MachineInstr *EndMI, unsigned RegNo) const { // Conservatively clear kill flag for the register if the instructions are in // different basic blocks and in SSA form, because the kill flag may no longer // be right. There is no need to bother with dead flags since defs with no // uses will be handled by DCE. MachineRegisterInfo &MRI = StartMI->getParent()->getParent()->getRegInfo(); if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) { MRI.clearKillFlags(RegNo); return; } // Instructions between [StartMI, EndMI] should be in same basic block. assert((StartMI->getParent() == EndMI->getParent()) && "Instructions are not in same basic block"); // If before RA, StartMI may be def through COPY, we need to adjust it to the // real def. See function getForwardingDefMI. if (MRI.isSSA()) { bool Reads, Writes; std::tie(Reads, Writes) = StartMI->readsWritesVirtualRegister(RegNo); if (!Reads && !Writes) { assert(Register::isVirtualRegister(RegNo) && "Must be a virtual register"); // Get real def and ignore copies. StartMI = MRI.getVRegDef(RegNo); } } bool IsKillSet = false; auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) { MachineOperand &MO = MI.getOperand(Index); if (MO.isReg() && MO.isUse() && MO.isKill() && getRegisterInfo().regsOverlap(MO.getReg(), RegNo)) MO.setIsKill(false); }; // Set killed flag for EndMI. // No need to do anything if EndMI defines RegNo. int UseIndex = EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo()); if (UseIndex != -1) { EndMI->getOperand(UseIndex).setIsKill(true); IsKillSet = true; // Clear killed flag for other EndMI operands related to RegNo. In some // upexpected cases, killed may be set multiple times for same register // operand in same MI. for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i) if (i != UseIndex) clearOperandKillInfo(*EndMI, i); } // Walking the inst in reverse order (EndMI -> StartMI]. MachineBasicBlock::reverse_iterator It = *EndMI; MachineBasicBlock::reverse_iterator E = EndMI->getParent()->rend(); // EndMI has been handled above, skip it here. It++; MachineOperand *MO = nullptr; for (; It != E; ++It) { // Skip insturctions which could not be a def/use of RegNo. if (It->isDebugInstr() || It->isPosition()) continue; // Clear killed flag for all It operands related to RegNo. In some // upexpected cases, killed may be set multiple times for same register // operand in same MI. for (int i = 0, e = It->getNumOperands(); i != e; ++i) clearOperandKillInfo(*It, i); // If killed is not set, set killed for its last use or set dead for its def // if no use found. if (!IsKillSet) { if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) { // Use found, set it killed. IsKillSet = true; MO->setIsKill(true); continue; } else if ((MO = It->findRegisterDefOperand(RegNo, false, true, &getRegisterInfo()))) { // No use found, set dead for its def. assert(&*It == StartMI && "No new def between StartMI and EndMI."); MO->setIsDead(true); break; } } if ((&*It) == StartMI) break; } // Ensure RegMo liveness is killed after EndMI. assert((IsKillSet || (MO && MO->isDead())) && "RegNo should be killed or dead"); } // This opt tries to convert the following imm form to an index form to save an // add for stack variables. // Return false if no such pattern found. // // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed) // // can be converted to: // // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm) // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed) // // In order to eliminate ADD instr, make sure that: // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in // new ADDI instr and ADDI can only take int16 Imm. // 2: ToBeChangedReg must be killed in ADD instr and there is no other use // between ADDI and ADD instr since its original def in ADDI will be changed // in new ADDI instr. And also there should be no new def for it between // ADD and Imm instr as ToBeChangedReg will be used in Index instr. // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use // between ADD and Imm instr since ADD instr will be eliminated. // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be // moved to Index instr. bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const { MachineFunction *MF = MI.getParent()->getParent(); MachineRegisterInfo *MRI = &MF->getRegInfo(); bool PostRA = !MRI->isSSA(); // Do this opt after PEI which is after RA. The reason is stack slot expansion // in PEI may expose such opportunities since in PEI, stack slot offsets to // frame base(OffsetAddi) are determined. if (!PostRA) return false; unsigned ToBeDeletedReg = 0; int64_t OffsetImm = 0; unsigned XFormOpcode = 0; ImmInstrInfo III; // Check if Imm instr meets requirement. if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm, III)) return false; bool OtherIntermediateUse = false; MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse); // Exit if there is other use between ADD and Imm instr or no def found. if (OtherIntermediateUse || !ADDMI) return false; // Check if ADD instr meets requirement. if (!isADDInstrEligibleForFolding(*ADDMI)) return false; unsigned ScaleRegIdx = 0; int64_t OffsetAddi = 0; MachineInstr *ADDIMI = nullptr; // Check if there is a valid ToBeChangedReg in ADDMI. // 1: It must be killed. // 2: Its definition must be a valid ADDIMI. // 3: It must satify int16 offset requirement. if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm)) ScaleRegIdx = 2; else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm)) ScaleRegIdx = 1; else return false; assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg."); unsigned ToBeChangedReg = ADDIMI->getOperand(0).getReg(); unsigned ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg(); auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start, MachineBasicBlock::iterator End) { for (auto It = ++Start; It != End; It++) if (It->modifiesRegister(Reg, &getRegisterInfo())) return true; return false; }; // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is // treated as special zero when ScaleReg is R0/X0 register. if (III.ZeroIsSpecialOrig == III.ImmOpNo && (ScaleReg == PPC::R0 || ScaleReg == PPC::X0)) return false; // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr // and Imm Instr. if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI)) return false; // Now start to do the transformation. LLVM_DEBUG(dbgs() << "Replace instruction: " << "\n"); LLVM_DEBUG(ADDIMI->dump()); LLVM_DEBUG(ADDMI->dump()); LLVM_DEBUG(MI.dump()); LLVM_DEBUG(dbgs() << "with: " << "\n"); // Update ADDI instr. ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm); // Update Imm instr. MI.setDesc(get(XFormOpcode)); MI.getOperand(III.ImmOpNo) .ChangeToRegister(ScaleReg, false, false, ADDMI->getOperand(ScaleRegIdx).isKill()); MI.getOperand(III.OpNoForForwarding) .ChangeToRegister(ToBeChangedReg, false, false, true); // Eliminate ADD instr. ADDMI->eraseFromParent(); LLVM_DEBUG(ADDIMI->dump()); LLVM_DEBUG(MI.dump()); return true; } bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI, int64_t &Imm) const { unsigned Opc = ADDIMI.getOpcode(); // Exit if the instruction is not ADDI. if (Opc != PPC::ADDI && Opc != PPC::ADDI8) return false; // The operand may not necessarily be an immediate - it could be a relocation. if (!ADDIMI.getOperand(2).isImm()) return false; Imm = ADDIMI.getOperand(2).getImm(); return true; } bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const { unsigned Opc = ADDMI.getOpcode(); // Exit if the instruction is not ADD. return Opc == PPC::ADD4 || Opc == PPC::ADD8; } bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI, unsigned &ToBeDeletedReg, unsigned &XFormOpcode, int64_t &OffsetImm, ImmInstrInfo &III) const { // Only handle load/store. if (!MI.mayLoadOrStore()) return false; unsigned Opc = MI.getOpcode(); XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc); // Exit if instruction has no index form. if (XFormOpcode == PPC::INSTRUCTION_LIST_END) return false; // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap. if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()), III, true)) return false; if (!III.IsSummingOperands) return false; MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo); MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding); // Only support imm operands, not relocation slots or others. if (!ImmOperand.isImm()) return false; assert(RegOperand.isReg() && "Instruction format is not right"); // There are other use for ToBeDeletedReg after Imm instr, can not delete it. if (!RegOperand.isKill()) return false; ToBeDeletedReg = RegOperand.getReg(); OffsetImm = ImmOperand.getImm(); return true; } bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index, MachineInstr *&ADDIMI, int64_t &OffsetAddi, int64_t OffsetImm) const { assert((Index == 1 || Index == 2) && "Invalid operand index for add."); MachineOperand &MO = ADDMI->getOperand(Index); if (!MO.isKill()) return false; bool OtherIntermediateUse = false; ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse); // Currently handle only one "add + Imminstr" pair case, exit if other // intermediate use for ToBeChangedReg found. // TODO: handle the cases where there are other "add + Imminstr" pairs // with same offset in Imminstr which is like: // // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed) // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed) // // can be converted to: // // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, // (OffsetAddi + OffsetImm) // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed) if (OtherIntermediateUse || !ADDIMI) return false; // Check if ADDI instr meets requirement. if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi)) return false; if (isInt<16>(OffsetAddi + OffsetImm)) return true; return false; } // If this instruction has an immediate form and one of its operands is a // result of a load-immediate or an add-immediate, convert it to // the immediate form if the constant is in range. bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI, MachineInstr **KilledDef) const { MachineFunction *MF = MI.getParent()->getParent(); MachineRegisterInfo *MRI = &MF->getRegInfo(); bool PostRA = !MRI->isSSA(); bool SeenIntermediateUse = true; unsigned ForwardingOperand = ~0U; MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand, SeenIntermediateUse); if (!DefMI) return false; assert(ForwardingOperand < MI.getNumOperands() && "The forwarding operand needs to be valid at this point"); bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill(); bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled; if (KilledDef && KillFwdDefMI) *KilledDef = DefMI; // If this is a imm instruction and its register operands is produced by ADDI, // put the imm into imm inst directly. if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) != PPC::INSTRUCTION_LIST_END && transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand)) return true; ImmInstrInfo III; bool IsVFReg = MI.getOperand(0).isReg() ? isVFRegister(MI.getOperand(0).getReg()) : false; bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA); // If this is a reg+reg instruction that has a reg+imm form, // and one of the operands is produced by an add-immediate, // try to convert it. if (HasImmForm && transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI, KillFwdDefMI)) return true; // If this is a reg+reg instruction that has a reg+imm form, // and one of the operands is produced by LI, convert it now. if (HasImmForm && transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI)) return true; // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI // can be simpified to LI. if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef)) return true; return false; } -// This function tries to combine two RLWINMs. We not only perform such -// optimization in SSA, but also after RA, since some RLWINM is generated after -// RA. -bool PPCInstrInfo::simplifyRotateAndMaskInstr(MachineInstr &MI, - MachineInstr *&ToErase) const { - bool Is64Bit = false; - switch (MI.getOpcode()) { - case PPC::RLWINM: - case PPC::RLWINM_rec: - break; - case PPC::RLWINM8: - case PPC::RLWINM8_rec: - Is64Bit = true; - break; - default: - return false; - } +bool PPCInstrInfo::combineRLWINM(MachineInstr &MI, + MachineInstr **ToErase) const { MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo(); - Register FoldingReg = MI.getOperand(1).getReg(); - MachineInstr *SrcMI = nullptr; - bool NoUse = false; - if (MRI->isSSA()) { - if (!Register::isVirtualRegister(FoldingReg)) - return false; - SrcMI = MRI->getVRegDef(FoldingReg); - } else { - bool OtherIntermediateUse = false; - SrcMI = getDefMIPostRA(FoldingReg, MI, OtherIntermediateUse); - NoUse = !OtherIntermediateUse && MI.getOperand(1).isKill(); - } - if (!SrcMI) + unsigned FoldingReg = MI.getOperand(1).getReg(); + if (!Register::isVirtualRegister(FoldingReg)) return false; - // TODO: The pairs of RLWINM8(RLWINM) or RLWINM(RLWINM8) never occur before - // RA, but after RA. And We can fold RLWINM8(RLWINM) -> RLWINM8, or - // RLWINM(RLWINM8) -> RLWINM. - switch (SrcMI->getOpcode()) { - case PPC::RLWINM: - case PPC::RLWINM_rec: - if (Is64Bit) - return false; - break; - case PPC::RLWINM8: - case PPC::RLWINM8_rec: - if (!Is64Bit) - return false; - break; - default: + MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg); + if (SrcMI->getOpcode() != PPC::RLWINM && + SrcMI->getOpcode() != PPC::RLWINM_rec && + SrcMI->getOpcode() != PPC::RLWINM8 && + SrcMI->getOpcode() != PPC::RLWINM8_rec) return false; - } assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() && MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() && SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) && "Invalid PPC::RLWINM Instruction!"); uint64_t SHSrc = SrcMI->getOperand(2).getImm(); uint64_t SHMI = MI.getOperand(2).getImm(); uint64_t MBSrc = SrcMI->getOperand(3).getImm(); uint64_t MBMI = MI.getOperand(3).getImm(); uint64_t MESrc = SrcMI->getOperand(4).getImm(); uint64_t MEMI = MI.getOperand(4).getImm(); assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) && "Invalid PPC::RLWINM Instruction!"); // If MBMI is bigger than MEMI, we always can not get run of ones. // RotatedSrcMask non-wrap: // 0........31|32........63 // RotatedSrcMask: B---E B---E // MaskMI: -----------|--E B------ // Result: ----- --- (Bad candidate) // // RotatedSrcMask wrap: // 0........31|32........63 // RotatedSrcMask: --E B----|--E B---- // MaskMI: -----------|--E B------ // Result: --- -----|--- ----- (Bad candidate) // // One special case is RotatedSrcMask is a full set mask. // RotatedSrcMask full: // 0........31|32........63 // RotatedSrcMask: ------EB---|-------EB--- // MaskMI: -----------|--E B------ // Result: -----------|--- ------- (Good candidate) // Mark special case. bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31); // For other MBMI > MEMI cases, just return. if ((MBMI > MEMI) && !SrcMaskFull) return false; // Handle MBMI <= MEMI cases. APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI); // In MI, we only need low 32 bits of SrcMI, just consider about low 32 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0, // while in PowerPC ISA, lowerest bit is at index 63. APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc); APInt RotatedSrcMask = MaskSrc.rotl(SHMI); APInt FinalMask = RotatedSrcMask & MaskMI; uint32_t NewMB, NewME; bool Simplified = false; // If final mask is 0, MI result should be 0 too. if (FinalMask.isNullValue()) { + bool Is64Bit = + (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec); Simplified = true; LLVM_DEBUG(dbgs() << "Replace Instr: "); LLVM_DEBUG(MI.dump()); if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) { // Replace MI with "LI 0" MI.RemoveOperand(4); MI.RemoveOperand(3); MI.RemoveOperand(2); MI.getOperand(1).ChangeToImmediate(0); MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI)); } else { // Replace MI with "ANDI_rec reg, 0" MI.RemoveOperand(4); MI.RemoveOperand(3); MI.getOperand(2).setImm(0); MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec)); MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg()); if (SrcMI->getOperand(1).isKill()) { MI.getOperand(1).setIsKill(true); SrcMI->getOperand(1).setIsKill(false); } else // About to replace MI.getOperand(1), clear its kill flag. MI.getOperand(1).setIsKill(false); } LLVM_DEBUG(dbgs() << "With: "); LLVM_DEBUG(MI.dump()); } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) && NewMB <= NewME) || SrcMaskFull) { // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger // than NewME. Otherwise we get a 64 bit value after folding, but MI // return a 32 bit value. Simplified = true; LLVM_DEBUG(dbgs() << "Converting Instr: "); LLVM_DEBUG(MI.dump()); uint16_t NewSH = (SHSrc + SHMI) % 32; MI.getOperand(2).setImm(NewSH); // If SrcMI mask is full, no need to update MBMI and MEMI. if (!SrcMaskFull) { MI.getOperand(3).setImm(NewMB); MI.getOperand(4).setImm(NewME); } MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg()); if (SrcMI->getOperand(1).isKill()) { MI.getOperand(1).setIsKill(true); SrcMI->getOperand(1).setIsKill(false); } else // About to replace MI.getOperand(1), clear its kill flag. MI.getOperand(1).setIsKill(false); LLVM_DEBUG(dbgs() << "To: "); LLVM_DEBUG(MI.dump()); } - if (Simplified && !SrcMI->hasImplicitDef()) { - // If SrcMI has no implicit def, and FoldingReg has no non-debug use or - // its flag is "killed", it's safe to delete SrcMI. Otherwise keep it. - if ((!MRI->isSSA() && NoUse) || - (MRI->isSSA() && MRI->use_nodbg_empty(FoldingReg))) { - ToErase = SrcMI; - LLVM_DEBUG(dbgs() << "Delete dead instruction: "); - LLVM_DEBUG(SrcMI->dump()); - } + if (Simplified & MRI->use_nodbg_empty(FoldingReg) && + !SrcMI->hasImplicitDef()) { + // If FoldingReg has no non-debug use and it has no implicit def (it + // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI. + // Otherwise keep it. + *ToErase = SrcMI; + LLVM_DEBUG(dbgs() << "Delete dead instruction: "); + LLVM_DEBUG(SrcMI->dump()); } return Simplified; } bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg, ImmInstrInfo &III, bool PostRA) const { // The vast majority of the instructions would need their operand 2 replaced // with an immediate when switching to the reg+imm form. A marked exception // are the update form loads/stores for which a constant operand 2 would need // to turn into a displacement and move operand 1 to the operand 2 position. III.ImmOpNo = 2; III.OpNoForForwarding = 2; III.ImmWidth = 16; III.ImmMustBeMultipleOf = 1; III.TruncateImmTo = 0; III.IsSummingOperands = false; switch (Opc) { default: return false; case PPC::ADD4: case PPC::ADD8: III.SignedImm = true; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 1; III.IsCommutative = true; III.IsSummingOperands = true; III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8; break; case PPC::ADDC: case PPC::ADDC8: III.SignedImm = true; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = true; III.IsSummingOperands = true; III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8; break; case PPC::ADDC_rec: III.SignedImm = true; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = true; III.IsSummingOperands = true; III.ImmOpcode = PPC::ADDIC_rec; break; case PPC::SUBFC: case PPC::SUBFC8: III.SignedImm = true; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = false; III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8; break; case PPC::CMPW: case PPC::CMPD: III.SignedImm = true; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = false; III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI; break; case PPC::CMPLW: case PPC::CMPLD: III.SignedImm = false; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = false; III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI; break; case PPC::AND_rec: case PPC::AND8_rec: case PPC::OR: case PPC::OR8: case PPC::XOR: case PPC::XOR8: III.SignedImm = false; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = true; switch(Opc) { default: llvm_unreachable("Unknown opcode"); case PPC::AND_rec: III.ImmOpcode = PPC::ANDI_rec; break; case PPC::AND8_rec: III.ImmOpcode = PPC::ANDI8_rec; break; case PPC::OR: III.ImmOpcode = PPC::ORI; break; case PPC::OR8: III.ImmOpcode = PPC::ORI8; break; case PPC::XOR: III.ImmOpcode = PPC::XORI; break; case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break; } break; case PPC::RLWNM: case PPC::RLWNM8: case PPC::RLWNM_rec: case PPC::RLWNM8_rec: case PPC::SLW: case PPC::SLW8: case PPC::SLW_rec: case PPC::SLW8_rec: case PPC::SRW: case PPC::SRW8: case PPC::SRW_rec: case PPC::SRW8_rec: case PPC::SRAW: case PPC::SRAW_rec: III.SignedImm = false; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = false; // This isn't actually true, but the instructions ignore any of the // upper bits, so any immediate loaded with an LI is acceptable. // This does not apply to shift right algebraic because a value // out of range will produce a -1/0. III.ImmWidth = 16; if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec || Opc == PPC::RLWNM8_rec) III.TruncateImmTo = 5; else III.TruncateImmTo = 6; switch(Opc) { default: llvm_unreachable("Unknown opcode"); case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break; case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break; case PPC::RLWNM_rec: III.ImmOpcode = PPC::RLWINM_rec; break; case PPC::RLWNM8_rec: III.ImmOpcode = PPC::RLWINM8_rec; break; case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break; case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break; case PPC::SLW_rec: III.ImmOpcode = PPC::RLWINM_rec; break; case PPC::SLW8_rec: III.ImmOpcode = PPC::RLWINM8_rec; break; case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break; case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break; case PPC::SRW_rec: III.ImmOpcode = PPC::RLWINM_rec; break; case PPC::SRW8_rec: III.ImmOpcode = PPC::RLWINM8_rec; break; case PPC::SRAW: III.ImmWidth = 5; III.TruncateImmTo = 0; III.ImmOpcode = PPC::SRAWI; break; case PPC::SRAW_rec: III.ImmWidth = 5; III.TruncateImmTo = 0; III.ImmOpcode = PPC::SRAWI_rec; break; } break; case PPC::RLDCL: case PPC::RLDCL_rec: case PPC::RLDCR: case PPC::RLDCR_rec: case PPC::SLD: case PPC::SLD_rec: case PPC::SRD: case PPC::SRD_rec: case PPC::SRAD: case PPC::SRAD_rec: III.SignedImm = false; III.ZeroIsSpecialOrig = 0; III.ZeroIsSpecialNew = 0; III.IsCommutative = false; // This isn't actually true, but the instructions ignore any of the // upper bits, so any immediate loaded with an LI is acceptable. // This does not apply to shift right algebraic because a value // out of range will produce a -1/0. III.ImmWidth = 16; if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR || Opc == PPC::RLDCR_rec) III.TruncateImmTo = 6; else III.TruncateImmTo = 7; switch(Opc) { default: llvm_unreachable("Unknown opcode"); case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break; case PPC::RLDCL_rec: III.ImmOpcode = PPC::RLDICL_rec; break; case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break; case PPC::RLDCR_rec: III.ImmOpcode = PPC::RLDICR_rec; break; case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break; case PPC::SLD_rec: III.ImmOpcode = PPC::RLDICR_rec; break; case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break; case PPC::SRD_rec: III.ImmOpcode = PPC::RLDICL_rec; break; case PPC::SRAD: III.ImmWidth = 6; III.TruncateImmTo = 0; III.ImmOpcode = PPC::SRADI; break; case PPC::SRAD_rec: III.ImmWidth = 6; III.TruncateImmTo = 0; III.ImmOpcode = PPC::SRADI_rec; break; } break; // Loads and stores: case PPC::LBZX: case PPC::LBZX8: case PPC::LHZX: case PPC::LHZX8: case PPC::LHAX: case PPC::LHAX8: case PPC::LWZX: case PPC::LWZX8: case PPC::LWAX: case PPC::LDX: case PPC::LFSX: case PPC::LFDX: case PPC::STBX: case PPC::STBX8: case PPC::STHX: case PPC::STHX8: case PPC::STWX: case PPC::STWX8: case PPC::STDX: case PPC::STFSX: case PPC::STFDX: III.SignedImm = true; III.ZeroIsSpecialOrig = 1; III.ZeroIsSpecialNew = 2; III.IsCommutative = true; III.IsSummingOperands = true; III.ImmOpNo = 1; III.OpNoForForwarding = 2; switch(Opc) { default: llvm_unreachable("Unknown opcode"); case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break; case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break; case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break; case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break; case PPC::LHAX: III.ImmOpcode = PPC::LHA; break; case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break; case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break; case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break; case PPC::LWAX: III.ImmOpcode = PPC::LWA; III.ImmMustBeMultipleOf = 4; break; case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break; case PPC::LFSX: III.ImmOpcode = PPC::LFS; break; case PPC::LFDX: III.ImmOpcode = PPC::LFD; break; case PPC::STBX: III.ImmOpcode = PPC::STB; break; case PPC::STBX8: III.ImmOpcode = PPC::STB8; break; case PPC::STHX: III.ImmOpcode = PPC::STH; break; case PPC::STHX8: III.ImmOpcode = PPC::STH8; break; case PPC::STWX: III.ImmOpcode = PPC::STW; break; case PPC::STWX8: III.ImmOpcode = PPC::STW8; break; case PPC::STDX: III.ImmOpcode = PPC::STD; III.ImmMustBeMultipleOf = 4; break; case PPC::STFSX: III.ImmOpcode = PPC::STFS; break; case PPC::STFDX: III.ImmOpcode = PPC::STFD; break; } break; case PPC::LBZUX: case PPC::LBZUX8: case PPC::LHZUX: case PPC::LHZUX8: case PPC::LHAUX: case PPC::LHAUX8: case PPC::LWZUX: case PPC::LWZUX8: case PPC::LDUX: case PPC::LFSUX: case PPC::LFDUX: case PPC::STBUX: case PPC::STBUX8: case PPC::STHUX: case PPC::STHUX8: case PPC::STWUX: case PPC::STWUX8: case PPC::STDUX: case PPC::STFSUX: case PPC::STFDUX: III.SignedImm = true; III.ZeroIsSpecialOrig = 2; III.ZeroIsSpecialNew = 3; III.IsCommutative = false; III.IsSummingOperands = true; III.ImmOpNo = 2; III.OpNoForForwarding = 3; switch(Opc) { default: llvm_unreachable("Unknown opcode"); case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break; case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break; case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break; case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break; case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break; case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break; case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break; case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break; case PPC::LDUX: III.ImmOpcode = PPC::LDU; III.ImmMustBeMultipleOf = 4; break; case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break; case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break; case PPC::STBUX: III.ImmOpcode = PPC::STBU; break; case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break; case PPC::STHUX: III.ImmOpcode = PPC::STHU; break; case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break; case PPC::STWUX: III.ImmOpcode = PPC::STWU; break; case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break; case PPC::STDUX: III.ImmOpcode = PPC::STDU; III.ImmMustBeMultipleOf = 4; break; case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break; case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break; } break; // Power9 and up only. For some of these, the X-Form version has access to all // 64 VSR's whereas the D-Form only has access to the VR's. We replace those // with pseudo-ops pre-ra and for post-ra, we check that the register loaded // into or stored from is one of the VR registers. case PPC::LXVX: case PPC::LXSSPX: case PPC::LXSDX: case PPC::STXVX: case PPC::STXSSPX: case PPC::STXSDX: case PPC::XFLOADf32: case PPC::XFLOADf64: case PPC::XFSTOREf32: case PPC::XFSTOREf64: if (!Subtarget.hasP9Vector()) return false; III.SignedImm = true; III.ZeroIsSpecialOrig = 1; III.ZeroIsSpecialNew = 2; III.IsCommutative = true; III.IsSummingOperands = true; III.ImmOpNo = 1; III.OpNoForForwarding = 2; III.ImmMustBeMultipleOf = 4; switch(Opc) { default: llvm_unreachable("Unknown opcode"); case PPC::LXVX: III.ImmOpcode = PPC::LXV; III.ImmMustBeMultipleOf = 16; break; case PPC::LXSSPX: if (PostRA) { if (IsVFReg) III.ImmOpcode = PPC::LXSSP; else { III.ImmOpcode = PPC::LFS; III.ImmMustBeMultipleOf = 1; } break; } LLVM_FALLTHROUGH; case PPC::XFLOADf32: III.ImmOpcode = PPC::DFLOADf32; break; case PPC::LXSDX: if (PostRA) { if (IsVFReg) III.ImmOpcode = PPC::LXSD; else { III.ImmOpcode = PPC::LFD; III.ImmMustBeMultipleOf = 1; } break; } LLVM_FALLTHROUGH; case PPC::XFLOADf64: III.ImmOpcode = PPC::DFLOADf64; break; case PPC::STXVX: III.ImmOpcode = PPC::STXV; III.ImmMustBeMultipleOf = 16; break; case PPC::STXSSPX: if (PostRA) { if (IsVFReg) III.ImmOpcode = PPC::STXSSP; else { III.ImmOpcode = PPC::STFS; III.ImmMustBeMultipleOf = 1; } break; } LLVM_FALLTHROUGH; case PPC::XFSTOREf32: III.ImmOpcode = PPC::DFSTOREf32; break; case PPC::STXSDX: if (PostRA) { if (IsVFReg) III.ImmOpcode = PPC::STXSD; else { III.ImmOpcode = PPC::STFD; III.ImmMustBeMultipleOf = 1; } break; } LLVM_FALLTHROUGH; case PPC::XFSTOREf64: III.ImmOpcode = PPC::DFSTOREf64; break; } break; } return true; } // Utility function for swaping two arbitrary operands of an instruction. static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) { assert(Op1 != Op2 && "Cannot swap operand with itself."); unsigned MaxOp = std::max(Op1, Op2); unsigned MinOp = std::min(Op1, Op2); MachineOperand MOp1 = MI.getOperand(MinOp); MachineOperand MOp2 = MI.getOperand(MaxOp); MI.RemoveOperand(std::max(Op1, Op2)); MI.RemoveOperand(std::min(Op1, Op2)); // If the operands we are swapping are the two at the end (the common case) // we can just remove both and add them in the opposite order. if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) { MI.addOperand(MOp2); MI.addOperand(MOp1); } else { // Store all operands in a temporary vector, remove them and re-add in the // right order. SmallVector MOps; unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops. for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) { MOps.push_back(MI.getOperand(i)); MI.RemoveOperand(i); } // MOp2 needs to be added next. MI.addOperand(MOp2); // Now add the rest. for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) { if (i == MaxOp) MI.addOperand(MOp1); else { MI.addOperand(MOps.back()); MOps.pop_back(); } } } } // Check if the 'MI' that has the index OpNoForForwarding // meets the requirement described in the ImmInstrInfo. bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding ) const { // As the algorithm of checking for PPC::ZERO/PPC::ZERO8 // would not work pre-RA, we can only do the check post RA. MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); if (MRI.isSSA()) return false; // Cannot do the transform if MI isn't summing the operands. if (!III.IsSummingOperands) return false; // The instruction we are trying to replace must have the ZeroIsSpecialOrig set. if (!III.ZeroIsSpecialOrig) return false; // We cannot do the transform if the operand we are trying to replace // isn't the same as the operand the instruction allows. if (OpNoForForwarding != III.OpNoForForwarding) return false; // Check if the instruction we are trying to transform really has // the special zero register as its operand. if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO && MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8) return false; // This machine instruction is convertible if it is, // 1. summing the operands. // 2. one of the operands is special zero register. // 3. the operand we are trying to replace is allowed by the MI. return true; } // Check if the DefMI is the add inst and set the ImmMO and RegMO // accordingly. bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI, const ImmInstrInfo &III, MachineOperand *&ImmMO, MachineOperand *&RegMO) const { unsigned Opc = DefMI.getOpcode(); if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8) return false; assert(DefMI.getNumOperands() >= 3 && "Add inst must have at least three operands"); RegMO = &DefMI.getOperand(1); ImmMO = &DefMI.getOperand(2); // Before RA, ADDI first operand could be a frame index. if (!RegMO->isReg()) return false; // This DefMI is elgible for forwarding if it is: // 1. add inst // 2. one of the operands is Imm/CPI/Global. return isAnImmediateOperand(*ImmMO); } bool PPCInstrInfo::isRegElgibleForForwarding( const MachineOperand &RegMO, const MachineInstr &DefMI, const MachineInstr &MI, bool KillDefMI, bool &IsFwdFeederRegKilled) const { // x = addi y, imm // ... // z = lfdx 0, x -> z = lfd imm(y) // The Reg "y" can be forwarded to the MI(z) only when there is no DEF // of "y" between the DEF of "x" and "z". // The query is only valid post RA. const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); if (MRI.isSSA()) return false; Register Reg = RegMO.getReg(); // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg. MachineBasicBlock::const_reverse_iterator It = MI; MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend(); It++; for (; It != E; ++It) { if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI) return false; else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI) IsFwdFeederRegKilled = true; // Made it to DefMI without encountering a clobber. if ((&*It) == &DefMI) break; } assert((&*It) == &DefMI && "DefMI is missing"); // If DefMI also defines the register to be forwarded, we can only forward it // if DefMI is being erased. if (DefMI.modifiesRegister(Reg, &getRegisterInfo())) return KillDefMI; return true; } bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO, const MachineInstr &DefMI, const ImmInstrInfo &III, int64_t &Imm, int64_t BaseImm) const { assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate"); if (DefMI.getOpcode() == PPC::ADDItocL) { // The operand for ADDItocL is CPI, which isn't imm at compiling time, // However, we know that, it is 16-bit width, and has the alignment of 4. // Check if the instruction met the requirement. if (III.ImmMustBeMultipleOf > 4 || III.TruncateImmTo || III.ImmWidth != 16) return false; // Going from XForm to DForm loads means that the displacement needs to be // not just an immediate but also a multiple of 4, or 16 depending on the // load. A DForm load cannot be represented if it is a multiple of say 2. // XForm loads do not have this restriction. if (ImmMO.isGlobal()) { const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout(); if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf) return false; } return true; } if (ImmMO.isImm()) { // It is Imm, we need to check if the Imm fit the range. // Sign-extend to 64-bits. // DefMI may be folded with another imm form instruction, the result Imm is // the sum of Imm of DefMI and BaseImm which is from imm form instruction. Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm); if (Imm % III.ImmMustBeMultipleOf) return false; if (III.TruncateImmTo) Imm &= ((1 << III.TruncateImmTo) - 1); if (III.SignedImm) { APInt ActualValue(64, Imm, true); if (!ActualValue.isSignedIntN(III.ImmWidth)) return false; } else { uint64_t UnsignedMax = (1 << III.ImmWidth) - 1; if ((uint64_t)Imm > UnsignedMax) return false; } } else return false; // This ImmMO is forwarded if it meets the requriement describle // in ImmInstrInfo return true; } bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding, MachineInstr **KilledDef) const { if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) || !DefMI.getOperand(1).isImm()) return false; MachineFunction *MF = MI.getParent()->getParent(); MachineRegisterInfo *MRI = &MF->getRegInfo(); bool PostRA = !MRI->isSSA(); int64_t Immediate = DefMI.getOperand(1).getImm(); // Sign-extend to 64-bits. int64_t SExtImm = SignExtend64<16>(Immediate); bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill(); Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg(); bool ReplaceWithLI = false; bool Is64BitLI = false; int64_t NewImm = 0; bool SetCR = false; unsigned Opc = MI.getOpcode(); switch (Opc) { default: return false; // FIXME: Any branches conditional on such a comparison can be made // unconditional. At this time, this happens too infrequently to be worth // the implementation effort, but if that ever changes, we could convert // such a pattern here. case PPC::CMPWI: case PPC::CMPLWI: case PPC::CMPDI: case PPC::CMPLDI: { // Doing this post-RA would require dataflow analysis to reliably find uses // of the CR register set by the compare. // No need to fixup killed/dead flag since this transformation is only valid // before RA. if (PostRA) return false; // If a compare-immediate is fed by an immediate and is itself an input of // an ISEL (the most common case) into a COPY of the correct register. bool Changed = false; Register DefReg = MI.getOperand(0).getReg(); int64_t Comparand = MI.getOperand(2).getImm(); int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0 ? (Comparand | 0xFFFFFFFFFFFF0000) : Comparand; for (auto &CompareUseMI : MRI->use_instructions(DefReg)) { unsigned UseOpc = CompareUseMI.getOpcode(); if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8) continue; unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg(); Register TrueReg = CompareUseMI.getOperand(1).getReg(); Register FalseReg = CompareUseMI.getOperand(2).getReg(); unsigned RegToCopy = selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg); if (RegToCopy == PPC::NoRegister) continue; // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0. if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) { CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI)); replaceInstrOperandWithImm(CompareUseMI, 1, 0); CompareUseMI.RemoveOperand(3); CompareUseMI.RemoveOperand(2); continue; } LLVM_DEBUG( dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n"); LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump()); LLVM_DEBUG(dbgs() << "Is converted to:\n"); // Convert to copy and remove unneeded operands. CompareUseMI.setDesc(get(PPC::COPY)); CompareUseMI.RemoveOperand(3); CompareUseMI.RemoveOperand(RegToCopy == TrueReg ? 2 : 1); CmpIselsConverted++; Changed = true; LLVM_DEBUG(CompareUseMI.dump()); } if (Changed) return true; // This may end up incremented multiple times since this function is called // during a fixed-point transformation, but it is only meant to indicate the // presence of this opportunity. MissedConvertibleImmediateInstrs++; return false; } // Immediate forms - may simply be convertable to an LI. case PPC::ADDI: case PPC::ADDI8: { // Does the sum fit in a 16-bit signed field? int64_t Addend = MI.getOperand(2).getImm(); if (isInt<16>(Addend + SExtImm)) { ReplaceWithLI = true; Is64BitLI = Opc == PPC::ADDI8; NewImm = Addend + SExtImm; break; } return false; } case PPC::SUBFIC: case PPC::SUBFIC8: { // Only transform this if the CARRY implicit operand is dead. if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead()) return false; int64_t Minuend = MI.getOperand(2).getImm(); if (isInt<16>(Minuend - SExtImm)) { ReplaceWithLI = true; Is64BitLI = Opc == PPC::SUBFIC8; NewImm = Minuend - SExtImm; break; } return false; } case PPC::RLDICL: case PPC::RLDICL_rec: case PPC::RLDICL_32: case PPC::RLDICL_32_64: { // Use APInt's rotate function. int64_t SH = MI.getOperand(2).getImm(); int64_t MB = MI.getOperand(3).getImm(); APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32, SExtImm, true); InVal = InVal.rotl(SH); uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1; InVal &= Mask; // Can't replace negative values with an LI as that will sign-extend // and not clear the left bits. If we're setting the CR bit, we will use // ANDI_rec which won't sign extend, so that's safe. if (isUInt<15>(InVal.getSExtValue()) || (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) { ReplaceWithLI = true; Is64BitLI = Opc != PPC::RLDICL_32; NewImm = InVal.getSExtValue(); SetCR = Opc == PPC::RLDICL_rec; break; } return false; } case PPC::RLWINM: case PPC::RLWINM8: case PPC::RLWINM_rec: case PPC::RLWINM8_rec: { int64_t SH = MI.getOperand(2).getImm(); int64_t MB = MI.getOperand(3).getImm(); int64_t ME = MI.getOperand(4).getImm(); APInt InVal(32, SExtImm, true); InVal = InVal.rotl(SH); APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB); InVal &= Mask; // Can't replace negative values with an LI as that will sign-extend // and not clear the left bits. If we're setting the CR bit, we will use // ANDI_rec which won't sign extend, so that's safe. bool ValueFits = isUInt<15>(InVal.getSExtValue()); ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) && isUInt<16>(InVal.getSExtValue())); if (ValueFits) { ReplaceWithLI = true; Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec; NewImm = InVal.getSExtValue(); SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec; break; } return false; } case PPC::ORI: case PPC::ORI8: case PPC::XORI: case PPC::XORI8: { int64_t LogicalImm = MI.getOperand(2).getImm(); int64_t Result = 0; if (Opc == PPC::ORI || Opc == PPC::ORI8) Result = LogicalImm | SExtImm; else Result = LogicalImm ^ SExtImm; if (isInt<16>(Result)) { ReplaceWithLI = true; Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8; NewImm = Result; break; } return false; } } if (ReplaceWithLI) { // We need to be careful with CR-setting instructions we're replacing. if (SetCR) { // We don't know anything about uses when we're out of SSA, so only // replace if the new immediate will be reproduced. bool ImmChanged = (SExtImm & NewImm) != NewImm; if (PostRA && ImmChanged) return false; if (!PostRA) { // If the defining load-immediate has no other uses, we can just replace // the immediate with the new immediate. if (MRI->hasOneUse(DefMI.getOperand(0).getReg())) DefMI.getOperand(1).setImm(NewImm); // If we're not using the GPR result of the CR-setting instruction, we // just need to and with zero/non-zero depending on the new immediate. else if (MRI->use_empty(MI.getOperand(0).getReg())) { if (NewImm) { assert(Immediate && "Transformation converted zero to non-zero?"); NewImm = Immediate; } } else if (ImmChanged) return false; } } LLVM_DEBUG(dbgs() << "Replacing instruction:\n"); LLVM_DEBUG(MI.dump()); LLVM_DEBUG(dbgs() << "Fed by:\n"); LLVM_DEBUG(DefMI.dump()); LoadImmediateInfo LII; LII.Imm = NewImm; LII.Is64Bit = Is64BitLI; LII.SetCR = SetCR; // If we're setting the CR, the original load-immediate must be kept (as an // operand to ANDI_rec/ANDI8_rec). if (KilledDef && SetCR) *KilledDef = nullptr; replaceInstrWithLI(MI, LII); // Fixup killed/dead flag after transformation. // Pattern: // ForwardingOperandReg = LI imm1 // y = op2 imm2, ForwardingOperandReg(killed) if (IsForwardingOperandKilled) fixupIsDeadOrKill(&DefMI, &MI, ForwardingOperandReg); LLVM_DEBUG(dbgs() << "With:\n"); LLVM_DEBUG(MI.dump()); return true; } return false; } bool PPCInstrInfo::transformToNewImmFormFedByAdd( MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const { MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo(); bool PostRA = !MRI->isSSA(); // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI // for post-ra. if (PostRA) return false; // Only handle load/store. if (!MI.mayLoadOrStore()) return false; unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode()); assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) && "MI must have x-form opcode"); // get Imm Form info. ImmInstrInfo III; bool IsVFReg = MI.getOperand(0).isReg() ? isVFRegister(MI.getOperand(0).getReg()) : false; if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA)) return false; if (!III.IsSummingOperands) return false; if (OpNoForForwarding != III.OpNoForForwarding) return false; MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo); if (!ImmOperandMI.isImm()) return false; // Check DefMI. MachineOperand *ImmMO = nullptr; MachineOperand *RegMO = nullptr; if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO)) return false; assert(ImmMO && RegMO && "Imm and Reg operand must have been set"); // Check Imm. // Set ImmBase from imm instruction as base and get new Imm inside // isImmElgibleForForwarding. int64_t ImmBase = ImmOperandMI.getImm(); int64_t Imm = 0; if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase)) return false; // Get killed info in case fixup needed after transformation. unsigned ForwardKilledOperandReg = ~0U; if (MI.getOperand(III.OpNoForForwarding).isKill()) ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg(); // Do the transform LLVM_DEBUG(dbgs() << "Replacing instruction:\n"); LLVM_DEBUG(MI.dump()); LLVM_DEBUG(dbgs() << "Fed by:\n"); LLVM_DEBUG(DefMI.dump()); MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg()); MI.getOperand(III.OpNoForForwarding).setIsKill(RegMO->isKill()); MI.getOperand(III.ImmOpNo).setImm(Imm); // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block. if (DefMI.getParent() == MI.getParent()) { // Check if reg is killed between MI and DefMI. auto IsKilledFor = [&](unsigned Reg) { MachineBasicBlock::const_reverse_iterator It = MI; MachineBasicBlock::const_reverse_iterator E = DefMI; It++; for (; It != E; ++It) { if (It->killsRegister(Reg)) return true; } return false; }; // Update kill flag if (RegMO->isKill() || IsKilledFor(RegMO->getReg())) fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg()); if (ForwardKilledOperandReg != ~0U) fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg); } LLVM_DEBUG(dbgs() << "With:\n"); LLVM_DEBUG(MI.dump()); return true; } // If an X-Form instruction is fed by an add-immediate and one of its operands // is the literal zero, attempt to forward the source of the add-immediate to // the corresponding D-Form instruction with the displacement coming from // the immediate being added. bool PPCInstrInfo::transformToImmFormFedByAdd( MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding, MachineInstr &DefMI, bool KillDefMI) const { // RegMO ImmMO // | | // x = addi reg, imm <----- DefMI // y = op 0 , x <----- MI // | // OpNoForForwarding // Check if the MI meet the requirement described in the III. if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding)) return false; // Check if the DefMI meet the requirement // described in the III. If yes, set the ImmMO and RegMO accordingly. MachineOperand *ImmMO = nullptr; MachineOperand *RegMO = nullptr; if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO)) return false; assert(ImmMO && RegMO && "Imm and Reg operand must have been set"); // As we get the Imm operand now, we need to check if the ImmMO meet // the requirement described in the III. If yes set the Imm. int64_t Imm = 0; if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm)) return false; bool IsFwdFeederRegKilled = false; // Check if the RegMO can be forwarded to MI. if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI, IsFwdFeederRegKilled)) return false; // Get killed info in case fixup needed after transformation. unsigned ForwardKilledOperandReg = ~0U; MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); bool PostRA = !MRI.isSSA(); if (PostRA && MI.getOperand(OpNoForForwarding).isKill()) ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg(); // We know that, the MI and DefMI both meet the pattern, and // the Imm also meet the requirement with the new Imm-form. // It is safe to do the transformation now. LLVM_DEBUG(dbgs() << "Replacing instruction:\n"); LLVM_DEBUG(MI.dump()); LLVM_DEBUG(dbgs() << "Fed by:\n"); LLVM_DEBUG(DefMI.dump()); // Update the base reg first. MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(), false, false, RegMO->isKill()); // Then, update the imm. if (ImmMO->isImm()) { // If the ImmMO is Imm, change the operand that has ZERO to that Imm // directly. replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm); } else { // Otherwise, it is Constant Pool Index(CPI) or Global, // which is relocation in fact. We need to replace the special zero // register with ImmMO. // Before that, we need to fixup the target flags for imm. // For some reason, we miss to set the flag for the ImmMO if it is CPI. if (DefMI.getOpcode() == PPC::ADDItocL) ImmMO->setTargetFlags(PPCII::MO_TOC_LO); // MI didn't have the interface such as MI.setOperand(i) though // it has MI.getOperand(i). To repalce the ZERO MachineOperand with // ImmMO, we need to remove ZERO operand and all the operands behind it, // and, add the ImmMO, then, move back all the operands behind ZERO. SmallVector MOps; for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) { MOps.push_back(MI.getOperand(i)); MI.RemoveOperand(i); } // Remove the last MO in the list, which is ZERO operand in fact. MOps.pop_back(); // Add the imm operand. MI.addOperand(*ImmMO); // Now add the rest back. for (auto &MO : MOps) MI.addOperand(MO); } // Update the opcode. MI.setDesc(get(III.ImmOpcode)); // Fix up killed/dead flag after transformation. // Pattern 1: // x = ADD KilledFwdFeederReg, imm // n = opn KilledFwdFeederReg(killed), regn // y = XOP 0, x // Pattern 2: // x = ADD reg(killed), imm // y = XOP 0, x if (IsFwdFeederRegKilled || RegMO->isKill()) fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg()); // Pattern 3: // ForwardKilledOperandReg = ADD reg, imm // y = XOP 0, ForwardKilledOperandReg(killed) if (ForwardKilledOperandReg != ~0U) fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg); LLVM_DEBUG(dbgs() << "With:\n"); LLVM_DEBUG(MI.dump()); return true; } bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI, const ImmInstrInfo &III, unsigned ConstantOpNo, MachineInstr &DefMI) const { // DefMI must be LI or LI8. if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) || !DefMI.getOperand(1).isImm()) return false; // Get Imm operand and Sign-extend to 64-bits. int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm()); MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); bool PostRA = !MRI.isSSA(); // Exit early if we can't convert this. if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative) return false; if (Imm % III.ImmMustBeMultipleOf) return false; if (III.TruncateImmTo) Imm &= ((1 << III.TruncateImmTo) - 1); if (III.SignedImm) { APInt ActualValue(64, Imm, true); if (!ActualValue.isSignedIntN(III.ImmWidth)) return false; } else { uint64_t UnsignedMax = (1 << III.ImmWidth) - 1; if ((uint64_t)Imm > UnsignedMax) return false; } // If we're post-RA, the instructions don't agree on whether register zero is // special, we can transform this as long as the register operand that will // end up in the location where zero is special isn't R0. if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) { unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig : III.ZeroIsSpecialNew + 1; Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg(); Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg(); // If R0 is in the operand where zero is special for the new instruction, // it is unsafe to transform if the constant operand isn't that operand. if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) && ConstantOpNo != III.ZeroIsSpecialNew) return false; if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) && ConstantOpNo != PosForOrigZero) return false; } // Get killed info in case fixup needed after transformation. unsigned ForwardKilledOperandReg = ~0U; if (PostRA && MI.getOperand(ConstantOpNo).isKill()) ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg(); unsigned Opc = MI.getOpcode(); bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec || Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SLW8 || Opc == PPC::SLW8_rec || Opc == PPC::SRW8 || Opc == PPC::SRW8_rec; bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec || Opc == PPC::SRD || Opc == PPC::SRD_rec; bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec || Opc == PPC::SLD_rec || Opc == PPC::SRD_rec; bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD || Opc == PPC::SRD_rec; MI.setDesc(get(III.ImmOpcode)); if (ConstantOpNo == III.OpNoForForwarding) { // Converting shifts to immediate form is a bit tricky since they may do // one of three things: // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero // 2. If the shift amount is zero, the result is unchanged (save for maybe // setting CR0) // 3. If the shift amount is in [1, OpSize), it's just a shift if (SpecialShift32 || SpecialShift64) { LoadImmediateInfo LII; LII.Imm = 0; LII.SetCR = SetCR; LII.Is64Bit = SpecialShift64; uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F); if (Imm & (SpecialShift32 ? 0x20 : 0x40)) replaceInstrWithLI(MI, LII); // Shifts by zero don't change the value. If we don't need to set CR0, // just convert this to a COPY. Can't do this post-RA since we've already // cleaned up the copies. else if (!SetCR && ShAmt == 0 && !PostRA) { MI.RemoveOperand(2); MI.setDesc(get(PPC::COPY)); } else { // The 32 bit and 64 bit instructions are quite different. if (SpecialShift32) { // Left shifts use (N, 0, 31-N). // Right shifts use (32-N, N, 31) if 0 < N < 32. // use (0, 0, 31) if N == 0. uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt; uint64_t MB = RightShift ? ShAmt : 0; uint64_t ME = RightShift ? 31 : 31 - ShAmt; replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH); MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB) .addImm(ME); } else { // Left shifts use (N, 63-N). // Right shifts use (64-N, N) if 0 < N < 64. // use (0, 0) if N == 0. uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt; uint64_t ME = RightShift ? ShAmt : 63 - ShAmt; replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH); MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME); } } } else replaceInstrOperandWithImm(MI, ConstantOpNo, Imm); } // Convert commutative instructions (switch the operands and convert the // desired one to an immediate. else if (III.IsCommutative) { replaceInstrOperandWithImm(MI, ConstantOpNo, Imm); swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding); } else llvm_unreachable("Should have exited early!"); // For instructions for which the constant register replaces a different // operand than where the immediate goes, we need to swap them. if (III.OpNoForForwarding != III.ImmOpNo) swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo); // If the special R0/X0 register index are different for original instruction // and new instruction, we need to fix up the register class in new // instruction. if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) { if (III.ZeroIsSpecialNew) { // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no // need to fix up register class. Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg(); if (Register::isVirtualRegister(RegToModify)) { const TargetRegisterClass *NewRC = MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ? &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass; MRI.setRegClass(RegToModify, NewRC); } } } // Fix up killed/dead flag after transformation. // Pattern: // ForwardKilledOperandReg = LI imm // y = XOP reg, ForwardKilledOperandReg(killed) if (ForwardKilledOperandReg != ~0U) fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg); return true; } const TargetRegisterClass * PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const { if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass) return &PPC::VSRCRegClass; return RC; } int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) { return PPC::getRecordFormOpcode(Opcode); } // This function returns true if the machine instruction // always outputs a value by sign-extending a 32 bit value, // i.e. 0 to 31-th bits are same as 32-th bit. static bool isSignExtendingOp(const MachineInstr &MI) { int Opcode = MI.getOpcode(); if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS || Opcode == PPC::LIS8 || Opcode == PPC::SRAW || Opcode == PPC::SRAW_rec || Opcode == PPC::SRAWI || Opcode == PPC::SRAWI_rec || Opcode == PPC::LWA || Opcode == PPC::LWAX || Opcode == PPC::LWA_32 || Opcode == PPC::LWAX_32 || Opcode == PPC::LHA || Opcode == PPC::LHAX || Opcode == PPC::LHA8 || Opcode == PPC::LHAX8 || Opcode == PPC::LBZ || Opcode == PPC::LBZX || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU || Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 || Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::EXTSB || Opcode == PPC::EXTSB_rec || Opcode == PPC::EXTSH || Opcode == PPC::EXTSH_rec || Opcode == PPC::EXTSB8 || Opcode == PPC::EXTSH8 || Opcode == PPC::EXTSW || Opcode == PPC::EXTSW_rec || Opcode == PPC::SETB || Opcode == PPC::SETB8 || Opcode == PPC::EXTSH8_32_64 || Opcode == PPC::EXTSW_32_64 || Opcode == PPC::EXTSB8_32_64) return true; if (Opcode == PPC::RLDICL && MI.getOperand(3).getImm() >= 33) return true; if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec || Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) && MI.getOperand(3).getImm() > 0 && MI.getOperand(3).getImm() <= MI.getOperand(4).getImm()) return true; return false; } // This function returns true if the machine instruction // always outputs zeros in higher 32 bits. static bool isZeroExtendingOp(const MachineInstr &MI) { int Opcode = MI.getOpcode(); // The 16-bit immediate is sign-extended in li/lis. // If the most significant bit is zero, all higher bits are zero. if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS || Opcode == PPC::LIS8) { int64_t Imm = MI.getOperand(1).getImm(); if (((uint64_t)Imm & ~0x7FFFuLL) == 0) return true; } // We have some variations of rotate-and-mask instructions // that clear higher 32-bits. if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec || Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec || Opcode == PPC::RLDICL_32_64) && MI.getOperand(3).getImm() >= 32) return true; if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) && MI.getOperand(3).getImm() >= 32 && MI.getOperand(3).getImm() <= 63 - MI.getOperand(2).getImm()) return true; if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec || Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec || Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) && MI.getOperand(3).getImm() <= MI.getOperand(4).getImm()) return true; // There are other instructions that clear higher 32-bits. if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec || Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec || Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8 || Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec || Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec || Opcode == PPC::POPCNTD || Opcode == PPC::POPCNTW || Opcode == PPC::SLW || Opcode == PPC::SLW_rec || Opcode == PPC::SRW || Opcode == PPC::SRW_rec || Opcode == PPC::SLW8 || Opcode == PPC::SRW8 || Opcode == PPC::SLWI || Opcode == PPC::SLWI_rec || Opcode == PPC::SRWI || Opcode == PPC::SRWI_rec || Opcode == PPC::LWZ || Opcode == PPC::LWZX || Opcode == PPC::LWZU || Opcode == PPC::LWZUX || Opcode == PPC::LWBRX || Opcode == PPC::LHBRX || Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LBZ || Opcode == PPC::LBZX || Opcode == PPC::LBZU || Opcode == PPC::LBZUX || Opcode == PPC::LWZ8 || Opcode == PPC::LWZX8 || Opcode == PPC::LWZU8 || Opcode == PPC::LWZUX8 || Opcode == PPC::LWBRX8 || Opcode == PPC::LHBRX8 || Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 || Opcode == PPC::ANDI_rec || Opcode == PPC::ANDIS_rec || Opcode == PPC::ROTRWI || Opcode == PPC::ROTRWI_rec || Opcode == PPC::EXTLWI || Opcode == PPC::EXTLWI_rec || Opcode == PPC::MFVSRWZ) return true; return false; } // This function returns true if the input MachineInstr is a TOC save // instruction. bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const { if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg()) return false; unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); unsigned StackOffset = MI.getOperand(1).getImm(); Register StackReg = MI.getOperand(2).getReg(); if (StackReg == PPC::X1 && StackOffset == TOCSaveOffset) return true; return false; } // We limit the max depth to track incoming values of PHIs or binary ops // (e.g. AND) to avoid excessive cost. const unsigned MAX_DEPTH = 1; bool PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt, const unsigned Depth) const { const MachineFunction *MF = MI.getParent()->getParent(); const MachineRegisterInfo *MRI = &MF->getRegInfo(); // If we know this instruction returns sign- or zero-extended result, // return true. if (SignExt ? isSignExtendingOp(MI): isZeroExtendingOp(MI)) return true; switch (MI.getOpcode()) { case PPC::COPY: { Register SrcReg = MI.getOperand(1).getReg(); // In both ELFv1 and v2 ABI, method parameters and the return value // are sign- or zero-extended. if (MF->getSubtarget().isSVR4ABI()) { const PPCFunctionInfo *FuncInfo = MF->getInfo(); // We check the ZExt/SExt flags for a method parameter. if (MI.getParent()->getBasicBlock() == &MF->getFunction().getEntryBlock()) { Register VReg = MI.getOperand(0).getReg(); if (MF->getRegInfo().isLiveIn(VReg)) return SignExt ? FuncInfo->isLiveInSExt(VReg) : FuncInfo->isLiveInZExt(VReg); } // For a method return value, we check the ZExt/SExt flags in attribute. // We assume the following code sequence for method call. // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1 // BL8_NOP @func,... // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1 // %5 = COPY %x3; G8RC:%5 if (SrcReg == PPC::X3) { const MachineBasicBlock *MBB = MI.getParent(); MachineBasicBlock::const_instr_iterator II = MachineBasicBlock::const_instr_iterator(&MI); if (II != MBB->instr_begin() && (--II)->getOpcode() == PPC::ADJCALLSTACKUP) { const MachineInstr &CallMI = *(--II); if (CallMI.isCall() && CallMI.getOperand(0).isGlobal()) { const Function *CalleeFn = dyn_cast(CallMI.getOperand(0).getGlobal()); if (!CalleeFn) return false; const IntegerType *IntTy = dyn_cast(CalleeFn->getReturnType()); const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttributes(); if (IntTy && IntTy->getBitWidth() <= 32) return Attrs.hasAttribute(SignExt ? Attribute::SExt : Attribute::ZExt); } } } } // If this is a copy from another register, we recursively check source. if (!Register::isVirtualRegister(SrcReg)) return false; const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); if (SrcMI != NULL) return isSignOrZeroExtended(*SrcMI, SignExt, Depth); return false; } case PPC::ANDI_rec: case PPC::ANDIS_rec: case PPC::ORI: case PPC::ORIS: case PPC::XORI: case PPC::XORIS: case PPC::ANDI8_rec: case PPC::ANDIS8_rec: case PPC::ORI8: case PPC::ORIS8: case PPC::XORI8: case PPC::XORIS8: { // logical operation with 16-bit immediate does not change the upper bits. // So, we track the operand register as we do for register copy. Register SrcReg = MI.getOperand(1).getReg(); if (!Register::isVirtualRegister(SrcReg)) return false; const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); if (SrcMI != NULL) return isSignOrZeroExtended(*SrcMI, SignExt, Depth); return false; } // If all incoming values are sign-/zero-extended, // the output of OR, ISEL or PHI is also sign-/zero-extended. case PPC::OR: case PPC::OR8: case PPC::ISEL: case PPC::PHI: { if (Depth >= MAX_DEPTH) return false; // The input registers for PHI are operand 1, 3, ... // The input registers for others are operand 1 and 2. unsigned E = 3, D = 1; if (MI.getOpcode() == PPC::PHI) { E = MI.getNumOperands(); D = 2; } for (unsigned I = 1; I != E; I += D) { if (MI.getOperand(I).isReg()) { Register SrcReg = MI.getOperand(I).getReg(); if (!Register::isVirtualRegister(SrcReg)) return false; const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); if (SrcMI == NULL || !isSignOrZeroExtended(*SrcMI, SignExt, Depth+1)) return false; } else return false; } return true; } // If at least one of the incoming values of an AND is zero extended // then the output is also zero-extended. If both of the incoming values // are sign-extended then the output is also sign extended. case PPC::AND: case PPC::AND8: { if (Depth >= MAX_DEPTH) return false; assert(MI.getOperand(1).isReg() && MI.getOperand(2).isReg()); Register SrcReg1 = MI.getOperand(1).getReg(); Register SrcReg2 = MI.getOperand(2).getReg(); if (!Register::isVirtualRegister(SrcReg1) || !Register::isVirtualRegister(SrcReg2)) return false; const MachineInstr *MISrc1 = MRI->getVRegDef(SrcReg1); const MachineInstr *MISrc2 = MRI->getVRegDef(SrcReg2); if (!MISrc1 || !MISrc2) return false; if(SignExt) return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) && isSignOrZeroExtended(*MISrc2, SignExt, Depth+1); else return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) || isSignOrZeroExtended(*MISrc2, SignExt, Depth+1); } default: break; } return false; } bool PPCInstrInfo::isBDNZ(unsigned Opcode) const { return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ)); } namespace { class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo { MachineInstr *Loop, *EndLoop, *LoopCount; MachineFunction *MF; const TargetInstrInfo *TII; int64_t TripCount; public: PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop, MachineInstr *LoopCount) : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount), MF(Loop->getParent()->getParent()), TII(MF->getSubtarget().getInstrInfo()) { // Inspect the Loop instruction up-front, as it may be deleted when we call // createTripCountGreaterCondition. if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI) TripCount = LoopCount->getOperand(1).getImm(); else TripCount = -1; } bool shouldIgnoreForPipelining(const MachineInstr *MI) const override { // Only ignore the terminator. return MI == EndLoop; } Optional createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB, SmallVectorImpl &Cond) override { if (TripCount == -1) { // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1, // so we don't need to generate any thing here. Cond.push_back(MachineOperand::CreateImm(0)); Cond.push_back(MachineOperand::CreateReg( MF->getSubtarget().isPPC64() ? PPC::CTR8 : PPC::CTR, true)); return {}; } return TripCount > TC; } void setPreheader(MachineBasicBlock *NewPreheader) override { // Do nothing. We want the LOOP setup instruction to stay in the *old* // preheader, so we can use BDZ in the prologs to adapt the loop trip count. } void adjustTripCount(int TripCountAdjust) override { // If the loop trip count is a compile-time value, then just change the // value. if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI) { int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust; LoopCount->getOperand(1).setImm(TripCount); return; } // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1, // so we don't need to generate any thing here. } void disposed() override { Loop->eraseFromParent(); // Ensure the loop setup instruction is deleted too. LoopCount->eraseFromParent(); } }; } // namespace std::unique_ptr PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const { // We really "analyze" only hardware loops right now. MachineBasicBlock::iterator I = LoopBB->getFirstTerminator(); MachineBasicBlock *Preheader = *LoopBB->pred_begin(); if (Preheader == LoopBB) Preheader = *std::next(LoopBB->pred_begin()); MachineFunction *MF = Preheader->getParent(); if (I != LoopBB->end() && isBDNZ(I->getOpcode())) { SmallPtrSet Visited; if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) { Register LoopCountReg = LoopInst->getOperand(0).getReg(); MachineRegisterInfo &MRI = MF->getRegInfo(); MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg); return std::make_unique(LoopInst, &*I, LoopCount); } } return nullptr; } MachineInstr *PPCInstrInfo::findLoopInstr( MachineBasicBlock &PreHeader, SmallPtrSet &Visited) const { unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop); // The loop set-up instruction should be in preheader for (auto &I : PreHeader.instrs()) if (I.getOpcode() == LOOPi) return &I; return nullptr; } // Return true if get the base operand, byte offset of an instruction and the // memory width. Width is the size of memory that is being loaded/stored. bool PPCInstrInfo::getMemOperandWithOffsetWidth( const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset, unsigned &Width, const TargetRegisterInfo *TRI) const { if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3) return false; // Handle only loads/stores with base register followed by immediate offset. if (!LdSt.getOperand(1).isImm() || (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI())) return false; if (!LdSt.getOperand(1).isImm() || (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI())) return false; if (!LdSt.hasOneMemOperand()) return false; Width = (*LdSt.memoperands_begin())->getSize(); Offset = LdSt.getOperand(1).getImm(); BaseReg = &LdSt.getOperand(2); return true; } bool PPCInstrInfo::areMemAccessesTriviallyDisjoint( const MachineInstr &MIa, const MachineInstr &MIb) const { assert(MIa.mayLoadOrStore() && "MIa must be a load or store."); assert(MIb.mayLoadOrStore() && "MIb must be a load or store."); if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() || MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) return false; // Retrieve the base register, offset from the base register and width. Width // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If // base registers are identical, and the offset of a lower memory access + // the width doesn't overlap the offset of a higher memory access, // then the memory accesses are different. const TargetRegisterInfo *TRI = &getRegisterInfo(); const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; int64_t OffsetA = 0, OffsetB = 0; unsigned int WidthA = 0, WidthB = 0; if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { if (BaseOpA->isIdenticalTo(*BaseOpB)) { int LowOffset = std::min(OffsetA, OffsetB); int HighOffset = std::max(OffsetA, OffsetB); int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; if (LowOffset + LowWidth <= HighOffset) return true; } } return false; } diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h index 0a2564a51fce..f3ada5a0febc 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h @@ -1,659 +1,658 @@ //===-- PPCInstrInfo.h - PowerPC Instruction Information --------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains the PowerPC implementation of the TargetInstrInfo class. // //===----------------------------------------------------------------------===// #ifndef LLVM_LIB_TARGET_POWERPC_PPCINSTRINFO_H #define LLVM_LIB_TARGET_POWERPC_PPCINSTRINFO_H #include "PPCRegisterInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" #define GET_INSTRINFO_HEADER #include "PPCGenInstrInfo.inc" namespace llvm { /// PPCII - This namespace holds all of the PowerPC target-specific /// per-instruction flags. These must match the corresponding definitions in /// PPC.td and PPCInstrFormats.td. namespace PPCII { enum { // PPC970 Instruction Flags. These flags describe the characteristics of the // PowerPC 970 (aka G5) dispatch groups and how they are formed out of // raw machine instructions. /// PPC970_First - This instruction starts a new dispatch group, so it will /// always be the first one in the group. PPC970_First = 0x1, /// PPC970_Single - This instruction starts a new dispatch group and /// terminates it, so it will be the sole instruction in the group. PPC970_Single = 0x2, /// PPC970_Cracked - This instruction is cracked into two pieces, requiring /// two dispatch pipes to be available to issue. PPC970_Cracked = 0x4, /// PPC970_Mask/Shift - This is a bitmask that selects the pipeline type that /// an instruction is issued to. PPC970_Shift = 3, PPC970_Mask = 0x07 << PPC970_Shift }; enum PPC970_Unit { /// These are the various PPC970 execution unit pipelines. Each instruction /// is one of these. PPC970_Pseudo = 0 << PPC970_Shift, // Pseudo instruction PPC970_FXU = 1 << PPC970_Shift, // Fixed Point (aka Integer/ALU) Unit PPC970_LSU = 2 << PPC970_Shift, // Load Store Unit PPC970_FPU = 3 << PPC970_Shift, // Floating Point Unit PPC970_CRU = 4 << PPC970_Shift, // Control Register Unit PPC970_VALU = 5 << PPC970_Shift, // Vector ALU PPC970_VPERM = 6 << PPC970_Shift, // Vector Permute Unit PPC970_BRU = 7 << PPC970_Shift // Branch Unit }; enum { /// Shift count to bypass PPC970 flags NewDef_Shift = 6, /// This instruction is an X-Form memory operation. XFormMemOp = 0x1 << NewDef_Shift, /// This instruction is prefixed. Prefixed = 0x1 << (NewDef_Shift+1) }; } // end namespace PPCII // Instructions that have an immediate form might be convertible to that // form if the correct input is a result of a load immediate. In order to // know whether the transformation is special, we might need to know some // of the details of the two forms. struct ImmInstrInfo { // Is the immediate field in the immediate form signed or unsigned? uint64_t SignedImm : 1; // Does the immediate need to be a multiple of some value? uint64_t ImmMustBeMultipleOf : 5; // Is R0/X0 treated specially by the original r+r instruction? // If so, in which operand? uint64_t ZeroIsSpecialOrig : 3; // Is R0/X0 treated specially by the new r+i instruction? // If so, in which operand? uint64_t ZeroIsSpecialNew : 3; // Is the operation commutative? uint64_t IsCommutative : 1; // The operand number to check for add-immediate def. uint64_t OpNoForForwarding : 3; // The operand number for the immediate. uint64_t ImmOpNo : 3; // The opcode of the new instruction. uint64_t ImmOpcode : 16; // The size of the immediate. uint64_t ImmWidth : 5; // The immediate should be truncated to N bits. uint64_t TruncateImmTo : 5; // Is the instruction summing the operand uint64_t IsSummingOperands : 1; }; // Information required to convert an instruction to just a materialized // immediate. struct LoadImmediateInfo { unsigned Imm : 16; unsigned Is64Bit : 1; unsigned SetCR : 1; }; // Index into the OpcodesForSpill array. enum SpillOpcodeKey { SOK_Int4Spill, SOK_Int8Spill, SOK_Float8Spill, SOK_Float4Spill, SOK_CRSpill, SOK_CRBitSpill, SOK_VRVectorSpill, SOK_VSXVectorSpill, SOK_VectorFloat8Spill, SOK_VectorFloat4Spill, SOK_SpillToVSR, SOK_SPESpill, SOK_LastOpcodeSpill // This must be last on the enum. }; // Define list of load and store spill opcodes. #define Pwr8LoadOpcodes \ { \ PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR, \ PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXVD2X, PPC::LXSDX, PPC::LXSSPX, \ PPC::SPILLTOVSR_LD, PPC::EVLDD \ } #define Pwr9LoadOpcodes \ { \ PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR, \ PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXV, PPC::DFLOADf64, \ PPC::DFLOADf32, PPC::SPILLTOVSR_LD \ } #define Pwr8StoreOpcodes \ { \ PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \ PPC::STVX, PPC::STXVD2X, PPC::STXSDX, PPC::STXSSPX, \ PPC::SPILLTOVSR_ST, PPC::EVSTDD \ } #define Pwr9StoreOpcodes \ { \ PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \ PPC::STVX, PPC::STXV, PPC::DFSTOREf64, PPC::DFSTOREf32, \ PPC::SPILLTOVSR_ST \ } // Initialize arrays for load and store spill opcodes on supported subtargets. #define StoreOpcodesForSpill \ { Pwr8StoreOpcodes, Pwr9StoreOpcodes } #define LoadOpcodesForSpill \ { Pwr8LoadOpcodes, Pwr9LoadOpcodes } class PPCSubtarget; class PPCInstrInfo : public PPCGenInstrInfo { PPCSubtarget &Subtarget; const PPCRegisterInfo RI; const unsigned StoreSpillOpcodesArray[2][SOK_LastOpcodeSpill] = StoreOpcodesForSpill; const unsigned LoadSpillOpcodesArray[2][SOK_LastOpcodeSpill] = LoadOpcodesForSpill; void StoreRegToStackSlot(MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx, const TargetRegisterClass *RC, SmallVectorImpl &NewMIs) const; void LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL, unsigned DestReg, int FrameIdx, const TargetRegisterClass *RC, SmallVectorImpl &NewMIs) const; // Replace the instruction with single LI if possible. \p DefMI must be LI or // LI8. bool simplifyToLI(MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding, MachineInstr **KilledDef) const; // If the inst is imm-form and its register operand is produced by a ADDI, put // the imm into the inst directly and remove the ADDI if possible. bool transformToNewImmFormFedByAdd(MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const; // If the inst is x-form and has imm-form and one of its operand is produced // by a LI, put the imm into the inst directly and remove the LI if possible. bool transformToImmFormFedByLI(MachineInstr &MI, const ImmInstrInfo &III, unsigned ConstantOpNo, MachineInstr &DefMI) const; // If the inst is x-form and has imm-form and one of its operand is produced // by an add-immediate, try to transform it when possible. bool transformToImmFormFedByAdd(MachineInstr &MI, const ImmInstrInfo &III, unsigned ConstantOpNo, MachineInstr &DefMI, bool KillDefMI) const; // Try to find that, if the instruction 'MI' contains any operand that // could be forwarded from some inst that feeds it. If yes, return the // Def of that operand. And OpNoForForwarding is the operand index in // the 'MI' for that 'Def'. If we see another use of this Def between // the Def and the MI, SeenIntermediateUse becomes 'true'. MachineInstr *getForwardingDefMI(MachineInstr &MI, unsigned &OpNoForForwarding, bool &SeenIntermediateUse) const; // Can the user MI have it's source at index \p OpNoForForwarding // forwarded from an add-immediate that feeds it? bool isUseMIElgibleForForwarding(MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding) const; bool isDefMIElgibleForForwarding(MachineInstr &DefMI, const ImmInstrInfo &III, MachineOperand *&ImmMO, MachineOperand *&RegMO) const; bool isImmElgibleForForwarding(const MachineOperand &ImmMO, const MachineInstr &DefMI, const ImmInstrInfo &III, int64_t &Imm, int64_t BaseImm = 0) const; bool isRegElgibleForForwarding(const MachineOperand &RegMO, const MachineInstr &DefMI, const MachineInstr &MI, bool KillDefMI, bool &IsFwdFeederRegKilled) const; unsigned getSpillTarget() const; const unsigned *getStoreOpcodesForSpillArray() const; const unsigned *getLoadOpcodesForSpillArray() const; int16_t getFMAOpIdxInfo(unsigned Opcode) const; void reassociateFMA(MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl &InsInstrs, SmallVectorImpl &DelInstrs, DenseMap &InstrIdxForVirtReg) const; virtual void anchor(); protected: /// Commutes the operands in the given instruction. /// The commutable operands are specified by their indices OpIdx1 and OpIdx2. /// /// Do not call this method for a non-commutable instruction or for /// non-commutable pair of operand indices OpIdx1 and OpIdx2. /// Even though the instruction is commutable, the method may still /// fail to commute the operands, null pointer is returned in such cases. /// /// For example, we can commute rlwimi instructions, but only if the /// rotate amt is zero. We also have to munge the immediates a bit. MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const override; public: explicit PPCInstrInfo(PPCSubtarget &STI); /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info. As /// such, whenever a client has an instance of instruction info, it should /// always be able to get register info as well (through this method). /// const PPCRegisterInfo &getRegisterInfo() const { return RI; } bool isXFormMemOp(unsigned Opcode) const { return get(Opcode).TSFlags & PPCII::XFormMemOp; } bool isPrefixed(unsigned Opcode) const { return get(Opcode).TSFlags & PPCII::Prefixed; } static bool isSameClassPhysRegCopy(unsigned Opcode) { unsigned CopyOpcodes[] = {PPC::OR, PPC::OR8, PPC::FMR, PPC::VOR, PPC::XXLOR, PPC::XXLORf, PPC::XSCPSGNDP, PPC::MCRF, PPC::CROR, PPC::EVOR, -1U}; for (int i = 0; CopyOpcodes[i] != -1U; i++) if (Opcode == CopyOpcodes[i]) return true; return false; } ScheduleHazardRecognizer * CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const override; ScheduleHazardRecognizer * CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const override; unsigned getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr &MI, unsigned *PredCost = nullptr) const override; int getOperandLatency(const InstrItineraryData *ItinData, const MachineInstr &DefMI, unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const override; int getOperandLatency(const InstrItineraryData *ItinData, SDNode *DefNode, unsigned DefIdx, SDNode *UseNode, unsigned UseIdx) const override { return PPCGenInstrInfo::getOperandLatency(ItinData, DefNode, DefIdx, UseNode, UseIdx); } bool hasLowDefLatency(const TargetSchedModel &SchedModel, const MachineInstr &DefMI, unsigned DefIdx) const override { // Machine LICM should hoist all instructions in low-register-pressure // situations; none are sufficiently free to justify leaving in a loop // body. return false; } bool useMachineCombiner() const override { return true; } /// When getMachineCombinerPatterns() finds patterns, this function generates /// the instructions that could replace the original code sequence void genAlternativeCodeSequence( MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl &InsInstrs, SmallVectorImpl &DelInstrs, DenseMap &InstrIdxForVirtReg) const override; /// Return true when there is potentially a faster code sequence for a fma /// chain ending in \p Root. All potential patterns are output in the \p /// P array. bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl &P) const; /// Return true when there is potentially a faster code sequence /// for an instruction chain ending in . All potential patterns are /// output in the array. bool getMachineCombinerPatterns( MachineInstr &Root, SmallVectorImpl &P) const override; bool isAssociativeAndCommutative(const MachineInstr &Inst) const override; /// On PowerPC, we try to reassociate FMA chain which will increase /// instruction size. Set extension resource length limit to 1 for edge case. /// Resource Length is calculated by scaled resource usage in getCycles(). /// Because of the division in getCycles(), it returns different cycles due to /// legacy scaled resource usage. So new resource length may be same with /// legacy or 1 bigger than legacy. /// We need to execlude the 1 bigger case even the resource length is not /// perserved for more FMA chain reassociations on PowerPC. int getExtendResourceLenLimit() const override { return 1; } void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, MachineInstr &NewMI1, MachineInstr &NewMI2) const override; void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const override; bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override; unsigned isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override; bool isReallyTriviallyReMaterializable(const MachineInstr &MI, AAResults *AA) const override; unsigned isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override; bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override; void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override; // Branch analysis. bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl &Cond, bool AllowModify) const override; unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved = nullptr) const override; unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef Cond, const DebugLoc &DL, int *BytesAdded = nullptr) const override; // Select analysis. bool canInsertSelect(const MachineBasicBlock &, ArrayRef Cond, Register, Register, Register, int &, int &, int &) const override; void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef Cond, Register TrueReg, Register FalseReg) const override; void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override; void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override; // Emits a register spill without updating the register class for vector // registers. This ensures that when we spill a vector register the // element order in the register is the same as it was in memory. void storeRegToStackSlotNoUpd(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const; void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override; // Emits a register reload without updating the register class for vector // registers. This ensures that when we reload a vector register the // element order in the register is the same as it was in memory. void loadRegFromStackSlotNoUpd(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const; unsigned getStoreOpcodeForSpill(const TargetRegisterClass *RC) const; unsigned getLoadOpcodeForSpill(const TargetRegisterClass *RC) const; bool reverseBranchCondition(SmallVectorImpl &Cond) const override; bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const override; bool onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg) const; // If conversion by predication (only supported by some branch instructions). // All of the profitability checks always return true; it is always // profitable to use the predicated branches. bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, unsigned ExtraPredCycles, BranchProbability Probability) const override { return true; } bool isProfitableToIfCvt(MachineBasicBlock &TMBB, unsigned NumT, unsigned ExtraT, MachineBasicBlock &FMBB, unsigned NumF, unsigned ExtraF, BranchProbability Probability) const override; bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, BranchProbability Probability) const override { return true; } bool isProfitableToUnpredicate(MachineBasicBlock &TMBB, MachineBasicBlock &FMBB) const override { return false; } // Predication support. bool isPredicated(const MachineInstr &MI) const override; bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override; bool PredicateInstruction(MachineInstr &MI, ArrayRef Pred) const override; bool SubsumesPredicate(ArrayRef Pred1, ArrayRef Pred2) const override; bool ClobbersPredicate(MachineInstr &MI, std::vector &Pred, bool SkipDead) const override; // Comparison optimization. bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int &Mask, int &Value) const override; bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int Mask, int Value, const MachineRegisterInfo *MRI) const override; /// Return true if get the base operand, byte offset of an instruction and /// the memory width. Width is the size of memory that is being /// loaded/stored (e.g. 1, 2, 4, 8). bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, unsigned &Width, const TargetRegisterInfo *TRI) const; /// Get the base operand and byte offset of an instruction that reads/writes /// memory. bool getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, const TargetRegisterInfo *TRI) const override; /// Returns true if the two given memory operations should be scheduled /// adjacent. bool shouldClusterMemOps(ArrayRef BaseOps1, ArrayRef BaseOps2, unsigned NumLoads, unsigned NumBytes) const override; /// Return true if two MIs access different memory addresses and false /// otherwise bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override; /// GetInstSize - Return the number of bytes of code the specified /// instruction may be. This returns the maximum number of bytes. /// unsigned getInstSizeInBytes(const MachineInstr &MI) const override; void getNoop(MCInst &NopInst) const override; std::pair decomposeMachineOperandsTargetFlags(unsigned TF) const override; ArrayRef> getSerializableDirectMachineOperandTargetFlags() const override; ArrayRef> getSerializableBitmaskMachineOperandTargetFlags() const override; // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction. bool expandVSXMemPseudo(MachineInstr &MI) const; // Lower pseudo instructions after register allocation. bool expandPostRAPseudo(MachineInstr &MI) const override; static bool isVFRegister(unsigned Reg) { return Reg >= PPC::VF0 && Reg <= PPC::VF31; } static bool isVRRegister(unsigned Reg) { return Reg >= PPC::V0 && Reg <= PPC::V31; } const TargetRegisterClass *updatedRC(const TargetRegisterClass *RC) const; static int getRecordFormOpcode(unsigned Opcode); bool isTOCSaveMI(const MachineInstr &MI) const; bool isSignOrZeroExtended(const MachineInstr &MI, bool SignExt, const unsigned PhiDepth) const; /// Return true if the output of the instruction is always a sign-extended, /// i.e. 0 to 31-th bits are same as 32-th bit. bool isSignExtended(const MachineInstr &MI, const unsigned depth = 0) const { return isSignOrZeroExtended(MI, true, depth); } /// Return true if the output of the instruction is always zero-extended, /// i.e. 0 to 31-th bits are all zeros bool isZeroExtended(const MachineInstr &MI, const unsigned depth = 0) const { return isSignOrZeroExtended(MI, false, depth); } bool convertToImmediateForm(MachineInstr &MI, MachineInstr **KilledDef = nullptr) const; bool foldFrameOffset(MachineInstr &MI) const; - bool simplifyRotateAndMaskInstr(MachineInstr &MI, - MachineInstr *&ToErase) const; + bool combineRLWINM(MachineInstr &MI, MachineInstr **ToErase = nullptr) const; bool isADDIInstrEligibleForFolding(MachineInstr &ADDIMI, int64_t &Imm) const; bool isADDInstrEligibleForFolding(MachineInstr &ADDMI) const; bool isImmInstrEligibleForFolding(MachineInstr &MI, unsigned &BaseReg, unsigned &XFormOpcode, int64_t &OffsetOfImmInstr, ImmInstrInfo &III) const; bool isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index, MachineInstr *&ADDIMI, int64_t &OffsetAddi, int64_t OffsetImm) const; /// Fixup killed/dead flag for register \p RegNo between instructions [\p /// StartMI, \p EndMI]. Some pre-RA or post-RA transformations may violate /// register killed/dead flags semantics, this function can be called to fix /// up. Before calling this function, /// 1. Ensure that \p RegNo liveness is killed after instruction \p EndMI. /// 2. Ensure that there is no new definition between (\p StartMI, \p EndMI) /// and possible definition for \p RegNo is \p StartMI or \p EndMI. For /// pre-RA cases, definition may be \p StartMI through COPY, \p StartMI /// will be adjust to true definition. /// 3. We can do accurate fixup for the case when all instructions between /// [\p StartMI, \p EndMI] are in same basic block. /// 4. For the case when \p StartMI and \p EndMI are not in same basic block, /// we conservatively clear kill flag for all uses of \p RegNo for pre-RA /// and for post-RA, we give an assertion as without reaching definition /// analysis post-RA, \p StartMI and \p EndMI are hard to keep right. void fixupIsDeadOrKill(MachineInstr *StartMI, MachineInstr *EndMI, unsigned RegNo) const; void replaceInstrWithLI(MachineInstr &MI, const LoadImmediateInfo &LII) const; void replaceInstrOperandWithImm(MachineInstr &MI, unsigned OpNo, int64_t Imm) const; bool instrHasImmForm(unsigned Opc, bool IsVFReg, ImmInstrInfo &III, bool PostRA) const; // In PostRA phase, try to find instruction defines \p Reg before \p MI. // \p SeenIntermediate is set to true if uses between DefMI and \p MI exist. MachineInstr *getDefMIPostRA(unsigned Reg, MachineInstr &MI, bool &SeenIntermediateUse) const; /// getRegNumForOperand - some operands use different numbering schemes /// for the same registers. For example, a VSX instruction may have any of /// vs0-vs63 allocated whereas an Altivec instruction could only have /// vs32-vs63 allocated (numbered as v0-v31). This function returns the actual /// register number needed for the opcode/operand number combination. /// The operand number argument will be useful when we need to extend this /// to instructions that use both Altivec and VSX numbering (for different /// operands). static unsigned getRegNumForOperand(const MCInstrDesc &Desc, unsigned Reg, unsigned OpNo) { int16_t regClass = Desc.OpInfo[OpNo].RegClass; switch (regClass) { // We store F0-F31, VF0-VF31 in MCOperand and it should be F0-F31, // VSX32-VSX63 during encoding/disassembling case PPC::VSSRCRegClassID: case PPC::VSFRCRegClassID: if (isVFRegister(Reg)) return PPC::VSX32 + (Reg - PPC::VF0); break; // We store VSL0-VSL31, V0-V31 in MCOperand and it should be VSL0-VSL31, // VSX32-VSX63 during encoding/disassembling case PPC::VSRCRegClassID: if (isVRRegister(Reg)) return PPC::VSX32 + (Reg - PPC::V0); break; // Other RegClass doesn't need mapping default: break; } return Reg; } /// Check \p Opcode is BDNZ (Decrement CTR and branch if it is still nonzero). bool isBDNZ(unsigned Opcode) const; /// Find the hardware loop instruction used to set-up the specified loop. /// On PPC, we have two instructions used to set-up the hardware loop /// (MTCTRloop, MTCTR8loop) with corresponding endloop (BDNZ, BDNZ8) /// instructions to indicate the end of a loop. MachineInstr * findLoopInstr(MachineBasicBlock &PreHeader, SmallPtrSet &Visited) const; /// Analyze loop L, which must be a single-basic-block loop, and if the /// conditions can be understood enough produce a PipelinerLoopInfo object. std::unique_ptr analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override; }; } #endif diff --git a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp index 827d3c4693b9..3fd02bc185c9 100644 --- a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp +++ b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp @@ -1,1526 +1,1526 @@ //===-------------- PPCMIPeephole.cpp - MI Peephole Cleanups -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // // This pass performs peephole optimizations to clean up ugly code // sequences at the MachineInstruction layer. It runs at the end of // the SSA phases, following VSX swap removal. A pass of dead code // elimination follows this one for quick clean-up of any dead // instructions introduced here. Although we could do this as callbacks // from the generic peephole pass, this would have a couple of bad // effects: it might remove optimization opportunities for VSX swap // removal, and it would miss cleanups made possible following VSX // swap removal. // //===---------------------------------------------------------------------===// #include "MCTargetDesc/PPCMCTargetDesc.h" #include "MCTargetDesc/PPCPredicates.h" #include "PPC.h" #include "PPCInstrBuilder.h" #include "PPCInstrInfo.h" #include "PPCMachineFunctionInfo.h" #include "PPCTargetMachine.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" using namespace llvm; #define DEBUG_TYPE "ppc-mi-peepholes" STATISTIC(RemoveTOCSave, "Number of TOC saves removed"); STATISTIC(MultiTOCSaves, "Number of functions with multiple TOC saves that must be kept"); STATISTIC(NumTOCSavesInPrologue, "Number of TOC saves placed in the prologue"); STATISTIC(NumEliminatedSExt, "Number of eliminated sign-extensions"); STATISTIC(NumEliminatedZExt, "Number of eliminated zero-extensions"); STATISTIC(NumOptADDLIs, "Number of optimized ADD instruction fed by LI"); STATISTIC(NumConvertedToImmediateForm, "Number of instructions converted to their immediate form"); STATISTIC(NumFunctionsEnteredInMIPeephole, "Number of functions entered in PPC MI Peepholes"); STATISTIC(NumFixedPointIterations, "Number of fixed-point iterations converting reg-reg instructions " "to reg-imm ones"); STATISTIC(NumRotatesCollapsed, "Number of pairs of rotate left, clear left/right collapsed"); STATISTIC(NumEXTSWAndSLDICombined, "Number of pairs of EXTSW and SLDI combined as EXTSWSLI"); STATISTIC(NumLoadImmZeroFoldedAndRemoved, "Number of LI(8) reg, 0 that are folded to r0 and removed"); static cl::opt FixedPointRegToImm("ppc-reg-to-imm-fixed-point", cl::Hidden, cl::init(true), cl::desc("Iterate to a fixed point when attempting to " "convert reg-reg instructions to reg-imm")); static cl::opt ConvertRegReg("ppc-convert-rr-to-ri", cl::Hidden, cl::init(true), cl::desc("Convert eligible reg+reg instructions to reg+imm")); static cl::opt EnableSExtElimination("ppc-eliminate-signext", cl::desc("enable elimination of sign-extensions"), cl::init(false), cl::Hidden); static cl::opt EnableZExtElimination("ppc-eliminate-zeroext", cl::desc("enable elimination of zero-extensions"), cl::init(false), cl::Hidden); namespace { struct PPCMIPeephole : public MachineFunctionPass { static char ID; const PPCInstrInfo *TII; MachineFunction *MF; MachineRegisterInfo *MRI; PPCMIPeephole() : MachineFunctionPass(ID) { initializePPCMIPeepholePass(*PassRegistry::getPassRegistry()); } private: MachineDominatorTree *MDT; MachinePostDominatorTree *MPDT; MachineBlockFrequencyInfo *MBFI; uint64_t EntryFreq; // Initialize class variables. void initialize(MachineFunction &MFParm); // Perform peepholes. bool simplifyCode(void); // Perform peepholes. bool eliminateRedundantCompare(void); bool eliminateRedundantTOCSaves(std::map &TOCSaves); bool combineSEXTAndSHL(MachineInstr &MI, MachineInstr *&ToErase); bool emitRLDICWhenLoweringJumpTables(MachineInstr &MI); void UpdateTOCSaves(std::map &TOCSaves, MachineInstr *MI); public: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.addPreserved(); AU.addPreserved(); AU.addPreserved(); MachineFunctionPass::getAnalysisUsage(AU); } // Main entry point for this pass. bool runOnMachineFunction(MachineFunction &MF) override { initialize(MF); // At this point, TOC pointer should not be used in a function that uses // PC-Relative addressing. assert((MF.getRegInfo().use_empty(PPC::X2) || !MF.getSubtarget().isUsingPCRelativeCalls()) && "TOC pointer used in a function using PC-Relative addressing!"); if (skipFunction(MF.getFunction())) return false; return simplifyCode(); } }; // Initialize class variables. void PPCMIPeephole::initialize(MachineFunction &MFParm) { MF = &MFParm; MRI = &MF->getRegInfo(); MDT = &getAnalysis(); MPDT = &getAnalysis(); MBFI = &getAnalysis(); EntryFreq = MBFI->getEntryFreq(); TII = MF->getSubtarget().getInstrInfo(); LLVM_DEBUG(dbgs() << "*** PowerPC MI peephole pass ***\n\n"); LLVM_DEBUG(MF->dump()); } static MachineInstr *getVRegDefOrNull(MachineOperand *Op, MachineRegisterInfo *MRI) { assert(Op && "Invalid Operand!"); if (!Op->isReg()) return nullptr; Register Reg = Op->getReg(); if (!Register::isVirtualRegister(Reg)) return nullptr; return MRI->getVRegDef(Reg); } // This function returns number of known zero bits in output of MI // starting from the most significant bit. static unsigned getKnownLeadingZeroCount(MachineInstr *MI, const PPCInstrInfo *TII) { unsigned Opcode = MI->getOpcode(); if (Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec || Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec) return MI->getOperand(3).getImm(); if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) && MI->getOperand(3).getImm() <= 63 - MI->getOperand(2).getImm()) return MI->getOperand(3).getImm(); if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec || Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec || Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) && MI->getOperand(3).getImm() <= MI->getOperand(4).getImm()) return 32 + MI->getOperand(3).getImm(); if (Opcode == PPC::ANDI_rec) { uint16_t Imm = MI->getOperand(2).getImm(); return 48 + countLeadingZeros(Imm); } if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec || Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec || Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8) // The result ranges from 0 to 32. return 58; if (Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec || Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec) // The result ranges from 0 to 64. return 57; if (Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8) return 48; if (Opcode == PPC::LBZ || Opcode == PPC::LBZX || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU || Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8) return 56; if (TII->isZeroExtended(*MI)) return 32; return 0; } // This function maintains a map for the pairs // Each time a new TOC save is encountered, it checks if any of the existing // ones are dominated by the new one. If so, it marks the existing one as // redundant by setting it's entry in the map as false. It then adds the new // instruction to the map with either true or false depending on if any // existing instructions dominated the new one. void PPCMIPeephole::UpdateTOCSaves( std::map &TOCSaves, MachineInstr *MI) { assert(TII->isTOCSaveMI(*MI) && "Expecting a TOC save instruction here"); assert(MF->getSubtarget().isELFv2ABI() && "TOC-save removal only supported on ELFv2"); PPCFunctionInfo *FI = MF->getInfo(); MachineBasicBlock *Entry = &MF->front(); uint64_t CurrBlockFreq = MBFI->getBlockFreq(MI->getParent()).getFrequency(); // If the block in which the TOC save resides is in a block that // post-dominates Entry, or a block that is hotter than entry (keep in mind // that early MachineLICM has already run so the TOC save won't be hoisted) // we can just do the save in the prologue. if (CurrBlockFreq > EntryFreq || MPDT->dominates(MI->getParent(), Entry)) FI->setMustSaveTOC(true); // If we are saving the TOC in the prologue, all the TOC saves can be removed // from the code. if (FI->mustSaveTOC()) { for (auto &TOCSave : TOCSaves) TOCSave.second = false; // Add new instruction to map. TOCSaves[MI] = false; return; } bool Keep = true; for (auto It = TOCSaves.begin(); It != TOCSaves.end(); It++ ) { MachineInstr *CurrInst = It->first; // If new instruction dominates an existing one, mark existing one as // redundant. if (It->second && MDT->dominates(MI, CurrInst)) It->second = false; // Check if the new instruction is redundant. if (MDT->dominates(CurrInst, MI)) { Keep = false; break; } } // Add new instruction to map. TOCSaves[MI] = Keep; } // Perform peephole optimizations. bool PPCMIPeephole::simplifyCode(void) { bool Simplified = false; MachineInstr* ToErase = nullptr; std::map TOCSaves; const TargetRegisterInfo *TRI = &TII->getRegisterInfo(); NumFunctionsEnteredInMIPeephole++; if (ConvertRegReg) { // Fixed-point conversion of reg/reg instructions fed by load-immediate // into reg/imm instructions. FIXME: This is expensive, control it with // an option. bool SomethingChanged = false; do { NumFixedPointIterations++; SomethingChanged = false; for (MachineBasicBlock &MBB : *MF) { for (MachineInstr &MI : MBB) { if (MI.isDebugInstr()) continue; if (TII->convertToImmediateForm(MI)) { // We don't erase anything in case the def has other uses. Let DCE // remove it if it can be removed. LLVM_DEBUG(dbgs() << "Converted instruction to imm form: "); LLVM_DEBUG(MI.dump()); NumConvertedToImmediateForm++; SomethingChanged = true; Simplified = true; continue; } } } } while (SomethingChanged && FixedPointRegToImm); } for (MachineBasicBlock &MBB : *MF) { for (MachineInstr &MI : MBB) { // If the previous instruction was marked for elimination, // remove it now. if (ToErase) { ToErase->eraseFromParent(); ToErase = nullptr; } // Ignore debug instructions. if (MI.isDebugInstr()) continue; // Per-opcode peepholes. switch (MI.getOpcode()) { default: break; case PPC::LI: case PPC::LI8: { // If we are materializing a zero, look for any use operands for which // zero means immediate zero. All such operands can be replaced with // PPC::ZERO. if (!MI.getOperand(1).isImm() || MI.getOperand(1).getImm() != 0) break; unsigned MIDestReg = MI.getOperand(0).getReg(); for (MachineInstr& UseMI : MRI->use_instructions(MIDestReg)) Simplified |= TII->onlyFoldImmediate(UseMI, MI, MIDestReg); if (MRI->use_nodbg_empty(MIDestReg)) { ++NumLoadImmZeroFoldedAndRemoved; ToErase = &MI; } break; } case PPC::STD: { MachineFrameInfo &MFI = MF->getFrameInfo(); if (MFI.hasVarSizedObjects() || !MF->getSubtarget().isELFv2ABI()) break; // When encountering a TOC save instruction, call UpdateTOCSaves // to add it to the TOCSaves map and mark any existing TOC saves // it dominates as redundant. if (TII->isTOCSaveMI(MI)) UpdateTOCSaves(TOCSaves, &MI); break; } case PPC::XXPERMDI: { // Perform simplifications of 2x64 vector swaps and splats. // A swap is identified by an immediate value of 2, and a splat // is identified by an immediate value of 0 or 3. int Immed = MI.getOperand(3).getImm(); if (Immed == 1) break; // For each of these simplifications, we need the two source // regs to match. Unfortunately, MachineCSE ignores COPY and // SUBREG_TO_REG, so for example we can see // XXPERMDI t, SUBREG_TO_REG(s), SUBREG_TO_REG(s), immed. // We have to look through chains of COPY and SUBREG_TO_REG // to find the real source values for comparison. unsigned TrueReg1 = TRI->lookThruCopyLike(MI.getOperand(1).getReg(), MRI); unsigned TrueReg2 = TRI->lookThruCopyLike(MI.getOperand(2).getReg(), MRI); if (!(TrueReg1 == TrueReg2 && Register::isVirtualRegister(TrueReg1))) break; MachineInstr *DefMI = MRI->getVRegDef(TrueReg1); if (!DefMI) break; unsigned DefOpc = DefMI->getOpcode(); // If this is a splat fed by a splatting load, the splat is // redundant. Replace with a copy. This doesn't happen directly due // to code in PPCDAGToDAGISel.cpp, but it can happen when converting // a load of a double to a vector of 64-bit integers. auto isConversionOfLoadAndSplat = [=]() -> bool { if (DefOpc != PPC::XVCVDPSXDS && DefOpc != PPC::XVCVDPUXDS) return false; unsigned FeedReg1 = TRI->lookThruCopyLike(DefMI->getOperand(1).getReg(), MRI); if (Register::isVirtualRegister(FeedReg1)) { MachineInstr *LoadMI = MRI->getVRegDef(FeedReg1); if (LoadMI && LoadMI->getOpcode() == PPC::LXVDSX) return true; } return false; }; if ((Immed == 0 || Immed == 3) && (DefOpc == PPC::LXVDSX || isConversionOfLoadAndSplat())) { LLVM_DEBUG(dbgs() << "Optimizing load-and-splat/splat " "to load-and-splat/copy: "); LLVM_DEBUG(MI.dump()); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::COPY), MI.getOperand(0).getReg()) .add(MI.getOperand(1)); ToErase = &MI; Simplified = true; } // If this is a splat or a swap fed by another splat, we // can replace it with a copy. if (DefOpc == PPC::XXPERMDI) { unsigned DefReg1 = DefMI->getOperand(1).getReg(); unsigned DefReg2 = DefMI->getOperand(2).getReg(); unsigned DefImmed = DefMI->getOperand(3).getImm(); // If the two inputs are not the same register, check to see if // they originate from the same virtual register after only // copy-like instructions. if (DefReg1 != DefReg2) { unsigned FeedReg1 = TRI->lookThruCopyLike(DefReg1, MRI); unsigned FeedReg2 = TRI->lookThruCopyLike(DefReg2, MRI); if (!(FeedReg1 == FeedReg2 && Register::isVirtualRegister(FeedReg1))) break; } if (DefImmed == 0 || DefImmed == 3) { LLVM_DEBUG(dbgs() << "Optimizing splat/swap or splat/splat " "to splat/copy: "); LLVM_DEBUG(MI.dump()); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::COPY), MI.getOperand(0).getReg()) .add(MI.getOperand(1)); ToErase = &MI; Simplified = true; } // If this is a splat fed by a swap, we can simplify modify // the splat to splat the other value from the swap's input // parameter. else if ((Immed == 0 || Immed == 3) && DefImmed == 2) { LLVM_DEBUG(dbgs() << "Optimizing swap/splat => splat: "); LLVM_DEBUG(MI.dump()); MI.getOperand(1).setReg(DefReg1); MI.getOperand(2).setReg(DefReg2); MI.getOperand(3).setImm(3 - Immed); Simplified = true; } // If this is a swap fed by a swap, we can replace it // with a copy from the first swap's input. else if (Immed == 2 && DefImmed == 2) { LLVM_DEBUG(dbgs() << "Optimizing swap/swap => copy: "); LLVM_DEBUG(MI.dump()); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::COPY), MI.getOperand(0).getReg()) .add(DefMI->getOperand(1)); ToErase = &MI; Simplified = true; } } else if ((Immed == 0 || Immed == 3) && DefOpc == PPC::XXPERMDIs && (DefMI->getOperand(2).getImm() == 0 || DefMI->getOperand(2).getImm() == 3)) { // Splat fed by another splat - switch the output of the first // and remove the second. DefMI->getOperand(0).setReg(MI.getOperand(0).getReg()); ToErase = &MI; Simplified = true; LLVM_DEBUG(dbgs() << "Removing redundant splat: "); LLVM_DEBUG(MI.dump()); } break; } case PPC::VSPLTB: case PPC::VSPLTH: case PPC::XXSPLTW: { unsigned MyOpcode = MI.getOpcode(); unsigned OpNo = MyOpcode == PPC::XXSPLTW ? 1 : 2; unsigned TrueReg = TRI->lookThruCopyLike(MI.getOperand(OpNo).getReg(), MRI); if (!Register::isVirtualRegister(TrueReg)) break; MachineInstr *DefMI = MRI->getVRegDef(TrueReg); if (!DefMI) break; unsigned DefOpcode = DefMI->getOpcode(); auto isConvertOfSplat = [=]() -> bool { if (DefOpcode != PPC::XVCVSPSXWS && DefOpcode != PPC::XVCVSPUXWS) return false; Register ConvReg = DefMI->getOperand(1).getReg(); if (!Register::isVirtualRegister(ConvReg)) return false; MachineInstr *Splt = MRI->getVRegDef(ConvReg); return Splt && (Splt->getOpcode() == PPC::LXVWSX || Splt->getOpcode() == PPC::XXSPLTW); }; bool AlreadySplat = (MyOpcode == DefOpcode) || (MyOpcode == PPC::VSPLTB && DefOpcode == PPC::VSPLTBs) || (MyOpcode == PPC::VSPLTH && DefOpcode == PPC::VSPLTHs) || (MyOpcode == PPC::XXSPLTW && DefOpcode == PPC::XXSPLTWs) || (MyOpcode == PPC::XXSPLTW && DefOpcode == PPC::LXVWSX) || (MyOpcode == PPC::XXSPLTW && DefOpcode == PPC::MTVSRWS)|| (MyOpcode == PPC::XXSPLTW && isConvertOfSplat()); // If the instruction[s] that feed this splat have already splat // the value, this splat is redundant. if (AlreadySplat) { LLVM_DEBUG(dbgs() << "Changing redundant splat to a copy: "); LLVM_DEBUG(MI.dump()); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::COPY), MI.getOperand(0).getReg()) .add(MI.getOperand(OpNo)); ToErase = &MI; Simplified = true; } // Splat fed by a shift. Usually when we align value to splat into // vector element zero. if (DefOpcode == PPC::XXSLDWI) { Register ShiftRes = DefMI->getOperand(0).getReg(); Register ShiftOp1 = DefMI->getOperand(1).getReg(); Register ShiftOp2 = DefMI->getOperand(2).getReg(); unsigned ShiftImm = DefMI->getOperand(3).getImm(); unsigned SplatImm = MI.getOperand(2).getImm(); if (ShiftOp1 == ShiftOp2) { unsigned NewElem = (SplatImm + ShiftImm) & 0x3; if (MRI->hasOneNonDBGUse(ShiftRes)) { LLVM_DEBUG(dbgs() << "Removing redundant shift: "); LLVM_DEBUG(DefMI->dump()); ToErase = DefMI; } Simplified = true; LLVM_DEBUG(dbgs() << "Changing splat immediate from " << SplatImm << " to " << NewElem << " in instruction: "); LLVM_DEBUG(MI.dump()); MI.getOperand(1).setReg(ShiftOp1); MI.getOperand(2).setImm(NewElem); } } break; } case PPC::XVCVDPSP: { // If this is a DP->SP conversion fed by an FRSP, the FRSP is redundant. unsigned TrueReg = TRI->lookThruCopyLike(MI.getOperand(1).getReg(), MRI); if (!Register::isVirtualRegister(TrueReg)) break; MachineInstr *DefMI = MRI->getVRegDef(TrueReg); // This can occur when building a vector of single precision or integer // values. if (DefMI && DefMI->getOpcode() == PPC::XXPERMDI) { unsigned DefsReg1 = TRI->lookThruCopyLike(DefMI->getOperand(1).getReg(), MRI); unsigned DefsReg2 = TRI->lookThruCopyLike(DefMI->getOperand(2).getReg(), MRI); if (!Register::isVirtualRegister(DefsReg1) || !Register::isVirtualRegister(DefsReg2)) break; MachineInstr *P1 = MRI->getVRegDef(DefsReg1); MachineInstr *P2 = MRI->getVRegDef(DefsReg2); if (!P1 || !P2) break; // Remove the passed FRSP/XSRSP instruction if it only feeds this MI // and set any uses of that FRSP/XSRSP (in this MI) to the source of // the FRSP/XSRSP. auto removeFRSPIfPossible = [&](MachineInstr *RoundInstr) { unsigned Opc = RoundInstr->getOpcode(); if ((Opc == PPC::FRSP || Opc == PPC::XSRSP) && MRI->hasOneNonDBGUse(RoundInstr->getOperand(0).getReg())) { Simplified = true; Register ConvReg1 = RoundInstr->getOperand(1).getReg(); Register FRSPDefines = RoundInstr->getOperand(0).getReg(); MachineInstr &Use = *(MRI->use_instr_begin(FRSPDefines)); for (int i = 0, e = Use.getNumOperands(); i < e; ++i) if (Use.getOperand(i).isReg() && Use.getOperand(i).getReg() == FRSPDefines) Use.getOperand(i).setReg(ConvReg1); LLVM_DEBUG(dbgs() << "Removing redundant FRSP/XSRSP:\n"); LLVM_DEBUG(RoundInstr->dump()); LLVM_DEBUG(dbgs() << "As it feeds instruction:\n"); LLVM_DEBUG(MI.dump()); LLVM_DEBUG(dbgs() << "Through instruction:\n"); LLVM_DEBUG(DefMI->dump()); RoundInstr->eraseFromParent(); } }; // If the input to XVCVDPSP is a vector that was built (even // partially) out of FRSP's, the FRSP(s) can safely be removed // since this instruction performs the same operation. if (P1 != P2) { removeFRSPIfPossible(P1); removeFRSPIfPossible(P2); break; } removeFRSPIfPossible(P1); } break; } case PPC::EXTSH: case PPC::EXTSH8: case PPC::EXTSH8_32_64: { if (!EnableSExtElimination) break; Register NarrowReg = MI.getOperand(1).getReg(); if (!Register::isVirtualRegister(NarrowReg)) break; MachineInstr *SrcMI = MRI->getVRegDef(NarrowReg); // If we've used a zero-extending load that we will sign-extend, // just do a sign-extending load. if (SrcMI->getOpcode() == PPC::LHZ || SrcMI->getOpcode() == PPC::LHZX) { if (!MRI->hasOneNonDBGUse(SrcMI->getOperand(0).getReg())) break; auto is64Bit = [] (unsigned Opcode) { return Opcode == PPC::EXTSH8; }; auto isXForm = [] (unsigned Opcode) { return Opcode == PPC::LHZX; }; auto getSextLoadOp = [] (bool is64Bit, bool isXForm) { if (is64Bit) if (isXForm) return PPC::LHAX8; else return PPC::LHA8; else if (isXForm) return PPC::LHAX; else return PPC::LHA; }; unsigned Opc = getSextLoadOp(is64Bit(MI.getOpcode()), isXForm(SrcMI->getOpcode())); LLVM_DEBUG(dbgs() << "Zero-extending load\n"); LLVM_DEBUG(SrcMI->dump()); LLVM_DEBUG(dbgs() << "and sign-extension\n"); LLVM_DEBUG(MI.dump()); LLVM_DEBUG(dbgs() << "are merged into sign-extending load\n"); SrcMI->setDesc(TII->get(Opc)); SrcMI->getOperand(0).setReg(MI.getOperand(0).getReg()); ToErase = &MI; Simplified = true; NumEliminatedSExt++; } break; } case PPC::EXTSW: case PPC::EXTSW_32: case PPC::EXTSW_32_64: { if (!EnableSExtElimination) break; Register NarrowReg = MI.getOperand(1).getReg(); if (!Register::isVirtualRegister(NarrowReg)) break; MachineInstr *SrcMI = MRI->getVRegDef(NarrowReg); // If we've used a zero-extending load that we will sign-extend, // just do a sign-extending load. if (SrcMI->getOpcode() == PPC::LWZ || SrcMI->getOpcode() == PPC::LWZX) { if (!MRI->hasOneNonDBGUse(SrcMI->getOperand(0).getReg())) break; auto is64Bit = [] (unsigned Opcode) { return Opcode == PPC::EXTSW || Opcode == PPC::EXTSW_32_64; }; auto isXForm = [] (unsigned Opcode) { return Opcode == PPC::LWZX; }; auto getSextLoadOp = [] (bool is64Bit, bool isXForm) { if (is64Bit) if (isXForm) return PPC::LWAX; else return PPC::LWA; else if (isXForm) return PPC::LWAX_32; else return PPC::LWA_32; }; unsigned Opc = getSextLoadOp(is64Bit(MI.getOpcode()), isXForm(SrcMI->getOpcode())); LLVM_DEBUG(dbgs() << "Zero-extending load\n"); LLVM_DEBUG(SrcMI->dump()); LLVM_DEBUG(dbgs() << "and sign-extension\n"); LLVM_DEBUG(MI.dump()); LLVM_DEBUG(dbgs() << "are merged into sign-extending load\n"); SrcMI->setDesc(TII->get(Opc)); SrcMI->getOperand(0).setReg(MI.getOperand(0).getReg()); ToErase = &MI; Simplified = true; NumEliminatedSExt++; } else if (MI.getOpcode() == PPC::EXTSW_32_64 && TII->isSignExtended(*SrcMI)) { // We can eliminate EXTSW if the input is known to be already // sign-extended. LLVM_DEBUG(dbgs() << "Removing redundant sign-extension\n"); Register TmpReg = MF->getRegInfo().createVirtualRegister(&PPC::G8RCRegClass); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::IMPLICIT_DEF), TmpReg); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::INSERT_SUBREG), MI.getOperand(0).getReg()) .addReg(TmpReg) .addReg(NarrowReg) .addImm(PPC::sub_32); ToErase = &MI; Simplified = true; NumEliminatedSExt++; } break; } case PPC::RLDICL: { // We can eliminate RLDICL (e.g. for zero-extension) // if all bits to clear are already zero in the input. // This code assume following code sequence for zero-extension. // %6 = COPY %5:sub_32; (optional) // %8 = IMPLICIT_DEF; // %7 = INSERT_SUBREG %8, %6, sub_32; if (!EnableZExtElimination) break; if (MI.getOperand(2).getImm() != 0) break; Register SrcReg = MI.getOperand(1).getReg(); if (!Register::isVirtualRegister(SrcReg)) break; MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); if (!(SrcMI && SrcMI->getOpcode() == PPC::INSERT_SUBREG && SrcMI->getOperand(0).isReg() && SrcMI->getOperand(1).isReg())) break; MachineInstr *ImpDefMI, *SubRegMI; ImpDefMI = MRI->getVRegDef(SrcMI->getOperand(1).getReg()); SubRegMI = MRI->getVRegDef(SrcMI->getOperand(2).getReg()); if (ImpDefMI->getOpcode() != PPC::IMPLICIT_DEF) break; SrcMI = SubRegMI; if (SubRegMI->getOpcode() == PPC::COPY) { Register CopyReg = SubRegMI->getOperand(1).getReg(); if (Register::isVirtualRegister(CopyReg)) SrcMI = MRI->getVRegDef(CopyReg); } unsigned KnownZeroCount = getKnownLeadingZeroCount(SrcMI, TII); if (MI.getOperand(3).getImm() <= KnownZeroCount) { LLVM_DEBUG(dbgs() << "Removing redundant zero-extension\n"); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::COPY), MI.getOperand(0).getReg()) .addReg(SrcReg); ToErase = &MI; Simplified = true; NumEliminatedZExt++; } break; } // TODO: Any instruction that has an immediate form fed only by a PHI // whose operands are all load immediate can be folded away. We currently // do this for ADD instructions, but should expand it to arithmetic and // binary instructions with immediate forms in the future. case PPC::ADD4: case PPC::ADD8: { auto isSingleUsePHI = [&](MachineOperand *PhiOp) { assert(PhiOp && "Invalid Operand!"); MachineInstr *DefPhiMI = getVRegDefOrNull(PhiOp, MRI); return DefPhiMI && (DefPhiMI->getOpcode() == PPC::PHI) && MRI->hasOneNonDBGUse(DefPhiMI->getOperand(0).getReg()); }; auto dominatesAllSingleUseLIs = [&](MachineOperand *DominatorOp, MachineOperand *PhiOp) { assert(PhiOp && "Invalid Operand!"); assert(DominatorOp && "Invalid Operand!"); MachineInstr *DefPhiMI = getVRegDefOrNull(PhiOp, MRI); MachineInstr *DefDomMI = getVRegDefOrNull(DominatorOp, MRI); // Note: the vregs only show up at odd indices position of PHI Node, // the even indices position save the BB info. for (unsigned i = 1; i < DefPhiMI->getNumOperands(); i += 2) { MachineInstr *LiMI = getVRegDefOrNull(&DefPhiMI->getOperand(i), MRI); if (!LiMI || (LiMI->getOpcode() != PPC::LI && LiMI->getOpcode() != PPC::LI8) || !MRI->hasOneNonDBGUse(LiMI->getOperand(0).getReg()) || !MDT->dominates(DefDomMI, LiMI)) return false; } return true; }; MachineOperand Op1 = MI.getOperand(1); MachineOperand Op2 = MI.getOperand(2); if (isSingleUsePHI(&Op2) && dominatesAllSingleUseLIs(&Op1, &Op2)) std::swap(Op1, Op2); else if (!isSingleUsePHI(&Op1) || !dominatesAllSingleUseLIs(&Op2, &Op1)) break; // We don't have an ADD fed by LI's that can be transformed // Now we know that Op1 is the PHI node and Op2 is the dominator Register DominatorReg = Op2.getReg(); const TargetRegisterClass *TRC = MI.getOpcode() == PPC::ADD8 ? &PPC::G8RC_and_G8RC_NOX0RegClass : &PPC::GPRC_and_GPRC_NOR0RegClass; MRI->setRegClass(DominatorReg, TRC); // replace LIs with ADDIs MachineInstr *DefPhiMI = getVRegDefOrNull(&Op1, MRI); for (unsigned i = 1; i < DefPhiMI->getNumOperands(); i += 2) { MachineInstr *LiMI = getVRegDefOrNull(&DefPhiMI->getOperand(i), MRI); LLVM_DEBUG(dbgs() << "Optimizing LI to ADDI: "); LLVM_DEBUG(LiMI->dump()); // There could be repeated registers in the PHI, e.g: %1 = // PHI %6, <%bb.2>, %8, <%bb.3>, %8, <%bb.6>; So if we've // already replaced the def instruction, skip. if (LiMI->getOpcode() == PPC::ADDI || LiMI->getOpcode() == PPC::ADDI8) continue; assert((LiMI->getOpcode() == PPC::LI || LiMI->getOpcode() == PPC::LI8) && "Invalid Opcode!"); auto LiImm = LiMI->getOperand(1).getImm(); // save the imm of LI LiMI->RemoveOperand(1); // remove the imm of LI LiMI->setDesc(TII->get(LiMI->getOpcode() == PPC::LI ? PPC::ADDI : PPC::ADDI8)); MachineInstrBuilder(*LiMI->getParent()->getParent(), *LiMI) .addReg(DominatorReg) .addImm(LiImm); // restore the imm of LI LLVM_DEBUG(LiMI->dump()); } // Replace ADD with COPY LLVM_DEBUG(dbgs() << "Optimizing ADD to COPY: "); LLVM_DEBUG(MI.dump()); BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(PPC::COPY), MI.getOperand(0).getReg()) .add(Op1); ToErase = &MI; Simplified = true; NumOptADDLIs++; break; } case PPC::RLDICR: { Simplified |= emitRLDICWhenLoweringJumpTables(MI) || combineSEXTAndSHL(MI, ToErase); break; } case PPC::RLWINM: case PPC::RLWINM_rec: case PPC::RLWINM8: case PPC::RLWINM8_rec: { - Simplified = TII->simplifyRotateAndMaskInstr(MI, ToErase); + Simplified = TII->combineRLWINM(MI, &ToErase); if (Simplified) ++NumRotatesCollapsed; break; } } } // If the last instruction was marked for elimination, // remove it now. if (ToErase) { ToErase->eraseFromParent(); ToErase = nullptr; } } // Eliminate all the TOC save instructions which are redundant. Simplified |= eliminateRedundantTOCSaves(TOCSaves); PPCFunctionInfo *FI = MF->getInfo(); if (FI->mustSaveTOC()) NumTOCSavesInPrologue++; // We try to eliminate redundant compare instruction. Simplified |= eliminateRedundantCompare(); return Simplified; } // helper functions for eliminateRedundantCompare static bool isEqOrNe(MachineInstr *BI) { PPC::Predicate Pred = (PPC::Predicate)BI->getOperand(0).getImm(); unsigned PredCond = PPC::getPredicateCondition(Pred); return (PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE); } static bool isSupportedCmpOp(unsigned opCode) { return (opCode == PPC::CMPLD || opCode == PPC::CMPD || opCode == PPC::CMPLW || opCode == PPC::CMPW || opCode == PPC::CMPLDI || opCode == PPC::CMPDI || opCode == PPC::CMPLWI || opCode == PPC::CMPWI); } static bool is64bitCmpOp(unsigned opCode) { return (opCode == PPC::CMPLD || opCode == PPC::CMPD || opCode == PPC::CMPLDI || opCode == PPC::CMPDI); } static bool isSignedCmpOp(unsigned opCode) { return (opCode == PPC::CMPD || opCode == PPC::CMPW || opCode == PPC::CMPDI || opCode == PPC::CMPWI); } static unsigned getSignedCmpOpCode(unsigned opCode) { if (opCode == PPC::CMPLD) return PPC::CMPD; if (opCode == PPC::CMPLW) return PPC::CMPW; if (opCode == PPC::CMPLDI) return PPC::CMPDI; if (opCode == PPC::CMPLWI) return PPC::CMPWI; return opCode; } // We can decrement immediate x in (GE x) by changing it to (GT x-1) or // (LT x) to (LE x-1) static unsigned getPredicateToDecImm(MachineInstr *BI, MachineInstr *CMPI) { uint64_t Imm = CMPI->getOperand(2).getImm(); bool SignedCmp = isSignedCmpOp(CMPI->getOpcode()); if ((!SignedCmp && Imm == 0) || (SignedCmp && Imm == 0x8000)) return 0; PPC::Predicate Pred = (PPC::Predicate)BI->getOperand(0).getImm(); unsigned PredCond = PPC::getPredicateCondition(Pred); unsigned PredHint = PPC::getPredicateHint(Pred); if (PredCond == PPC::PRED_GE) return PPC::getPredicate(PPC::PRED_GT, PredHint); if (PredCond == PPC::PRED_LT) return PPC::getPredicate(PPC::PRED_LE, PredHint); return 0; } // We can increment immediate x in (GT x) by changing it to (GE x+1) or // (LE x) to (LT x+1) static unsigned getPredicateToIncImm(MachineInstr *BI, MachineInstr *CMPI) { uint64_t Imm = CMPI->getOperand(2).getImm(); bool SignedCmp = isSignedCmpOp(CMPI->getOpcode()); if ((!SignedCmp && Imm == 0xFFFF) || (SignedCmp && Imm == 0x7FFF)) return 0; PPC::Predicate Pred = (PPC::Predicate)BI->getOperand(0).getImm(); unsigned PredCond = PPC::getPredicateCondition(Pred); unsigned PredHint = PPC::getPredicateHint(Pred); if (PredCond == PPC::PRED_GT) return PPC::getPredicate(PPC::PRED_GE, PredHint); if (PredCond == PPC::PRED_LE) return PPC::getPredicate(PPC::PRED_LT, PredHint); return 0; } // This takes a Phi node and returns a register value for the specified BB. static unsigned getIncomingRegForBlock(MachineInstr *Phi, MachineBasicBlock *MBB) { for (unsigned I = 2, E = Phi->getNumOperands() + 1; I != E; I += 2) { MachineOperand &MO = Phi->getOperand(I); if (MO.getMBB() == MBB) return Phi->getOperand(I-1).getReg(); } llvm_unreachable("invalid src basic block for this Phi node\n"); return 0; } // This function tracks the source of the register through register copy. // If BB1 and BB2 are non-NULL, we also track PHI instruction in BB2 // assuming that the control comes from BB1 into BB2. static unsigned getSrcVReg(unsigned Reg, MachineBasicBlock *BB1, MachineBasicBlock *BB2, MachineRegisterInfo *MRI) { unsigned SrcReg = Reg; while (1) { unsigned NextReg = SrcReg; MachineInstr *Inst = MRI->getVRegDef(SrcReg); if (BB1 && Inst->getOpcode() == PPC::PHI && Inst->getParent() == BB2) { NextReg = getIncomingRegForBlock(Inst, BB1); // We track through PHI only once to avoid infinite loop. BB1 = nullptr; } else if (Inst->isFullCopy()) NextReg = Inst->getOperand(1).getReg(); if (NextReg == SrcReg || !Register::isVirtualRegister(NextReg)) break; SrcReg = NextReg; } return SrcReg; } static bool eligibleForCompareElimination(MachineBasicBlock &MBB, MachineBasicBlock *&PredMBB, MachineBasicBlock *&MBBtoMoveCmp, MachineRegisterInfo *MRI) { auto isEligibleBB = [&](MachineBasicBlock &BB) { auto BII = BB.getFirstInstrTerminator(); // We optimize BBs ending with a conditional branch. // We check only for BCC here, not BCCLR, because BCCLR // will be formed only later in the pipeline. if (BB.succ_size() == 2 && BII != BB.instr_end() && (*BII).getOpcode() == PPC::BCC && (*BII).getOperand(1).isReg()) { // We optimize only if the condition code is used only by one BCC. Register CndReg = (*BII).getOperand(1).getReg(); if (!Register::isVirtualRegister(CndReg) || !MRI->hasOneNonDBGUse(CndReg)) return false; MachineInstr *CMPI = MRI->getVRegDef(CndReg); // We assume compare and branch are in the same BB for ease of analysis. if (CMPI->getParent() != &BB) return false; // We skip this BB if a physical register is used in comparison. for (MachineOperand &MO : CMPI->operands()) if (MO.isReg() && !Register::isVirtualRegister(MO.getReg())) return false; return true; } return false; }; // If this BB has more than one successor, we can create a new BB and // move the compare instruction in the new BB. // So far, we do not move compare instruction to a BB having multiple // successors to avoid potentially increasing code size. auto isEligibleForMoveCmp = [](MachineBasicBlock &BB) { return BB.succ_size() == 1; }; if (!isEligibleBB(MBB)) return false; unsigned NumPredBBs = MBB.pred_size(); if (NumPredBBs == 1) { MachineBasicBlock *TmpMBB = *MBB.pred_begin(); if (isEligibleBB(*TmpMBB)) { PredMBB = TmpMBB; MBBtoMoveCmp = nullptr; return true; } } else if (NumPredBBs == 2) { // We check for partially redundant case. // So far, we support cases with only two predecessors // to avoid increasing the number of instructions. MachineBasicBlock::pred_iterator PI = MBB.pred_begin(); MachineBasicBlock *Pred1MBB = *PI; MachineBasicBlock *Pred2MBB = *(PI+1); if (isEligibleBB(*Pred1MBB) && isEligibleForMoveCmp(*Pred2MBB)) { // We assume Pred1MBB is the BB containing the compare to be merged and // Pred2MBB is the BB to which we will append a compare instruction. // Hence we can proceed as is. } else if (isEligibleBB(*Pred2MBB) && isEligibleForMoveCmp(*Pred1MBB)) { // We need to swap Pred1MBB and Pred2MBB to canonicalize. std::swap(Pred1MBB, Pred2MBB); } else return false; // Here, Pred2MBB is the BB to which we need to append a compare inst. // We cannot move the compare instruction if operands are not available // in Pred2MBB (i.e. defined in MBB by an instruction other than PHI). MachineInstr *BI = &*MBB.getFirstInstrTerminator(); MachineInstr *CMPI = MRI->getVRegDef(BI->getOperand(1).getReg()); for (int I = 1; I <= 2; I++) if (CMPI->getOperand(I).isReg()) { MachineInstr *Inst = MRI->getVRegDef(CMPI->getOperand(I).getReg()); if (Inst->getParent() == &MBB && Inst->getOpcode() != PPC::PHI) return false; } PredMBB = Pred1MBB; MBBtoMoveCmp = Pred2MBB; return true; } return false; } // This function will iterate over the input map containing a pair of TOC save // instruction and a flag. The flag will be set to false if the TOC save is // proven redundant. This function will erase from the basic block all the TOC // saves marked as redundant. bool PPCMIPeephole::eliminateRedundantTOCSaves( std::map &TOCSaves) { bool Simplified = false; int NumKept = 0; for (auto TOCSave : TOCSaves) { if (!TOCSave.second) { TOCSave.first->eraseFromParent(); RemoveTOCSave++; Simplified = true; } else { NumKept++; } } if (NumKept > 1) MultiTOCSaves++; return Simplified; } // If multiple conditional branches are executed based on the (essentially) // same comparison, we merge compare instructions into one and make multiple // conditional branches on this comparison. // For example, // if (a == 0) { ... } // else if (a < 0) { ... } // can be executed by one compare and two conditional branches instead of // two pairs of a compare and a conditional branch. // // This method merges two compare instructions in two MBBs and modifies the // compare and conditional branch instructions if needed. // For the above example, the input for this pass looks like: // cmplwi r3, 0 // beq 0, .LBB0_3 // cmpwi r3, -1 // bgt 0, .LBB0_4 // So, before merging two compares, we need to modify these instructions as // cmpwi r3, 0 ; cmplwi and cmpwi yield same result for beq // beq 0, .LBB0_3 // cmpwi r3, 0 ; greather than -1 means greater or equal to 0 // bge 0, .LBB0_4 bool PPCMIPeephole::eliminateRedundantCompare(void) { bool Simplified = false; for (MachineBasicBlock &MBB2 : *MF) { MachineBasicBlock *MBB1 = nullptr, *MBBtoMoveCmp = nullptr; // For fully redundant case, we select two basic blocks MBB1 and MBB2 // as an optimization target if // - both MBBs end with a conditional branch, // - MBB1 is the only predecessor of MBB2, and // - compare does not take a physical register as a operand in both MBBs. // In this case, eligibleForCompareElimination sets MBBtoMoveCmp nullptr. // // As partially redundant case, we additionally handle if MBB2 has one // additional predecessor, which has only one successor (MBB2). // In this case, we move the compare instruction originally in MBB2 into // MBBtoMoveCmp. This partially redundant case is typically appear by // compiling a while loop; here, MBBtoMoveCmp is the loop preheader. // // Overview of CFG of related basic blocks // Fully redundant case Partially redundant case // -------- ---------------- -------- // | MBB1 | (w/ 2 succ) | MBBtoMoveCmp | | MBB1 | (w/ 2 succ) // -------- ---------------- -------- // | \ (w/ 1 succ) \ | \ // | \ \ | \ // | \ | // -------- -------- // | MBB2 | (w/ 1 pred | MBB2 | (w/ 2 pred // -------- and 2 succ) -------- and 2 succ) // | \ | \ // | \ | \ // if (!eligibleForCompareElimination(MBB2, MBB1, MBBtoMoveCmp, MRI)) continue; MachineInstr *BI1 = &*MBB1->getFirstInstrTerminator(); MachineInstr *CMPI1 = MRI->getVRegDef(BI1->getOperand(1).getReg()); MachineInstr *BI2 = &*MBB2.getFirstInstrTerminator(); MachineInstr *CMPI2 = MRI->getVRegDef(BI2->getOperand(1).getReg()); bool IsPartiallyRedundant = (MBBtoMoveCmp != nullptr); // We cannot optimize an unsupported compare opcode or // a mix of 32-bit and 64-bit comaprisons if (!isSupportedCmpOp(CMPI1->getOpcode()) || !isSupportedCmpOp(CMPI2->getOpcode()) || is64bitCmpOp(CMPI1->getOpcode()) != is64bitCmpOp(CMPI2->getOpcode())) continue; unsigned NewOpCode = 0; unsigned NewPredicate1 = 0, NewPredicate2 = 0; int16_t Imm1 = 0, NewImm1 = 0, Imm2 = 0, NewImm2 = 0; bool SwapOperands = false; if (CMPI1->getOpcode() != CMPI2->getOpcode()) { // Typically, unsigned comparison is used for equality check, but // we replace it with a signed comparison if the comparison // to be merged is a signed comparison. // In other cases of opcode mismatch, we cannot optimize this. // We cannot change opcode when comparing against an immediate // if the most significant bit of the immediate is one // due to the difference in sign extension. auto CmpAgainstImmWithSignBit = [](MachineInstr *I) { if (!I->getOperand(2).isImm()) return false; int16_t Imm = (int16_t)I->getOperand(2).getImm(); return Imm < 0; }; if (isEqOrNe(BI2) && !CmpAgainstImmWithSignBit(CMPI2) && CMPI1->getOpcode() == getSignedCmpOpCode(CMPI2->getOpcode())) NewOpCode = CMPI1->getOpcode(); else if (isEqOrNe(BI1) && !CmpAgainstImmWithSignBit(CMPI1) && getSignedCmpOpCode(CMPI1->getOpcode()) == CMPI2->getOpcode()) NewOpCode = CMPI2->getOpcode(); else continue; } if (CMPI1->getOperand(2).isReg() && CMPI2->getOperand(2).isReg()) { // In case of comparisons between two registers, these two registers // must be same to merge two comparisons. unsigned Cmp1Operand1 = getSrcVReg(CMPI1->getOperand(1).getReg(), nullptr, nullptr, MRI); unsigned Cmp1Operand2 = getSrcVReg(CMPI1->getOperand(2).getReg(), nullptr, nullptr, MRI); unsigned Cmp2Operand1 = getSrcVReg(CMPI2->getOperand(1).getReg(), MBB1, &MBB2, MRI); unsigned Cmp2Operand2 = getSrcVReg(CMPI2->getOperand(2).getReg(), MBB1, &MBB2, MRI); if (Cmp1Operand1 == Cmp2Operand1 && Cmp1Operand2 == Cmp2Operand2) { // Same pair of registers in the same order; ready to merge as is. } else if (Cmp1Operand1 == Cmp2Operand2 && Cmp1Operand2 == Cmp2Operand1) { // Same pair of registers in different order. // We reverse the predicate to merge compare instructions. PPC::Predicate Pred = (PPC::Predicate)BI2->getOperand(0).getImm(); NewPredicate2 = (unsigned)PPC::getSwappedPredicate(Pred); // In case of partial redundancy, we need to swap operands // in another compare instruction. SwapOperands = true; } else continue; } else if (CMPI1->getOperand(2).isImm() && CMPI2->getOperand(2).isImm()) { // In case of comparisons between a register and an immediate, // the operand register must be same for two compare instructions. unsigned Cmp1Operand1 = getSrcVReg(CMPI1->getOperand(1).getReg(), nullptr, nullptr, MRI); unsigned Cmp2Operand1 = getSrcVReg(CMPI2->getOperand(1).getReg(), MBB1, &MBB2, MRI); if (Cmp1Operand1 != Cmp2Operand1) continue; NewImm1 = Imm1 = (int16_t)CMPI1->getOperand(2).getImm(); NewImm2 = Imm2 = (int16_t)CMPI2->getOperand(2).getImm(); // If immediate are not same, we try to adjust by changing predicate; // e.g. GT imm means GE (imm+1). if (Imm1 != Imm2 && (!isEqOrNe(BI2) || !isEqOrNe(BI1))) { int Diff = Imm1 - Imm2; if (Diff < -2 || Diff > 2) continue; unsigned PredToInc1 = getPredicateToIncImm(BI1, CMPI1); unsigned PredToDec1 = getPredicateToDecImm(BI1, CMPI1); unsigned PredToInc2 = getPredicateToIncImm(BI2, CMPI2); unsigned PredToDec2 = getPredicateToDecImm(BI2, CMPI2); if (Diff == 2) { if (PredToInc2 && PredToDec1) { NewPredicate2 = PredToInc2; NewPredicate1 = PredToDec1; NewImm2++; NewImm1--; } } else if (Diff == 1) { if (PredToInc2) { NewImm2++; NewPredicate2 = PredToInc2; } else if (PredToDec1) { NewImm1--; NewPredicate1 = PredToDec1; } } else if (Diff == -1) { if (PredToDec2) { NewImm2--; NewPredicate2 = PredToDec2; } else if (PredToInc1) { NewImm1++; NewPredicate1 = PredToInc1; } } else if (Diff == -2) { if (PredToDec2 && PredToInc1) { NewPredicate2 = PredToDec2; NewPredicate1 = PredToInc1; NewImm2--; NewImm1++; } } } // We cannot merge two compares if the immediates are not same. if (NewImm2 != NewImm1) continue; } LLVM_DEBUG(dbgs() << "Optimize two pairs of compare and branch:\n"); LLVM_DEBUG(CMPI1->dump()); LLVM_DEBUG(BI1->dump()); LLVM_DEBUG(CMPI2->dump()); LLVM_DEBUG(BI2->dump()); // We adjust opcode, predicates and immediate as we determined above. if (NewOpCode != 0 && NewOpCode != CMPI1->getOpcode()) { CMPI1->setDesc(TII->get(NewOpCode)); } if (NewPredicate1) { BI1->getOperand(0).setImm(NewPredicate1); } if (NewPredicate2) { BI2->getOperand(0).setImm(NewPredicate2); } if (NewImm1 != Imm1) { CMPI1->getOperand(2).setImm(NewImm1); } if (IsPartiallyRedundant) { // We touch up the compare instruction in MBB2 and move it to // a previous BB to handle partially redundant case. if (SwapOperands) { Register Op1 = CMPI2->getOperand(1).getReg(); Register Op2 = CMPI2->getOperand(2).getReg(); CMPI2->getOperand(1).setReg(Op2); CMPI2->getOperand(2).setReg(Op1); } if (NewImm2 != Imm2) CMPI2->getOperand(2).setImm(NewImm2); for (int I = 1; I <= 2; I++) { if (CMPI2->getOperand(I).isReg()) { MachineInstr *Inst = MRI->getVRegDef(CMPI2->getOperand(I).getReg()); if (Inst->getParent() != &MBB2) continue; assert(Inst->getOpcode() == PPC::PHI && "We cannot support if an operand comes from this BB."); unsigned SrcReg = getIncomingRegForBlock(Inst, MBBtoMoveCmp); CMPI2->getOperand(I).setReg(SrcReg); } } auto I = MachineBasicBlock::iterator(MBBtoMoveCmp->getFirstTerminator()); MBBtoMoveCmp->splice(I, &MBB2, MachineBasicBlock::iterator(CMPI2)); DebugLoc DL = CMPI2->getDebugLoc(); Register NewVReg = MRI->createVirtualRegister(&PPC::CRRCRegClass); BuildMI(MBB2, MBB2.begin(), DL, TII->get(PPC::PHI), NewVReg) .addReg(BI1->getOperand(1).getReg()).addMBB(MBB1) .addReg(BI2->getOperand(1).getReg()).addMBB(MBBtoMoveCmp); BI2->getOperand(1).setReg(NewVReg); } else { // We finally eliminate compare instruction in MBB2. BI2->getOperand(1).setReg(BI1->getOperand(1).getReg()); CMPI2->eraseFromParent(); } BI2->getOperand(1).setIsKill(true); BI1->getOperand(1).setIsKill(false); LLVM_DEBUG(dbgs() << "into a compare and two branches:\n"); LLVM_DEBUG(CMPI1->dump()); LLVM_DEBUG(BI1->dump()); LLVM_DEBUG(BI2->dump()); if (IsPartiallyRedundant) { LLVM_DEBUG(dbgs() << "The following compare is moved into " << printMBBReference(*MBBtoMoveCmp) << " to handle partial redundancy.\n"); LLVM_DEBUG(CMPI2->dump()); } Simplified = true; } return Simplified; } // We miss the opportunity to emit an RLDIC when lowering jump tables // since ISEL sees only a single basic block. When selecting, the clear // and shift left will be in different blocks. bool PPCMIPeephole::emitRLDICWhenLoweringJumpTables(MachineInstr &MI) { if (MI.getOpcode() != PPC::RLDICR) return false; Register SrcReg = MI.getOperand(1).getReg(); if (!Register::isVirtualRegister(SrcReg)) return false; MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); if (SrcMI->getOpcode() != PPC::RLDICL) return false; MachineOperand MOpSHSrc = SrcMI->getOperand(2); MachineOperand MOpMBSrc = SrcMI->getOperand(3); MachineOperand MOpSHMI = MI.getOperand(2); MachineOperand MOpMEMI = MI.getOperand(3); if (!(MOpSHSrc.isImm() && MOpMBSrc.isImm() && MOpSHMI.isImm() && MOpMEMI.isImm())) return false; uint64_t SHSrc = MOpSHSrc.getImm(); uint64_t MBSrc = MOpMBSrc.getImm(); uint64_t SHMI = MOpSHMI.getImm(); uint64_t MEMI = MOpMEMI.getImm(); uint64_t NewSH = SHSrc + SHMI; uint64_t NewMB = MBSrc - SHMI; if (NewMB > 63 || NewSH > 63) return false; // The bits cleared with RLDICL are [0, MBSrc). // The bits cleared with RLDICR are (MEMI, 63]. // After the sequence, the bits cleared are: // [0, MBSrc-SHMI) and (MEMI, 63). // // The bits cleared with RLDIC are [0, NewMB) and (63-NewSH, 63]. if ((63 - NewSH) != MEMI) return false; LLVM_DEBUG(dbgs() << "Converting pair: "); LLVM_DEBUG(SrcMI->dump()); LLVM_DEBUG(MI.dump()); MI.setDesc(TII->get(PPC::RLDIC)); MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg()); MI.getOperand(2).setImm(NewSH); MI.getOperand(3).setImm(NewMB); MI.getOperand(1).setIsKill(SrcMI->getOperand(1).isKill()); SrcMI->getOperand(1).setIsKill(false); LLVM_DEBUG(dbgs() << "To: "); LLVM_DEBUG(MI.dump()); NumRotatesCollapsed++; // If SrcReg has no non-debug use it's safe to delete its def SrcMI. if (MRI->use_nodbg_empty(SrcReg)) { assert(!SrcMI->hasImplicitDef() && "Not expecting an implicit def with this instr."); SrcMI->eraseFromParent(); } return true; } // For case in LLVM IR // entry: // %iconv = sext i32 %index to i64 // br i1 undef label %true, label %false // true: // %ptr = getelementptr inbounds i32, i32* null, i64 %iconv // ... // PPCISelLowering::combineSHL fails to combine, because sext and shl are in // different BBs when conducting instruction selection. We can do a peephole // optimization to combine these two instructions into extswsli after // instruction selection. bool PPCMIPeephole::combineSEXTAndSHL(MachineInstr &MI, MachineInstr *&ToErase) { if (MI.getOpcode() != PPC::RLDICR) return false; if (!MF->getSubtarget().isISA3_0()) return false; assert(MI.getNumOperands() == 4 && "RLDICR should have 4 operands"); MachineOperand MOpSHMI = MI.getOperand(2); MachineOperand MOpMEMI = MI.getOperand(3); if (!(MOpSHMI.isImm() && MOpMEMI.isImm())) return false; uint64_t SHMI = MOpSHMI.getImm(); uint64_t MEMI = MOpMEMI.getImm(); if (SHMI + MEMI != 63) return false; Register SrcReg = MI.getOperand(1).getReg(); if (!Register::isVirtualRegister(SrcReg)) return false; MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); if (SrcMI->getOpcode() != PPC::EXTSW && SrcMI->getOpcode() != PPC::EXTSW_32_64) return false; // If the register defined by extsw has more than one use, combination is not // needed. if (!MRI->hasOneNonDBGUse(SrcReg)) return false; assert(SrcMI->getNumOperands() == 2 && "EXTSW should have 2 operands"); assert(SrcMI->getOperand(1).isReg() && "EXTSW's second operand should be a register"); if (!Register::isVirtualRegister(SrcMI->getOperand(1).getReg())) return false; LLVM_DEBUG(dbgs() << "Combining pair: "); LLVM_DEBUG(SrcMI->dump()); LLVM_DEBUG(MI.dump()); MachineInstr *NewInstr = BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), SrcMI->getOpcode() == PPC::EXTSW ? TII->get(PPC::EXTSWSLI) : TII->get(PPC::EXTSWSLI_32_64), MI.getOperand(0).getReg()) .add(SrcMI->getOperand(1)) .add(MOpSHMI); (void)NewInstr; LLVM_DEBUG(dbgs() << "TO: "); LLVM_DEBUG(NewInstr->dump()); ++NumEXTSWAndSLDICombined; ToErase = &MI; // SrcMI, which is extsw, is of no use now, erase it. SrcMI->eraseFromParent(); return true; } } // end default namespace INITIALIZE_PASS_BEGIN(PPCMIPeephole, DEBUG_TYPE, "PowerPC MI Peephole Optimization", false, false) INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree) INITIALIZE_PASS_END(PPCMIPeephole, DEBUG_TYPE, "PowerPC MI Peephole Optimization", false, false) char PPCMIPeephole::ID = 0; FunctionPass* llvm::createPPCMIPeepholePass() { return new PPCMIPeephole(); } diff --git a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp index 8e4a50c1b33d..7d1282df369b 100644 --- a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp +++ b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp @@ -1,513 +1,504 @@ //===--------- PPCPreEmitPeephole.cpp - Late peephole optimizations -------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // A pre-emit peephole for catching opportunities introduced by late passes such // as MachineBlockPlacement. // //===----------------------------------------------------------------------===// #include "PPC.h" #include "PPCInstrInfo.h" #include "PPCSubtarget.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" using namespace llvm; #define DEBUG_TYPE "ppc-pre-emit-peephole" STATISTIC(NumRRConvertedInPreEmit, "Number of r+r instructions converted to r+i in pre-emit peephole"); STATISTIC(NumRemovedInPreEmit, "Number of instructions deleted in pre-emit peephole"); STATISTIC(NumberOfSelfCopies, "Number of self copy instructions eliminated"); STATISTIC(NumFrameOffFoldInPreEmit, "Number of folding frame offset by using r+r in pre-emit peephole"); -STATISTIC(NumRotateInstrFoldInPreEmit, - "Number of folding Rotate instructions in pre-emit peephole"); static cl::opt EnablePCRelLinkerOpt("ppc-pcrel-linker-opt", cl::Hidden, cl::init(true), cl::desc("enable PC Relative linker optimization")); static cl::opt RunPreEmitPeephole("ppc-late-peephole", cl::Hidden, cl::init(true), cl::desc("Run pre-emit peephole optimizations.")); namespace { static bool hasPCRelativeForm(MachineInstr &Use) { switch (Use.getOpcode()) { default: return false; case PPC::LBZ: case PPC::LBZ8: case PPC::LHA: case PPC::LHA8: case PPC::LHZ: case PPC::LHZ8: case PPC::LWZ: case PPC::LWZ8: case PPC::STB: case PPC::STB8: case PPC::STH: case PPC::STH8: case PPC::STW: case PPC::STW8: case PPC::LD: case PPC::STD: case PPC::LWA: case PPC::LXSD: case PPC::LXSSP: case PPC::LXV: case PPC::STXSD: case PPC::STXSSP: case PPC::STXV: case PPC::LFD: case PPC::LFS: case PPC::STFD: case PPC::STFS: case PPC::DFLOADf32: case PPC::DFLOADf64: case PPC::DFSTOREf32: case PPC::DFSTOREf64: return true; } } class PPCPreEmitPeephole : public MachineFunctionPass { public: static char ID; PPCPreEmitPeephole() : MachineFunctionPass(ID) { initializePPCPreEmitPeepholePass(*PassRegistry::getPassRegistry()); } void getAnalysisUsage(AnalysisUsage &AU) const override { MachineFunctionPass::getAnalysisUsage(AU); } MachineFunctionProperties getRequiredProperties() const override { return MachineFunctionProperties().set( MachineFunctionProperties::Property::NoVRegs); } // This function removes any redundant load immediates. It has two level // loops - The outer loop finds the load immediates BBI that could be used // to replace following redundancy. The inner loop scans instructions that // after BBI to find redundancy and update kill/dead flags accordingly. If // AfterBBI is the same as BBI, it is redundant, otherwise any instructions // that modify the def register of BBI would break the scanning. // DeadOrKillToUnset is a pointer to the previous operand that had the // kill/dead flag set. It keeps track of the def register of BBI, the use // registers of AfterBBIs and the def registers of AfterBBIs. bool removeRedundantLIs(MachineBasicBlock &MBB, const TargetRegisterInfo *TRI) { LLVM_DEBUG(dbgs() << "Remove redundant load immediates from MBB:\n"; MBB.dump(); dbgs() << "\n"); DenseSet InstrsToErase; for (auto BBI = MBB.instr_begin(); BBI != MBB.instr_end(); ++BBI) { // Skip load immediate that is marked to be erased later because it // cannot be used to replace any other instructions. if (InstrsToErase.find(&*BBI) != InstrsToErase.end()) continue; // Skip non-load immediate. unsigned Opc = BBI->getOpcode(); if (Opc != PPC::LI && Opc != PPC::LI8 && Opc != PPC::LIS && Opc != PPC::LIS8) continue; // Skip load immediate, where the operand is a relocation (e.g., $r3 = // LI target-flags(ppc-lo) %const.0). if (!BBI->getOperand(1).isImm()) continue; assert(BBI->getOperand(0).isReg() && "Expected a register for the first operand"); LLVM_DEBUG(dbgs() << "Scanning after load immediate: "; BBI->dump();); Register Reg = BBI->getOperand(0).getReg(); int64_t Imm = BBI->getOperand(1).getImm(); MachineOperand *DeadOrKillToUnset = nullptr; if (BBI->getOperand(0).isDead()) { DeadOrKillToUnset = &BBI->getOperand(0); LLVM_DEBUG(dbgs() << " Kill flag of " << *DeadOrKillToUnset << " from load immediate " << *BBI << " is a unsetting candidate\n"); } // This loop scans instructions after BBI to see if there is any // redundant load immediate. for (auto AfterBBI = std::next(BBI); AfterBBI != MBB.instr_end(); ++AfterBBI) { // Track the operand that kill Reg. We would unset the kill flag of // the operand if there is a following redundant load immediate. int KillIdx = AfterBBI->findRegisterUseOperandIdx(Reg, true, TRI); // We can't just clear implicit kills, so if we encounter one, stop // looking further. if (KillIdx != -1 && AfterBBI->getOperand(KillIdx).isImplicit()) { LLVM_DEBUG(dbgs() << "Encountered an implicit kill, cannot proceed: "); LLVM_DEBUG(AfterBBI->dump()); break; } if (KillIdx != -1) { assert(!DeadOrKillToUnset && "Shouldn't kill same register twice"); DeadOrKillToUnset = &AfterBBI->getOperand(KillIdx); LLVM_DEBUG(dbgs() << " Kill flag of " << *DeadOrKillToUnset << " from " << *AfterBBI << " is a unsetting candidate\n"); } if (!AfterBBI->modifiesRegister(Reg, TRI)) continue; // Finish scanning because Reg is overwritten by a non-load // instruction. if (AfterBBI->getOpcode() != Opc) break; assert(AfterBBI->getOperand(0).isReg() && "Expected a register for the first operand"); // Finish scanning because Reg is overwritten by a relocation or a // different value. if (!AfterBBI->getOperand(1).isImm() || AfterBBI->getOperand(1).getImm() != Imm) break; // It loads same immediate value to the same Reg, which is redundant. // We would unset kill flag in previous Reg usage to extend live range // of Reg first, then remove the redundancy. if (DeadOrKillToUnset) { LLVM_DEBUG(dbgs() << " Unset dead/kill flag of " << *DeadOrKillToUnset << " from " << *DeadOrKillToUnset->getParent()); if (DeadOrKillToUnset->isDef()) DeadOrKillToUnset->setIsDead(false); else DeadOrKillToUnset->setIsKill(false); } DeadOrKillToUnset = AfterBBI->findRegisterDefOperand(Reg, true, true, TRI); if (DeadOrKillToUnset) LLVM_DEBUG(dbgs() << " Dead flag of " << *DeadOrKillToUnset << " from " << *AfterBBI << " is a unsetting candidate\n"); InstrsToErase.insert(&*AfterBBI); LLVM_DEBUG(dbgs() << " Remove redundant load immediate: "; AfterBBI->dump()); } } for (MachineInstr *MI : InstrsToErase) { MI->eraseFromParent(); } NumRemovedInPreEmit += InstrsToErase.size(); return !InstrsToErase.empty(); } // Check if this instruction is a PLDpc that is part of a GOT indirect // access. bool isGOTPLDpc(MachineInstr &Instr) { if (Instr.getOpcode() != PPC::PLDpc) return false; // The result must be a register. const MachineOperand &LoadedAddressReg = Instr.getOperand(0); if (!LoadedAddressReg.isReg()) return false; // Make sure that this is a global symbol. const MachineOperand &SymbolOp = Instr.getOperand(1); if (!SymbolOp.isGlobal()) return false; // Finally return true only if the GOT flag is present. return (SymbolOp.getTargetFlags() & PPCII::MO_GOT_FLAG); } bool addLinkerOpt(MachineBasicBlock &MBB, const TargetRegisterInfo *TRI) { MachineFunction *MF = MBB.getParent(); // If the linker opt is disabled then just return. if (!EnablePCRelLinkerOpt) return false; // Add this linker opt only if we are using PC Relative memops. if (!MF->getSubtarget().isUsingPCRelativeCalls()) return false; // Struct to keep track of one def/use pair for a GOT indirect access. struct GOTDefUsePair { MachineBasicBlock::iterator DefInst; MachineBasicBlock::iterator UseInst; Register DefReg; Register UseReg; bool StillValid; }; // Vector of def/ues pairs in this basic block. SmallVector CandPairs; SmallVector ValidPairs; bool MadeChange = false; // Run through all of the instructions in the basic block and try to // collect potential pairs of GOT indirect access instructions. for (auto BBI = MBB.instr_begin(); BBI != MBB.instr_end(); ++BBI) { // Look for the initial GOT indirect load. if (isGOTPLDpc(*BBI)) { GOTDefUsePair CurrentPair{BBI, MachineBasicBlock::iterator(), BBI->getOperand(0).getReg(), PPC::NoRegister, true}; CandPairs.push_back(CurrentPair); continue; } // We haven't encountered any new PLD instructions, nothing to check. if (CandPairs.empty()) continue; // Run through the candidate pairs and see if any of the registers // defined in the PLD instructions are used by this instruction. // Note: the size of CandPairs can change in the loop. for (unsigned Idx = 0; Idx < CandPairs.size(); Idx++) { GOTDefUsePair &Pair = CandPairs[Idx]; // The instruction does not use or modify this PLD's def reg, // ignore it. if (!BBI->readsRegister(Pair.DefReg, TRI) && !BBI->modifiesRegister(Pair.DefReg, TRI)) continue; // The use needs to be used in the address compuation and not // as the register being stored for a store. const MachineOperand *UseOp = hasPCRelativeForm(*BBI) ? &BBI->getOperand(2) : nullptr; // Check for a valid use. if (UseOp && UseOp->isReg() && UseOp->getReg() == Pair.DefReg && UseOp->isUse() && UseOp->isKill()) { Pair.UseInst = BBI; Pair.UseReg = BBI->getOperand(0).getReg(); ValidPairs.push_back(Pair); } CandPairs.erase(CandPairs.begin() + Idx); } } // Go through all of the pairs and check for any more valid uses. for (auto Pair = ValidPairs.begin(); Pair != ValidPairs.end(); Pair++) { // We shouldn't be here if we don't have a valid pair. assert(Pair->UseInst.isValid() && Pair->StillValid && "Kept an invalid def/use pair for GOT PCRel opt"); // We have found a potential pair. Search through the instructions // between the def and the use to see if it is valid to mark this as a // linker opt. MachineBasicBlock::iterator BBI = Pair->DefInst; ++BBI; for (; BBI != Pair->UseInst; ++BBI) { if (BBI->readsRegister(Pair->UseReg, TRI) || BBI->modifiesRegister(Pair->UseReg, TRI)) { Pair->StillValid = false; break; } } if (!Pair->StillValid) continue; // The load/store instruction that uses the address from the PLD will // either use a register (for a store) or define a register (for the // load). That register will be added as an implicit def to the PLD // and as an implicit use on the second memory op. This is a precaution // to prevent future passes from using that register between the two // instructions. MachineOperand ImplDef = MachineOperand::CreateReg(Pair->UseReg, true, true); MachineOperand ImplUse = MachineOperand::CreateReg(Pair->UseReg, false, true); Pair->DefInst->addOperand(ImplDef); Pair->UseInst->addOperand(ImplUse); // Create the symbol. MCContext &Context = MF->getContext(); MCSymbol *Symbol = Context.createTempSymbol(Twine("pcrel"), false, false); MachineOperand PCRelLabel = MachineOperand::CreateMCSymbol(Symbol, PPCII::MO_PCREL_OPT_FLAG); Pair->DefInst->addOperand(*MF, PCRelLabel); Pair->UseInst->addOperand(*MF, PCRelLabel); MadeChange |= true; } return MadeChange; } bool runOnMachineFunction(MachineFunction &MF) override { if (skipFunction(MF.getFunction()) || !RunPreEmitPeephole) { // Remove UNENCODED_NOP even when this pass is disabled. // This needs to be done unconditionally so we don't emit zeros // in the instruction stream. SmallVector InstrsToErase; for (MachineBasicBlock &MBB : MF) for (MachineInstr &MI : MBB) if (MI.getOpcode() == PPC::UNENCODED_NOP) InstrsToErase.push_back(&MI); for (MachineInstr *MI : InstrsToErase) MI->eraseFromParent(); return false; } bool Changed = false; const PPCInstrInfo *TII = MF.getSubtarget().getInstrInfo(); const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); SmallVector InstrsToErase; for (MachineBasicBlock &MBB : MF) { Changed |= removeRedundantLIs(MBB, TRI); Changed |= addLinkerOpt(MBB, TRI); for (MachineInstr &MI : MBB) { unsigned Opc = MI.getOpcode(); if (Opc == PPC::UNENCODED_NOP) { InstrsToErase.push_back(&MI); continue; } // Detect self copies - these can result from running AADB. if (PPCInstrInfo::isSameClassPhysRegCopy(Opc)) { const MCInstrDesc &MCID = TII->get(Opc); if (MCID.getNumOperands() == 3 && MI.getOperand(0).getReg() == MI.getOperand(1).getReg() && MI.getOperand(0).getReg() == MI.getOperand(2).getReg()) { NumberOfSelfCopies++; LLVM_DEBUG(dbgs() << "Deleting self-copy instruction: "); LLVM_DEBUG(MI.dump()); InstrsToErase.push_back(&MI); continue; } else if (MCID.getNumOperands() == 2 && MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) { NumberOfSelfCopies++; LLVM_DEBUG(dbgs() << "Deleting self-copy instruction: "); LLVM_DEBUG(MI.dump()); InstrsToErase.push_back(&MI); continue; } } MachineInstr *DefMIToErase = nullptr; if (TII->convertToImmediateForm(MI, &DefMIToErase)) { Changed = true; NumRRConvertedInPreEmit++; LLVM_DEBUG(dbgs() << "Converted instruction to imm form: "); LLVM_DEBUG(MI.dump()); if (DefMIToErase) { InstrsToErase.push_back(DefMIToErase); } } if (TII->foldFrameOffset(MI)) { Changed = true; NumFrameOffFoldInPreEmit++; LLVM_DEBUG(dbgs() << "Frame offset folding by using index form: "); LLVM_DEBUG(MI.dump()); } - MachineInstr *ToErase = nullptr; - if (TII->simplifyRotateAndMaskInstr(MI, ToErase)) { - Changed = true; - NumRotateInstrFoldInPreEmit++; - if (ToErase) - InstrsToErase.push_back(ToErase); - } } // Eliminate conditional branch based on a constant CR bit by // CRSET or CRUNSET. We eliminate the conditional branch or // convert it into an unconditional branch. Also, if the CR bit // is not used by other instructions, we eliminate CRSET as well. auto I = MBB.getFirstInstrTerminator(); if (I == MBB.instr_end()) continue; MachineInstr *Br = &*I; if (Br->getOpcode() != PPC::BC && Br->getOpcode() != PPC::BCn) continue; MachineInstr *CRSetMI = nullptr; Register CRBit = Br->getOperand(0).getReg(); unsigned CRReg = getCRFromCRBit(CRBit); bool SeenUse = false; MachineBasicBlock::reverse_iterator It = Br, Er = MBB.rend(); for (It++; It != Er; It++) { if (It->modifiesRegister(CRBit, TRI)) { if ((It->getOpcode() == PPC::CRUNSET || It->getOpcode() == PPC::CRSET) && It->getOperand(0).getReg() == CRBit) CRSetMI = &*It; break; } if (It->readsRegister(CRBit, TRI)) SeenUse = true; } if (!CRSetMI) continue; unsigned CRSetOp = CRSetMI->getOpcode(); if ((Br->getOpcode() == PPC::BCn && CRSetOp == PPC::CRSET) || (Br->getOpcode() == PPC::BC && CRSetOp == PPC::CRUNSET)) { // Remove this branch since it cannot be taken. InstrsToErase.push_back(Br); MBB.removeSuccessor(Br->getOperand(1).getMBB()); } else { // This conditional branch is always taken. So, remove all branches // and insert an unconditional branch to the destination of this. MachineBasicBlock::iterator It = Br, Er = MBB.end(); for (; It != Er; It++) { if (It->isDebugInstr()) continue; assert(It->isTerminator() && "Non-terminator after a terminator"); InstrsToErase.push_back(&*It); } if (!MBB.isLayoutSuccessor(Br->getOperand(1).getMBB())) { ArrayRef NoCond; TII->insertBranch(MBB, Br->getOperand(1).getMBB(), nullptr, NoCond, Br->getDebugLoc()); } for (auto &Succ : MBB.successors()) if (Succ != Br->getOperand(1).getMBB()) { MBB.removeSuccessor(Succ); break; } } // If the CRBit is not used by another instruction, we can eliminate // CRSET/CRUNSET instruction. if (!SeenUse) { // We need to check use of the CRBit in successors. for (auto &SuccMBB : MBB.successors()) if (SuccMBB->isLiveIn(CRBit) || SuccMBB->isLiveIn(CRReg)) { SeenUse = true; break; } if (!SeenUse) InstrsToErase.push_back(CRSetMI); } } for (MachineInstr *MI : InstrsToErase) { LLVM_DEBUG(dbgs() << "PPC pre-emit peephole: erasing instruction: "); LLVM_DEBUG(MI->dump()); MI->eraseFromParent(); NumRemovedInPreEmit++; } return Changed; } }; } INITIALIZE_PASS(PPCPreEmitPeephole, DEBUG_TYPE, "PowerPC Pre-Emit Peephole", false, false) char PPCPreEmitPeephole::ID = 0; FunctionPass *llvm::createPPCPreEmitPeepholePass() { return new PPCPreEmitPeephole(); } diff --git a/llvm/test/CodeGen/PowerPC/fold-rlwinm-after-ra.mir b/llvm/test/CodeGen/PowerPC/fold-rlwinm-after-ra.mir deleted file mode 100644 index b26487e6f136..000000000000 --- a/llvm/test/CodeGen/PowerPC/fold-rlwinm-after-ra.mir +++ /dev/null @@ -1,104 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -stop-after \ -# RUN: ppc-pre-emit-peephole %s -o - | FileCheck %s - ---- -name: testFoldRLWINM -tracksRegLiveness: true -body: | - bb.0.entry: - liveins: $r3 - ; CHECK-LABEL: name: testFoldRLWINM - ; CHECK: liveins: $r3 - ; CHECK: renamable $r3 = RLWINM killed renamable $r3, 14, 0, 12, implicit-def $x3 - ; CHECK: BLR8 implicit $lr8, implicit $rm, implicit killed $x3 - $r3 = RLWINM killed $r3, 27, 5, 31 - dead renamable $r3 = RLWINM killed renamable $r3, 19, 0, 12, implicit-def $x3 - BLR8 implicit $lr8, implicit $rm, implicit killed $x3 -... ---- -name: testFoldRLWINMSrcFullMask -tracksRegLiveness: true -body: | - bb.0.entry: - liveins: $r3 - ; CHECK-LABEL: name: testFoldRLWINMSrcFullMask - ; CHECK: liveins: $r3 - ; CHECK: renamable $r3 = RLWINM killed renamable $r3, 14, 0, 12, implicit-def $x3 - ; CHECK: BLR8 implicit $lr8, implicit $rm, implicit killed $x3 - $r3 = RLWINM killed $r3, 27, 0, 31 - dead renamable $r3 = RLWINM killed renamable $r3, 19, 0, 12, implicit-def $x3 - BLR8 implicit $lr8, implicit $rm, implicit killed $x3 -... ---- -name: testFoldRLWINMSrcWrapped -tracksRegLiveness: true -body: | - bb.0.entry: - liveins: $r3 - ; CHECK-LABEL: name: testFoldRLWINMSrcWrapped - ; CHECK: liveins: $r3 - ; CHECK: renamable $r3 = RLWINM killed renamable $r3, 14, 11, 12, implicit-def $x3 - ; CHECK: BLR8 implicit $lr8, implicit $rm, implicit killed $x3 - $r3 = RLWINM killed $r3, 27, 30, 10 - dead renamable $r3 = RLWINM killed renamable $r3, 19, 0, 12, implicit-def $x3 - BLR8 implicit $lr8, implicit $rm, implicit killed $x3 -... ---- -name: testFoldRLWINMToZero -tracksRegLiveness: true -body: | - bb.0.entry: - liveins: $r3 - ; CHECK-LABEL: name: testFoldRLWINMToZero - ; CHECK: liveins: $r3 - ; CHECK: renamable $r3 = LI 0, implicit-def $x3 - ; CHECK: BLR8 implicit $lr8, implicit $rm, implicit killed $x3 - $r3 = RLWINM killed $r3, 27, 5, 10 - dead renamable $r3 = RLWINM killed renamable $r3, 8, 5, 10, implicit-def $x3 - BLR8 implicit $lr8, implicit $rm, implicit killed $x3 -... ---- -name: testFoldRLWINM_recToZero -tracksRegLiveness: true -body: | - bb.0.entry: - liveins: $r3 - ; CHECK-LABEL: name: testFoldRLWINM_recToZero - ; CHECK: liveins: $r3 - ; CHECK: dead renamable $r3 = ANDI_rec killed renamable $r3, 0, implicit-def $cr0 - ; CHECK: BLR8 implicit $lr8, implicit $rm, implicit killed $cr0 - $r3 = RLWINM killed $r3, 27, 5, 10 - dead renamable $r3 = RLWINM_rec killed renamable $r3, 8, 5, 10, implicit-def $cr0 - BLR8 implicit $lr8, implicit $rm, implicit killed $cr0 -... ---- -name: testFoldRLWINMoToZeroSrcCanNotBeDeleted -tracksRegLiveness: true -body: | - bb.0.entry: - liveins: $r3 - ; CHECK-LABEL: name: testFoldRLWINMoToZeroSrcCanNotBeDeleted - ; CHECK: liveins: $r3 - ; CHECK: $r3 = RLWINM_rec $r3, 27, 5, 10, implicit-def dead $cr0 - ; CHECK: dead renamable $r3 = ANDI_rec killed renamable $r3, 0, implicit-def $cr0 - ; CHECK: BLR8 implicit $lr8, implicit $rm, implicit killed $cr0 - $r3 = RLWINM_rec $r3, 27, 5, 10, implicit-def $cr0 - dead renamable $r3 = RLWINM_rec killed renamable $r3, 8, 5, 10, implicit-def $cr0 - BLR8 implicit $lr8, implicit $rm, implicit killed $cr0 -... ---- -name: testFoldRLWINMInvalidMask -tracksRegLiveness: true -body: | - bb.0.entry: - liveins: $r3 - ; CHECK-LABEL: name: testFoldRLWINMInvalidMask - ; CHECK: liveins: $r3 - ; CHECK: $r3 = RLWINM killed $r3, 20, 5, 31 - ; CHECK: renamable $r3 = RLWINM killed renamable $r3, 19, 10, 20, implicit-def $x3 - ; CHECK: BLR8 implicit $lr8, implicit $rm, implicit killed $x3 - $r3 = RLWINM killed $r3, 20, 5, 31 - dead renamable $r3 = RLWINM killed renamable $r3, 19, 10, 20, implicit-def $x3 - BLR8 implicit $lr8, implicit $rm, implicit killed $x3 -... diff --git a/llvm/test/CodeGen/PowerPC/sms-phi-5.ll b/llvm/test/CodeGen/PowerPC/sms-phi-5.ll index c147ddf8182b..bdc773de8aaf 100644 --- a/llvm/test/CodeGen/PowerPC/sms-phi-5.ll +++ b/llvm/test/CodeGen/PowerPC/sms-phi-5.ll @@ -1,55 +1,56 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\ ; RUN: -mcpu=pwr9 --ppc-enable-pipeliner 2>&1 | FileCheck %s define void @phi5() unnamed_addr { ; CHECK-LABEL: phi5: ; CHECK: # %bb.0: ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: li 3, 0 ; CHECK-NEXT: li 4, 1 ; CHECK-NEXT: slw 3, 4, 3 ; CHECK-NEXT: andi. 3, 3, 6336 ; CHECK-NEXT: beqlr 0 ; CHECK-NEXT: # %bb.2: ; CHECK-NEXT: lhz 3, 0(3) ; CHECK-NEXT: slwi 3, 3, 15 -; CHECK-NEXT: li 4, 0 -; CHECK-NEXT: ori 3, 4, 0 +; CHECK-NEXT: clrlwi 3, 3, 31 +; CHECK-NEXT: rlwinm 4, 3, 31, 17, 31 +; CHECK-NEXT: or 3, 3, 4 ; CHECK-NEXT: rlwimi 3, 3, 15, 0, 16 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: blr switch i12 undef, label %21 [ i12 6, label %1 i12 7, label %1 i12 12, label %1 i12 11, label %1 ] 1: ; preds = %0, %0, %0, %0 %2 = load i16, i16* undef, align 2 br label %3 3: ; preds = %3, %1 %4 = phi i16 [ %18, %3 ], [ undef, %1 ] %5 = phi i16 [ %13, %3 ], [ undef, %1 ] %6 = phi i16 [ %11, %3 ], [ undef, %1 ] %7 = phi i16 [ undef, %3 ], [ %2, %1 ] %8 = phi i32 [ %19, %3 ], [ undef, %1 ] %9 = lshr i16 %6, 1 %10 = shl i16 %7, 15 %11 = or i16 %10, %9 %12 = shl i16 %6, 15 %13 = or i16 %12, 0 %14 = and i16 %4, 1 %15 = lshr i16 %4, 1 %16 = shl i16 %5, 15 %17 = or i16 %14, %15 %18 = or i16 %17, %16 %19 = add i32 %8, -1 %20 = icmp eq i32 %19, 0 br i1 %20, label %21, label %3 21: ; preds = %3, %0 ret void } diff --git a/llvm/test/CodeGen/PowerPC/vsx_builtins.ll b/llvm/test/CodeGen/PowerPC/vsx_builtins.ll index b40a84a7e95c..0aae50af2649 100644 --- a/llvm/test/CodeGen/PowerPC/vsx_builtins.ll +++ b/llvm/test/CodeGen/PowerPC/vsx_builtins.ll @@ -1,141 +1,142 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -relocation-model=static -verify-machineinstrs -mcpu=pwr9 \ ; RUN: -mtriple=powerpc64le-unknown-linux-gnu \ ; RUN: -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names < %s | FileCheck %s ; Function Attrs: nounwind readnone define <4 x i32> @test1(i8* %a) { ; CHECK-LABEL: test1: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: lxvw4x v2, 0, r3 ; CHECK-NEXT: blr entry: %0 = tail call <4 x i32> @llvm.ppc.vsx.lxvw4x.be(i8* %a) ret <4 x i32> %0 } ; Function Attrs: nounwind readnone declare <4 x i32> @llvm.ppc.vsx.lxvw4x.be(i8*) ; Function Attrs: nounwind readnone define <2 x double> @test2(i8* %a) { ; CHECK-LABEL: test2: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: lxvd2x v2, 0, r3 ; CHECK-NEXT: blr entry: %0 = tail call <2 x double> @llvm.ppc.vsx.lxvd2x.be(i8* %a) ret <2 x double> %0 } ; Function Attrs: nounwind readnone declare <2 x double> @llvm.ppc.vsx.lxvd2x.be(i8*) ; Function Attrs: nounwind readnone define void @test3(<4 x i32> %a, i8* %b) { ; CHECK-LABEL: test3: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: stxvw4x v2, 0, r5 ; CHECK-NEXT: blr entry: tail call void @llvm.ppc.vsx.stxvw4x.be(<4 x i32> %a, i8* %b) ret void } ; Function Attrs: nounwind readnone declare void @llvm.ppc.vsx.stxvw4x.be(<4 x i32>, i8*) ; Function Attrs: nounwind readnone define void @test4(<2 x double> %a, i8* %b) { ; CHECK-LABEL: test4: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: stxvd2x v2, 0, r5 ; CHECK-NEXT: blr entry: tail call void @llvm.ppc.vsx.stxvd2x.be(<2 x double> %a, i8* %b) ret void } ; Function Attrs: nounwind readnone declare void @llvm.ppc.vsx.stxvd2x.be(<2 x double>, i8*) define i32 @test_vec_test_swdiv(<2 x double> %a, <2 x double> %b) { ; CHECK-LABEL: test_vec_test_swdiv: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: xvtdivdp cr0, v2, v3 ; CHECK-NEXT: mfocrf r3, 128 ; CHECK-NEXT: srwi r3, r3, 28 ; CHECK-NEXT: blr entry: %0 = tail call i32 @llvm.ppc.vsx.xvtdivdp(<2 x double> %a, <2 x double> %b) ret i32 %0 } declare i32 @llvm.ppc.vsx.xvtdivdp(<2 x double>, <2 x double>) define i32 @test_vec_test_swdivs(<4 x float> %a, <4 x float> %b) { ; CHECK-LABEL: test_vec_test_swdivs: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: xvtdivsp cr0, v2, v3 ; CHECK-NEXT: mfocrf r3, 128 ; CHECK-NEXT: srwi r3, r3, 28 ; CHECK-NEXT: blr entry: %0 = tail call i32 @llvm.ppc.vsx.xvtdivsp(<4 x float> %a, <4 x float> %b) ret i32 %0 } declare i32 @llvm.ppc.vsx.xvtdivsp(<4 x float>, <4 x float>) define i32 @test_vec_test_swsqrt(<2 x double> %a) { ; CHECK-LABEL: test_vec_test_swsqrt: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: xvtsqrtdp cr0, v2 ; CHECK-NEXT: mfocrf r3, 128 ; CHECK-NEXT: srwi r3, r3, 28 ; CHECK-NEXT: blr entry: %0 = tail call i32 @llvm.ppc.vsx.xvtsqrtdp(<2 x double> %a) ret i32 %0 } declare i32 @llvm.ppc.vsx.xvtsqrtdp(<2 x double>) define i32 @test_vec_test_swsqrts(<4 x float> %a) { ; CHECK-LABEL: test_vec_test_swsqrts: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: xvtsqrtsp cr0, v2 ; CHECK-NEXT: mfocrf r3, 128 ; CHECK-NEXT: srwi r3, r3, 28 ; CHECK-NEXT: blr entry: %0 = tail call i32 @llvm.ppc.vsx.xvtsqrtsp(<4 x float> %a) ret i32 %0 } declare i32 @llvm.ppc.vsx.xvtsqrtsp(<4 x float>) define i32 @xvtdivdp_andi(<2 x double> %a, <2 x double> %b) { ; CHECK-LABEL: xvtdivdp_andi: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: xvtdivdp cr0, v2, v3 ; CHECK-NEXT: li r4, 222 ; CHECK-NEXT: mfocrf r3, 128 ; CHECK-NEXT: srwi r3, r3, 28 ; CHECK-NEXT: andi. r3, r3, 2 ; CHECK-NEXT: li r3, 22 ; CHECK-NEXT: iseleq r3, r4, r3 ; CHECK-NEXT: blr entry: %0 = tail call i32 @llvm.ppc.vsx.xvtdivdp(<2 x double> %a, <2 x double> %b) %1 = and i32 %0, 2 %cmp.not = icmp eq i32 %1, 0 %retval.0 = select i1 %cmp.not, i32 222, i32 22 ret i32 %retval.0 } define i32 @xvtdivdp_shift(<2 x double> %a, <2 x double> %b) { ; CHECK-LABEL: xvtdivdp_shift: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: xvtdivdp cr0, v2, v3 ; CHECK-NEXT: mfocrf r3, 128 -; CHECK-NEXT: li r3, 0 +; CHECK-NEXT: srwi r3, r3, 28 +; CHECK-NEXT: rlwinm r3, r3, 28, 31, 31 ; CHECK-NEXT: blr entry: %0 = tail call i32 @llvm.ppc.vsx.xvtdivdp(<2 x double> %a, <2 x double> %b) %1 = lshr i32 %0, 4 %.lobit = and i32 %1, 1 ret i32 %.lobit }