Index: lib/Target/AArch64/AArch64BranchRelaxation.cpp =================================================================== --- lib/Target/AArch64/AArch64BranchRelaxation.cpp +++ lib/Target/AArch64/AArch64BranchRelaxation.cpp @@ -30,18 +30,6 @@ BranchRelaxation("aarch64-branch-relax", cl::Hidden, cl::init(true), cl::desc("Relax out of range conditional branches")); -static cl::opt -TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), - cl::desc("Restrict range of TB[N]Z instructions (DEBUG)")); - -static cl::opt -CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), - cl::desc("Restrict range of CB[N]Z instructions (DEBUG)")); - -static cl::opt -BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), - cl::desc("Restrict range of Bcc instructions (DEBUG)")); - STATISTIC(NumSplit, "Number of basic blocks split"); STATISTIC(NumRelaxed, "Number of conditional branches relaxed"); @@ -90,10 +78,11 @@ void scanFunction(); MachineBasicBlock *splitBlockBeforeInstr(MachineInstr &MI); void adjustBlockOffsets(MachineBasicBlock &MBB); - bool isBlockInRange(MachineInstr &MI, MachineBasicBlock &BB, unsigned Disp); + bool isBlockInRange(const MachineInstr &MI, const MachineBasicBlock &BB) const; + void invertConditionalBranch(MachineInstr &MI) const; bool fixupConditionalBranch(MachineInstr &MI); void computeBlockSize(const MachineBasicBlock &MBB); - unsigned getInstrOffset(MachineInstr &MI) const; + unsigned getInstrOffset(const MachineInstr &MI) const; void dumpBBs(); void verify(); @@ -184,8 +173,8 @@ /// getInstrOffset - Return the current offset of the specified machine /// instruction from the start of the function. This offset changes as stuff is /// moved around inside the function. -unsigned AArch64BranchRelaxation::getInstrOffset(MachineInstr &MI) const { - MachineBasicBlock *MBB = MI.getParent(); +unsigned AArch64BranchRelaxation::getInstrOffset(const MachineInstr &MI) const { + const MachineBasicBlock *MBB = MI.getParent(); // The offset is composed of two things: the sum of the sizes of all MBB's // before this instruction's block, and the offset from the start of the block @@ -193,7 +182,7 @@ unsigned Offset = BlockInfo[MBB->getNumber()].Offset; // Sum instructions before MI in MBB. - for (MachineBasicBlock::iterator I = MBB->begin(); &*I != &MI; ++I) { + for (MachineBasicBlock::const_iterator I = MBB->begin(); &*I != &MI; ++I) { assert(I != MBB->end() && "Didn't find MI in its own basic block?"); Offset += TII->GetInstSizeInBytes(*I); } @@ -263,43 +252,26 @@ /// isBlockInRange - Returns true if the distance between specific MI and /// specific BB can fit in MI's displacement field. -bool AArch64BranchRelaxation::isBlockInRange(MachineInstr &MI, - MachineBasicBlock &DestBB, - unsigned Bits) { - unsigned MaxOffs = ((1 << (Bits - 1)) - 1) << 2; +bool AArch64BranchRelaxation::isBlockInRange( + const MachineInstr &MI, const MachineBasicBlock &DestBB) const { unsigned BrOffset = getInstrOffset(MI); unsigned DestOffset = BlockInfo[DestBB.getNumber()].Offset; - DEBUG(dbgs() << "Branch of destination BB#" << DestBB.getNumber() - << " from BB#" << MI.getParent()->getNumber() - << " max delta=" << MaxOffs << " from " << BrOffset - << " to " << DestOffset << " offset " - << static_cast(DestOffset - BrOffset) << '\t' << MI); + if (TII->isBranchInRange(MI.getOpcode(), BrOffset, DestOffset)) + return true; - // Branch before the Dest. - if (BrOffset <= DestOffset) - return (DestOffset - BrOffset <= MaxOffs); - return (BrOffset - DestOffset <= MaxOffs); -} + DEBUG( + dbgs() << "Out of range branch to destination BB#" << DestBB.getNumber() + << " from BB#" << MI.getParent()->getNumber() + << " to " << DestOffset + << " offset " << static_cast(DestOffset - BrOffset) + << '\t' << MI + ); -static bool isConditionalBranch(unsigned Opc) { - switch (Opc) { - default: - return false; - case AArch64::TBZW: - case AArch64::TBNZW: - case AArch64::TBZX: - case AArch64::TBNZX: - case AArch64::CBZW: - case AArch64::CBNZW: - case AArch64::CBZX: - case AArch64::CBNZX: - case AArch64::Bcc: - return true; - } + return false; } -static MachineBasicBlock *getDestBlock(MachineInstr &MI) { +static MachineBasicBlock *getDestBlock(const MachineInstr &MI) { switch (MI.getOpcode()) { default: llvm_unreachable("unexpected opcode!"); @@ -333,30 +305,30 @@ } } -static unsigned getBranchDisplacementBits(unsigned Opc) { - switch (Opc) { - default: - llvm_unreachable("unexpected opcode!"); - case AArch64::TBNZW: - case AArch64::TBZW: - case AArch64::TBNZX: - case AArch64::TBZX: - return TBZDisplacementBits; - case AArch64::CBNZW: - case AArch64::CBZW: - case AArch64::CBNZX: - case AArch64::CBZX: - return CBZDisplacementBits; - case AArch64::Bcc: - return BCCDisplacementBits; - } -} - static inline void invertBccCondition(MachineInstr &MI) { assert(MI.getOpcode() == AArch64::Bcc && "Unexpected opcode!"); - AArch64CC::CondCode CC = (AArch64CC::CondCode)MI.getOperand(0).getImm(); - CC = AArch64CC::getInvertedCondCode(CC); - MI.getOperand(0).setImm((int64_t)CC); + MachineOperand &CCOp = MI.getOperand(0); + + AArch64CC::CondCode CC = static_cast(CCOp.getImm()); + CCOp.setImm(AArch64CC::getInvertedCondCode(CC)); +} + +/// Invert the branch condition of \p MI and change the destination to \p NewB +void AArch64BranchRelaxation::invertConditionalBranch(MachineInstr &MI) const { + MI.setDesc(TII->get(getOppositeConditionOpcode(MI.getOpcode()))); + + if (MI.getOpcode() == AArch64::Bcc) + invertBccCondition(MI); +} + +static void changeBranchDestBlock(MachineInstr &MI, + MachineBasicBlock &NewDestBB) { + unsigned Opc = MI.getOpcode(); + unsigned OpNum = (Opc == AArch64::TBZW || + Opc == AArch64::TBNZW || + Opc == AArch64::TBZX || + Opc == AArch64::TBNZX) ? 2 : 1; + MI.getOperand(OpNum).setMBB(&NewDestBB); } /// fixupConditionalBranch - Fix up a conditional branch whose destination is @@ -383,8 +355,8 @@ if (BMI != &MI) { if (std::next(MachineBasicBlock::iterator(MI)) == std::prev(MBB->getLastNonDebugInstr()) && - BMI->getOpcode() == AArch64::B) { - // Last MI in the BB is an unconditional branch. Can we simply invert the + BMI->isUnconditionalBranch()) { + // Last MI in the BB is an unconditional branch. We can simply invert the // condition and swap destinations: // beq L1 // b L2 @@ -392,21 +364,12 @@ // bne L2 // b L1 MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB(); - if (isBlockInRange(MI, *NewDest, - getBranchDisplacementBits(MI.getOpcode()))) { + if (isBlockInRange(MI, *NewDest)) { DEBUG(dbgs() << " Invert condition and swap its destination with " << *BMI); BMI->getOperand(0).setMBB(DestBB); - unsigned OpNum = (MI.getOpcode() == AArch64::TBZW || - MI.getOpcode() == AArch64::TBNZW || - MI.getOpcode() == AArch64::TBZX || - MI.getOpcode() == AArch64::TBNZX) - ? 2 - : 1; - MI.getOperand(OpNum).setMBB(NewDest); - MI.setDesc(TII->get(getOppositeConditionOpcode(MI.getOpcode()))); - if (MI.getOpcode() == AArch64::Bcc) - invertBccCondition(MI); + invertConditionalBranch(MI); + changeBranchDestBlock(MI, *NewDest); return true; } } @@ -431,11 +394,11 @@ MBB->replaceSuccessor(FBB, NewBB); NewBB->addSuccessor(FBB); } - MachineBasicBlock *NextBB = &*std::next(MachineFunction::iterator(MBB)); + MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB)); DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber() << ", invert condition and change dest. to BB#" - << NextBB->getNumber() << "\n"); + << NextBB.getNumber() << '\n'); unsigned OppositeCondOpc = getOppositeConditionOpcode(MI.getOpcode()); // Insert a new conditional branch and a new unconditional branch. @@ -447,13 +410,13 @@ MIB.addOperand(MI.getOperand(1)); if (MI.getOpcode() == AArch64::Bcc) invertBccCondition(*MIB); - MIB.addMBB(NextBB); + MIB.addMBB(&NextBB); BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(MBB->back()); BuildMI(MBB, DebugLoc(), TII->get(AArch64::B)).addMBB(DestBB); BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(MBB->back()); // Remove the old conditional branch. It may or may not still be in MBB. - BlockInfo[MI.getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI); + BlockInfo[MBB->getNumber()].Size -= TII->GetInstSizeInBytes(MI); MI.eraseFromParent(); // Finally, keep the block offsets up to date. @@ -472,9 +435,7 @@ continue; MachineInstr &MI = *J; - if (isConditionalBranch(MI.getOpcode()) && - !isBlockInRange(MI, *getDestBlock(MI), - getBranchDisplacementBits(MI.getOpcode()))) { + if (MI.isConditionalBranch() && !isBlockInRange(MI, *getDestBlock(MI))) { fixupConditionalBranch(MI); ++NumRelaxed; Changed = true; @@ -492,7 +453,7 @@ DEBUG(dbgs() << "***** AArch64BranchRelaxation *****\n"); - TII = (const AArch64InstrInfo *)MF->getSubtarget().getInstrInfo(); + TII = MF->getSubtarget().getInstrInfo(); // Renumber all of the machine basic blocks in the function, guaranteeing that // the numbers agree with the position of the block in the function. Index: lib/Target/AArch64/AArch64InstrInfo.h =================================================================== --- lib/Target/AArch64/AArch64InstrInfo.h +++ lib/Target/AArch64/AArch64InstrInfo.h @@ -141,6 +141,12 @@ MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS = nullptr) const override; + /// \returns true if a branch from an instruction with opcode \p BranchOpc + /// located at \p BrOffset bytes is capable of jumping to a position at \p + /// DestOffset. + bool isBranchInRange(unsigned BranchOpc, uint64_t BrOffset, + uint64_t DestOffset) const; + bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl &Cond, Index: lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- lib/Target/AArch64/AArch64InstrInfo.cpp +++ lib/Target/AArch64/AArch64InstrInfo.cpp @@ -32,6 +32,18 @@ static LLVM_CONSTEXPR MachineMemOperand::Flags MOSuppressPair = MachineMemOperand::MOTargetFlag1; +static cl::opt +TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), + cl::desc("Restrict range of TB[N]Z instructions (DEBUG)")); + +static cl::opt +CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), + cl::desc("Restrict range of CB[N]Z instructions (DEBUG)")); + +static cl::opt +BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), + cl::desc("Restrict range of Bcc instructions (DEBUG)")); + AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI) : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP), RI(STI.getTargetTriple()), Subtarget(STI) {} @@ -92,6 +104,67 @@ } } +static unsigned getBranchDisplacementBits(unsigned Opc) { + switch (Opc) { + default: + llvm_unreachable("unexpected opcode!"); + case AArch64::TBNZW: + case AArch64::TBZW: + case AArch64::TBNZX: + case AArch64::TBZX: + return TBZDisplacementBits; + case AArch64::CBNZW: + case AArch64::CBZW: + case AArch64::CBNZX: + case AArch64::CBZX: + return CBZDisplacementBits; + case AArch64::Bcc: + return BCCDisplacementBits; + } +} + +static unsigned getBranchMaxDisplacementBytes(unsigned Opc) { + if (Opc == AArch64::B) + return -1; + + unsigned Bits = getBranchDisplacementBits(Opc); + unsigned MaxOffs = ((1 << (Bits - 1)) - 1) << 2; + return MaxOffs; +} + +static unsigned getOppositeConditionOpcode(unsigned Opc) { + switch (Opc) { + default: + llvm_unreachable("unexpected opcode!"); + case AArch64::TBNZW: return AArch64::TBZW; + case AArch64::TBNZX: return AArch64::TBZX; + case AArch64::TBZW: return AArch64::TBNZW; + case AArch64::TBZX: return AArch64::TBNZX; + case AArch64::CBNZW: return AArch64::CBZW; + case AArch64::CBNZX: return AArch64::CBZX; + case AArch64::CBZW: return AArch64::CBNZW; + case AArch64::CBZX: return AArch64::CBNZX; + case AArch64::Bcc: return AArch64::Bcc; // Condition is an operand for Bcc. + } +} + +static inline void invertBccCondition(MachineInstr &MI) { + assert(MI.getOpcode() == AArch64::Bcc && "Unexpected opcode!"); + AArch64CC::CondCode CC = (AArch64CC::CondCode)MI.getOperand(0).getImm(); + CC = AArch64CC::getInvertedCondCode(CC); + MI.getOperand(0).setImm((int64_t)CC); +} + +bool AArch64InstrInfo::isBranchInRange(unsigned BranchOp, uint64_t BrOffset, + uint64_t DestOffset) const { + unsigned MaxOffs = getBranchMaxDisplacementBytes(BranchOp); + + // Branch before the Dest. + if (BrOffset <= DestOffset) + return (DestOffset - BrOffset <= MaxOffs); + return (BrOffset - DestOffset <= MaxOffs); +} + // Branch analysis. bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,