Index: lib/Target/AArch64/AArch64BranchRelaxation.cpp
===================================================================
--- lib/Target/AArch64/AArch64BranchRelaxation.cpp
+++ lib/Target/AArch64/AArch64BranchRelaxation.cpp
@@ -30,18 +30,6 @@
BranchRelaxation("aarch64-branch-relax", cl::Hidden, cl::init(true),
cl::desc("Relax out of range conditional branches"));
-static cl::opt<unsigned>
-TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
- cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
-
-static cl::opt<unsigned>
-CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
- cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
-
-static cl::opt<unsigned>
-BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
- cl::desc("Restrict range of Bcc instructions (DEBUG)"));
-
STATISTIC(NumSplit, "Number of basic blocks split");
STATISTIC(NumRelaxed, "Number of conditional branches relaxed");
@@ -90,10 +78,11 @@
void scanFunction();
MachineBasicBlock *splitBlockBeforeInstr(MachineInstr &MI);
void adjustBlockOffsets(MachineBasicBlock &MBB);
- bool isBlockInRange(MachineInstr &MI, MachineBasicBlock &BB, unsigned Disp);
+ bool isBlockInRange(const MachineInstr &MI, const MachineBasicBlock &BB) const;
+ void invertConditionalBranch(MachineInstr &MI) const;
bool fixupConditionalBranch(MachineInstr &MI);
void computeBlockSize(const MachineBasicBlock &MBB);
- unsigned getInstrOffset(MachineInstr &MI) const;
+ unsigned getInstrOffset(const MachineInstr &MI) const;
void dumpBBs();
void verify();
@@ -184,8 +173,8 @@
/// getInstrOffset - Return the current offset of the specified machine
/// instruction from the start of the function. This offset changes as stuff is
/// moved around inside the function.
-unsigned AArch64BranchRelaxation::getInstrOffset(MachineInstr &MI) const {
- MachineBasicBlock *MBB = MI.getParent();
+unsigned AArch64BranchRelaxation::getInstrOffset(const MachineInstr &MI) const {
+ const MachineBasicBlock *MBB = MI.getParent();
// The offset is composed of two things: the sum of the sizes of all MBB's
// before this instruction's block, and the offset from the start of the block
@@ -193,7 +182,7 @@
unsigned Offset = BlockInfo[MBB->getNumber()].Offset;
// Sum instructions before MI in MBB.
- for (MachineBasicBlock::iterator I = MBB->begin(); &*I != &MI; ++I) {
+ for (MachineBasicBlock::const_iterator I = MBB->begin(); &*I != &MI; ++I) {
assert(I != MBB->end() && "Didn't find MI in its own basic block?");
Offset += TII->GetInstSizeInBytes(*I);
}
@@ -263,43 +252,26 @@
/// isBlockInRange - Returns true if the distance between specific MI and
/// specific BB can fit in MI's displacement field.
-bool AArch64BranchRelaxation::isBlockInRange(MachineInstr &MI,
- MachineBasicBlock &DestBB,
- unsigned Bits) {
- unsigned MaxOffs = ((1 << (Bits - 1)) - 1) << 2;
+bool AArch64BranchRelaxation::isBlockInRange(
+ const MachineInstr &MI, const MachineBasicBlock &DestBB) const {
unsigned BrOffset = getInstrOffset(MI);
unsigned DestOffset = BlockInfo[DestBB.getNumber()].Offset;
- DEBUG(dbgs() << "Branch of destination BB#" << DestBB.getNumber()
- << " from BB#" << MI.getParent()->getNumber()
- << " max delta=" << MaxOffs << " from " << BrOffset
- << " to " << DestOffset << " offset "
- << static_cast<int>(DestOffset - BrOffset) << '\t' << MI);
+ if (TII->isBranchInRange(MI.getOpcode(), BrOffset, DestOffset))
+ return true;
- // Branch before the Dest.
- if (BrOffset <= DestOffset)
- return (DestOffset - BrOffset <= MaxOffs);
- return (BrOffset - DestOffset <= MaxOffs);
-}
+ DEBUG(
+ dbgs() << "Out of range branch to destination BB#" << DestBB.getNumber()
+ << " from BB#" << MI.getParent()->getNumber()
+ << " to " << DestOffset
+ << " offset " << static_cast<int>(DestOffset - BrOffset)
+ << '\t' << MI
+ );
-static bool isConditionalBranch(unsigned Opc) {
- switch (Opc) {
- default:
- return false;
- case AArch64::TBZW:
- case AArch64::TBNZW:
- case AArch64::TBZX:
- case AArch64::TBNZX:
- case AArch64::CBZW:
- case AArch64::CBNZW:
- case AArch64::CBZX:
- case AArch64::CBNZX:
- case AArch64::Bcc:
- return true;
- }
+ return false;
}
-static MachineBasicBlock *getDestBlock(MachineInstr &MI) {
+static MachineBasicBlock *getDestBlock(const MachineInstr &MI) {
switch (MI.getOpcode()) {
default:
llvm_unreachable("unexpected opcode!");
@@ -333,30 +305,30 @@
}
}
-static unsigned getBranchDisplacementBits(unsigned Opc) {
- switch (Opc) {
- default:
- llvm_unreachable("unexpected opcode!");
- case AArch64::TBNZW:
- case AArch64::TBZW:
- case AArch64::TBNZX:
- case AArch64::TBZX:
- return TBZDisplacementBits;
- case AArch64::CBNZW:
- case AArch64::CBZW:
- case AArch64::CBNZX:
- case AArch64::CBZX:
- return CBZDisplacementBits;
- case AArch64::Bcc:
- return BCCDisplacementBits;
- }
-}
-
static inline void invertBccCondition(MachineInstr &MI) {
assert(MI.getOpcode() == AArch64::Bcc && "Unexpected opcode!");
- AArch64CC::CondCode CC = (AArch64CC::CondCode)MI.getOperand(0).getImm();
- CC = AArch64CC::getInvertedCondCode(CC);
- MI.getOperand(0).setImm((int64_t)CC);
+ MachineOperand &CCOp = MI.getOperand(0);
+
+ AArch64CC::CondCode CC = static_cast<AArch64CC::CondCode>(CCOp.getImm());
+ CCOp.setImm(AArch64CC::getInvertedCondCode(CC));
+}
+
+/// Invert the branch condition of \p MI and change the destination to \p NewB
+void AArch64BranchRelaxation::invertConditionalBranch(MachineInstr &MI) const {
+ MI.setDesc(TII->get(getOppositeConditionOpcode(MI.getOpcode())));
+
+ if (MI.getOpcode() == AArch64::Bcc)
+ invertBccCondition(MI);
+}
+
+static void changeBranchDestBlock(MachineInstr &MI,
+ MachineBasicBlock &NewDestBB) {
+ unsigned Opc = MI.getOpcode();
+ unsigned OpNum = (Opc == AArch64::TBZW ||
+ Opc == AArch64::TBNZW ||
+ Opc == AArch64::TBZX ||
+ Opc == AArch64::TBNZX) ? 2 : 1;
+ MI.getOperand(OpNum).setMBB(&NewDestBB);
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
@@ -383,8 +355,8 @@
if (BMI != &MI) {
if (std::next(MachineBasicBlock::iterator(MI)) ==
std::prev(MBB->getLastNonDebugInstr()) &&
- BMI->getOpcode() == AArch64::B) {
- // Last MI in the BB is an unconditional branch. Can we simply invert the
+ BMI->isUnconditionalBranch()) {
+ // Last MI in the BB is an unconditional branch. We can simply invert the
// condition and swap destinations:
// beq L1
// b L2
@@ -392,21 +364,12 @@
// bne L2
// b L1
MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB();
- if (isBlockInRange(MI, *NewDest,
- getBranchDisplacementBits(MI.getOpcode()))) {
+ if (isBlockInRange(MI, *NewDest)) {
DEBUG(dbgs() << " Invert condition and swap its destination with "
<< *BMI);
BMI->getOperand(0).setMBB(DestBB);
- unsigned OpNum = (MI.getOpcode() == AArch64::TBZW ||
- MI.getOpcode() == AArch64::TBNZW ||
- MI.getOpcode() == AArch64::TBZX ||
- MI.getOpcode() == AArch64::TBNZX)
- ? 2
- : 1;
- MI.getOperand(OpNum).setMBB(NewDest);
- MI.setDesc(TII->get(getOppositeConditionOpcode(MI.getOpcode())));
- if (MI.getOpcode() == AArch64::Bcc)
- invertBccCondition(MI);
+ invertConditionalBranch(MI);
+ changeBranchDestBlock(MI, *NewDest);
return true;
}
}
@@ -431,11 +394,11 @@
MBB->replaceSuccessor(FBB, NewBB);
NewBB->addSuccessor(FBB);
}
- MachineBasicBlock *NextBB = &*std::next(MachineFunction::iterator(MBB));
+ MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber()
<< ", invert condition and change dest. to BB#"
- << NextBB->getNumber() << "\n");
+ << NextBB.getNumber() << '\n');
unsigned OppositeCondOpc = getOppositeConditionOpcode(MI.getOpcode());
// Insert a new conditional branch and a new unconditional branch.
@@ -447,13 +410,13 @@
MIB.addOperand(MI.getOperand(1));
if (MI.getOpcode() == AArch64::Bcc)
invertBccCondition(*MIB);
- MIB.addMBB(NextBB);
+ MIB.addMBB(&NextBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(MBB->back());
BuildMI(MBB, DebugLoc(), TII->get(AArch64::B)).addMBB(DestBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(MBB->back());
// Remove the old conditional branch. It may or may not still be in MBB.
- BlockInfo[MI.getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI);
+ BlockInfo[MBB->getNumber()].Size -= TII->GetInstSizeInBytes(MI);
MI.eraseFromParent();
// Finally, keep the block offsets up to date.
@@ -472,9 +435,7 @@
continue;
MachineInstr &MI = *J;
- if (isConditionalBranch(MI.getOpcode()) &&
- !isBlockInRange(MI, *getDestBlock(MI),
- getBranchDisplacementBits(MI.getOpcode()))) {
+ if (MI.isConditionalBranch() && !isBlockInRange(MI, *getDestBlock(MI))) {
fixupConditionalBranch(MI);
++NumRelaxed;
Changed = true;
@@ -492,7 +453,7 @@
DEBUG(dbgs() << "***** AArch64BranchRelaxation *****\n");
- TII = (const AArch64InstrInfo *)MF->getSubtarget().getInstrInfo();
+ TII = MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
// Renumber all of the machine basic blocks in the function, guaranteeing that
// the numbers agree with the position of the block in the function.
Index: lib/Target/AArch64/AArch64InstrInfo.h
===================================================================
--- lib/Target/AArch64/AArch64InstrInfo.h
+++ lib/Target/AArch64/AArch64InstrInfo.h
@@ -141,6 +141,12 @@
MachineBasicBlock::iterator InsertPt, int FrameIndex,
LiveIntervals *LIS = nullptr) const override;
+ /// \returns true if a branch from an instruction with opcode \p BranchOpc
+ /// located at \p BrOffset bytes is capable of jumping to a position at \p
+ /// DestOffset.
+ bool isBranchInRange(unsigned BranchOpc, uint64_t BrOffset,
+ uint64_t DestOffset) const;
+
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
MachineBasicBlock *&FBB,
SmallVectorImpl<MachineOperand> &Cond,
Index: lib/Target/AArch64/AArch64InstrInfo.cpp
===================================================================
--- lib/Target/AArch64/AArch64InstrInfo.cpp
+++ lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -32,6 +32,18 @@
static LLVM_CONSTEXPR MachineMemOperand::Flags MOSuppressPair =
MachineMemOperand::MOTargetFlag1;
+static cl::opt<unsigned>
+TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
+ cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
+
+static cl::opt<unsigned>
+CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
+ cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
+
+static cl::opt<unsigned>
+BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
+ cl::desc("Restrict range of Bcc instructions (DEBUG)"));
+
AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
: AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP),
RI(STI.getTargetTriple()), Subtarget(STI) {}
@@ -92,6 +104,67 @@
}
}
+static unsigned getBranchDisplacementBits(unsigned Opc) {
+ switch (Opc) {
+ default:
+ llvm_unreachable("unexpected opcode!");
+ case AArch64::TBNZW:
+ case AArch64::TBZW:
+ case AArch64::TBNZX:
+ case AArch64::TBZX:
+ return TBZDisplacementBits;
+ case AArch64::CBNZW:
+ case AArch64::CBZW:
+ case AArch64::CBNZX:
+ case AArch64::CBZX:
+ return CBZDisplacementBits;
+ case AArch64::Bcc:
+ return BCCDisplacementBits;
+ }
+}
+
+static unsigned getBranchMaxDisplacementBytes(unsigned Opc) {
+ if (Opc == AArch64::B)
+ return -1;
+
+ unsigned Bits = getBranchDisplacementBits(Opc);
+ unsigned MaxOffs = ((1 << (Bits - 1)) - 1) << 2;
+ return MaxOffs;
+}
+
+static unsigned getOppositeConditionOpcode(unsigned Opc) {
+ switch (Opc) {
+ default:
+ llvm_unreachable("unexpected opcode!");
+ case AArch64::TBNZW: return AArch64::TBZW;
+ case AArch64::TBNZX: return AArch64::TBZX;
+ case AArch64::TBZW: return AArch64::TBNZW;
+ case AArch64::TBZX: return AArch64::TBNZX;
+ case AArch64::CBNZW: return AArch64::CBZW;
+ case AArch64::CBNZX: return AArch64::CBZX;
+ case AArch64::CBZW: return AArch64::CBNZW;
+ case AArch64::CBZX: return AArch64::CBNZX;
+ case AArch64::Bcc: return AArch64::Bcc; // Condition is an operand for Bcc.
+ }
+}
+
+static inline void invertBccCondition(MachineInstr &MI) {
+ assert(MI.getOpcode() == AArch64::Bcc && "Unexpected opcode!");
+ AArch64CC::CondCode CC = (AArch64CC::CondCode)MI.getOperand(0).getImm();
+ CC = AArch64CC::getInvertedCondCode(CC);
+ MI.getOperand(0).setImm((int64_t)CC);
+}
+
+bool AArch64InstrInfo::isBranchInRange(unsigned BranchOp, uint64_t BrOffset,
+ uint64_t DestOffset) const {
+ unsigned MaxOffs = getBranchMaxDisplacementBytes(BranchOp);
+
+ // Branch before the Dest.
+ if (BrOffset <= DestOffset)
+ return (DestOffset - BrOffset <= MaxOffs);
+ return (BrOffset - DestOffset <= MaxOffs);
+}
+
// Branch analysis.
bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
MachineBasicBlock *&TBB,