diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -859,7 +859,7 @@ /// Succ, can be split. If this returns true a subsequent call to /// SplitCriticalEdge is guaranteed to return a valid basic block if /// no changes occurred in the meantime. - bool canSplitCriticalEdge(const MachineBasicBlock *Succ) const; + bool canSplitCriticalEdge(const MachineBasicBlock *Succ); void pop_front() { Insts.pop_front(); } void pop_back() { Insts.pop_back(); } diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -994,6 +994,13 @@ return false; } + /// Return the pointer to the jump table, if given instr has it, + /// otherwise return nullptr. + virtual MachineJumpTableInfo * + getJumpTableFromIndirectJump(MachineInstr &MI) const { + return nullptr; + } + protected: /// Target-dependent implementation for IsCopyInstr. /// If the specific machine instruction is a instruction that moves/copies diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -18,6 +18,7 @@ #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SlotIndexes.h" @@ -35,6 +36,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include +#include using namespace llvm; #define DEBUG_TYPE "codegen" @@ -673,6 +675,14 @@ getParent()->splice(++NewBefore->getIterator(), getIterator()); } +static MachineJumpTableInfo *findIndirectJumpTable(MachineBasicBlock &MBB) { + MachineBasicBlock::iterator TerminatorI = MBB.getFirstTerminator(); + if (TerminatorI == MBB.end()) + return nullptr; + const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo(); + return TII->getJumpTableFromIndirectJump(*TerminatorI); +} + void MachineBasicBlock::updateTerminator( MachineBasicBlock *PreviousLayoutSuccessor) { LLVM_DEBUG(dbgs() << "Updating terminators on " << printMBBReference(*this) @@ -688,7 +698,8 @@ DebugLoc DL = findBranchDebugLoc(); bool B = TII->analyzeBranch(*this, TBB, FBB, Cond); (void) B; - assert(!B && "UpdateTerminators requires analyzable predecessors!"); + assert((!B || findIndirectJumpTable(*this)) && + "UpdateTerminators requires analyzable predecessors!"); if (Cond.empty()) { if (TBB) { // The block has an unconditional branch. If its successor is now its @@ -1042,13 +1053,24 @@ MachineBasicBlock *Succ, Pass &P, std::vector> *LiveInSets) { if (!canSplitCriticalEdge(Succ)) - return nullptr; + return nullptr; // this is not a critical edge MachineFunction *MF = getParent(); + const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); + DebugLoc DL; MachineBasicBlock *PrevFallthrough = getNextNode(); - DebugLoc DL; // FIXME: this is nowhere - MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock(); + + if (MachineJumpTableInfo *JTI = findIndirectJumpTable(*this)) { + // Check if there is a indirect jump with jump table + MachineInstr &MI = *getFirstTerminator(); + const MachineOperand &MO = MI.getOperand(3); + JTI->ReplaceMBBInJumpTable(MO.getIndex(), Succ, NMBB); + + // insert an unconditional branch at the end of the new split BB + TII.insertUnconditionalBranch(*NMBB, Succ, DL); + } + MF->insert(std::next(MachineFunction::iterator(this)), NMBB); LLVM_DEBUG(dbgs() << "Splitting critical edge: " << printMBBReference(*this) << " -- " << printMBBReference(*NMBB) << " -- " @@ -1272,10 +1294,7 @@ return NMBB; } -bool MachineBasicBlock::canSplitCriticalEdge( - const MachineBasicBlock *Succ) const { - // Splitting the critical edge to a landing pad block is non-trivial. Don't do - // it in this generic function. +bool MachineBasicBlock::canSplitCriticalEdge(const MachineBasicBlock *Succ) { if (Succ->isEHPad()) return false; @@ -1284,17 +1303,21 @@ if (Succ->isInlineAsmBrIndirectTarget()) return false; - const MachineFunction *MF = getParent(); + const MachineFunction &MF = *Succ->getParent(); // Performance might be harmed on HW that implements branching using exec mask // where both sides of the branches are always executed. - if (MF->getTarget().requiresStructuredCFG()) + if (MF.getTarget().requiresStructuredCFG()) return false; + // Check whether we have indirect jump with a jumptable that we can rewrite. + if (findIndirectJumpTable(*this) != nullptr) + return true; + const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); // We may need to update this's terminator, but we can't do that if - // analyzeBranch fails. If this uses a jump table, we won't touch it. - const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); + // analyzeBranch fails. MachineBasicBlock *TBB = nullptr, *FBB = nullptr; SmallVector Cond; + // AnalyzeBanch should modify this, since we did not allow modification. if (TII->analyzeBranch(*const_cast(this), TBB, FBB, Cond, /*AllowModify*/ false)) diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h --- a/llvm/lib/Target/X86/X86InstrInfo.h +++ b/llvm/lib/Target/X86/X86InstrInfo.h @@ -328,6 +328,10 @@ SmallVectorImpl &Cond, bool AllowModify) const override; + /// Return the pointer to the jump table, if given instr has it. + MachineJumpTableInfo * + getJumpTableFromIndirectJump(MachineInstr &MI) const override; + Optional getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override; diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -3033,8 +3033,9 @@ if (I->getOpcode() == X86::JMP_1) { UnCondBrIter = I; + // TBB is used to indicate the unconditional destination. + TBB = I->getOperand(0).getMBB(); if (!AllowModify) { - TBB = I->getOperand(0).getMBB(); continue; } @@ -3050,11 +3051,8 @@ I->eraseFromParent(); I = MBB.end(); UnCondBrIter = MBB.end(); - continue; } - // TBB is used to indicate the unconditional destination. - TBB = I->getOperand(0).getMBB(); continue; } @@ -3179,6 +3177,21 @@ return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify); } +MachineJumpTableInfo * +X86InstrInfo::getJumpTableFromIndirectJump(MachineInstr &MI) const { + // Currently we only take care of JMP64m jump instrs. + if (MI.getOpcode() != X86::JMP64m) + return nullptr; + + // For a JMP64m instr, the 4th operand is the jump table. + MachineOperand &MO = MI.getOperand(3); + if (!MO.isJTI()) + return nullptr; + + MachineFunction *MF = MI.getParent()->getParent(); + return MF->getJumpTableInfo(); +} + bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const { diff --git a/llvm/test/CodeGen/X86/switch-jmp-edge-split.mir b/llvm/test/CodeGen/X86/switch-jmp-edge-split.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/switch-jmp-edge-split.mir @@ -0,0 +1,138 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=x86_64-none-linux-gnu -run-pass=machine-sink -verify-machineinstrs -o - %s | FileCheck %s + +# This test was originally generated from the following sample: +# +# int foo(int i) { +# int j = 2; +# switch(i) { +# case 0: +# j+= 1; +# break; +# case 1: +# j*=3; +# break; +# case 2: +# j/=2; +# break; +# case 3: +# j-=10; +# default: +# j+=100; +# } +# +# return j; +# } + +# The added code is to split the critical edge formed by the switch jumps. + +--- +name: _Z3fooi +tracksRegLiveness: true +jumpTable: + kind: block-address + entries: + - id: 0 + blocks: [ '%bb.5', '%bb.1', '%bb.2', '%bb.3' ] +body: | + ; CHECK-LABEL: name: _Z3fooi + ; CHECK: bb.0.entry: + ; CHECK-NEXT: successors: %bb.7(0x19999998), %bb.1(0x66666668) + ; CHECK-NEXT: liveins: $edi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi + ; CHECK-NEXT: [[SUB32ri8_:%[0-9]+]]:gr32 = SUB32ri8 [[COPY]], 3, implicit-def $eflags + ; CHECK-NEXT: JCC_1 %bb.1, 6, implicit $eflags + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.7: + ; CHECK-NEXT: successors: %bb.5(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 102 + ; CHECK-NEXT: JMP_1 %bb.5 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.entry: + ; CHECK-NEXT: successors: %bb.8(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32rr:%[0-9]+]]:gr32 = MOV32rr [[COPY]] + ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[MOV32rr]], %subreg.sub_32bit + ; CHECK-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG]], %jump-table.0, $noreg :: (load (s64) from jump-table) + ; CHECK-NEXT: JMP_1 %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.8: + ; CHECK-NEXT: successors: %bb.6(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri1:%[0-9]+]]:gr32 = MOV32ri 3 + ; CHECK-NEXT: JMP_1 %bb.6 + ; CHECK-NEXT: JMP_1 %bb.6 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: successors: %bb.6(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri2:%[0-9]+]]:gr32 = MOV32ri 6 + ; CHECK-NEXT: JMP_1 %bb.6 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3: + ; CHECK-NEXT: successors: %bb.6(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri3:%[0-9]+]]:gr32 = MOV32ri 1 + ; CHECK-NEXT: JMP_1 %bb.6 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.4: + ; CHECK-NEXT: successors: %bb.5(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOV32ri4:%[0-9]+]]:gr32 = MOV32ri 92 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.5: + ; CHECK-NEXT: successors: %bb.6(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.7, [[MOV32ri]], %bb.7, [[MOV32ri4]], %bb.4 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.6: + ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gr32 = PHI [[MOV32ri1]], %bb.8, [[MOV32ri3]], %bb.3, [[MOV32ri2]], %bb.2, [[PHI]], %bb.5 + ; CHECK-NEXT: $eax = COPY [[PHI1]] + ; CHECK-NEXT: RET 0, $eax + bb.0.entry: + successors: %bb.4(0x19999998), %bb.6(0x66666668) + liveins: $edi + + %2:gr32 = COPY $edi + %4:gr32 = MOV32ri 3 + %3:gr32 = MOV32ri 102 + %6:gr32 = MOV32rr %2 + %5:gr64_nosp = SUBREG_TO_REG 0, killed %6, %subreg.sub_32bit + %7:gr32 = SUB32ri8 %2, 3, implicit-def $eflags + JCC_1 %bb.4, 7, implicit $eflags + + bb.6.entry: + successors: %bb.5(0x20000000), %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000) + + JMP64m $noreg, 8, %5, %jump-table.0, $noreg :: (load (s64) from jump-table) + + bb.1: + successors: %bb.5(0x80000000) + + %10:gr32 = MOV32ri 6 + JMP_1 %bb.5 + + bb.2: + successors: %bb.5(0x80000000) + + %9:gr32 = MOV32ri 1 + JMP_1 %bb.5 + + bb.3: + successors: %bb.4(0x80000000) + + %8:gr32 = MOV32ri 92 + + bb.4: + successors: %bb.5(0x80000000) + + %0:gr32 = PHI %3, %bb.0, %3, %bb.0, %8, %bb.3 + + bb.5: + %1:gr32 = PHI %4, %bb.6, %9, %bb.2, %10, %bb.1, %0, %bb.4 + $eax = COPY %1 + RET 0, $eax + +...