diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -839,6 +839,21 @@ The above example generates a pointer to the source jump table index. +G_INVOKE_REGION_START +^^^^^^^^^^^^^^^^^^^^^ + +A marker instruction that acts as a pseudo-terminator for regions of code that may +throw exceptions. Being a terminator, it prevents code from being inserted after +it during passes like legalization. This is needed because calls to exception +throw routines do not return, so no code that must be on an executable path must +be placed after throwing. + +G_INVOKE_REGION_END +^^^^^^^^^^^^^^^^^^^ + +Marks the end of an invoke region. Instructions generated after this may be +unreachable. + G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -815,6 +815,11 @@ /// instr_iterator instead. instr_iterator getFirstInstrTerminator(); + /// Finds the first terminator in a block by scanning forward. This can handle + /// cases in GlobalISel where there may be non-terminator instructions between + /// terminators, for which getFirstTerminator() will not work correctly. + iterator getFirstTerminatorForward(); + /// Returns an iterator to the first non-debug instruction in the basic block, /// or end(). Skip any pseudo probe operation if \c SkipPseudoOp is true. /// Pseudo probes are like debug instructions which do not turn into real diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -402,6 +402,12 @@ /// Generic indirect branch instruction. HANDLE_TARGET_OPCODE(G_BRINDIRECT) +/// Begin an invoke region marker. +HANDLE_TARGET_OPCODE(G_INVOKE_REGION_START) + +/// End an invoke region marker. +HANDLE_TARGET_OPCODE(G_INVOKE_REGION_END) + /// Generic intrinsic use (without side effects). HANDLE_TARGET_OPCODE(G_INTRINSIC) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1270,6 +1270,22 @@ let isIndirectBranch = true; } +// A marker to signal the following code is an invoke region, that may throw +// an exception and therefore not return. +def G_INVOKE_REGION_START : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let isTerminator = true; + let hasSideEffects = false; +} + +// End an invoke region. +def G_INVOKE_REGION_END : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let hasSideEffects = true; +} + def G_READ_REGISTER : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$register); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2580,6 +2580,7 @@ // the region covered by the try. MCSymbol *BeginSymbol = nullptr; if (NeedEHLabel) { + MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START); BeginSymbol = Context.createTempSymbol(); MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol); } @@ -2594,6 +2595,7 @@ if (NeedEHLabel) { EndSymbol = Context.createTempSymbol(); MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol); + MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_END); } SmallVector, 1> UnwindDests; diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp --- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp +++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp @@ -184,6 +184,12 @@ continue; } + if (MI.getOpcode() == TargetOpcode::G_INVOKE_REGION_START || + MI.getOpcode() == TargetOpcode::G_INVOKE_REGION_END) { + MI.eraseFromParent(); + continue; + } + if (!ISel->select(MI)) { // FIXME: It would be nice to dump all inserted instructions. It's // not obvious how, esp. considering select() can insert after MI. diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -1173,7 +1173,7 @@ Observer.changingInstr(MI); for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB(); - MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); + MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, SrcRegs[i / 2]); } @@ -2458,7 +2458,7 @@ Observer.changingInstr(MI); for (unsigned I = 1; I < MI.getNumOperands(); I += 2) { MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); - MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); + MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT); } @@ -3777,7 +3777,7 @@ for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands(); UseIdx += 2, ++UseNo) { MachineBasicBlock &OpMBB = *MI.getOperand(UseIdx + 1).getMBB(); - MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); + MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); extractVectorParts(MI.getReg(UseIdx), NumElts, InputOpsPieces[UseNo]); } diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -254,6 +254,13 @@ return I; } +MachineBasicBlock::iterator MachineBasicBlock::getFirstTerminatorForward() { + iterator I = begin(), E = end(); + while (I != E && !I->isTerminator()) + ++I; + return I; +} + MachineBasicBlock::iterator MachineBasicBlock::getFirstNonDebugInstr(bool SkipPseudoOp) { // Skip over begin-of-block dbg_value instructions. diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -830,8 +830,12 @@ if (!FirstTerminator) FirstTerminator = MI; } else if (FirstTerminator) { - report("Non-terminator instruction after the first terminator", MI); - errs() << "First terminator was:\t" << *FirstTerminator; + // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to + // precede non-terminators. + if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) { + report("Non-terminator instruction after the first terminator", MI); + errs() << "First terminator was:\t" << *FirstTerminator; + } } } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/invoke-region.ll b/llvm/test/CodeGen/AArch64/GlobalISel/invoke-region.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/invoke-region.ll @@ -0,0 +1,60 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +; RUN: llc -O0 -mtriple=aarch64-apple-ios -global-isel -stop-after=legalizer %s -o - | FileCheck %s + +@global_var = external global i32 +declare i32 @__gxx_personality_v0(...) +declare void @may_throw() + +; This test checks that the widened G_CONSTANT operand to the phi in "continue" bb +; is placed before the potentially throwing call in the entry block. +define i1 @test_lpad_phi_widen_into_pred() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { + ; CHECK-LABEL: name: test_lpad_phi_widen_into_pred + ; CHECK: bb.1 (%ir-block.0): + ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 + ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @global_var + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11 + ; CHECK-NEXT: G_STORE [[C]](s32), [[GV]](p0) :: (store (s32) into @global_var) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 1 + ; CHECK-NEXT: G_INVOKE_REGION_START + ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp + ; CHECK-NEXT: BL @may_throw, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp + ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp + ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_INVOKE_REGION_END + ; CHECK-NEXT: G_BR %bb.3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.lpad (landing-pad): + ; CHECK-NEXT: successors: %bb.3(0x80000000) + ; CHECK-NEXT: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.1 + ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_STORE [[PHI]](s32), [[GV]](p0) :: (store (s32) into @global_var) + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 + ; CHECK-NEXT: G_BR %bb.3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3.continue: + ; CHECK-NEXT: [[PHI1:%[0-9]+]]:_(s16) = G_PHI [[C2]](s16), %bb.1, [[C3]](s16), %bb.2 + ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[PHI1]](s16) + ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]] + ; CHECK-NEXT: $w0 = COPY [[AND]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + store i32 42, i32* @global_var + invoke void @may_throw() + to label %continue unwind label %lpad + +lpad: ; preds = %entry + %p = phi i32 [ 11, %0 ] + %1 = landingpad { i8*, i32 } + catch i8* null + store i32 %p, i32* @global_var + br label %continue + +continue: ; preds = %entry, %lpad + %r.0 = phi i1 [ 1, %0 ], [ 0, %lpad ] + ret i1 %r.0 +} diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll @@ -15,6 +15,7 @@ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-NEXT: G_INVOKE_REGION_START ; CHECK-NEXT: EH_LABEL ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: $w0 = COPY [[C]](s32) @@ -22,6 +23,7 @@ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_INVOKE_REGION_END ; CHECK-NEXT: G_BR %bb.3 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2.broken (landing-pad): @@ -61,11 +63,13 @@ ; CHECK-NEXT: liveins: $x0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64(p0) = COPY $x0 + ; CHECK-NEXT: G_INVOKE_REGION_START ; CHECK-NEXT: EH_LABEL ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: BLR [[COPY]](p0), csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_INVOKE_REGION_END ; CHECK-NEXT: G_BR %bb.3 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2.broken (landing-pad): @@ -99,6 +103,7 @@ ; CHECK-NEXT: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0 ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 + ; CHECK-NEXT: G_INVOKE_REGION_START ; CHECK-NEXT: EH_LABEL ; CHECK-NEXT: ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $sp @@ -113,6 +118,7 @@ ; CHECK-NEXT: BL @printf, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0 ; CHECK-NEXT: ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_INVOKE_REGION_END ; CHECK-NEXT: G_BR %bb.3 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2.broken (landing-pad): @@ -151,11 +157,13 @@ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 13 ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 55 ; CHECK-NEXT: G_STORE [[C]](s32), [[GV]](p0) :: (store (s32) into @global_var) + ; CHECK-NEXT: G_INVOKE_REGION_START ; CHECK-NEXT: EH_LABEL ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: BL @may_throw, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_INVOKE_REGION_END ; CHECK-NEXT: G_BR %bb.3 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2.lpad (landing-pad): diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unwind-inline-asm.ll @@ -20,9 +20,11 @@ ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @.str.2 ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY [[GV]](p0) + ; CHECK-NEXT: G_INVOKE_REGION_START ; CHECK-NEXT: EH_LABEL ; CHECK-NEXT: INLINEASM &"bl trap", 1 /* sideeffect attdialect */ ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_INVOKE_REGION_END ; CHECK-NEXT: G_BR %bb.2 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2.invoke.cont: diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll @@ -16,12 +16,14 @@ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.exn.slot ; CHECK-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.ehselector.slot + ; CHECK-NEXT: G_INVOKE_REGION_START ; CHECK-NEXT: EH_LABEL ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: $w0 = COPY [[C]](s32) ; CHECK-NEXT: BL @foo, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit-def $w0 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp ; CHECK-NEXT: EH_LABEL + ; CHECK-NEXT: G_INVOKE_REGION_END ; CHECK-NEXT: G_BR %bb.3 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2.cleanup (landing-pad): diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -248,6 +248,12 @@ # DEBUG-NEXT: G_BRINDIRECT (opcode {{[0-9]+}}): 1 type index, 0 imm indices # DEBUG-NEXT: .. the first uncovered type index: 1, OK # DEBUG-NEXT: .. the first uncovered imm index: 0, OK +# DEBUG-NEXT: G_INVOKE_REGION_START (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_INVOKE_REGION_END (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_INTRINSIC (opcode {{[0-9]+}}): 0 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined