diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -620,6 +620,10 @@ def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>; def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>; +// This intrinsic is intentionally undocumented and users shouldn't call it; +// it's produced then quickly consumed during codegen. +def int_callbr_landingpad : Intrinsic<[llvm_any_ty], [], [IntrNoMerge]>; + //===------------------- Standard C Library Intrinsics --------------------===// // diff --git a/llvm/lib/CodeGen/CallBrPrepare.cpp b/llvm/lib/CodeGen/CallBrPrepare.cpp --- a/llvm/lib/CodeGen/CallBrPrepare.cpp +++ b/llvm/lib/CodeGen/CallBrPrepare.cpp @@ -22,13 +22,16 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/CFG.h" #include "llvm/CodeGen/Passes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -94,6 +97,23 @@ return Changed; } +static bool InsertIntrinsicCalls(ArrayRef CBRs) { + bool Changed = false; + SmallPtrSet Visited; + IRBuilder<> Builder(CBRs[0]->getContext()); + for (CallBrInst *CBR : CBRs) { + for (BasicBlock *IndDest : CBR->getIndirectDests()) { + if (!Visited.insert(IndDest).second) + continue; + Builder.SetInsertPoint(&*IndDest->begin()); + Builder.CreateIntrinsic(CBR->getType(), + llvm::Intrinsic::callbr_landingpad, {}); + Changed = true; + } + } + return Changed; +} + bool CallBrPrepare::runOnFunction(Function &Fn) { bool Changed = false; SmallVector CBRs = FindCallBrs(Fn); @@ -120,5 +140,8 @@ if (SplitCriticalEdges(CBRs, *DT)) Changed = true; + if (InsertIntrinsicCalls(CBRs)) + Changed = true; + return Changed; } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -5793,6 +5793,33 @@ "isdata argument to llvm.aarch64.prefetch must be 0 or 1", Call); break; } + case Intrinsic::callbr_landingpad: { + const BasicBlock *LandingPadBB = Call.getParent(); + const BasicBlock *CallBrBB = LandingPadBB->getUniquePredecessor(); + if (!CallBrBB) { + CheckFailed("Intrinsic in block must have 1 unique predecessor", &Call); + break; + } + const auto *CBR = dyn_cast(CallBrBB->getTerminator()); + if (!CBR) { + CheckFailed("Intrinsic must have corresponding callbr in predecessor", + &Call); + break; + } + Check(llvm::any_of(CBR->getIndirectDests(), + [LandingPadBB](const BasicBlock *IndDest) { + return IndDest == LandingPadBB; + }), + "Intrinsic's corresponding callbr must have intrinsic's parent basic " + "block in indirect destination list", + &Call); + Check(!CBR->getType()->isVoidTy(), + "intrinsic's corresponding callbr must return a value", &Call); + const Instruction &First = *LandingPadBB->begin(); + Check(&First == &Call, "No other instructions may proceed intrinsic", + &Call); + break; + } }; } diff --git a/llvm/test/CodeGen/AArch64/callbr-prepare.ll b/llvm/test/CodeGen/AArch64/callbr-prepare.ll --- a/llvm/test/CodeGen/AArch64/callbr-prepare.ll +++ b/llvm/test/CodeGen/AArch64/callbr-prepare.ll @@ -7,11 +7,13 @@ ; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"() ; CHECK-NEXT: to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge] ; CHECK: entry.indirect_crit_edge: +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32() ; CHECK-NEXT: br label [[INDIRECT:%.*]] ; CHECK: direct: ; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"() ; CHECK-NEXT: to label [[DIRECT2:%.*]] [label %direct.indirect_crit_edge] ; CHECK: direct.indirect_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32() ; CHECK-NEXT: br label [[INDIRECT]] ; CHECK: direct2: ; CHECK-NEXT: ret i32 0 @@ -67,6 +69,7 @@ ; CHECK: x: ; CHECK-NEXT: ret i32 42 ; CHECK: y: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32() ; CHECK-NEXT: ret i32 [[TMP0]] ; entry: @@ -138,12 +141,13 @@ ; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i"() ; CHECK-NEXT: to label [[X:%.*]] [label %entry.y_crit_edge] ; CHECK: entry.y_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32() ; CHECK-NEXT: br label [[Y:%.*]] ; CHECK: x: ; CHECK-NEXT: br label [[Y]] ; CHECK: y: -; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ] -; CHECK-NEXT: ret i32 [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ] +; CHECK-NEXT: ret i32 [[TMP2]] ; entry: %0 = callbr i32 asm "", "=r,!i"() @@ -168,12 +172,13 @@ ; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"() ; CHECK-NEXT: to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge] ; CHECK: w.v_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32() ; CHECK-NEXT: br label [[V]] ; CHECK: x: ; CHECK-NEXT: ret i32 42 ; CHECK: v: -; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ] -; CHECK-NEXT: ret i32 [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ] +; CHECK-NEXT: ret i32 [[TMP2]] ; entry: br i1 %z, label %w, label %v @@ -200,12 +205,13 @@ ; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"() ; CHECK-NEXT: to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge] ; CHECK: w.v_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32() ; CHECK-NEXT: br label [[V]] ; CHECK: x: ; CHECK-NEXT: ret i32 42 ; CHECK: v: -; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ] -; CHECK-NEXT: ret i32 [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ] +; CHECK-NEXT: ret i32 [[TMP2]] ; entry: br i1 %z, label %w, label %v diff --git a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll --- a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll +++ b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s +; RUN: opt < %s -S -passes="simplifycfg" \ +; RUN: -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s define void @callbr_duplicate_dest() { ; CHECK-LABEL: @callbr_duplicate_dest( @@ -57,3 +58,49 @@ bb3: ret void } + +; Validate that callbr landingpad intrinsics do not get merged (via the +; IntrNoMerge attribute). +define i32 @callbr_landingpad_nomerge() { +; CHECK-LABEL: @callbr_landingpad_nomerge( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge] +; CHECK: entry.indirect_crit_edge: +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32() +; CHECK-NEXT: br label [[COMMON_RET:%.*]] +; CHECK: direct: +; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[COMMON_RET]] [label %direct.indirect_crit_edge] +; CHECK: direct.indirect_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32() +; CHECK-NEXT: br label [[COMMON_RET]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[DIRECT]] ], [ [[TMP0]], [[ENTRY_INDIRECT_CRIT_EDGE:%.*]] ], [ [[TMP1]], [[DIRECT_INDIRECT_CRIT_EDGE:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] +; +entry: + %out = callbr i32 asm "# $0", "=r,!i"() + to label %direct [label %entry.indirect_crit_edge] + +entry.indirect_crit_edge: + %0 = call i32 @llvm.callbr.landingpad.i32() + br label %indirect + +direct: + %out2 = callbr i32 asm "# $0", "=r,!i"() + to label %direct2 [label %direct.indirect_crit_edge] + +direct.indirect_crit_edge: + %1 = call i32 @llvm.callbr.landingpad.i32() + br label %indirect + +direct2: + ret i32 0 + +indirect: + %out3 = phi i32 [ %0, %entry.indirect_crit_edge ], [ %1, %direct.indirect_crit_edge ] + ret i32 %out3 +} + +declare i32 @llvm.callbr.landingpad.i32() diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll --- a/llvm/test/Verifier/callbr.ll +++ b/llvm/test/Verifier/callbr.ll @@ -68,3 +68,78 @@ abnormal: ret i32 %ret } + +;; Tests of the callbr.landingpad intrinsic function. +; The intrinsic does not accept parameters. +declare i32 @llvm.callbr.landingpad.i64(i64 %x) +define void @callbrpad_bad_type() { +entry: +; CHECK: Intrinsic has incorrect argument type! +; CHECK-NEXT: ptr @llvm.callbr.landingpad.i64 + %foo = call i32 @llvm.callbr.landingpad.i64(i64 42) + ret void +} + +declare i32 @llvm.callbr.landingpad.i32() +define i32 @callbrpad_multi_preds() { +entry: + %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect] +direct: + br label %indirect +indirect: +; CHECK-NEXT: Intrinsic in block must have 1 unique predecessor +; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32() + %out = call i32 @llvm.callbr.landingpad.i32() + ret i32 %out +} + +define void @callbrpad_no_callbr() { +entry: + br label %foo +foo: +; CHECK-NEXT: Intrinsic must have corresponding callbr in predecessor +; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32() + %out = call i32 @llvm.callbr.landingpad.i32() + ret void +} + +define void @callbrpad_wrong_callbr() { +entry: + %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect] +direct: +; CHECK-NEXT: Intrinsic's corresponding callbr must have intrinsic's parent basic block in indirect destination list +; CHECK-NEXT: %x = call i32 @llvm.callbr.landingpad.i32() + %x = call i32 @llvm.callbr.landingpad.i32() + ret void +indirect: + ret void +} + +define void @callbrpad_void_callbr() { +entry: + callbr void asm "", "!i"() to label %direct [label %indirect] +direct: + ret void +indirect: +; CHECK-NEXT: intrinsic's corresponding callbr must return a value +; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32() + %out = call i32 @llvm.callbr.landingpad.i32() + ret void +} + +declare i32 @foo(i32) +define i32 @test_callbr_landingpad_not_first_inst() { +entry: + %0 = callbr i32 asm "", "=r,!i"() + to label %asm.fallthrough [label %landingpad] + +asm.fallthrough: + ret i32 42 + +landingpad: + %foo = call i32 @foo(i32 42) +; CHECK-NEXT: No other instructions may proceed intrinsic +; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32() + %out = call i32 @llvm.callbr.landingpad.i32() + ret i32 %out +}