diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -620,6 +620,11 @@ def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>; def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>; +// This intrinsic is intentionally undocumented and users shouldn't call it; +// it's produced then quickly consumed during codegen. +def int_callbr_landingpad : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], + [IntrNoMerge]>; + //===------------------- Standard C Library Intrinsics --------------------===// // diff --git a/llvm/lib/CodeGen/CallBrPrepare.cpp b/llvm/lib/CodeGen/CallBrPrepare.cpp --- a/llvm/lib/CodeGen/CallBrPrepare.cpp +++ b/llvm/lib/CodeGen/CallBrPrepare.cpp @@ -32,13 +32,17 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/CFG.h" #include "llvm/CodeGen/Passes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -104,6 +108,23 @@ return Changed; } +static bool InsertIntrinsicCalls(ArrayRef CBRs) { + bool Changed = false; + SmallPtrSet Visited; + IRBuilder<> Builder(CBRs[0]->getContext()); + for (CallBrInst *CBR : CBRs) { + for (BasicBlock *IndDest : CBR->getIndirectDests()) { + if (!Visited.insert(IndDest).second) + continue; + Builder.SetInsertPoint(&*IndDest->begin()); + Builder.CreateIntrinsic(CBR->getType(), Intrinsic::callbr_landingpad, + {CBR}); + Changed = true; + } + } + return Changed; +} + bool CallBrPrepare::runOnFunction(Function &Fn) { bool Changed = false; SmallVector CBRs = FindCallBrs(Fn); @@ -130,5 +151,8 @@ if (SplitCriticalEdges(CBRs, *DT)) Changed = true; + if (InsertIntrinsicCalls(CBRs)) + Changed = true; + return Changed; } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -5824,6 +5824,35 @@ "isdata argument to llvm.aarch64.prefetch must be 0 or 1", Call); break; } + case Intrinsic::callbr_landingpad: { + const auto *CBR = dyn_cast(Call.getOperand(0)); + Check(CBR, "intrinstic requires callbr operand", &Call); + if (!CBR) + break; + + const BasicBlock *LandingPadBB = Call.getParent(); + const BasicBlock *PredBB = LandingPadBB->getUniquePredecessor(); + if (!PredBB) { + CheckFailed("Intrinsic in block must have 1 unique predecessor", &Call); + break; + } + if (!isa(PredBB->getTerminator())) { + CheckFailed("Intrinsic must have corresponding callbr in predecessor", + &Call); + break; + } + Check(llvm::any_of(CBR->getIndirectDests(), + [LandingPadBB](const BasicBlock *IndDest) { + return IndDest == LandingPadBB; + }), + "Intrinsic's corresponding callbr must have intrinsic's parent basic " + "block in indirect destination list", + &Call); + const Instruction &First = *LandingPadBB->begin(); + Check(&First == &Call, "No other instructions may proceed intrinsic", + &Call); + break; + } }; // Verify that there aren't any unmediated control transfers between funclets. diff --git a/llvm/test/CodeGen/AArch64/callbr-prepare.ll b/llvm/test/CodeGen/AArch64/callbr-prepare.ll --- a/llvm/test/CodeGen/AArch64/callbr-prepare.ll +++ b/llvm/test/CodeGen/AArch64/callbr-prepare.ll @@ -7,11 +7,13 @@ ; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"() ; CHECK-NEXT: to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge] ; CHECK: entry.indirect_crit_edge: +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT]]) ; CHECK-NEXT: br label [[INDIRECT:%.*]] ; CHECK: direct: ; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"() ; CHECK-NEXT: to label [[DIRECT2:%.*]] [label %direct.indirect_crit_edge] ; CHECK: direct.indirect_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT2]]) ; CHECK-NEXT: br label [[INDIRECT]] ; CHECK: direct2: ; CHECK-NEXT: ret i32 0 @@ -67,6 +69,7 @@ ; CHECK: x: ; CHECK-NEXT: ret i32 42 ; CHECK: y: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]]) ; CHECK-NEXT: ret i32 [[TMP0]] ; entry: @@ -138,12 +141,13 @@ ; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i"() ; CHECK-NEXT: to label [[X:%.*]] [label %entry.y_crit_edge] ; CHECK: entry.y_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]]) ; CHECK-NEXT: br label [[Y:%.*]] ; CHECK: x: ; CHECK-NEXT: br label [[Y]] ; CHECK: y: -; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ] -; CHECK-NEXT: ret i32 [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ] +; CHECK-NEXT: ret i32 [[TMP2]] ; entry: %0 = callbr i32 asm "", "=r,!i"() @@ -168,12 +172,13 @@ ; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"() ; CHECK-NEXT: to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge] ; CHECK: w.v_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]]) ; CHECK-NEXT: br label [[V]] ; CHECK: x: ; CHECK-NEXT: ret i32 42 ; CHECK: v: -; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ] -; CHECK-NEXT: ret i32 [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ] +; CHECK-NEXT: ret i32 [[TMP2]] ; entry: br i1 %z, label %w, label %v @@ -200,12 +205,13 @@ ; CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"() ; CHECK-NEXT: to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge] ; CHECK: w.v_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]]) ; CHECK-NEXT: br label [[V]] ; CHECK: x: ; CHECK-NEXT: ret i32 42 ; CHECK: v: -; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ] -; CHECK-NEXT: ret i32 [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ] +; CHECK-NEXT: ret i32 [[TMP2]] ; entry: br i1 %z, label %w, label %v diff --git a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll --- a/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll +++ b/llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s +; RUN: opt < %s -S -passes="simplifycfg" \ +; RUN: -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s define void @callbr_duplicate_dest() { ; CHECK-LABEL: @callbr_duplicate_dest( @@ -57,3 +58,49 @@ bb3: ret void } + +; Validate that callbr landingpad intrinsics do not get merged (via the +; IntrNoMerge attribute). +define i32 @callbr_landingpad_nomerge() { +; CHECK-LABEL: @callbr_landingpad_nomerge( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge] +; CHECK: entry.indirect_crit_edge: +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT]]) +; CHECK-NEXT: br label [[COMMON_RET:%.*]] +; CHECK: direct: +; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[COMMON_RET]] [label %direct.indirect_crit_edge] +; CHECK: direct.indirect_crit_edge: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT2]]) +; CHECK-NEXT: br label [[COMMON_RET]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[DIRECT]] ], [ [[TMP0]], [[ENTRY_INDIRECT_CRIT_EDGE:%.*]] ], [ [[TMP1]], [[DIRECT_INDIRECT_CRIT_EDGE:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] +; +entry: + %out = callbr i32 asm "# $0", "=r,!i"() + to label %direct [label %entry.indirect_crit_edge] + +entry.indirect_crit_edge: + %0 = call i32 @llvm.callbr.landingpad.i32(i32 %out) + br label %indirect + +direct: + %out2 = callbr i32 asm "# $0", "=r,!i"() + to label %direct2 [label %direct.indirect_crit_edge] + +direct.indirect_crit_edge: + %1 = call i32 @llvm.callbr.landingpad.i32(i32 %out2) + br label %indirect + +direct2: + ret i32 0 + +indirect: + %out3 = phi i32 [ %0, %entry.indirect_crit_edge ], [ %1, %direct.indirect_crit_edge ] + ret i32 %out3 +} + +declare i32 @llvm.callbr.landingpad.i32(i32) diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll --- a/llvm/test/Verifier/callbr.ll +++ b/llvm/test/Verifier/callbr.ll @@ -68,3 +68,55 @@ abnormal: ret i32 %ret } + +;; Tests of the callbr.landingpad intrinsic function. +declare i32 @llvm.callbr.landingpad.i64(i64) +define void @callbrpad_bad_type() { +entry: +; CHECK: Intrinsic has incorrect argument type! +; CHECK-NEXT: ptr @llvm.callbr.landingpad.i64 + %foo = call i32 @llvm.callbr.landingpad.i64(i64 42) + ret void +} + +declare i32 @llvm.callbr.landingpad.i32(i32) +define i32 @callbrpad_multi_preds() { +entry: + %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect] +direct: + br label %indirect +indirect: +; CHECK-NEXT: Intrinsic in block must have 1 unique predecessor +; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %foo) + %out = call i32 @llvm.callbr.landingpad.i32(i32 %foo) + ret i32 %out +} + +define void @callbrpad_wrong_callbr() { +entry: + %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect] +direct: +; CHECK-NEXT: Intrinsic's corresponding callbr must have intrinsic's parent basic block in indirect destination list +; CHECK-NEXT: %x = call i32 @llvm.callbr.landingpad.i32(i32 %foo) + %x = call i32 @llvm.callbr.landingpad.i32(i32 %foo) + ret void +indirect: + ret void +} + +declare i32 @foo(i32) +define i32 @test_callbr_landingpad_not_first_inst() { +entry: + %0 = callbr i32 asm "", "=r,!i"() + to label %asm.fallthrough [label %landingpad] + +asm.fallthrough: + ret i32 42 + +landingpad: + %foo = call i32 @foo(i32 42) +; CHECK-NEXT: No other instructions may proceed intrinsic +; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %0) + %out = call i32 @llvm.callbr.landingpad.i32(i32 %0) + ret i32 %out +}