diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h --- a/llvm/include/llvm/IR/Instruction.h +++ b/llvm/include/llvm/IR/Instruction.h @@ -715,7 +715,8 @@ CompareIgnoringAlignment = 1<<0, /// Check for equivalence treating a type and a vector of that type /// as equivalent. - CompareUsingScalarTypes = 1<<1 + CompareUsingScalarTypes = 1<<1, + CompareIgnoringTailCall = 1<<2 }; /// This function determines if the specified instruction executes the same diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -459,7 +459,8 @@ /// kept in sync with FunctionComparator::cmpOperations in /// lib/Transforms/IPO/MergeFunctions.cpp. static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, - bool IgnoreAlignment = false) { + bool IgnoreAlignment = false, + bool IgnoreTailCall = false) { assert(I1->getOpcode() == I2->getOpcode() && "Can not compare special state of different instructions"); @@ -482,7 +483,8 @@ if (const CmpInst *CI = dyn_cast(I1)) return CI->getPredicate() == cast(I2)->getPredicate(); if (const CallInst *CI = dyn_cast(I1)) - return CI->isTailCall() == cast(I2)->isTailCall() && + return (CI->isTailCall() == cast(I2)->isTailCall() + || IgnoreTailCall) && CI->getCallingConv() == cast(I2)->getCallingConv() && CI->getAttributes() == cast(I2)->getAttributes() && CI->hasIdenticalOperandBundleSchema(*cast(I2)); @@ -561,6 +563,7 @@ unsigned flags) const { bool IgnoreAlignment = flags & CompareIgnoringAlignment; bool UseScalarTypes = flags & CompareUsingScalarTypes; + bool IgnoreTailCall = flags & CompareIgnoringTailCall; if (getOpcode() != I->getOpcode() || getNumOperands() != I->getNumOperands() || @@ -578,7 +581,7 @@ getOperand(i)->getType() != I->getOperand(i)->getType()) return false; - return haveSameSpecialState(this, I, IgnoreAlignment); + return haveSameSpecialState(this, I, IgnoreAlignment, IgnoreTailCall); } bool Instruction::isUsedOutsideOfBlock(const BasicBlock *BB) const { diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -1778,6 +1778,31 @@ return !isa(I); } +// Compute if there is a common tail call kind that could be used for +// a merged call instruction. Assumes that all `Insts` are CallInst. +// Allow identical tail call kinds or a mix of TCK_None and TCK_Tail. +static Optional +computeCommonTailCallKind(ArrayRef Insts) { + using TCK = CallInst::TailCallKind; + const auto MustTail = 1u << TCK::TCK_MustTail; + const auto NoTail = 1u << TCK::TCK_NoTail; + const auto Tail = 1u << TCK::TCK_Tail; + const auto TCKNone = 1u << TCK::TCK_None; + unsigned Present = 0u; + for (auto *I : Insts) { + auto *Call = cast(I); + Present |= 1u << Call->getTailCallKind(); + } + switch (Present) { + case MustTail: return {TCK::TCK_MustTail}; + case NoTail: return {TCK::TCK_NoTail}; + case Tail: return {TCK::TCK_Tail}; + case TCKNone: return {TCK::TCK_None}; + case Tail | TCKNone: return {TCK::TCK_None}; + default: return None; + } +} + // All instructions in Insts belong to different blocks that all unconditionally // branch to a common successor. Analyze each instruction and return true if it // would be possible to sink them into their successor, creating one common @@ -1818,9 +1843,13 @@ const Instruction *I0 = Insts.front(); for (auto *I : Insts) - if (!I->isSameOperationAs(I0)) + if (!I->isSameOperationAs(I0, Instruction::CompareIgnoringTailCall)) return false; + // Do not sink if tail call kinds are incompatible + if (isa(I0) && !computeCommonTailCallKind(Insts)) + return false; + // All instructions in Insts are known to be the same opcode. If they have a // use, check that the only user is a PHI or in the same block as the // instruction, because if a user is in the same block as an instruction we're @@ -1989,6 +2018,11 @@ I0->andIRFlags(I); } + if (auto *Call = dyn_cast(I0)) { + auto TailCallKind = computeCommonTailCallKind(Insts); + Call->setTailCallKind(TailCallKind.value()); + } + if (!I0->user_empty()) { // canSinkLastInstruction checked that all instructions were used by // one and only one PHI node. Find that now, RAUW it to our common diff --git a/llvm/test/Transforms/SimplifyCFG/sink-tail-call-kind.ll b/llvm/test/Transforms/SimplifyCFG/sink-tail-call-kind.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SimplifyCFG/sink-tail-call-kind.ll @@ -0,0 +1,176 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -opaque-pointers -simplifycfg -sink-common-insts -S %s | FileCheck %s + +; Verify that calls to function @tailfn are sunk when tail call kinds +; are compatible (and correct tail call kind is specified for a sunk +; call). Test the full matrix (not applicable for musttail, as it is +; always followed by ret): + +; | | none | tail | notail | +; |--------+---------+---------+--------| +; | none | sink | | | +; | tail | sink | sink | | +; | notail | no sink | no sink | sink | + +declare i32 @magic() +declare void @tailfn() + +define dso_local void @tail_vs_none() { +; CHECK-LABEL: @tail_vs_none( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @magic() +; CHECK-NEXT: call void @tailfn() +; CHECK-NEXT: ret void +; +entry: + %call = call i32 @magic() + %tobool = icmp ne i32 %call, 0 + br i1 %tobool, label %if.then, label %if.else + +if.then: ; preds = %entry + tail call void @tailfn() + br label %if.end + +if.else: ; preds = %entry + call void @tailfn() + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void +} + +define dso_local void @none_vs_none() { +; CHECK-LABEL: @none_vs_none( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @magic() +; CHECK-NEXT: call void @tailfn() +; CHECK-NEXT: ret void +; +entry: + %call = call i32 @magic() + %tobool = icmp ne i32 %call, 0 + br i1 %tobool, label %if.then, label %if.else + +if.then: ; preds = %entry + call void @tailfn() + br label %if.end + +if.else: ; preds = %entry + call void @tailfn() + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void +} + +define dso_local void @tail_vs_tail() { +; CHECK-LABEL: @tail_vs_tail( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @magic() +; CHECK-NEXT: tail call void @tailfn() +; CHECK-NEXT: ret void +; +entry: + %call = call i32 @magic() + %tobool = icmp ne i32 %call, 0 + br i1 %tobool, label %if.then, label %if.else + +if.then: ; preds = %entry + tail call void @tailfn() + br label %if.end + +if.else: ; preds = %entry + tail call void @tailfn() + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void +} + +define dso_local void @no_vs_no() { +; CHECK-LABEL: @no_vs_no( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @magic() +; CHECK-NEXT: notail call void @tailfn() +; CHECK-NEXT: ret void +; +entry: + %call = call i32 @magic() + %tobool = icmp ne i32 %call, 0 + br i1 %tobool, label %if.then, label %if.else + +if.then: ; preds = %entry + notail call void @tailfn() + br label %if.end + +if.else: ; preds = %entry + notail call void @tailfn() + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void +} + +define dso_local void @no_vs_none() { +; CHECK-LABEL: @no_vs_none( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @magic() +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[CALL]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: notail call void @tailfn() +; CHECK-NEXT: br label [[IF_END:%.*]] +; CHECK: if.else: +; CHECK-NEXT: call void @tailfn() +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + %call = call i32 @magic() + %tobool = icmp ne i32 %call, 0 + br i1 %tobool, label %if.then, label %if.else + +if.then: ; preds = %entry + notail call void @tailfn() + br label %if.end + +if.else: ; preds = %entry + call void @tailfn() + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void +} + +define dso_local void @no_vs_tail() { +; CHECK-LABEL: @no_vs_tail( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @magic() +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[CALL]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: notail call void @tailfn() +; CHECK-NEXT: br label [[IF_END:%.*]] +; CHECK: if.else: +; CHECK-NEXT: tail call void @tailfn() +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + %call = call i32 @magic() + %tobool = icmp ne i32 %call, 0 + br i1 %tobool, label %if.then, label %if.else + +if.then: ; preds = %entry + notail call void @tailfn() + br label %if.end + +if.else: ; preds = %entry + tail call void @tailfn() + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void +}