diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h --- a/bolt/include/bolt/Core/BinaryBasicBlock.h +++ b/bolt/include/bolt/Core/BinaryBasicBlock.h @@ -144,6 +144,9 @@ /// blocks may contain out of date or incorrect information. bool IsValid{true}; + /// Whether this block serves as a tail call trampoline. + bool IsTailCallBlock{false}; + /// Last computed hash value. mutable uint64_t Hash{0}; @@ -675,6 +678,10 @@ void markValid(const bool Valid) { IsValid = Valid; } + bool isTailCallBlock() const { return IsTailCallBlock; } + + void setIsTailCallBlock(const bool IsTC) { IsTailCallBlock = IsTC; } + FragmentNum getFragmentNum() const { return Fragment; } void setFragmentNum(const FragmentNum Value) { Fragment = Value; } diff --git a/bolt/include/bolt/Passes/Instrumentation.h b/bolt/include/bolt/Passes/Instrumentation.h --- a/bolt/include/bolt/Passes/Instrumentation.h +++ b/bolt/include/bolt/Passes/Instrumentation.h @@ -55,7 +55,7 @@ const BinaryFunction &FromFunction, uint32_t From, uint32_t FromNodeID, const BinaryFunction &ToFunction, uint32_t To, - bool IsInvoke); + bool IsInvoke, bool IsCondTailCall); bool createEdgeDescription(FunctionDescription &FuncDesc, const BinaryFunction &FromFunction, uint32_t From, uint32_t FromNodeID, @@ -88,8 +88,8 @@ BinaryBasicBlock &FromBB, uint32_t From, BinaryFunction &ToFunc, BinaryBasicBlock *TargetBB, uint32_t ToOffset, bool IsLeaf, bool IsInvoke, - FunctionDescription *FuncDesc, uint32_t FromNodeID, - uint32_t ToNodeID = 0); + bool IsCondTailCall, FunctionDescription *FuncDesc, + uint32_t FromNodeID, uint32_t ToNodeID = 0); void instrumentLeafNode(BinaryBasicBlock &BB, BinaryBasicBlock::iterator Iter, bool IsLeaf, FunctionDescription &FuncDesc, diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -2285,6 +2285,7 @@ TailCallBB->setOffset(BB.getInputOffset()); TailCallBB->addInstruction(TailCallInstr); TailCallBB->setCFIState(CFIStateBeforeCTC); + TailCallBB->setIsTailCallBlock(true); // Add CFG edge with profile info from BB to TailCallBB. BB.addSuccessor(TailCallBB.get(), CTCTakenCount, CTCMispredCount); diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp --- a/bolt/lib/Passes/Instrumentation.cpp +++ b/bolt/lib/Passes/Instrumentation.cpp @@ -100,13 +100,15 @@ const BinaryFunction &FromFunction, uint32_t From, uint32_t FromNodeID, const BinaryFunction &ToFunction, - uint32_t To, bool IsInvoke) { + uint32_t To, bool IsInvoke, + bool IsCondTailCall) { CallDescription CD; // Ordinarily, we don't augment direct calls with an explicit counter, except // when forced to do so or when we know this callee could be throwing // exceptions, in which case there is no other way to accurately record its // frequency. - bool ForceInstrumentation = opts::ConservativeInstrumentation || IsInvoke; + bool ForceInstrumentation = + opts::ConservativeInstrumentation || IsInvoke || IsCondTailCall; CD.FromLoc.FuncString = getFunctionNameIndex(FromFunction); CD.FromLoc.Offset = From; CD.FromNode = FromNodeID; @@ -227,14 +229,16 @@ BinaryBasicBlock::iterator &Iter, BinaryFunction &FromFunction, BinaryBasicBlock &FromBB, uint32_t From, BinaryFunction &ToFunc, BinaryBasicBlock *TargetBB, uint32_t ToOffset, bool IsLeaf, bool IsInvoke, - FunctionDescription *FuncDesc, uint32_t FromNodeID, uint32_t ToNodeID) { + bool IsCondTailCall, FunctionDescription *FuncDesc, uint32_t FromNodeID, + uint32_t ToNodeID) { BinaryContext &BC = FromFunction.getBinaryContext(); { auto L = BC.scopeLock(); bool Created = true; if (!TargetBB) - Created = createCallDescription(*FuncDesc, FromFunction, From, FromNodeID, - ToFunc, ToOffset, IsInvoke); + Created = + createCallDescription(*FuncDesc, FromFunction, From, FromNodeID, + ToFunc, ToOffset, IsInvoke, IsCondTailCall); else Created = createEdgeDescription(*FuncDesc, FromFunction, From, FromNodeID, ToFunc, ToOffset, ToNodeID, @@ -246,7 +250,7 @@ InstructionListType CounterInstrs = createInstrumentationSnippet(BC, IsLeaf); const MCInst &Inst = *Iter; - if (BC.MIB->isCall(Inst)) { + if (BC.MIB->isCall(Inst) || IsCondTailCall) { // This code handles both // - (regular) inter-function calls (cross-function control transfer), // - (rare) intra-function calls (function-local control transfer) @@ -387,10 +391,21 @@ const uint32_t FromOffset = *BC.MIB->getOffset(Inst); const MCSymbol *Target = BC.MIB->getTargetSymbol(Inst); BinaryBasicBlock *TargetBB = Function.getBasicBlockForLabel(Target); + // Handle conditional tail call-produced blocks (.LTC) + bool IsCondTailCall = false; + if (TargetBB && TargetBB->isTailCallBlock()) { + const MCInst *TC = TargetBB->getLastNonPseudoInstr(); + assert(TC && "Tail call block must have a non-pseudo instruction"); + assert(BC.MIB->isTailCall(*TC) && + "Tail call block must end with a tail call"); + Target = BC.MIB->getTargetSymbol(*TC); + TargetBB = nullptr; + IsCondTailCall = true; + } uint32_t ToOffset = TargetBB ? TargetBB->getInputOffset() : 0; BinaryFunction *TargetFunc = TargetBB ? &Function : BC.getFunctionForSymbol(Target); - if (TargetFunc && BC.MIB->isCall(Inst)) { + if (TargetFunc && (BC.MIB->isCall(Inst) || IsCondTailCall)) { if (opts::InstrumentCalls) { const BinaryBasicBlock *ForeignBB = TargetFunc->getBasicBlockForLabel(Target); @@ -398,8 +413,8 @@ ToOffset = ForeignBB->getInputOffset(); instrumentOneTarget(SplitWorklist, SplitInstrs, I, Function, BB, FromOffset, *TargetFunc, TargetBB, ToOffset, - IsLeafFunction, IsInvokeBlock, FuncDesc, - BBToID[&BB]); + IsLeafFunction, IsInvokeBlock, IsCondTailCall, + FuncDesc, BBToID[&BB]); } continue; } @@ -414,8 +429,8 @@ } instrumentOneTarget(SplitWorklist, SplitInstrs, I, Function, BB, FromOffset, *TargetFunc, TargetBB, ToOffset, - IsLeafFunction, IsInvokeBlock, FuncDesc, - BBToID[&BB], BBToID[TargetBB]); + IsLeafFunction, IsInvokeBlock, IsCondTailCall, + FuncDesc, BBToID[&BB], BBToID[TargetBB]); continue; } @@ -432,7 +447,7 @@ instrumentOneTarget( SplitWorklist, SplitInstrs, I, Function, BB, FromOffset, Function, &*Succ, Succ->getInputOffset(), IsLeafFunction, IsInvokeBlock, - FuncDesc, BBToID[&BB], BBToID[&*Succ]); + IsCondTailCall, FuncDesc, BBToID[&BB], BBToID[&*Succ]); } continue; } @@ -472,10 +487,10 @@ /*Instrumented=*/false); continue; } - instrumentOneTarget(SplitWorklist, SplitInstrs, I, Function, BB, - FromOffset, Function, FTBB, FTBB->getInputOffset(), - IsLeafFunction, IsInvokeBlock, FuncDesc, BBToID[&BB], - BBToID[FTBB]); + instrumentOneTarget( + SplitWorklist, SplitInstrs, I, Function, BB, FromOffset, Function, + FTBB, FTBB->getInputOffset(), IsLeafFunction, IsInvokeBlock, + /*IsCondTailCall=*/false, FuncDesc, BBToID[&BB], BBToID[FTBB]); } } // End of BBs loop diff --git a/bolt/test/runtime/X86/instrumentation-tail-call.s b/bolt/test/runtime/X86/instrumentation-tail-call.s --- a/bolt/test/runtime/X86/instrumentation-tail-call.s +++ b/bolt/test/runtime/X86/instrumentation-tail-call.s @@ -14,6 +14,10 @@ # CHECK: leaq 0x80(%rsp), %rsp +# Check that BOLT properly instruments conditional tail calls +# RUN: FileCheck %s --input-file %t.fdata --check-prefix=CHECK-FDATA +# CHECK-FDATA: 1 main {{.*}} 1 targetFunc 0 0 1 + .text .globl main .type main, %function @@ -32,7 +36,8 @@ movq %rbp, %rsp pop %rbp mov -0x10(%rsp),%rax - jmp targetFunc + test %rsp, %rsp + jne targetFunc .LBBerror: addq $0x20, %rsp