diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -110,6 +110,13 @@ return AnnotationInst; } + void removeAnnotationInst(MCInst &Inst) const { + assert(getAnnotationInst(Inst) && "Expected annotation instruction."); + Inst.erase(std::prev(Inst.end())); + assert(!getAnnotationInst(Inst) && + "More than one annotation instruction detected."); + } + void setAnnotationOpValue(MCInst &Inst, unsigned Index, int64_t Value, AllocatorIdTy AllocatorId = 0) { MCInst *AnnotationInst = getAnnotationInst(Inst); @@ -163,6 +170,18 @@ /// MCPlusBuilder classes must use convert/lower/create* interfaces instead. void setTailCall(MCInst &Inst); + /// Transfer annotations from \p SrcInst to \p DstInst. + void moveAnnotations(MCInst &&SrcInst, MCInst &DstInst) const { + assert(!getAnnotationInst(DstInst) && + "Destination instruction should not have annotations."); + const MCInst *AnnotationInst = getAnnotationInst(SrcInst); + if (!AnnotationInst) + return; + + DstInst.addOperand(MCOperand::createInst(AnnotationInst)); + removeAnnotationInst(SrcInst); + } + public: class InstructionIterator { public: @@ -1864,9 +1883,8 @@ void stripAnnotations(MCInst &Inst, bool KeepTC = false); virtual InstructionListType - createInstrumentedIndirectCall(const MCInst &CallInst, bool TailCall, - MCSymbol *HandlerFuncAddr, int CallSiteID, - MCContext *Ctx) { + createInstrumentedIndirectCall(MCInst &&CallInst, MCSymbol *HandlerFuncAddr, + int CallSiteID, MCContext *Ctx) { llvm_unreachable("not implemented"); return InstructionListType(); } diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp --- a/bolt/lib/Core/MCPlusBuilder.cpp +++ b/bolt/lib/Core/MCPlusBuilder.cpp @@ -298,7 +298,8 @@ // Preserve TailCall annotation. auto IsTC = hasAnnotation(Inst, MCAnnotation::kTailCall); - Inst.erase(std::prev(Inst.end())); + removeAnnotationInst(Inst); + if (KeepTC && IsTC) setTailCall(Inst); } diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp --- a/bolt/lib/Passes/Instrumentation.cpp +++ b/bolt/lib/Passes/Instrumentation.cpp @@ -215,7 +215,7 @@ BinaryContext &BC = FromFunction.getBinaryContext(); bool IsTailCall = BC.MIB->isTailCall(*Iter); InstructionListType CounterInstrs = BC.MIB->createInstrumentedIndirectCall( - *Iter, IsTailCall, + std::move(*Iter), IsTailCall ? IndTailCallHandlerExitBBFunction->getSymbol() : IndCallHandlerExitBBFunction->getSymbol(), IndCallSiteID, &*BC.Ctx); diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -3085,8 +3085,7 @@ Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg } - InstructionListType createInstrumentedIndirectCall(const MCInst &CallInst, - bool TailCall, + InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, MCSymbol *HandlerFuncAddr, int CallSiteID, MCContext *Ctx) override { @@ -3137,14 +3136,13 @@ createLoadImmediate(Insts.back(), TempReg, CallSiteID); Insts.emplace_back(); createPushRegister(Insts.back(), TempReg, 8); - Insts.emplace_back(); - createDirectCall(Insts.back(), HandlerFuncAddr, Ctx, - /*TailCall=*/TailCall); - // Carry over metadata - for (int I = MCPlus::getNumPrimeOperands(CallInst), - E = CallInst.getNumOperands(); - I != E; ++I) - Insts.back().addOperand(CallInst.getOperand(I)); + + MCInst &NewCallInst = Insts.emplace_back(); + createDirectCall(NewCallInst, HandlerFuncAddr, Ctx, isTailCall(CallInst)); + + // Carry over metadata including tail call marker if present. + stripAnnotations(NewCallInst); + moveAnnotations(std::move(CallInst), NewCallInst); return Insts; }