diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -61,7 +61,8 @@ private: void ExpandICallBranchFunnel(MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI); - + void expandCALL_RVMARKER(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI); bool ExpandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI); bool ExpandMBB(MachineBasicBlock &MBB); }; @@ -173,6 +174,49 @@ JTMBB->erase(JTInst); } +void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) { + // Expand CALL_RVMARKER pseudo to call instruction, followed by the special + //"movl %ebp, %ebp" marker. + // TODO: Mark the sequence as bundle, to avoid passes moving other code + // in between. + MachineInstr &MI = *MBBI; + + MachineInstr *OriginalCall; + MachineOperand &CallTarget = MI.getOperand(0); + assert((CallTarget.isGlobal() || CallTarget.isReg()) && + "invalid operand for regular call"); + unsigned Opc = + MI.getOpcode() == X86::CALL32_RVMARKER ? X86::CALL32r : X86::CALL64r; + if (MI.getOpcode() == X86::CALL64m_RVMARKER) + Opc = X86::CALL64m; + else if (CallTarget.isGlobal() || CallTarget.isSymbol()) + Opc = MI.getOpcode() == X86::CALL32_RVMARKER ? X86::CALLpcrel32 + : X86::CALL64pcrel32; + OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr(); + OriginalCall->addOperand(CallTarget); + + unsigned RegMaskStartIdx = 1; + // Skip register arguments. Those are added during ISel, but are not + // needed for the concrete branch. + while (!MI.getOperand(RegMaskStartIdx).isRegMask()) { + assert(MI.getOperand(RegMaskStartIdx).isReg() && + "should only skip register operands"); + RegMaskStartIdx++; + } + for (; RegMaskStartIdx < MI.getNumOperands(); ++RegMaskStartIdx) + OriginalCall->addOperand(MI.getOperand(RegMaskStartIdx)); + + // Emit marker "movl %ebp, %ebp". + auto *Marker = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::MOV32rr)) + .addReg(X86::EBP, RegState::Define) + .addReg(X86::EBP) + .getInstr(); + if (MI.shouldUpdateCallSiteInfo()) + MBB.getParent()->moveCallSiteInfo(&MI, Marker); + MI.eraseFromParent(); +} + /// If \p MBBI is a pseudo instruction, this method expands /// it to the corresponding (sequence of) actual instruction(s). /// \returns true if \p MBBI has been expanded. @@ -500,6 +544,11 @@ MI.setDesc(TII->get(X86::TILEZERO)); return true; } + case X86::CALL32_RVMARKER: + case X86::CALL64_RVMARKER: + case X86::CALL64m_RVMARKER: + expandCALL_RVMARKER(MBB, MBBI); + return true; } llvm_unreachable("Previous switch has a fallthrough?"); } diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -76,6 +76,10 @@ /// Same as call except it adds the NoTrack prefix. NT_CALL, + // Pseudo for a OBJC call that gets emitted together with a special + // marker instruction. + CALL_RVMARKER, + /// X86 compare and logical compare instructions. CMP, FCMP, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -4399,6 +4399,15 @@ } else { Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); } + + // Calls marked with "rv_marker" are special. They should be expanded to the + // call, directly followed by a special marker sequence. Use the CALL_RVMARKER + // to do that. + if (CLI.CB && CLI.CB->hasRetAttr("rv_marker")) { + assert(!isTailCall && "tail calls cannot be marked with rv_marker"); + Chain = DAG.getNode(X86ISD::CALL_RVMARKER, dl, NodeTys, Ops); + } + InFlag = Chain.getValue(1); DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); @@ -30971,6 +30980,7 @@ NODE_NAME_CASE(FLD) NODE_NAME_CASE(FST) NODE_NAME_CASE(CALL) + NODE_NAME_CASE(CALL_RVMARKER) NODE_NAME_CASE(BT) NODE_NAME_CASE(CMP) NODE_NAME_CASE(FCMP) diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td --- a/llvm/lib/Target/X86/X86InstrControl.td +++ b/llvm/lib/Target/X86/X86InstrControl.td @@ -415,6 +415,22 @@ } } +let isPseudo = 1, isCall = 1, isCodeGenOnly = 1, + Uses = [RSP, SSP], + SchedRW = [WriteJump] in { + def CALL64m_RVMARKER : + PseudoI<(outs), (ins i64mem:$dst), [(X86call_rvmarker (loadi64 addr:$dst))]>, + Requires<[In64BitMode]>; + + def CALL32_RVMARKER : + PseudoI<(outs), (ins GR32:$dst), [(X86call_rvmarker GR32:$dst)]>, + Requires<[Not64BitMode]>; + + def CALL64_RVMARKER : + PseudoI<(outs), (ins GR64:$dst), [(X86call_rvmarker GR64:$dst)]>, + Requires<[In64BitMode]>; +} + // Conditional tail calls are similar to the above, but they are branches // rather than barriers, and they use EFLAGS. let isCall = 1, isTerminator = 1, isReturn = 1, isBranch = 1, diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td --- a/llvm/lib/Target/X86/X86InstrInfo.td +++ b/llvm/lib/Target/X86/X86InstrInfo.td @@ -204,6 +204,11 @@ [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, SDNPVariadic]>; +def X86call_rvmarker : SDNode<"X86ISD::CALL_RVMARKER", SDT_X86Call, + [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, + SDNPVariadic]>; + + def X86NoTrackCall : SDNode<"X86ISD::NT_CALL", SDT_X86Call, [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, SDNPVariadic]>; diff --git a/llvm/test/CodeGen/X86/call-rv-marker.ll b/llvm/test/CodeGen/X86/call-rv-marker.ll --- a/llvm/test/CodeGen/X86/call-rv-marker.ll +++ b/llvm/test/CodeGen/X86/call-rv-marker.ll @@ -28,6 +28,7 @@ ; CHECK: pushq %rax ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: callq _foo1 +; CHECK-NEXT: movl %ebp, %ebp ; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq ; @@ -44,6 +45,7 @@ ; CHECK-NEXT: movl $1, %edi ; CHECK-NEXT: adcl $0, %edi ; CHECK-NEXT: callq _foo0 +; CHECK-NEXT: movl %ebp, %ebp ; CHECK-NEXT: movq %rax, %rdi ; CHECK-NEXT: popq %rax ; CHECK-NEXT: jmp _foo2 @@ -67,6 +69,7 @@ ; CHECK-NEXT: .cfi_offset %rbx, -24 ; CHECK-NEXT: .cfi_offset %r14, -16 ; CHECK-NEXT: callq _foo1 +; CHECK-NEXT: movl %ebp, %ebp ; CHECK-NEXT: movq %rax, %rbx ; CHECK-NEXT: Ltmp0: ; @@ -98,6 +101,7 @@ ; CHECK-NEXT: .cfi_offset %r14, -16 ; CHECK-NEXT: Ltmp3: ; CHECK-NEXT: callq _foo1 +; CHECK-NEXT: movl %ebp, %ebp ; CHECK-NEXT: Ltmp4: ; entry: @@ -135,12 +139,15 @@ resume { i8*, i32 } %.pn } +; TODO: This should use "callq *_fptr(%rip)". define i8* @rv_marker_5_indirect_call() { ; CHECK-LABEL: rv_marker_5_indirect_call ; CHECK: pushq %rbx ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset %rbx, -16 -; CHECK-NEXT: callq *_fptr(%rip) +; CHECK-NEXT: movq _fptr(%rip), %rax +; CHECK-NEXT: callq *%rax +; CHECK-NEXT: movl %ebp, %ebp ; CHECK-NEXT: movq %rax, %rbx ; CHECK-NEXT: movq %rax, %rdi ; CHECK-NEXT: callq _foo2 @@ -165,6 +172,7 @@ ; CHECK-NEXT: movq %rdx, %rdi ; CHECK-NEXT: movq %rax, %rdx ; CHECK-NEXT: callq _foo +; CHECK-NEXT: movl %ebp, %ebp ; CHECK-NEXT: popq %rax ; CHECK-NEXT: retq ;