diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -61,7 +61,8 @@ private: void ExpandICallBranchFunnel(MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI); - + void expandCALL_RVMARKER(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI); bool ExpandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI); bool ExpandMBB(MachineBasicBlock &MBB); }; @@ -173,6 +174,62 @@ JTMBB->erase(JTInst); } +void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) { + // Expand CALL_RVMARKER pseudo to call instruction, followed by the special + //"movl %ebp, %ebp" marker. + // TODO: Mark the sequence as bundle, to avoid passes moving other code + // in between. + MachineInstr &MI = *MBBI; + + MachineInstr *OriginalCall; + unsigned Sel = MI.getOperand(0).getImm(); + MachineOperand &CallTarget = MI.getOperand(1); + assert((CallTarget.isGlobal() || CallTarget.isReg()) && + "invalid operand for regular call"); + unsigned Opc = + MI.getOpcode() == X86::CALL32_RVMARKER ? X86::CALL32r : X86::CALL64r; + if (MI.getOpcode() == X86::CALL64m_RVMARKER) + Opc = X86::CALL64m; + else if (CallTarget.isGlobal() || CallTarget.isSymbol()) + Opc = MI.getOpcode() == X86::CALL32_RVMARKER ? X86::CALLpcrel32 + : X86::CALL64pcrel32; + OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr(); + OriginalCall->addOperand(CallTarget); + + unsigned RegMaskStartIdx = 2; + // Skip register arguments. Those are added during ISel, but are not + // needed for the concrete branch. + while (!MI.getOperand(RegMaskStartIdx).isRegMask()) { + assert(MI.getOperand(RegMaskStartIdx).isReg() && + "should only skip register operands"); + RegMaskStartIdx++; + } + for (; RegMaskStartIdx < MI.getNumOperands(); ++RegMaskStartIdx) + OriginalCall->addOperand(MI.getOperand(RegMaskStartIdx)); + + // Emit marker "movq %rax, %rdi". + auto *Marker = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::MOV64rr)) + .addReg(X86::RAX, RegState::Define) + .addReg(X86::RDI) + .getInstr(); + if (MI.shouldUpdateCallSiteInfo()) + MBB.getParent()->moveCallSiteInfo(&MI, Marker); + + Module *M = MBB.getParent()->getFunction().getParent(); + auto &Context = M->getContext(); + auto *I8PtrTy = PointerType::get(IntegerType::get(Context, 8), 0); + FunctionCallee FCache = M->getOrInsertFunction( + Sel == 0 ? "objc_retainAutoreleasedReturnValue" + : "objc_unsafeClaimAutoreleasedReturnValue", + FunctionType::get(I8PtrTy, I8PtrTy)); + + BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32)) + .addGlobalAddress(cast(FCache.getCallee()), 0, 0) + .getInstr(); + MI.eraseFromParent(); +} + /// If \p MBBI is a pseudo instruction, this method expands /// it to the corresponding (sequence of) actual instruction(s). /// \returns true if \p MBBI has been expanded. @@ -487,6 +544,11 @@ MI.setDesc(TII->get(X86::TILEZERO)); return true; } + case X86::CALL32_RVMARKER: + case X86::CALL64_RVMARKER: + case X86::CALL64m_RVMARKER: + expandCALL_RVMARKER(MBB, MBBI); + return true; } llvm_unreachable("Previous switch has a fallthrough?"); } diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -76,6 +76,10 @@ /// Same as call except it adds the NoTrack prefix. NT_CALL, + // Pseudo for a OBJC call that gets emitted together with a special + // marker instruction. + CALL_RVMARKER, + /// X86 compare and logical compare instructions. CMP, FCMP, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -28,6 +28,7 @@ #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/EHPersonalities.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/IntrinsicLowering.h" @@ -4408,9 +4409,19 @@ if (HasNoCfCheck && IsCFProtectionSupported && IsIndirectCall) { Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops); + } else if (CLI.CB && objcarc::hasRVOpBundle(CLI.CB)) { + // Calls marked with "rv_marker" are special. They should be expanded to the + // call, directly followed by a special marker sequence. Use the + // CALL_RVMARKER to do that. + assert(!isTailCall && "tail calls cannot be marked with clang.arc.rv"); + auto Last = Ops.back(); + unsigned Sel = objcarc::hasRVOpBundle(CLI.CB, true) ? 0 : 1; + Ops.insert(Ops.begin() + 1, DAG.getTargetConstant(Sel, dl, MVT::i32)); + Chain = DAG.getNode(X86ISD::CALL_RVMARKER, dl, NodeTys, Ops); } else { Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); } + InFlag = Chain.getValue(1); DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); @@ -31011,6 +31022,7 @@ NODE_NAME_CASE(FLD) NODE_NAME_CASE(FST) NODE_NAME_CASE(CALL) + NODE_NAME_CASE(CALL_RVMARKER) NODE_NAME_CASE(BT) NODE_NAME_CASE(CMP) NODE_NAME_CASE(FCMP) diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td --- a/llvm/lib/Target/X86/X86InstrControl.td +++ b/llvm/lib/Target/X86/X86InstrControl.td @@ -415,6 +415,22 @@ } } +let isPseudo = 1, isCall = 1, isCodeGenOnly = 1, + Uses = [RSP, SSP], + SchedRW = [WriteJump] in { + def CALL64m_RVMARKER : + PseudoI<(outs), (ins i32imm:$sel, i64mem:$dst), [(X86call_rvmarker timm:$sel, (loadi64 addr:$dst))]>, + Requires<[In64BitMode]>; + + def CALL32_RVMARKER : + PseudoI<(outs), (ins i32imm:$sel, GR32:$dst), [(X86call_rvmarker timm:$sel,GR32:$dst)]>, + Requires<[Not64BitMode]>; + + def CALL64_RVMARKER : + PseudoI<(outs), (ins i32imm:$sel, GR64:$dst), [(X86call_rvmarker timm:$sel, GR64:$dst)]>, + Requires<[In64BitMode]>; +} + // Conditional tail calls are similar to the above, but they are branches // rather than barriers, and they use EFLAGS. let isCall = 1, isTerminator = 1, isReturn = 1, isBranch = 1, diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td --- a/llvm/lib/Target/X86/X86InstrInfo.td +++ b/llvm/lib/Target/X86/X86InstrInfo.td @@ -204,6 +204,11 @@ [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, SDNPVariadic]>; +def X86call_rvmarker : SDNode<"X86ISD::CALL_RVMARKER", SDT_X86Call, + [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, + SDNPVariadic]>; + + def X86NoTrackCall : SDNode<"X86ISD::NT_CALL", SDT_X86Call, [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, SDNPVariadic]>; diff --git a/llvm/test/CodeGen/X86/call-rv-marker.ll b/llvm/test/CodeGen/X86/call-rv-marker.ll --- a/llvm/test/CodeGen/X86/call-rv-marker.ll +++ b/llvm/test/CodeGen/X86/call-rv-marker.ll @@ -23,19 +23,37 @@ @g = global i8* null, align 8 @fptr = global i8* ()* null, align 8 -define i8* @rv_marker_1() { -; CHECK-LABEL: rv_marker_1: +define i8* @rv_marker_1_retain() { +; CHECK-LABEL: rv_marker_1_retain: ; CHECK: pushq %rax ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: callq _foo1 +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue ; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq ; entry: - %call = call "rv_marker" i8* @foo1() + %call = call i8* @foo1() [ "clang.arc.rv"(i64 0) ] ret i8* %call } +define i8* @rv_marker_1_claim() { +; CHECK-LABEL: rv_marker_1_claim: +; CHECK: pushq %rax +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: callq _foo1 +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: callq _objc_unsafeClaimAutoreleasedReturnValue +; CHECK-NEXT: popq %rcx +; CHECK-NEXT: retq +; +entry: + %call = call i8* @foo1() [ "clang.arc.rv"(i64 1) ] + ret i8* %call +} + + define void @rv_marker_2_select(i32 %c) { ; CHECK-LABEL: rv_marker_2_select: ; CHECK: pushq %rax @@ -44,6 +62,8 @@ ; CHECK-NEXT: movl $1, %edi ; CHECK-NEXT: adcl $0, %edi ; CHECK-NEXT: callq _foo0 +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue ; CHECK-NEXT: movq %rax, %rdi ; CHECK-NEXT: popq %rax ; CHECK-NEXT: jmp _foo2 @@ -51,7 +71,7 @@ entry: %tobool.not = icmp eq i32 %c, 0 %.sink = select i1 %tobool.not, i32 2, i32 1 - %call1 = call "rv_marker" i8* @foo0(i32 %.sink) + %call1 = call i8* @foo0(i32 %.sink) [ "clang.arc.rv"(i64 0) ] tail call void @foo2(i8* %call1) ret void } @@ -67,11 +87,13 @@ ; CHECK-NEXT: .cfi_offset %rbx, -24 ; CHECK-NEXT: .cfi_offset %r14, -16 ; CHECK-NEXT: callq _foo1 +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue ; CHECK-NEXT: movq %rax, %rbx ; CHECK-NEXT: Ltmp0: ; entry: - %call = call "rv_marker" i8* @foo1() + %call = call i8* @foo1() [ "clang.arc.rv"(i64 0) ] invoke void @objc_object(i8* %call) #5 to label %invoke.cont unwind label %lpad @@ -98,13 +120,15 @@ ; CHECK-NEXT: .cfi_offset %r14, -16 ; CHECK-NEXT: Ltmp3: ; CHECK-NEXT: callq _foo1 +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue ; CHECK-NEXT: Ltmp4: ; entry: %s = alloca %struct.S, align 1 %0 = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 0 call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %0) #2 - %call = invoke "rv_marker" i8* @foo1() + %call = invoke i8* @foo1() [ "clang.arc.rv"(i64 0) ] to label %invoke.cont unwind label %lpad invoke.cont: ; preds = %entry @@ -135,12 +159,16 @@ resume { i8*, i32 } %.pn } +; TODO: This should use "callq *_fptr(%rip)". define i8* @rv_marker_5_indirect_call() { ; CHECK-LABEL: rv_marker_5_indirect_call ; CHECK: pushq %rbx ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset %rbx, -16 -; CHECK-NEXT: callq *_fptr(%rip) +; CHECK-NEXT: movq _fptr(%rip), %rax +; CHECK-NEXT: callq *%rax +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue ; CHECK-NEXT: movq %rax, %rbx ; CHECK-NEXT: movq %rax, %rdi ; CHECK-NEXT: callq _foo2 @@ -149,8 +177,8 @@ ; CHECK-NEXT: retq ; entry: - %0 = load i8* ()*, i8* ()** @fptr, align 8 - %call = call "rv_marker" i8* %0() + %lv = load i8* ()*, i8* ()** @fptr, align 8 + %call = call i8* %lv() [ "clang.arc.rv"(i64 0) ] tail call void @foo2(i8* %call) ret i8* %call } @@ -165,10 +193,12 @@ ; CHECK-NEXT: movq %rdx, %rdi ; CHECK-NEXT: movq %rax, %rdx ; CHECK-NEXT: callq _foo +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue ; CHECK-NEXT: popq %rax ; CHECK-NEXT: retq ; - call "rv_marker" void @foo(i64 %c, i64 %b, i64 %a) + call void @foo(i64 %c, i64 %b, i64 %a) [ "clang.arc.rv"(i64 0) ] ret void }