diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -191,8 +191,6 @@ MachineBasicBlock::iterator MBBI) { // Expand CALL_RVMARKER pseudo to call instruction, followed by the special //"movq %rax, %rdi" marker. - // TODO: Mark the sequence as bundle, to avoid passes moving other code - // in between. MachineInstr &MI = *MBBI; MachineInstr *OriginalCall; @@ -236,15 +234,23 @@ // Emit call to ObjC runtime. const uint32_t *RegMask = TRI->getCallPreservedMask(*MBB.getParent(), CallingConv::C); - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32)) - .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0) - .addRegMask(RegMask) - .addReg(X86::RAX, - RegState::Implicit | - (RAXImplicitDead ? (RegState::Dead | RegState::Define) - : RegState::Define)) - .getInstr(); + MachineInstr *RtCall = + BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32)) + .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0) + .addRegMask(RegMask) + .addReg(X86::RAX, + RegState::Implicit | + (RAXImplicitDead ? (RegState::Dead | RegState::Define) + : RegState::Define)) + .getInstr(); MI.eraseFromParent(); + + auto &TM = MBB.getParent()->getTarget(); + // On Darwin platforms, wrap the expanded sequence in a bundle to prevent + // later optimizations from breaking up the sequence. + if (TM.getTargetTriple().isOSDarwin()) + finalizeBundle(MBB, OriginalCall->getIterator(), + std::next(RtCall->getIterator())); } /// If \p MBBI is a pseudo instruction, this method expands diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -588,6 +588,18 @@ // Insert pseudo probe annotation for callsite profiling addPass(createPseudoProbeInserter()); + + // On Darwin platforms, BLR_RVMARKER pseudo instructions are lowered to + // bundles. + if (TT.isOSDarwin()) + addPass(createUnpackMachineBundles([](const MachineFunction &MF) { + // Only run bundle expansion if there are relevant ObjC runtime functions + // present in the module. + const Function &F = MF.getFunction(); + const Module *M = F.getParent(); + return M->getFunction("objc_retainAutoreleasedReturnValue") || + M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"); + })); } bool X86PassConfig::addPostFastRegAllocRewrite() { diff --git a/llvm/test/CodeGen/X86/call-rv-marker.ll b/llvm/test/CodeGen/X86/call-rv-marker.ll --- a/llvm/test/CodeGen/X86/call-rv-marker.ll +++ b/llvm/test/CodeGen/X86/call-rv-marker.ll @@ -232,14 +232,16 @@ ; CHECK-NEXT: ## %bb.1: ; CHECK-NEXT: callq _fn1 +; CHECK-NEXT: movq %rax, %rdi +; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue ; CHECK-NEXT: jmp LBB8_3 ; CHECK-NEXT: LBB8_2: ; CHECK-NEXT: callq _fn2 - -; CHECK-NEXT: LBB8_3: ; CHECK-NEXT: movq %rax, %rdi ; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue + +; CHECK-NEXT: LBB8_3: ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq diff --git a/llvm/test/CodeGen/X86/expand-call-rvmarker.mir b/llvm/test/CodeGen/X86/expand-call-rvmarker.mir --- a/llvm/test/CodeGen/X86/expand-call-rvmarker.mir +++ b/llvm/test/CodeGen/X86/expand-call-rvmarker.mir @@ -30,9 +30,11 @@ # CHECK: bb.0 # CHECK-NEXT: frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp # CHECK-NEXT: CFI_INSTRUCTION def_cfa_offset 16 -# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax -# CHECK-NEXT: $rdi = MOV64rr $rax -# CHECK-NEXT: CALL64pcrel32 @objc_retainAutoreleasedReturnValue, csr_64, implicit $rsp, implicit $ssp, implicit-def $rax +# CHECK-NEXT: BUNDLE +# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax +# CHECK-NEXT: $rdi = MOV64rr internal $rax +# CHECK-NEXT: CALL64pcrel32 @objc_retainAutoreleasedReturnValue, csr_64, implicit internal $rsp, implicit internal $ssp, implicit-def $rax +# CHECK-NEXT: } # CHECK-NEXT: $rcx = frame-destroy POP64r implicit-def $rsp, implicit $rsp # CHECK-NEXT: RET64 # @@ -62,9 +64,11 @@ # CHECK: bb.0 # CHECK-NEXT: frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp # CHECK-NEXT: CFI_INSTRUCTION def_cfa_offset 16 -# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax -# CHECK-NEXT: $rdi = MOV64rr $rax -# CHECK-NEXT: CALL64pcrel32 @objc_unsafeClaimAutoreleasedReturnValue, csr_64, implicit $rsp, implicit $ssp, implicit-def $rax +# CHECK-NEXT: BUNDLE +# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax +# CHECK-NEXT: $rdi = MOV64rr internal $rax +# CHECK-NEXT: CALL64pcrel32 @objc_unsafeClaimAutoreleasedReturnValue, csr_64, implicit internal $rsp, implicit internal $ssp, implicit-def $rax +# CHECK-NEXT: } # CHECK-NEXT: $rcx = frame-destroy POP64r implicit-def $rsp, implicit $rsp # CHECK-NEXT: RET64 # @@ -95,9 +99,11 @@ # CHECK-NEXT: $rax = MOV64rr $rdi # CHECK-NEXT: $rdi = MOV64rr killed $rdx # CHECK-NEXT: $rdx = MOV64rr killed $rax -# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit-def $rsp, implicit-def $ssp, implicit-def $rax -# CHECK-NEXT: $rdi = MOV64rr $rax -# CHECK-NEXT: CALL64pcrel32 @objc_retainAutoreleasedReturnValue, csr_64, implicit $rsp, implicit $ssp, implicit-def dead $rax +# CHECK-NEXT: BUNDLE +# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit-def $rsp, implicit-def $ssp, implicit-def $rax +# CHECK-NEXT: $rdi = MOV64rr internal $rax +# CHECK-NEXT: CALL64pcrel32 @objc_retainAutoreleasedReturnValue, csr_64, implicit internal $rsp, implicit internal $ssp, implicit-def dead $rax +# CHECK-NEXT: } # CHECK-NEXT: $rax = frame-destroy POP64r implicit-def $rsp, implicit $rsp # CHECK-NEXT: RET64 # @@ -129,9 +135,11 @@ # CHECK: bb.0 # CHECK-NEXT: frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp # CHECK-NEXT: CFI_INSTRUCTION def_cfa_offset 16 -# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax -# CHECK-NEXT: $rdi = MOV64rr $rax -# CHECK-NEXT: CALL64pcrel32 @objc_retainAutoreleasedReturnValue, csr_64, implicit $rsp, implicit $ssp, implicit-def dead $rax +# CHECK-NEXT: BUNDLE +# CHECK-NEXT: CALL64pcrel32 @fn, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax +# CHECK-NEXT: $rdi = MOV64rr internal $rax +# CHECK-NEXT: CALL64pcrel32 @objc_retainAutoreleasedReturnValue, csr_64, implicit internal $rsp, implicit internal $ssp, implicit-def dead $rax +# CHECK-NEXT: } # CHECK-NEXT: $rax = frame-destroy POP64r implicit-def $rsp, implicit $rsp # CHECK-NEXT: RET64 #