diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp --- a/clang/lib/CodeGen/CGObjC.cpp +++ b/clang/lib/CodeGen/CGObjC.cpp @@ -23,6 +23,7 @@ #include "clang/Basic/Diagnostic.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/InlineAsm.h" @@ -2078,6 +2079,15 @@ EmitNounwindRuntimeCall(fn, values); } +/// Emit a call to "clang.arc.noop.use", which consumes the result of a call +/// that has operand bundle "clang.arc.rv". +void CodeGenFunction::EmitARCNoopIntrinsicUse(ArrayRef values) { + llvm::Function *&fn = CGM.getObjCEntrypoints().clang_arc_noop_use; + if (!fn) + fn = CGM.getIntrinsic(llvm::Intrinsic::objc_clang_arc_noop_use); + EmitNounwindRuntimeCall(fn, values); +} + static void setARCRuntimeFunctionLinkage(CodeGenModule &CGM, llvm::Value *RTF) { if (auto *F = dyn_cast(RTF)) { // If the target runtime doesn't naturally support ARC, emit weak @@ -2304,10 +2314,11 @@ // with this marker yet, so leave a breadcrumb for the ARC // optimizer to pick up. } else { - const char *markerKey = "clang.arc.retainAutoreleasedReturnValueMarker"; - if (!CGF.CGM.getModule().getModuleFlag(markerKey)) { + const char *retainRVMarkerKey = llvm::objcarc::getRVMarkerModuleFlagStr(); + if (!CGF.CGM.getModule().getModuleFlag(retainRVMarkerKey)) { auto *str = llvm::MDString::get(CGF.getLLVMContext(), assembly); - CGF.CGM.getModule().addModuleFlag(llvm::Module::Error, markerKey, str); + CGF.CGM.getModule().addModuleFlag(llvm::Module::Error, + retainRVMarkerKey, str); } } } @@ -2317,6 +2328,46 @@ CGF.Builder.CreateCall(marker, None, CGF.getBundlesForFunclet(marker)); } +static llvm::Value *emitOptimizedARCReturnCall(llvm::Value *value, + bool IsRetainRV, + CodeGenFunction &CGF) { + emitAutoreleasedReturnValueMarker(CGF); + + // Add operand bundle "clang.arc.rv" to the call instead of emitting retainRV + // or claimRV calls in the IR. We currently do this only when the optimization + // level isn't -O0 since global-isel, which is currently run at -O0, doesn't + // know about the operand bundle. + + // FIXME: Do this when the target isn't aarch64. + if (CGF.CGM.getCodeGenOpts().OptimizationLevel > 0 && + CGF.CGM.getTarget().getTriple().isAArch64()) { + llvm::Value *bundleArgs[] = {llvm::ConstantInt::get( + CGF.Int64Ty, llvm::objcarc::getRVOperandBundleEnum(IsRetainRV))}; + SmallVector bundles; + bundles.emplace_back("clang.arc.rv", bundleArgs); + auto *oldCall = cast(value); + llvm::CallBase *newCall = llvm::CallBase::Create(oldCall, bundles, oldCall); + newCall->copyMetadata(*oldCall); + oldCall->replaceAllUsesWith(newCall); + oldCall->eraseFromParent(); + CGF.EmitARCNoopIntrinsicUse(newCall); + return newCall; + } + + bool isNoTail = + CGF.CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail(); + llvm::CallInst::TailCallKind tailKind = + isNoTail ? llvm::CallInst::TCK_NoTail : llvm::CallInst::TCK_None; + ObjCEntrypoints &EPs = CGF.CGM.getObjCEntrypoints(); + llvm::Function *&EP = IsRetainRV + ? EPs.objc_retainAutoreleasedReturnValue + : EPs.objc_unsafeClaimAutoreleasedReturnValue; + llvm::Intrinsic::ID IID = + IsRetainRV ? llvm::Intrinsic::objc_retainAutoreleasedReturnValue + : llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue; + return emitARCValueOperation(CGF, value, nullptr, EP, IID, tailKind); +} + /// Retain the given object which is the result of a function call. /// call i8* \@objc_retainAutoreleasedReturnValue(i8* %value) /// @@ -2324,15 +2375,7 @@ /// call with completely different semantics. llvm::Value * CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) { - emitAutoreleasedReturnValueMarker(*this); - llvm::CallInst::TailCallKind tailKind = - CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail() - ? llvm::CallInst::TCK_NoTail - : llvm::CallInst::TCK_None; - return emitARCValueOperation( - *this, value, nullptr, - CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue, - llvm::Intrinsic::objc_retainAutoreleasedReturnValue, tailKind); + return emitOptimizedARCReturnCall(value, true, *this); } /// Claim a possibly-autoreleased return value at +0. This is only @@ -2344,15 +2387,7 @@ /// call i8* \@objc_unsafeClaimAutoreleasedReturnValue(i8* %value) llvm::Value * CodeGenFunction::EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value) { - emitAutoreleasedReturnValueMarker(*this); - llvm::CallInst::TailCallKind tailKind = - CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail() - ? llvm::CallInst::TCK_NoTail - : llvm::CallInst::TCK_None; - return emitARCValueOperation( - *this, value, nullptr, - CGM.getObjCEntrypoints().objc_unsafeClaimAutoreleasedReturnValue, - llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue, tailKind); + return emitOptimizedARCReturnCall(value, false, *this); } /// Release the given object. diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4202,6 +4202,8 @@ void EmitARCIntrinsicUse(ArrayRef values); + void EmitARCNoopIntrinsicUse(ArrayRef values); + static Destroyer destroyARCStrongImprecise; static Destroyer destroyARCStrongPrecise; static Destroyer destroyARCWeak; diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -210,6 +210,9 @@ /// void clang.arc.use(...); llvm::Function *clang_arc_use; + + /// void clang.arc.noop.use(...); + llvm::Function *clang_arc_noop_use; }; /// This class records statistics on instrumentation based profiling. diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -4535,7 +4535,6 @@ llvm::Type *newRetTy = newFn->getReturnType(); SmallVector newArgs; - SmallVector newBundles; for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end(); ui != ue; ) { @@ -4592,6 +4591,7 @@ newArgs.append(callSite->arg_begin(), callSite->arg_begin() + argNo); // Copy over any operand bundles. + SmallVector newBundles; callSite->getOperandBundlesAsDefs(newBundles); llvm::CallBase *newCall; diff --git a/clang/test/CodeGenObjC/arc-rv-attr.m b/clang/test/CodeGenObjC/arc-rv-attr.m new file mode 100644 --- /dev/null +++ b/clang/test/CodeGenObjC/arc-rv-attr.m @@ -0,0 +1,177 @@ +// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK + +@class A; + +A *makeA(void); + +void test_assign() { + __unsafe_unretained id x; + x = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_assign() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A:.*]]* @makeA() [ "clang.arc.rv"(i64 1) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_assign_assign() { + __unsafe_unretained id x, y; + x = y = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_assign_assign() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[Y:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: store i8* [[T1]], i8** [[Y]] +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_strong_assign_assign() { + __strong id x; + __unsafe_unretained id y; + x = y = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_strong_assign_assign() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[Y:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: store i8* [[T1]], i8** [[Y]] +// CHECK-NEXT: [[OLD:%.*]] = load i8*, i8** [[X]] +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: call void @llvm.objc.release(i8* [[OLD]] +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: [[T0:%.*]] = load i8*, i8** [[X]] +// CHECK-NEXT: call void @llvm.objc.release(i8* [[T0]]) +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_assign_strong_assign() { + __unsafe_unretained id x; + __strong id y; + x = y = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_assign_strong_assign() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[Y:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: [[OLD:%.*]] = load i8*, i8** [[Y]] +// CHECK-NEXT: store i8* [[T1]], i8** [[Y]] +// CHECK-NEXT: call void @llvm.objc.release(i8* [[OLD]] +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: [[T0:%.*]] = load i8*, i8** [[Y]] +// CHECK-NEXT: call void @llvm.objc.release(i8* [[T0]]) +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_init() { + __unsafe_unretained id x = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_init() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_init_assignment() { + __unsafe_unretained id x; + __unsafe_unretained id y = x = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_init_assignment() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[Y:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: store i8* [[T1]], i8** [[Y]] +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_strong_init_assignment() { + __unsafe_unretained id x; + __strong id y = x = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_strong_init_assignment() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[Y:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: store i8* [[T1]], i8** [[Y]] +// CHECK-NEXT: [[T0:%.*]] = load i8*, i8** [[Y]] +// CHECK-NEXT: call void @llvm.objc.release(i8* [[T0]]) +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_init_strong_assignment() { + __strong id x; + __unsafe_unretained id y = x = makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_init_strong_assignment() +// CHECK: [[X:%.*]] = alloca i8* +// CHECK: [[Y:%.*]] = alloca i8* +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 0) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8* +// CHECK-NEXT: [[OLD:%.*]] = load i8*, i8** [[X]] +// CHECK-NEXT: store i8* [[T1]], i8** [[X]] +// CHECK-NEXT: call void @llvm.objc.release(i8* [[OLD]]) +// CHECK-NEXT: store i8* [[T1]], i8** [[Y]] +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: [[T0:%.*]] = load i8*, i8** [[X]] +// CHECK-NEXT: call void @llvm.objc.release(i8* [[T0]]) +// CHECK-NEXT: bitcast +// CHECK-NEXT: lifetime.end +// CHECK-NEXT: ret void + +void test_ignored() { + makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_ignored() +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: ret void + +void test_cast_to_void() { + (void) makeA(); +} +// CHECK-LABEL: define{{.*}} void @test_cast_to_void() +// CHECK: [[T0:%.*]] = call [[A]]* @makeA() [ "clang.arc.rv"(i64 1) ] +// CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use({{.*}} [[T0]]) +// CHECK-NEXT: ret void + +// This is always at the end of the module. + +// CHECK-OPTIMIZED: !llvm.module.flags = !{!0, +// CHECK-OPTIMIZED: !0 = !{i32 1, !"clang.arc.retainAutoreleasedReturnValueMarker", !"mov{{.*}}marker for objc_retainAutoreleaseReturnValue"} diff --git a/clang/test/CodeGenObjC/arc-unsafeclaim.m b/clang/test/CodeGenObjC/arc-unsafeclaim.m --- a/clang/test/CodeGenObjC/arc-unsafeclaim.m +++ b/clang/test/CodeGenObjC/arc-unsafeclaim.m @@ -4,11 +4,10 @@ // Make sure it works on x86-32. // RUN: %clang_cc1 -triple i386-apple-darwin11 -fobjc-runtime=macosx-fragile-10.11 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL -// Make sure it works on ARM. +// Make sure it works on ARM64. // RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL -// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED -check-prefix=CALL -// Make sure it works on ARM64. +// Make sure it works on ARM. // RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL // RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED -check-prefix=CALL diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -2325,6 +2325,18 @@ :ref:`stackmap entry `. See the intrinsic description for further details. +ObjC ARC RetainRV/ClaimRV Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A ``"clang.arc.rv"`` operand bundle on a call indicates the call is implicitly +followed by a marker instruction and a call to an ObjC runtime function that +uses the result of the call. If the argument passed to the operand bundle is 0, +``@objc_retainAutoreleasedReturnValue`` is called. If 1 is passed, +``@objc_unsafeClaimAutoreleasedReturnValue`` is called. + +The operand bundle is needed to ensure the call is immediately followed by the +marker instruction or the ObjC runtime call in the final output. + .. _moduleasm: Module-Level Inline Assembly diff --git a/llvm/include/llvm/Analysis/ObjCARCUtil.h b/llvm/include/llvm/Analysis/ObjCARCUtil.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Analysis/ObjCARCUtil.h @@ -0,0 +1,48 @@ +//===- ObjCARCUtil.h - ObjC ARC Utility Functions ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file defines ARC utility functions which are used by various parts of +/// the compiler. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_ANALYSIS_OBJCARCUTIL_H +#define LLVM_LIB_ANALYSIS_OBJCARCUTIL_H + +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/LLVMContext.h" + +namespace llvm { +namespace objcarc { + +static inline const char *getRVMarkerModuleFlagStr() { + return "clang.arc.retainAutoreleasedReturnValueMarker"; +} + +enum RVOperandBundle : unsigned { RVOB_Retain, RVOB_Claim }; + +static RVOperandBundle getRVOperandBundleEnum(bool IsRetain) { + return IsRetain ? RVOB_Retain : RVOB_Claim; +} + +static inline bool hasRVOpBundle(const CallBase *CB, bool IsRetain) { + auto B = CB->getOperandBundle(LLVMContext::OB_clang_arc_rv); + if (!B.hasValue()) + return false; + return cast(B->Inputs[0])->getZExtValue() == + getRVOperandBundleEnum(IsRetain); +} + +static inline bool hasRVOpBundle(const CallBase *CB) { + return CB->getOperandBundle(LLVMContext::OB_clang_arc_rv).hasValue(); +} + +} // end namespace objcarc +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1214,6 +1214,15 @@ static CallBase *Create(CallBase *CB, ArrayRef Bundles, Instruction *InsertPt = nullptr); + /// Create a clone of \p CB with operand bundle \p OB added. + static CallBase *addOperandBundle(CallBase *CB, uint32_t ID, + OperandBundleDef OB, + Instruction *InsertPt = nullptr); + + /// Create a clone of \p CB with operand bundle \p ID removed. + static CallBase *removeOperandBundle(CallBase *CB, uint32_t ID, + Instruction *InsertPt = nullptr); + static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Call || I->getOpcode() == Instruction::Invoke || diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -446,6 +446,9 @@ llvm_ptr_ty]>; def int_objc_clang_arc_use : Intrinsic<[], [llvm_vararg_ty]>; +def int_objc_clang_arc_noop_use : DefaultAttrsIntrinsic<[], + [llvm_vararg_ty], + [IntrInaccessibleMemOnly]>; def int_objc_unsafeClaimAutoreleasedReturnValue : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty]>; def int_objc_retainedObject : Intrinsic<[llvm_ptr_ty], diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -93,6 +93,7 @@ OB_cfguardtarget = 3, // "cfguardtarget" OB_preallocated = 4, // "preallocated" OB_gc_live = 5, // "gc-live" + OB_clang_arc_rv = 6, // "clang.arc.rv" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. diff --git a/llvm/lib/Analysis/ObjCARCInstKind.cpp b/llvm/lib/Analysis/ObjCARCInstKind.cpp --- a/llvm/lib/Analysis/ObjCARCInstKind.cpp +++ b/llvm/lib/Analysis/ObjCARCInstKind.cpp @@ -140,6 +140,7 @@ return ARCInstKind::User; case Intrinsic::objc_sync_exit: return ARCInstKind::User; + case Intrinsic::objc_clang_arc_noop_use: case Intrinsic::objc_arc_annotation_topdown_bbstart: case Intrinsic::objc_arc_annotation_topdown_bbend: case Intrinsic::objc_arc_annotation_bottomup_bbstart: diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2782,11 +2782,10 @@ // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't // have to do anything here to lower funclet bundles. - assert(!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, - LLVMContext::OB_gc_transition, - LLVMContext::OB_gc_live, - LLVMContext::OB_funclet, - LLVMContext::OB_cfguardtarget}) && + assert(!I.hasOperandBundlesOtherThan( + {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, + LLVMContext::OB_gc_live, LLVMContext::OB_funclet, + LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_rv}) && "Cannot lower invokes with arbitrary operand bundles yet!"); const Value *Callee(I.getCalledOperand()); @@ -7872,7 +7871,8 @@ // CFGuardTarget bundles are lowered in LowerCallTo. assert(!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, - LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated}) && + LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, + LLVMContext::OB_clang_arc_rv}) && "Cannot lower calls with arbitrary operand bundles!"); SDValue Callee = getValue(I.getCalledOperand()); diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -14,14 +14,15 @@ #include "llvm/IR/AutoUpgrade.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/InstVisitor.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsAArch64.h" @@ -3996,7 +3997,7 @@ /// returns true if module is modified. static bool UpgradeRetainReleaseMarker(Module &M) { bool Changed = false; - const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker"; + const char *MarkerKey = objcarc::getRVMarkerModuleFlagStr(); NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey); if (ModRetainReleaseMarker) { MDNode *Op = ModRetainReleaseMarker->getOperand(0); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -424,6 +424,35 @@ return *Current; } +CallBase *CallBase::addOperandBundle(CallBase *CB, uint32_t ID, + OperandBundleDef OB, + Instruction *InsertPt) { + if (CB->getOperandBundle(ID)) + return CB; + + SmallVector Bundles; + CB->getOperandBundlesAsDefs(Bundles); + Bundles.push_back(OB); + return Create(CB, Bundles, InsertPt); +} + +CallBase *CallBase::removeOperandBundle(CallBase *CB, uint32_t ID, + Instruction *InsertPt) { + SmallVector Bundles; + bool CreateNew = false; + + for (unsigned I = 0, E = CB->getNumOperandBundles(); I != E; ++I) { + auto Bundle = CB->getOperandBundleAt(I); + if (Bundle.getTagID() == ID) { + CreateNew = true; + continue; + } + Bundles.emplace_back(Bundle); + } + + return CreateNew ? Create(CB, Bundles, InsertPt) : CB; +} + //===----------------------------------------------------------------------===// // CallInst Implementation //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -78,6 +78,11 @@ "gc-transition operand bundle id drifted!"); (void)GCLiveEntry; + auto *ClangARCRVEntry = pImpl->getOrInsertBundleTag("clang.arc.rv"); + assert(ClangARCRVEntry->second == LLVMContext::OB_clang_arc_rv && + "clang.arc.rv operand bundle id drifted!"); + (void)ClangARCRVEntry; + SyncScope::ID SingleThreadSSID = pImpl->getOrInsertSyncScopeID("singlethread"); assert(SingleThreadSSID == SyncScope::SingleThread && diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -29,6 +29,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -5653,11 +5654,11 @@ } unsigned CallOpc = AArch64ISD::CALL; - // Calls marked with "rv_marker" are special. They should be expanded to the - // call, directly followed by a special marker sequence. Use the CALL_RVMARKER - // to do that. - if (CLI.CB && CLI.CB->hasRetAttr("rv_marker")) { - assert(!IsTailCall && "tail calls cannot be marked with rv_marker"); + // Calls with operand bundle "clang.arc.rv" are special. They should be + // expanded to the call, directly followed by a special marker sequence. Use + // the CALL_RVMARKER to do that. + if (CLI.CB && objcarc::hasRVOpBundle(CLI.CB)) { + assert(!IsTailCall && "tail calls cannot be marked with clang.arc.rv"); CallOpc = AArch64ISD::CALL_RVMARKER; } diff --git a/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h b/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h --- a/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h +++ b/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h @@ -42,6 +42,7 @@ Autorelease, StoreStrong, RetainRV, + ClaimRV, RetainAutorelease, RetainAutoreleaseRV, }; @@ -61,6 +62,7 @@ Autorelease = nullptr; StoreStrong = nullptr; RetainRV = nullptr; + ClaimRV = nullptr; RetainAutorelease = nullptr; RetainAutoreleaseRV = nullptr; } @@ -85,6 +87,9 @@ case ARCRuntimeEntryPointKind::RetainRV: return getIntrinsicEntryPoint(RetainRV, Intrinsic::objc_retainAutoreleasedReturnValue); + case ARCRuntimeEntryPointKind::ClaimRV: + return getIntrinsicEntryPoint( + ClaimRV, Intrinsic::objc_unsafeClaimAutoreleasedReturnValue); case ARCRuntimeEntryPointKind::RetainAutorelease: return getIntrinsicEntryPoint(RetainAutorelease, Intrinsic::objc_retainAutorelease); @@ -121,6 +126,9 @@ /// Declaration for objc_retainAutoreleasedReturnValue(). Function *RetainRV = nullptr; + /// Declaration for objc_unsafeClaimAutoreleasedReturnValue(). + Function *ClaimRV = nullptr; + /// Declaration for objc_retainAutorelease(). Function *RetainAutorelease = nullptr; diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARC.h b/llvm/lib/Transforms/ObjCARC/ObjCARC.h --- a/llvm/lib/Transforms/ObjCARC/ObjCARC.h +++ b/llvm/lib/Transforms/ObjCARC/ObjCARC.h @@ -22,7 +22,10 @@ #ifndef LLVM_LIB_TRANSFORMS_OBJCARC_OBJCARC_H #define LLVM_LIB_TRANSFORMS_OBJCARC_OBJCARC_H +#include "ARCRuntimeEntryPoints.h" +#include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/ObjCARCAnalysisUtils.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/Transforms/Utils/Local.h" namespace llvm { @@ -87,6 +90,67 @@ } } +static inline MDString *getRVInstMarker(Module &M) { + const char *MarkerKey = getRVMarkerModuleFlagStr(); + return dyn_cast_or_null(M.getModuleFlag(MarkerKey)); +} + +/// Create a call instruction with the correct funclet token. This should be +/// called instead of calling CallInst::Create directly unless the call is +/// going to be removed from the IR before WinEHPrepare. +CallInst *createCallInstWithColors( + FunctionCallee Func, ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore, + const DenseMap &BlockColors); + +class BundledRetainClaimRVs { +public: + BundledRetainClaimRVs(ARCRuntimeEntryPoints &P, bool ContractPass) + : EP(P), ContractPass(ContractPass) {} + ~BundledRetainClaimRVs(); + + /// Insert a retainRV/claimRV call to the normal destination blocks of invokes + /// with operand bundle "clang.arc.rv". If the edge to the normal destination + /// block is a critical edge, split it. + std::pair insertAfterInvokes(Function &F, DominatorTree *DT); + + /// Insert a retainRV/claimRV call. + CallInst *insertRVCall(Instruction *InsertPt, CallBase *AnnotatedCall); + + /// Insert a retainRV/claimRV call with colors. + CallInst *insertRVCallWithColors( + Instruction *InsertPt, CallBase *AnnotatedCall, + const DenseMap &BlockColors); + + /// See if an instruction is a bundled retainRV/claimRV call. + bool contains(const Instruction *I) const { + if (auto *CI = dyn_cast(I)) + return RVCalls.count(CI); + return false; + } + + /// Remove a retainRV/claimRV call entirely. + void eraseInst(CallInst *CI) { + auto It = RVCalls.find(CI); + if (It != RVCalls.end()) { + auto *NewCall = CallBase::removeOperandBundle( + It->second, LLVMContext::OB_clang_arc_rv, It->second); + NewCall->copyMetadata(*It->second); + It->second->replaceAllUsesWith(NewCall); + It->second->eraseFromParent(); + RVCalls.erase(It); + } + EraseInstruction(CI); + } + +private: + /// A map of inserted retainRV/claimRV calls to annotated calls/invokes. + DenseMap RVCalls; + + ARCRuntimeEntryPoints &EP; + bool ContractPass; +}; + } // end namespace objcarc } // end namespace llvm diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp --- a/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp +++ b/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp @@ -14,7 +14,12 @@ #include "ObjCARC.h" #include "llvm-c/Initialization.h" +#include "llvm/Analysis/ObjCARCUtil.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/Instructions.h" #include "llvm/InitializePasses.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" namespace llvm { class PassRegistry; @@ -37,3 +42,91 @@ void LLVMInitializeObjCARCOpts(LLVMPassRegistryRef R) { initializeObjCARCOpts(*unwrap(R)); } + +CallInst *objcarc::createCallInstWithColors( + FunctionCallee Func, ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore, + const DenseMap &BlockColors) { + FunctionType *FTy = Func.getFunctionType(); + Value *Callee = Func.getCallee(); + SmallVector OpBundles; + + if (!BlockColors.empty()) { + const ColorVector &CV = BlockColors.find(InsertBefore->getParent())->second; + assert(CV.size() == 1 && "non-unique color for block!"); + Instruction *EHPad = CV.front()->getFirstNonPHI(); + if (EHPad->isEHPad()) + OpBundles.emplace_back("funclet", EHPad); + } + + return CallInst::Create(FTy, Callee, Args, OpBundles, NameStr, InsertBefore); +} + +std::pair +BundledRetainClaimRVs::insertAfterInvokes(Function &F, DominatorTree *DT) { + bool Changed = false, CFGChanged = false; + + for (BasicBlock &BB : F) { + auto *I = dyn_cast(BB.getTerminator()); + + if (!I) + continue; + + if (!objcarc::hasRVOpBundle(I)) + continue; + + BasicBlock *DestBB = I->getNormalDest(); + + if (!DestBB->getSinglePredecessor()) { + assert(I->getSuccessor(0) == DestBB && + "the normal dest is expected to be the first successor"); + DestBB = SplitCriticalEdge(I, 0, CriticalEdgeSplittingOptions(DT)); + CFGChanged = true; + } + + // We don't have to call insertRVCallWithColors since DestBB is the normal + // destination of the invoke. + insertRVCall(&*DestBB->getFirstInsertionPt(), I); + Changed = true; + } + + return std::make_pair(Changed, CFGChanged); +} + +CallInst *BundledRetainClaimRVs::insertRVCall(Instruction *InsertPt, + CallBase *AnnotatedCall) { + DenseMap BlockColors; + return insertRVCallWithColors(InsertPt, AnnotatedCall, BlockColors); +} + +CallInst *BundledRetainClaimRVs::insertRVCallWithColors( + Instruction *InsertPt, CallBase *AnnotatedCall, + const DenseMap &BlockColors) { + IRBuilder<> Builder(InsertPt); + bool IsRetainRV = objcarc::hasRVOpBundle(AnnotatedCall, true); + Function *Func = EP.get(IsRetainRV ? ARCRuntimeEntryPointKind::RetainRV + : ARCRuntimeEntryPointKind::ClaimRV); + Type *ParamTy = Func->getArg(0)->getType(); + Value *CallArg = Builder.CreateBitCast(AnnotatedCall, ParamTy); + auto *Call = + createCallInstWithColors(Func, CallArg, "", InsertPt, BlockColors); + RVCalls[Call] = AnnotatedCall; + return Call; +} + +BundledRetainClaimRVs::~BundledRetainClaimRVs() { + if (ContractPass) { + // At this point, we know that the annotated calls can't be tail calls as + // they are followed by marker instructions and retainRV/claimRV calls. Mark + // them as notail, so that the backend knows these calls can't be tail + // calls. + for (auto P : RVCalls) + if (auto *CI = dyn_cast(P.second)) + CI->setTailCallKind(CallInst::TCK_NoTail); + } else { + for (auto P : RVCalls) + EraseInstruction(P.first); + } + + RVCalls.clear(); +} diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp --- a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp +++ b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp @@ -32,6 +32,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/EHPersonalities.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/InstIterator.h" @@ -63,13 +64,12 @@ class ObjCARCContract { bool Changed; + bool CFGChanged; AAResults *AA; DominatorTree *DT; ProvenanceAnalysis PA; ARCRuntimeEntryPoints EP; - - /// A flag indicating whether this optimization pass should run. - bool Run; + BundledRetainClaimRVs *BundledInsts = nullptr; /// The inline asm string to insert between calls and RetainRV calls to make /// the optimization work on targets which need it. @@ -98,6 +98,7 @@ public: bool init(Module &M); bool run(Function &F, AAResults *AA, DominatorTree *DT); + bool hasCFGChanged() const { return CFGChanged; } }; class ObjCARCContractLegacyPass : public FunctionPass { @@ -304,32 +305,6 @@ return Retain; } -/// Create a call instruction with the correct funclet token. Should be used -/// instead of calling CallInst::Create directly. -static CallInst * -createCallInst(FunctionType *FTy, Value *Func, ArrayRef Args, - const Twine &NameStr, Instruction *InsertBefore, - const DenseMap &BlockColors) { - SmallVector OpBundles; - if (!BlockColors.empty()) { - const ColorVector &CV = BlockColors.find(InsertBefore->getParent())->second; - assert(CV.size() == 1 && "non-unique color for block!"); - Instruction *EHPad = CV.front()->getFirstNonPHI(); - if (EHPad->isEHPad()) - OpBundles.emplace_back("funclet", EHPad); - } - - return CallInst::Create(FTy, Func, Args, OpBundles, NameStr, InsertBefore); -} - -static CallInst * -createCallInst(FunctionCallee Func, ArrayRef Args, const Twine &NameStr, - Instruction *InsertBefore, - const DenseMap &BlockColors) { - return createCallInst(Func.getFunctionType(), Func.getCallee(), Args, NameStr, - InsertBefore, BlockColors); -} - /// Attempt to merge an objc_release with a store, load, and objc_retain to form /// an objc_storeStrong. An objc_storeStrong: /// @@ -411,7 +386,8 @@ if (Args[1]->getType() != I8X) Args[1] = new BitCastInst(Args[1], I8X, "", Store); Function *Decl = EP.get(ARCRuntimeEntryPointKind::StoreStrong); - CallInst *StoreStrong = createCallInst(Decl, Args, "", Store, BlockColors); + CallInst *StoreStrong = + objcarc::createCallInstWithColors(Decl, Args, "", Store, BlockColors); StoreStrong->setDoesNotThrow(); StoreStrong->setDebugLoc(Store->getDebugLoc()); @@ -456,9 +432,14 @@ case ARCInstKind::RetainRV: case ARCInstKind::ClaimRV: { // If we're compiling for a target which needs a special inline-asm - // marker to do the return value optimization, insert it now. + // marker to do the return value optimization and the retainRV/claimRV call + // wasn't bundled with a call, insert the marker now. if (!RVInstMarker) return false; + + if (BundledInsts->contains(Inst)) + return false; + BasicBlock::iterator BBI = Inst->getIterator(); BasicBlock *InstParent = Inst->getParent(); @@ -486,7 +467,7 @@ RVInstMarker->getString(), /*Constraints=*/"", /*hasSideEffects=*/true); - createCallInst(IA, None, "", Inst, BlockColors); + objcarc::createCallInstWithColors(IA, None, "", Inst, BlockColors); } decline_rv_optimization: return false; @@ -525,6 +506,12 @@ Inst->eraseFromParent(); return true; default: + if (auto *CI = dyn_cast(Inst)) + if (CI->getIntrinsicID() == Intrinsic::objc_clang_arc_noop_use) { + // Remove calls to @llvm.objc.clang.arc.noop.use(...). + Changed = true; + CI->eraseFromParent(); + } return true; } } @@ -534,16 +521,10 @@ //===----------------------------------------------------------------------===// bool ObjCARCContract::init(Module &M) { - // If nothing in the Module uses ARC, don't do anything. - Run = ModuleHasARC(M); - if (!Run) - return false; - EP.init(&M); // Initialize RVInstMarker. - const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker"; - RVInstMarker = dyn_cast_or_null(M.getModuleFlag(MarkerKey)); + RVInstMarker = getRVInstMarker(M); return false; } @@ -552,14 +533,16 @@ if (!EnableARCOpts) return false; - // If nothing in the Module uses ARC, don't do anything. - if (!Run) - return false; - - Changed = false; + Changed = CFGChanged = false; AA = A; DT = D; PA.setAA(A); + BundledRetainClaimRVs BRV(EP, true); + BundledInsts = &BRV; + + std::pair R = BundledInsts->insertAfterInvokes(F, DT); + Changed |= R.first; + CFGChanged |= R.second; DenseMap BlockColors; if (F.hasPersonalityFn() && @@ -584,6 +567,13 @@ LLVM_DEBUG(dbgs() << "Visiting: " << *Inst << "\n"); + if (auto *CI = dyn_cast(Inst)) + if (objcarc::hasRVOpBundle(CI)) { + BundledInsts->insertRVCallWithColors(&*I, CI, BlockColors); + --I; + Changed = true; + } + // First try to peephole Inst. If there is nothing further we can do in // terms of undoing objc-arc-expand, process the next inst. if (tryToPeepholeInstruction(F, Inst, I, TailOkForStoreStrongs, @@ -733,7 +723,6 @@ void ObjCARCContractLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); AU.addRequired(); - AU.setPreservesCFG(); } Pass *llvm::createObjCARCContractPass() { @@ -757,9 +746,11 @@ bool Changed = OCAC.run(F, &AM.getResult(F), &AM.getResult(F)); + bool CFGChanged = OCAC.hasCFGChanged(); if (Changed) { PreservedAnalyses PA; - PA.preserveSet(); + if (!CFGChanged) + PA.preserveSet(); return PA; } return PreservedAnalyses::all(); diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp --- a/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp +++ b/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp @@ -41,6 +41,7 @@ #include "llvm/Analysis/ObjCARCAliasAnalysis.h" #include "llvm/Analysis/ObjCARCAnalysisUtils.h" #include "llvm/Analysis/ObjCARCInstKind.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" @@ -483,6 +484,7 @@ /// The main ARC optimization pass. class ObjCARCOpt { bool Changed; + bool CFGChanged; ProvenanceAnalysis PA; /// A cache of references to runtime entry point constants. @@ -492,8 +494,7 @@ /// MDKind identifiers. ARCMDKindCache MDKindCache; - /// A flag indicating whether this optimization pass should run. - bool Run; + BundledRetainClaimRVs *BundledInsts = nullptr; /// A flag indicating whether the optimization that removes or moves /// retain/release pairs should be performed. @@ -573,6 +574,7 @@ void init(Module &M); bool run(Function &F, AAResults &AA); void releaseMemory(); + bool hasCFGChanged() const { return CFGChanged; } }; /// The main ARC optimization pass. @@ -610,8 +612,6 @@ void ObjCARCOptLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); AU.addRequired(); - // ARC optimization doesn't currently split critical edges. - AU.setPreservesCFG(); } /// Turn objc_retainAutoreleasedReturnValue into objc_retain if the operand is @@ -640,6 +640,9 @@ } } + assert(!BundledInsts->contains(RetainRV) && + "a bundled retainRV's argument should be a call"); + // Turn it to a plain objc_retain. Changed = true; ++NumPeeps; @@ -661,6 +664,9 @@ Function &F, DenseMap &BlockColors, Instruction *Inst, const Value *&Arg, ARCInstKind Class, Instruction *AutoreleaseRV, const Value *&AutoreleaseRVArg) { + if (BundledInsts->contains(Inst)) + return false; + // Must be in the same basic block. assert(Inst->getParent() == AutoreleaseRV->getParent()); @@ -844,6 +850,12 @@ for (inst_iterator I = inst_begin(&F), E = inst_end(&F); I != E; ) { Instruction *Inst = &*I++; + if (auto *CI = dyn_cast(Inst)) + if (objcarc::hasRVOpBundle(CI)) { + BundledInsts->insertRVCall(&*I, CI); + Changed = true; + } + ARCInstKind Class = GetBasicARCInstKind(Inst); // Skip this loop if this instruction isn't itself an ARC intrinsic. @@ -922,6 +934,11 @@ // We can delete this call if it takes an inert value. SmallPtrSet VisitedPhis; + if (BundledInsts->contains(Inst)) { + UsedInThisFunction |= 1 << unsigned(Class); + return; + } + if (IsNoopOnGlobal(Class)) if (isInertARCValue(Inst->getOperand(0), VisitedPhis)) { if (!Inst->getType()->isVoidTy()) @@ -1542,7 +1559,7 @@ if (Ptr == Arg) continue; // Handled above. TopDownPtrState &S = MI->second; - if (S.HandlePotentialAlterRefCount(Inst, Ptr, PA, Class)) + if (S.HandlePotentialAlterRefCount(Inst, Ptr, PA, Class, *BundledInsts)) continue; S.HandlePotentialUse(Inst, Ptr, PA, Class); @@ -2343,7 +2360,7 @@ ++NumRets; LLVM_DEBUG(dbgs() << "Erasing: " << *Retain << "\nErasing: " << *Autorelease << "\n"); - EraseInstruction(Retain); + BundledInsts->eraseInst(Retain); EraseInstruction(Autorelease); } } @@ -2376,11 +2393,6 @@ if (!EnableARCOpts) return; - // If nothing in the Module uses ARC, don't do anything. - Run = ModuleHasARC(M); - if (!Run) - return; - // Intuitively, objc_retain and others are nocapture, however in practice // they are not, because they return their argument value. And objc_release // calls finalizers which can have arbitrary side effects. @@ -2394,16 +2406,18 @@ if (!EnableARCOpts) return false; - // If nothing in the Module uses ARC, don't do anything. - if (!Run) - return false; - - Changed = false; + Changed = CFGChanged = false; + BundledRetainClaimRVs BRV(EP, false); + BundledInsts = &BRV; LLVM_DEBUG(dbgs() << "<<< ObjCARCOpt: Visiting Function: " << F.getName() << " >>>" "\n"); + std::pair R = BundledInsts->insertAfterInvokes(F, nullptr); + Changed |= R.first; + CFGChanged |= R.second; + PA.setAA(&AA); #ifndef NDEBUG @@ -2468,9 +2482,11 @@ OCAO.init(*F.getParent()); bool Changed = OCAO.run(F, AM.getResult(F)); + bool CFGChanged = OCAO.hasCFGChanged(); if (Changed) { PreservedAnalyses PA; - PA.preserveSet(); + if (!CFGChanged) + PA.preserveSet(); return PA; } return PreservedAnalyses::all(); diff --git a/llvm/lib/Transforms/ObjCARC/PtrState.h b/llvm/lib/Transforms/ObjCARC/PtrState.h --- a/llvm/lib/Transforms/ObjCARC/PtrState.h +++ b/llvm/lib/Transforms/ObjCARC/PtrState.h @@ -31,6 +31,7 @@ namespace objcarc { class ARCMDKindCache; +class BundledRetainClaimRVs; class ProvenanceAnalysis; /// \enum Sequence @@ -202,7 +203,8 @@ ProvenanceAnalysis &PA, ARCInstKind Class); bool HandlePotentialAlterRefCount(Instruction *Inst, const Value *Ptr, - ProvenanceAnalysis &PA, ARCInstKind Class); + ProvenanceAnalysis &PA, ARCInstKind Class, + const BundledRetainClaimRVs &BundledRVs); }; } // end namespace objcarc diff --git a/llvm/lib/Transforms/ObjCARC/PtrState.cpp b/llvm/lib/Transforms/ObjCARC/PtrState.cpp --- a/llvm/lib/Transforms/ObjCARC/PtrState.cpp +++ b/llvm/lib/Transforms/ObjCARC/PtrState.cpp @@ -11,6 +11,7 @@ #include "ObjCARC.h" #include "llvm/Analysis/ObjCARCAnalysisUtils.h" #include "llvm/Analysis/ObjCARCInstKind.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" @@ -280,6 +281,12 @@ InsertAfter = skipDebugIntrinsics(InsertAfter); InsertReverseInsertPt(&*InsertAfter); + + // Don't insert anything between a call/invoke with operand bundle + // "clang.arc.rv" and the retainRV/claimRV call that uses the call result. + if (auto *CB = dyn_cast(Inst)) + if (objcarc::hasRVOpBundle(CB)) + SetCFGHazardAfflicted(true); }; // Check for possible direct uses. @@ -377,10 +384,9 @@ llvm_unreachable("Sequence unknown enum value"); } -bool TopDownPtrState::HandlePotentialAlterRefCount(Instruction *Inst, - const Value *Ptr, - ProvenanceAnalysis &PA, - ARCInstKind Class) { +bool TopDownPtrState::HandlePotentialAlterRefCount( + Instruction *Inst, const Value *Ptr, ProvenanceAnalysis &PA, + ARCInstKind Class, const BundledRetainClaimRVs &BundledRVs) { // Check for possible releases. Treat clang.arc.use as a releasing instruction // to prevent sinking a retain past it. if (!CanDecrementRefCount(Inst, Ptr, PA, Class) && @@ -396,6 +402,11 @@ assert(!HasReverseInsertPts()); InsertReverseInsertPt(Inst); + // Don't insert anything between a call/invoke with operand bundle + // "clang.arc.rv" and the retainRV/claimRV call that uses the call result. + if (BundledRVs.contains(Inst)) + SetCFGHazardAfflicted(true); + // One call can't cause a transition from S_Retain to S_CanRelease // and S_CanRelease to S_Use. If we've made the first transition, // we're done. diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp --- a/llvm/lib/Transforms/Scalar/SCCP.cpp +++ b/llvm/lib/Transforms/Scalar/SCCP.cpp @@ -144,9 +144,8 @@ /// represented here for efficient lookup. SmallPtrSet MRVFunctionsTracked; - /// MustTailFunctions - Each function here is a callee of non-removable - /// musttail call site. - SmallPtrSet MustTailCallees; + /// A list of functions whose return values cannot be zapped. + SmallPtrSet DontZapReturnFunctions; /// TrackingIncomingArguments - This is the set of functions for whose /// arguments we make optimistic assumptions about and try to prove as @@ -238,17 +237,13 @@ TrackedRetVals.insert(std::make_pair(F, ValueLatticeElement())); } - /// AddMustTailCallee - If the SCCP solver finds that this function is called - /// from non-removable musttail call site. - void AddMustTailCallee(Function *F) { - MustTailCallees.insert(F); + /// Add function to the list of functions whose return value cannot be zapped. + void addDontZapReturnFunction(Function *F) { + DontZapReturnFunctions.insert(F); } - /// Returns true if the given function is called from non-removable musttail - /// call site. - bool isMustTailCallee(Function *F) { - return MustTailCallees.count(F); - } + /// Returns true if the return value of the given function cannot be zapped. + bool dontZapReturn(Function *F) { return DontZapReturnFunctions.count(F); } void AddArgumentTrackedFunction(Function *F) { TrackingIncomingArguments.insert(F); @@ -319,12 +314,6 @@ return MRVFunctionsTracked; } - /// getMustTailCallees - Get the set of functions which are called - /// from non-removable musttail call sites. - const SmallPtrSet getMustTailCallees() { - return MustTailCallees; - } - /// markOverdefined - Mark the specified value overdefined. This /// works with both scalars and structs. void markOverdefined(Value *V) { @@ -1650,16 +1639,21 @@ assert(Const && "Constant is nullptr here!"); // Replacing `musttail` instructions with constant breaks `musttail` invariant - // unless the call itself can be removed - CallInst *CI = dyn_cast(V); - if (CI && CI->isMustTailCall() && !CI->isSafeToRemove()) { - Function *F = CI->getCalledFunction(); + // unless the call itself can be removed. + // If the call has operand bundle "clang.arc.rv", don't replace the call value + // with a constant. Doing so would enable dead argument elimination to change + // the function return type to `void`, which is illegal since the result of + // "clang.arc.rv" is used by an implicit ObjC runtime call. + CallBase *CB = dyn_cast(V); + if (CB && ((CB->isMustTailCall() && !CB->isSafeToRemove()) || + CB->getOperandBundle(LLVMContext::OB_clang_arc_rv))) { + Function *F = CB->getCalledFunction(); // Don't zap returns of the callee if (F) - Solver.AddMustTailCallee(F); + Solver.addDontZapReturnFunction(F); - LLVM_DEBUG(dbgs() << " Can\'t treat the result of musttail call : " << *CI + LLVM_DEBUG(dbgs() << " Can\'t treat the result of call " << *CB << " as a constant\n"); return false; } @@ -1821,11 +1815,10 @@ if (!Solver.isArgumentTrackedFunction(&F)) return; - // There is a non-removable musttail call site of this function. Zapping - // returns is not allowed. - if (Solver.isMustTailCallee(&F)) { + if (Solver.dontZapReturn(&F)) { LLVM_DEBUG(dbgs() << "Can't zap returns of the function : " << F.getName() - << " due to present musttail call of it\n"); + << " due to present musttail or \"clang.arc.rv\" call of " + "it\n"); return; } diff --git a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp --- a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp +++ b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp @@ -247,7 +247,9 @@ isa(&I)) continue; - bool IsNoTail = CI->isNoTailCall() || CI->hasOperandBundles(); + // Special-case operand bundle "clang.arc.rv". + bool IsNoTail = CI->isNoTailCall() || CI->hasOperandBundlesOtherThan( + LLVMContext::OB_clang_arc_rv); if (!IsNoTail && CI->doesNotAccessMemory()) { // A call to a readnone function whose arguments are all things computed diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -27,8 +27,9 @@ #include "llvm/Analysis/CaptureTracking.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/ObjCARCAnalysisUtils.h" +#include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/Analysis/ProfileSummaryInfo.h" -#include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/Argument.h" @@ -61,6 +62,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/ValueMapper.h" #include #include @@ -1650,6 +1652,98 @@ } } +/// An operand bundle "clang.arc.rv" on a call indicates the call result is +/// implicitly consumed by a call to retainRV or claimRV immediately after the +/// call. This function inlines the retainRV/claimRV calls. +/// +/// There are three cases to consider: +/// +/// 1. If there is a call to autoreleaseRV that takes a pointer to the returned +/// object in the callee return block, the autoreleaseRV call and the +/// retainRV/claimRV call in the caller cancel out. If the call in the caller +/// is a claimRV call, a call to objc_release is emitted. +/// +/// 2. If there is a call in the callee return block that doesn't have operand +/// bundle "clang.arc.rv", the operand bundle on the original call is +/// transferred to the call in the callee. +/// +/// 3. Otherwise, a call to objc_retain is inserted if the call in the caller is +/// a retainRV call. +static void +inlineRetainOrClaimRVCalls(CallBase &CB, + const SmallVectorImpl &Returns) { + Module *Mod = CB.getModule(); + bool IsRetainRV = objcarc::hasRVOpBundle(&CB, true), IsClaimRV = !IsRetainRV; + + for (auto *RI : Returns) { + Value *RetOpnd = objcarc::GetRCIdentityRoot(RI->getOperand(0)); + BasicBlock::reverse_iterator I = ++(RI->getIterator().getReverse()); + BasicBlock::reverse_iterator EI = RI->getParent()->rend(); + bool InsertRetainCall = IsRetainRV; + IRBuilder<> Builder(RI->getContext()); + + // Walk backwards through the basic block looking for either a matching + // autoreleaseRV call or an unannotated call. + for (; I != EI;) { + auto CurI = I++; + + // Ignore casts. + if (isa(*CurI)) + continue; + + if (auto *II = dyn_cast(&*CurI)) { + if (II->getIntrinsicID() == Intrinsic::objc_autoreleaseReturnValue && + II->hasNUses(0) && + objcarc::GetRCIdentityRoot(II->getOperand(0)) == RetOpnd) { + // If we've found a matching authoreleaseRV call: + // - If the call is annotated with claimRV, insert a call to + // objc_release and erase the autoreleaseRV call. + // - If the call is annotated with retainRV, just erase the + // autoreleaseRV call. + if (IsClaimRV) { + Builder.SetInsertPoint(II); + Function *IFn = + Intrinsic::getDeclaration(Mod, Intrinsic::objc_release); + Value *BC = + Builder.CreateBitCast(RetOpnd, IFn->getArg(0)->getType()); + Builder.CreateCall(IFn, BC, ""); + } + II->eraseFromParent(); + InsertRetainCall = false; + } + } else if (auto *CI = dyn_cast(&*CurI)) { + if (objcarc::GetRCIdentityRoot(CI) == RetOpnd && + !objcarc::hasRVOpBundle(CI)) { + // If we've found an unannotated call that defines RetOpnd, add a + // "clang.arc.rv" operand bundle. + Value *BundleArgs[] = { + ConstantInt::get(Builder.getInt64Ty(), + objcarc::getRVOperandBundleEnum(IsRetainRV))}; + OperandBundleDef OB("clang.arc.rv", BundleArgs); + auto *NewCall = CallBase::addOperandBundle( + CI, LLVMContext::OB_clang_arc_rv, OB, CI); + NewCall->copyMetadata(*CI); + CI->replaceAllUsesWith(NewCall); + CI->eraseFromParent(); + InsertRetainCall = false; + } + } + + break; + } + + if (InsertRetainCall) { + // The call has operand bundle "clang.arc.rv"="retain" and we've failed to + // find a matching autoreleaseRV or an annotated call in the callee. Emit + // a call to objc_retain. + Builder.SetInsertPoint(RI); + Function *IFn = Intrinsic::getDeclaration(Mod, Intrinsic::objc_retain); + Value *BC = Builder.CreateBitCast(RetOpnd, IFn->getArg(0)->getType()); + Builder.CreateCall(IFn, BC, ""); + } + } +} + /// This function inlines the called function into the basic block of the /// caller. This returns false if it is not possible to inline this call. /// The program is still in a well defined state if this occurs though. @@ -1687,6 +1781,8 @@ // ... and "funclet" operand bundles. if (Tag == LLVMContext::OB_funclet) continue; + if (Tag == LLVMContext::OB_clang_arc_rv) + continue; return InlineResult::failure("unsupported operand bundle"); } @@ -1853,6 +1949,10 @@ // Remember the first block that is newly cloned over. FirstNewBlock = LastBlock; ++FirstNewBlock; + // Insert retainRV/clainRV runtime calls. + if (objcarc::hasRVOpBundle(&CB)) + inlineRetainOrClaimRVCalls(CB, Returns); + if (IFI.CallerBFI != nullptr && IFI.CalleeBFI != nullptr) // Update the BFI of blocks cloned into the caller. updateCallerBFI(OrigBB, VMap, IFI.CallerBFI, IFI.CalleeBFI, diff --git a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll --- a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll +++ b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll @@ -9,6 +9,7 @@ ; CHECK-NEXT: . @@ -108,6 +110,21 @@ ret void } +; ARC optimizer should be able to safely remove the retain/release pair as the +; call to @llvm.objc.clang.arc.noop.use is a no-op. + +; CHECK-LABEL: define void @test_arc_noop_use( +; CHECK-NEXT: call void @can_release(i8* %x) +; CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use( +; CHECK-NEXT: ret void + +define void @test_arc_noop_use(i8** %out, i8* %x) { + call i8* @llvm.objc.retain(i8* %x) + call void @can_release(i8* %x) + call void (...) @llvm.objc.clang.arc.noop.use(i8* %x) + call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0 + ret void +} !0 = !{} diff --git a/llvm/test/Transforms/ObjCARC/rv.ll b/llvm/test/Transforms/ObjCARC/rv.ll --- a/llvm/test/Transforms/ObjCARC/rv.ll +++ b/llvm/test/Transforms/ObjCARC/rv.ll @@ -452,6 +452,29 @@ ret i8* %v3 } +; Remove operand bundle "clang.arc.rv" and the autoreleaseRV call if the call +; is a tail call. + +; CHECK-LABEL: define i8* @test31( +; CHECK: %[[CALL:.*]] = tail call i8* @returner() +; CHECK: ret i8* %[[CALL]] + +define i8* @test31() { + %call = tail call i8* @returner() [ "clang.arc.rv"(i64 0) ] + %1 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %call) + ret i8* %1 +} + +; CHECK-LABEL: define i8* @test32( +; CHECK: %[[CALL:.*]] = call i8* @returner() [ "clang.arc.rv"(i64 0) ] +; CHECK: call i8* @llvm.objc.autoreleaseReturnValue(i8* %[[CALL]]) + +define i8* @test32() { + %call = call i8* @returner() [ "clang.arc.rv"(i64 0) ] + %1 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %call) + ret i8* %1 +} + !0 = !{} ; CHECK: attributes [[NUW]] = { nounwind } diff --git a/llvm/test/Transforms/SCCP/clang-arc-rv.ll b/llvm/test/Transforms/SCCP/clang-arc-rv.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SCCP/clang-arc-rv.ll @@ -0,0 +1,24 @@ +; RUN: opt < %s -ipsccp -S | FileCheck %s +; Return value can't be zapped if there is a call that has operand bundle +; "clang.arc.rv". + +@g0 = global i8 zeroinitializer, align 1 + +; CHECK-LABEL: @foo( +; CHECK: ret i8* @g0 + +define internal i8* @foo() { + ret i8* @g0 +} + +; CHECK-LABEL: @test( +; CHECK: %[[R:.*]] = call i8* @foo() +; CHECK call void (...) @llvm.objc.clang.arc.noop.use(i8* %[[R]]) + +define void @test() { + %r = call i8* @foo() [ "clang.arc.rv"(i64 1) ] + call void (...) @llvm.objc.clang.arc.noop.use(i8* %r) + ret void +} + +declare void @llvm.objc.clang.arc.noop.use(...) diff --git a/llvm/test/Transforms/TailCallElim/deopt-bundle.ll b/llvm/test/Transforms/TailCallElim/deopt-bundle.ll --- a/llvm/test/Transforms/TailCallElim/deopt-bundle.ll +++ b/llvm/test/Transforms/TailCallElim/deopt-bundle.ll @@ -55,3 +55,13 @@ exit: ret void } + +; CHECK-LABEL: @test_clang_arc_rv( +; CHECK: tail call i8* @getObj( + +declare i8* @getObj() + +define i8* @test_clang_arc_rv() { + %r = call i8* @getObj() [ "clang.arc.rv"(i64 0) ] + ret i8* %r +}