[analyzer] Support for OSObjects out parameters in RetainCountChecker

[george.karpenkov]

Moved logic into the summary manager.

[george.karpenkov]

Moved attributes into a separate revision.

[george.karpenkov]

Support custom diagnostics.

[NoQ]

clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp

535

Mmm, what does L stand for here?

541–545

std::any_of()?

clang/lib/StaticAnalyzer/Core/RetainSummaryManager.cpp

867

Typo: "because".

clang/test/Analysis/osobject-retain-release.cpp

114

Should we somehow indicate that we assumed that the function returns a success value? I.e., bug visitor / tag.

Also what is the intended behavior if we replace write_into_out_param_on_success with write_into_out_param_on_failure here?

114

That is, i really want this test added, and then it's good to commit, other nits are minor.

[george.karpenkov]

clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp

535

vaLue.
Other letters are usually taken =/ Is there a different convention ?

541–545

That's super verbose with iterators and requires putting those values in the array.
Unfortunately, libc++ does not have a

<template T, template Others...>
bool isAnyOf(T Val, Others... OtherVals)

function

[NoQ]

Yay thx.

clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp

535

L usually stands for Loc; V is more common for raw SVals.

clang/include/clang/Basic/Attr.td

 def OSConsumed : InheritableParamAttr {
   let Spellings = [Clang<"os_consumed">];
   let Subjects = SubjectList<[ParmVar]>;
   let Documentation = [RetainBehaviorDocs];
 }
 
 def OSReturnsRetained : InheritableAttr {
   let Spellings = [Clang<"os_returns_retained">];
-  let Subjects = SubjectList<[Function, ObjCMethod, ObjCProperty]>;
+  let Subjects = SubjectList<[Function, ObjCMethod, ObjCProperty, ParmVar]>;
   let Documentation = [RetainBehaviorDocs];
 }
 
 def OSReturnsNotRetained : InheritableAttr {
   let Spellings = [Clang<"os_returns_not_retained">];
-  let Subjects = SubjectList<[Function, ObjCMethod, ObjCProperty]>;
+  let Subjects = SubjectList<[Function, ObjCMethod, ObjCProperty, ParmVar]>;
   let Documentation = [RetainBehaviorDocs];
 }
 
 def OSConsumesThis : InheritableAttr {
   let Spellings = [Clang<"os_consumes_this">];
   let Subjects = SubjectList<[NonStaticCXXMethod]>;
   let Documentation = [RetainBehaviorDocs];
 }

clang/include/clang/StaticAnalyzer/Core/RetainSummaryManager.h

enum ArgEffectKind {
   /// The argument has its reference count decreased by 1 to model
   /// a transferred bridge cast under ARC.
   DecRefBridgedTransferred,
 
   /// The argument has its reference count increased by 1.
   IncRef,
 
   /// The argument is a pointer to a retain-counted object; on exit, the new
-  /// value of the pointer is a +0 value or NULL.
+  /// value of the pointer is a +0 value.
   UnretainedOutParameter,
 
   /// The argument is a pointer to a retain-counted object; on exit, the new
-  /// value of the pointer is a +1 value or NULL.
+  /// value of the pointer is a +1 value.
   RetainedOutParameter,
 
+  /// The argument is a pointer to a retain-counted object; on exit, the new
+  /// value of the pointer is a +1 value iff the return code is zero.
+  RetainedOutParameterOnZero,
+
+  /// The argument is a pointer to a retain-counted object; on exit, the new
+  /// value of the pointer is a +1 value iff the return code is non-zero.
+  RetainedOutParameterOnNonZero,
+
   /// The argument is treated as potentially escaping, meaning that
   /// even when its reference count hits 0 it should be treated as still
   /// possibly being alive as someone else *may* be holding onto the object.
   MayEscape,
 
   /// All typestate tracking of the object ceases.  This is usually employed
   /// when the effect of the call is completely unknown.
   StopTracking,

clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp

void RetainCountChecker::processSummaryOfInlined(const RetainSummary &Summ,
   if (SymbolRef Sym = CallOrMsg.getReturnValue().getAsSymbol()) {
     if (RE.getKind() == RetEffect::NoRetHard)
       state = removeRefBinding(state, Sym);
   }
 
   C.addTransition(state);
 }
 
-static ProgramStateRef updateOutParameter(ProgramStateRef State,
-                                          SVal ArgVal,
-                                          ArgEffectKind Effect) {
+static SmallVector<ProgramStateRef, 2>
+updateOutParameter(ProgramStateRef State,
+                   const RetainSummary &Summ,
+                   const CallEvent &CE) {
+  SVal L = CE.getReturnValue();
+
+  // Splitting is required to support out parameters,
+  // as out parameters might be created only on the "success" branch.
+  // We want to avoid eagerly splitting unless out parameters are actually
+  // needed.
+  bool SplitNecessary = false;
+  for (auto &P : Summ.getArgEffects())
+    if (P.second.getKind() == RetainedOutParameterOnNonZero ||
+        P.second.getKind() == RetainedOutParameterOnZero)
+      SplitNecessary = true;
+
+  ProgramStateRef AssumeNonZero = State;
+  ProgramStateRef AssumeZero = State;
+
+  if (SplitNecessary) {
+    if (auto DL = L.getAs<DefinedOrUnknownSVal>()) {
+      AssumeNonZero = AssumeNonZero->assume(*DL, true);
+      AssumeZero = AssumeZero->assume(*DL, false);
+    }
+  }
+
+  for (unsigned idx = 0, e = CE.getNumArgs(); idx != e; ++idx) {
+    SVal ArgVal = CE.getArgSVal(idx);
+    ArgEffect AE = Summ.getArg(idx);
+
     auto *ArgRegion = dyn_cast_or_null<TypedValueRegion>(ArgVal.getAsRegion());
     if (!ArgRegion)
-    return State;
+      continue;
 
     QualType PointeeTy = ArgRegion->getValueType();
-  if (!coreFoundation::isCFObjectRef(PointeeTy))
-    return State;
-
     SVal PointeeVal = State->getSVal(ArgRegion);
     SymbolRef Pointee = PointeeVal.getAsLocSymbol();
     if (!Pointee)
-    return State;
+      continue;
 
-  switch (Effect) {
+    auto makeNotOwned = [&](ProgramStateRef St) {
+      return setRefBinding(St, Pointee,
+                           RefVal::makeNotOwned(AE.getObjKind(), PointeeTy));
+    };
+    auto makeOwned = [&](ProgramStateRef St) {
+      return setRefBinding(St, Pointee,
+                           RefVal::makeOwned(ObjKind::OS, PointeeTy));
+    };
+
+    switch (AE.getKind()) {
     case UnretainedOutParameter:
-    State = setRefBinding(State, Pointee,
-                          RefVal::makeNotOwned(ObjKind::CF, PointeeTy));
+      AssumeNonZero = makeNotOwned(AssumeNonZero);
+      AssumeZero = makeNotOwned(AssumeZero);
       break;
     case RetainedOutParameter:
-    // Do nothing. Retained out parameters will either point to a +1 reference
-    // or NULL, but the way you check for failure differs depending on the API.
-    // Consequently, we don't have a good way to track them yet.
+      AssumeNonZero = makeOwned(AssumeNonZero);
+      AssumeZero = makeOwned(AssumeZero);
+      break;
+    case RetainedOutParameterOnNonZero:
+      AssumeNonZero = makeOwned(AssumeNonZero);
+      break;
+    case RetainedOutParameterOnZero:
+      AssumeZero = makeOwned(AssumeZero);
       break;
-
     default:
-    llvm_unreachable("only for out parameters");
+      break;
+    }
   }
 
-  return State;
+  if (SplitNecessary) {
+    return {AssumeNonZero, AssumeZero};
+  } else {
+    assert(AssumeZero == AssumeNonZero);
+    return {AssumeZero};
+  }
 }
 
 void RetainCountChecker::checkSummary(const RetainSummary &Summ,
                                       const CallEvent &CallOrMsg,
                                       CheckerContext &C) const {
   ProgramStateRef state = C.getState();
 
   // Evaluate the effect of the arguments.
   RefVal::Kind hasErr = (RefVal::Kind) 0;
   SourceRange ErrorRange;
   SymbolRef ErrorSym = nullptr;
 
   // Helper tag for providing diagnostics: indicate whether dealloc was sent
   // at this location.
   static CheckerProgramPointTag DeallocSentTag(this, DeallocTagDescription);
   bool DeallocSent = false;
 
   for (unsigned idx = 0, e = CallOrMsg.getNumArgs(); idx != e; ++idx) {
     SVal V = CallOrMsg.getArgSVal(idx);
 
     ArgEffect Effect = Summ.getArg(idx);
-    if (Effect.getKind() == RetainedOutParameter ||
-        Effect.getKind() == UnretainedOutParameter) {
-      state = updateOutParameter(state, V, Effect.getKind());
-    } else if (SymbolRef Sym = V.getAsLocSymbol()) {
+    if (SymbolRef Sym = V.getAsLocSymbol()) {
       if (const RefVal *T = getRefBinding(state, Sym)) {
 
         if (shouldEscapeOSArgumentOnCall(CallOrMsg, idx, T))
           Effect = ArgEffect(StopTrackingHard, ObjKind::OS);
 
         state = updateSymbol(state, Sym, *T, Effect, hasErr, C);
         if (hasErr) {
           ErrorRange = CallOrMsg.getArgSourceRange(idx);
if (RE.notOwned()) {
       const Expr *Ex = CallOrMsg.getOriginExpr();
       assert(Ex);
       ResultTy = GetReturnType(Ex, C.getASTContext());
     }
     if (Optional<RefVal> updatedRefVal = refValFromRetEffect(RE, ResultTy))
       state = setRefBinding(state, Sym, *updatedRefVal);
   }
 
+  SmallVector<ProgramStateRef, 2> Out =
+      updateOutParameter(state, Summ, CallOrMsg);
+
+  for (ProgramStateRef St : Out) {
     if (DeallocSent) {
-    C.addTransition(state, C.getPredecessor(), &DeallocSentTag);
+      C.addTransition(St, C.getPredecessor(), &DeallocSentTag);
     } else {
-    C.addTransition(state);
+      C.addTransition(St);
+    }
   }
 }
 
 ProgramStateRef RetainCountChecker::updateSymbol(ProgramStateRef state,
                                                  SymbolRef sym, RefVal V,
                                                  ArgEffect AE,
                                                  RefVal::Kind &hasErr,
                                                  CheckerContext &C) const {
if (V.getKind() == RefVal::Released) {
     V = V ^ RefVal::ErrorUseAfterRelease;
     hasErr = V.getKind();
     return setRefBinding(state, sym, V);
   }
 
   switch (AE.getKind()) {
     case UnretainedOutParameter:
     case RetainedOutParameter:
+    case RetainedOutParameterOnZero:
+    case RetainedOutParameterOnNonZero:
       llvm_unreachable("Applies to pointer-to-pointer parameters, which should "
                        "not have ref state.");
 
     case Dealloc: // NB. we only need to add a note in a non-error case.
       switch (V.getKind()) {
         default:
           llvm_unreachable("Invalid RefVal state for an explicit dealloc.");
         case RefVal::Owned:

clang/lib/StaticAnalyzer/Core/RetainSummaryManager.cpp

 static ArgEffect getStopTrackingHardEquivalent(ArgEffect E) {
   switch (E.getKind()) {
   case DoNothing:
   case Autorelease:
   case DecRefBridgedTransferred:
   case IncRef:
   case UnretainedOutParameter:
   case RetainedOutParameter:
+  case RetainedOutParameterOnZero:
+  case RetainedOutParameterOnNonZero:
   case MayEscape:
   case StopTracking:
   case StopTrackingHard:
     return E.withKind(StopTrackingHard);
   case DecRef:
   case DecRefAndStopTrackingHard:
     return E.withKind(DecRefAndStopTrackingHard);
   case Dealloc:
RetainSummaryManager::getRetEffectFromAnnotations(QualType RetTy,
   if (const auto *MD = dyn_cast<CXXMethodDecl>(D))
     for (const auto *PD : MD->overridden_methods())
       if (auto RE = getRetEffectFromAnnotations(RetTy, PD))
         return RE;
 
   return None;
 }
 
+/// \return Whether the chain of typedefs starting from {@code QT}
+/// has a typedef with a given name {@code Name}.
+static bool hasTypedefNamed(QualType QT,
+                            StringRef Name) {
+  while (auto *T = dyn_cast<TypedefType>(QT)) {
+    const auto &Context = T->getDecl()->getASTContext();
+    if (T->getDecl()->getIdentifier() == &Context.Idents.get(Name))
+      return true;
+    QT = T->getDecl()->getUnderlyingType();
+  }
+  return false;
+}
+
+static QualType getCallableReturnType(const NamedDecl *ND) {
+  if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
+    return FD->getReturnType();
+  } else if (const auto *MD = dyn_cast<ObjCMethodDecl>(ND)) {
+    return MD->getReturnType();
+  } else {
+    llvm_unreachable("Unexpected decl");
+  }
+}
+
 bool RetainSummaryManager::applyParamAnnotationEffect(
     const ParmVarDecl *pd, unsigned parm_idx, const NamedDecl *FD,
     RetainSummaryTemplate &Template) {
   QualType QT = pd->getType();
   if (auto K =
           hasAnyEnabledAttrOf<NSConsumedAttr, CFConsumedAttr, OSConsumedAttr,
                               GeneralizedConsumedAttr>(pd, QT)) {
     Template->addArg(AF, parm_idx, ArgEffect(DecRef, *K));
     return true;
   } else if (auto K =
                  hasAnyEnabledAttrOf<CFReturnsRetainedAttr,
+                                     OSReturnsRetainedAttr,
                                      GeneralizedReturnsRetainedAttr>(pd, QT)) {
-    Template->addArg(AF, parm_idx, ArgEffect(RetainedOutParameter, *K));
+
+    // For OSObjects, we try to guess whether the object is created based
+    // on the return value.
+    if (K == ObjKind::OS) {
+      QualType QT = getCallableReturnType(FD);
+
+      // The usual convention is to create an object on non-zero return, but
+      // it's reverted if the typedef chain has a typedef kern_return_t,
+      // becauese kReturnSuccess constant is defined as zero.
+      bool SuccessOnZero = hasTypedefNamed(QT, "kern_return_t");
+      bool ShouldSplit = !QT.isNull() && !QT->isVoidType();
+      ArgEffectKind AK = RetainedOutParameter;
+      if (ShouldSplit && SuccessOnZero) {
+        AK = RetainedOutParameterOnZero;
+      } else if (ShouldSplit && !SuccessOnZero) {
+        AK = RetainedOutParameterOnNonZero;
+      }
+      Template->addArg(AF, parm_idx, ArgEffect(AK, *K));
+    }
+
+    // For others:
+    // Do nothing. Retained out parameters will either point to a +1 reference
+    // or NULL, but the way you check for failure differs depending on the
+    // API. Consequently, we don't have a good way to track them yet.
     return true;
-  } else if (auto K = hasAnyEnabledAttrOf<CFReturnsNotRetainedAttr>(pd, QT)) {
+  } else if (auto K = hasAnyEnabledAttrOf<CFReturnsNotRetainedAttr,
+                                          OSReturnsNotRetainedAttr,
+                                          GeneralizedReturnsNotRetainedAttr>(
+                 pd, QT)) {
     Template->addArg(AF, parm_idx, ArgEffect(UnretainedOutParameter, *K));
     return true;
-  } else {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
-      for (const auto *OD : MD->overridden_methods()) {
-        const ParmVarDecl *OP = OD->parameters()[parm_idx];
-        if (applyParamAnnotationEffect(OP, parm_idx, OD, Template))
-          return true;
-      }
-    }
-  }
+  }
+
 
   return false;
 }
 
 void
 RetainSummaryManager::updateSummaryFromAnnotations(const RetainSummary *&Summ,
                                                    const FunctionDecl *FD) {
   if (!FD)

clang/test/Analysis/osobject-retain-release.cpp

 struct OSMetaClassBase {
   static OSObject *safeMetaCast(const OSObject *inst, const OSMetaClass *meta);
 };
 
 void escape(void *);
 void escape_with_source(void *p) {}
 bool coin();
 
+bool write_into_out_param_on_success(OS_RETURNS_RETAINED OSObject **obj);
+
+void use_out_param() {
+  OSObject *obj;
+  if (write_into_out_param_on_success(&obj)) {
+    obj->release();
+  }
+}
+
+void use_out_param_leak() {
+  OSObject *obj;
+  write_into_out_param_on_success(&obj); // expected-note{{Call to function 'write_into_out_param_on_success' returns an OSObject of type OSObject with a +1 retain count}}
+} // expected-warning{{Potential leak of an object stored into 'obj'}}
+ // expected-note@-1{{Object leaked: object allocated and stored into 'obj' is not referenced later in this execution path and has a retain count of +1}}
+
+typedef int kern_return_t;
+typedef kern_return_t IOReturn;
+typedef kern_return_t OSReturn;
+#define kOSReturnSuccess  0
+#define kIOReturnSuccess 0
+
+bool write_into_two_out_params(OS_RETURNS_RETAINED OSObject **a,
+                               OS_RETURNS_RETAINED OSObject **b);
+
+void use_write_into_two_out_params() {
+  OSObject *obj1;
+  OSObject *obj2;
+  if (write_into_two_out_params(&obj1, &obj2)) {
+    obj1->release();
+    obj2->release();
+  }
+}
+
+void use_write_two_out_params_leak() {
+  OSObject *obj1;
+  OSObject *obj2;
+  write_into_two_out_params(&obj1, &obj2); // expected-note{{Call to function 'write_into_two_out_params' returns an OSObject of type OSObject with a +1 retain count}}
+                                           // expected-note@-1{{Call to function 'write_into_two_out_params' returns an OSObject of type OSObject with a +1 retain count}}
+} // expected-warning{{Potential leak of an object stored into 'obj1'}}
+  // expected-warning@-1{{Potential leak of an object stored into 'obj2'}}
+  // expected-note@-2{{Object leaked: object allocated and stored into 'obj1' is not referenced later in this execution path and has a retain count of +1}}
+  // expected-note@-3{{Object leaked: object allocated and stored into 'obj2' is not referenced later in this execution path and has a retain count of +1}}
+
+void always_write_into_two_out_params(OS_RETURNS_RETAINED OSObject **a,
+                                      OS_RETURNS_RETAINED OSObject **b);
+
+void use_always_write_into_two_out_params() {
+  OSObject *obj1;
+  OSObject *obj2;
+  always_write_into_two_out_params(&obj1, &obj2);
+  obj1->release();
+  obj2->release();
+}
+
+void use_always_write_into_two_out_params_leak() {
+  OSObject *obj1;
+  OSObject *obj2;
+  always_write_into_two_out_params(&obj1, &obj2); // expected-note{{Call to function 'always_write_into_two_out_params' returns an OSObject of type OSObject with a +1 retain count}}
+                                                  // expected-note@-1{{Call to function 'always_write_into_two_out_params' returns an OSObject of type OSObject with a +1 retain count}}
+} // expected-warning{{Potential leak of an object stored into 'obj1'}}
+  // expected-warning@-1{{Potential leak of an object stored into 'obj2'}}
+  // expected-note@-2{{Object leaked: object allocated and stored into 'obj1' is not referenced later in this execution path and has a retain count of +1}}
+  // expected-note@-3{{Object leaked: object allocated and stored into 'obj2' is not referenced later in this execution path and has a retain count of +1}}
+
+char *write_into_out_param_on_nonnull(OS_RETURNS_RETAINED OSObject **obj);
+
+void use_out_param_osreturn_on_nonnull() {
+  OSObject *obj;
+  if (write_into_out_param_on_nonnull(&obj)) {
+    obj->release();
+  }
+}
+
+void use_out_param_leak_osreturn_on_nonnull() {
+  OSObject *obj;
+  write_into_out_param_on_nonnull(&obj); // expected-note{{Call to function 'write_into_out_param_on_nonnull' returns an OSObject of type OSObject with a +1 retain count}}
+} // expected-warning{{Potential leak of an object stored into 'obj'}}
+  // expected-note@-1{{Object leaked: object allocated and stored into 'obj' is not referenced later in this execution path and has a retain count of +1}}
+
+bool write_optional_out_param(OS_RETURNS_RETAINED OSObject **obj=nullptr);
+
+void use_optional_out_param() {
+  if (write_optional_out_param()) {};
+}
+
+OSReturn write_into_out_param_on_os_success(OS_RETURNS_RETAINED OSObject **obj);
+
+void write_into_non_retained_out_param(OS_RETURNS_NOT_RETAINED OSObject **obj);
+
+void use_write_into_non_retained_out_param() {
+  OSObject *obj;
+  write_into_non_retained_out_param(&obj);
+}
+
+void use_write_into_non_retained_out_param_uaf() {
+  OSObject *obj;
+  write_into_non_retained_out_param(&obj); // expected-note{{Call to function 'write_into_non_retained_out_param' returns an OSObject of type OSObject with a +0 retain count}}
+  obj->release(); // expected-warning{{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
+                  // expected-note@-1{{Incorrect decrement of the reference count of an object that is not owned at this point by the caller}}
+}
+
+void always_write_into_out_param(OS_RETURNS_RETAINED OSObject **obj);
+
+void use_void_out_param_osreturn() {
+  OSObject *obj;
+  always_write_into_out_param(&obj);
+  obj->release();
+}
+
+void use_void_out_param_osreturn_leak() {
+  OSObject *obj;
+  always_write_into_out_param(&obj); // expected-note{{Call to function 'always_write_into_out_param' returns an OSObject of type OSObject with a +1 retain count}}
+} // expected-warning{{Potential leak of an object stored into 'obj'}}
+  // expected-note@-1{{Object leaked: object allocated and stored into 'obj' is not referenced later in this execution path and has a retain count of +1}}
+
+void use_out_param_osreturn() {
+  OSObject *obj;
+  if (write_into_out_param_on_os_success(&obj) == kOSReturnSuccess) {
+    obj->release();
+  }
+}
+
+void use_out_param_leak_osreturn() {
+  OSObject *obj;
+  write_into_out_param_on_os_success(&obj); // expected-note{{Call to function 'write_into_out_param_on_os_success' returns an OSObject of type OSObject with a +1 retain count}}
+} // expected-warning{{Potential leak of an object stored into 'obj'}}
+  // expected-note@-1{{Object leaked: object allocated and stored into 'obj' is not referenced later in this execution path and has a retain count of +1}}
+
 bool os_consume_violation_two_args(OS_CONSUME OSObject *obj, bool extra) {
   if (coin()) { // expected-note{{Assuming the condition is false}}
                 // expected-note@-1{{Taking false branch}}
     escape(obj);
     return true;
   }
   return false; // expected-note{{Parameter 'obj' is marked as consuming, but the function did not consume the reference}}
 }

 }
 
 OSObject *getObject();
 typedef bool (^Blk)(OSObject *);
 
 void test_escape_to_unknown_block(Blk blk) {
   blk(getObject()); // no-crash
 }
-