[AST] Only store data for the NRVO candidate in ReturnStmt if needed.

[riccibruno]

Summary

Only store the NRVO candidate if needed in ReturnStmt.
A good chuck of all of the ReturnStmt have no NRVO candidate.
For all of them this saves one pointer.

This has no impact on children().

include/clang/AST/Stmt.h

class BreakStmtBitfields {
     SourceLocation BreakLoc;
   };
 
   class ReturnStmtBitfields {
     friend class ReturnStmt;
 
     unsigned : NumStmtBits;
 
+    /// True if this ReturnStmt has storage for an NRVO candidate.
+    unsigned HasNRVOCandidate : 1;
+
     /// The location of the "return".
     SourceLocation RetLoc;
   };
 
   class SwitchCaseBitfields {
     friend class SwitchCase;
     friend class CaseStmt;
 

 /// ReturnStmt - This represents a return, optionally of an expression:
 ///   return;
 ///   return 4;
 ///
 /// Note that GCC allows return with no argument in a function declared to
 /// return a value, and it allows returning a value in functions declared to
 /// return void.  We explicitly model this in the AST, which means you can't
 /// depend on the return type of the function and the presence of an argument.
-class ReturnStmt : public Stmt {
+class ReturnStmt final
+    : public Stmt,
+      private llvm::TrailingObjects<ReturnStmt, const VarDecl *> {
+  friend TrailingObjects;
+
+  /// The return expression.
   Stmt *RetExpr;
-  const VarDecl *NRVOCandidate;
 
-public:
-  explicit ReturnStmt(SourceLocation RL) : ReturnStmt(RL, nullptr, nullptr) {}
+  // ReturnStmt is followed optionally by a trailing "const VarDecl *"
+  // for the NRVO candidate. Present if and only if hasNRVOCandidate().
+
+  /// True if this ReturnStmt has storage for an NRVO candidate.
+  bool hasNRVOCandidate() const { return ReturnStmtBits.HasNRVOCandidate; }
 
-  ReturnStmt(SourceLocation RL, Expr *E, const VarDecl *NRVOCandidate)
-      : Stmt(ReturnStmtClass), RetExpr((Stmt *)E),
-        NRVOCandidate(NRVOCandidate) {
-    ReturnStmtBits.RetLoc = RL;
+  unsigned numTrailingObjects(OverloadToken<const VarDecl *>) const {
+    return hasNRVOCandidate();
   }
 
-  /// Build an empty return expression.
-  explicit ReturnStmt(EmptyShell Empty) : Stmt(ReturnStmtClass, Empty) {}
+  /// Build a return statement.
+  ReturnStmt(SourceLocation RL, Expr *E, const VarDecl *NRVOCandidate);
 
-  const Expr *getRetValue() const;
-  Expr *getRetValue();
-  void setRetValue(Expr *E) { RetExpr = reinterpret_cast<Stmt*>(E); }
+  /// Build an empty return statement.
+  explicit ReturnStmt(EmptyShell Empty, bool HasNRVOCandidate);
 
-  SourceLocation getReturnLoc() const { return ReturnStmtBits.RetLoc; }
-  void setReturnLoc(SourceLocation L) { ReturnStmtBits.RetLoc = L; }
+public:
+  /// Create a return statement.
+  static ReturnStmt *Create(const ASTContext &Ctx, SourceLocation RL, Expr *E,
+                            const VarDecl *NRVOCandidate);
+
+  /// Create an empty return statement, optionally with
+  /// storage for an NRVO candidate.
+  static ReturnStmt *CreateEmpty(const ASTContext &Ctx, bool HasNRVOCandidate);
+
+  Expr *getRetValue() { return reinterpret_cast<Expr *>(RetExpr); }
+  const Expr *getRetValue() const { return reinterpret_cast<Expr *>(RetExpr); }
+  void setRetValue(Expr *E) { RetExpr = reinterpret_cast<Stmt *>(E); }
 
   /// Retrieve the variable that might be used for the named return
   /// value optimization.
   ///
   /// The optimization itself can only be performed if the variable is
   /// also marked as an NRVO object.
-  const VarDecl *getNRVOCandidate() const { return NRVOCandidate; }
-  void setNRVOCandidate(const VarDecl *Var) { NRVOCandidate = Var; }
+  const VarDecl *getNRVOCandidate() const {
+    return hasNRVOCandidate() ? *getTrailingObjects<const VarDecl *>()
+                              : nullptr;
+  }
 
-  SourceLocation getBeginLoc() const { return getReturnLoc(); }
+  /// Set the variable that might be used for the named return value
+  /// optimization. The return statement must have storage for it,
+  /// which is the case if and only if hasNRVOCandidate() is true.
+  void setNRVOCandidate(const VarDecl *Var) {
+    assert(hasNRVOCandidate() &&
+           "This return statement has no storage for an NRVO candidate!");
+    *getTrailingObjects<const VarDecl *>() = Var;
+  }
 
-  SourceLocation getEndLoc() const {
+  SourceLocation getReturnLoc() const { return ReturnStmtBits.RetLoc; }
+  void setReturnLoc(SourceLocation L) { ReturnStmtBits.RetLoc = L; }
+
+  SourceLocation getBeginLoc() const { return getReturnLoc(); }
+  SourceLocation getEndLoc() const LLVM_READONLY {
     return RetExpr ? RetExpr->getEndLoc() : getReturnLoc();
   }
 
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == ReturnStmtClass;
   }
 
   // Iterators
   child_range children() {
-    if (RetExpr) return child_range(&RetExpr, &RetExpr+1);
+    if (RetExpr)
+      return child_range(&RetExpr, &RetExpr + 1);
     return child_range(child_iterator(), child_iterator());
   }
 };
 
 /// AsmStmt is the base class for GCCAsmStmt and MSAsmStmt.
 class AsmStmt : public Stmt {
 protected:
   friend class ASTStmtReader;

lib/AST/ASTImporter.cpp

ExpectedStmt ASTNodeImporter::VisitReturnStmt(ReturnStmt *S) {
   if (!Imp)
     return Imp.takeError();
 
   SourceLocation ToReturnLoc;
   Expr *ToRetValue;
   const VarDecl *ToNRVOCandidate;
   std::tie(ToReturnLoc, ToRetValue, ToNRVOCandidate) = *Imp;
 
-  return new (Importer.getToContext()) ReturnStmt(
-      ToReturnLoc, ToRetValue, ToNRVOCandidate);
+  return ReturnStmt::Create(Importer.getToContext(), ToReturnLoc, ToRetValue,
+                            ToNRVOCandidate);
 }
 
 ExpectedStmt ASTNodeImporter::VisitCXXCatchStmt(CXXCatchStmt *S) {
   auto Imp = importSeq(
       S->getCatchLoc(), S->getExceptionDecl(), S->getHandlerBlock());
   if (!Imp)
     return Imp.takeError();
 

lib/AST/Stmt.cpp

 // IndirectGotoStmt
 LabelDecl *IndirectGotoStmt::getConstantTarget() {
   if (auto *E = dyn_cast<AddrLabelExpr>(getTarget()->IgnoreParenImpCasts()))
     return E->getLabel();
   return nullptr;
 }
 
 // ReturnStmt
-const Expr* ReturnStmt::getRetValue() const {
-  return cast_or_null<Expr>(RetExpr);
-}
-Expr* ReturnStmt::getRetValue() {
-  return cast_or_null<Expr>(RetExpr);
+ReturnStmt::ReturnStmt(SourceLocation RL, Expr *E, const VarDecl *NRVOCandidate)
+    : Stmt(ReturnStmtClass), RetExpr(E) {
+  bool HasNRVOCandidate = NRVOCandidate != nullptr;
+  ReturnStmtBits.HasNRVOCandidate = HasNRVOCandidate;
+  if (HasNRVOCandidate)
+    setNRVOCandidate(NRVOCandidate);
+  setReturnLoc(RL);
+}
+
+ReturnStmt::ReturnStmt(EmptyShell Empty, bool HasNRVOCandidate)
+    : Stmt(ReturnStmtClass, Empty) {
+  ReturnStmtBits.HasNRVOCandidate = HasNRVOCandidate;
+}
+
+ReturnStmt *ReturnStmt::Create(const ASTContext &Ctx, SourceLocation RL,
+                               Expr *E, const VarDecl *NRVOCandidate) {
+  bool HasNRVOCandidate = NRVOCandidate != nullptr;
+  void *Mem = Ctx.Allocate(totalSizeToAlloc<const VarDecl *>(HasNRVOCandidate),
+                           alignof(ReturnStmt));
+  return new (Mem) ReturnStmt(RL, E, NRVOCandidate);
+}
+
+ReturnStmt *ReturnStmt::CreateEmpty(const ASTContext &Ctx,
+                                    bool HasNRVOCandidate) {
+  void *Mem = Ctx.Allocate(totalSizeToAlloc<const VarDecl *>(HasNRVOCandidate),
+                           alignof(ReturnStmt));
+  return new (Mem) ReturnStmt(EmptyShell(), HasNRVOCandidate);
 }
 
 // CaseStmt
 CaseStmt *CaseStmt::Create(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
                            SourceLocation caseLoc, SourceLocation ellipsisLoc,
                            SourceLocation colonLoc) {
   bool CaseStmtIsGNURange = rhs != nullptr;
   void *Mem = Ctx.Allocate(

lib/Analysis/BodyFarm.cpp

 ObjCIvarRefExpr *ASTMaker::makeObjCIvarRef(const Expr *Base,
                                            const ObjCIvarDecl *IVar) {
   return new (C) ObjCIvarRefExpr(const_cast<ObjCIvarDecl*>(IVar),
                                  IVar->getType(), SourceLocation(),
                                  SourceLocation(), const_cast<Expr*>(Base),
                                  /*arrow=*/true, /*free=*/false);
 }
 
-
 ReturnStmt *ASTMaker::makeReturn(const Expr *RetVal) {
-  return new (C) ReturnStmt(SourceLocation(), const_cast<Expr*>(RetVal),
-                            nullptr);
+  return ReturnStmt::Create(C, SourceLocation(), const_cast<Expr *>(RetVal),
+                            /* NRVOCandidate=*/nullptr);
 }
 
 IntegerLiteral *ASTMaker::makeIntegerLiteral(uint64_t Value, QualType Ty) {
   llvm::APInt APValue = llvm::APInt(C.getTypeSize(Ty), Value);
   return IntegerLiteral::Create(C, APValue, Ty, SourceLocation());
 }
 
 MemberExpr *ASTMaker::makeMemberExpression(Expr *base, ValueDecl *MemberDecl,

lib/CodeGen/CGObjC.cpp

 void
 CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
                                         const ObjCPropertyImplDecl *propImpl,
                                         const ObjCMethodDecl *GetterMethodDecl,
                                         llvm::Constant *AtomicHelperFn) {
   // If there's a non-trivial 'get' expression, we just have to emit that.
   if (!hasTrivialGetExpr(propImpl)) {
     if (!AtomicHelperFn) {
-      ReturnStmt ret(SourceLocation(), propImpl->getGetterCXXConstructor(),
-                     /*nrvo*/ nullptr);
-      EmitReturnStmt(ret);
+      auto *ret = ReturnStmt::Create(getContext(), SourceLocation(),
+                                     propImpl->getGetterCXXConstructor(),
+                                     /* NRVOCandidate=*/nullptr);
+      EmitReturnStmt(*ret);
     }
     else {
       ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();
       emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(),
                                     ivar, AtomicHelperFn);
     }
     return;
   }

lib/Sema/SemaStmt.cpp

if (ExprEvalContexts.back().Context ==
           ExpressionEvaluationContext::DiscardedStatement &&
       (HasDeducedReturnType || CurCap->HasImplicitReturnType)) {
     if (RetValExp) {
       ExprResult ER = ActOnFinishFullExpr(RetValExp, ReturnLoc);
       if (ER.isInvalid())
         return StmtError();
       RetValExp = ER.get();
     }
-    return new (Context) ReturnStmt(ReturnLoc, RetValExp, nullptr);
+    return ReturnStmt::Create(Context, ReturnLoc, RetValExp,
+                              /* NRVOCandidate=*/nullptr);
   }
 
   if (HasDeducedReturnType) {
     // In C++1y, the return type may involve 'auto'.
     // FIXME: Blocks might have a return type of 'auto' explicitly specified.
     FunctionDecl *FD = CurLambda->CallOperator;
     if (CurCap->ReturnType.isNull())
       CurCap->ReturnType = FD->getReturnType();
Sema::ActOnCapScopeReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
   }
 
   if (RetValExp) {
     ExprResult ER = ActOnFinishFullExpr(RetValExp, ReturnLoc);
     if (ER.isInvalid())
       return StmtError();
     RetValExp = ER.get();
   }
-  ReturnStmt *Result = new (Context) ReturnStmt(ReturnLoc, RetValExp,
-                                                NRVOCandidate);
+  auto *Result =
+      ReturnStmt::Create(Context, ReturnLoc, RetValExp, NRVOCandidate);
 
   // If we need to check for the named return value optimization,
   // or if we need to infer the return type,
   // save the return statement in our scope for later processing.
   if (CurCap->HasImplicitReturnType || NRVOCandidate)
     FunctionScopes.back()->Returns.push_back(Result);
 
   if (FunctionScopes.back()->FirstReturnLoc.isInvalid())
if (ExprEvalContexts.back().Context ==
           ExpressionEvaluationContext::DiscardedStatement &&
       FnRetType->getContainedAutoType()) {
     if (RetValExp) {
       ExprResult ER = ActOnFinishFullExpr(RetValExp, ReturnLoc);
       if (ER.isInvalid())
         return StmtError();
       RetValExp = ER.get();
     }
-    return new (Context) ReturnStmt(ReturnLoc, RetValExp, nullptr);
+    return ReturnStmt::Create(Context, ReturnLoc, RetValExp,
+                              /* NRVOCandidate=*/nullptr);
   }
 
   // FIXME: Add a flag to the ScopeInfo to indicate whether we're performing
   // deduction.
   if (getLangOpts().CPlusPlus14) {
     if (AutoType *AT = FnRetType->getContainedAutoType()) {
       FunctionDecl *FD = cast<FunctionDecl>(CurContext);
       if (DeduceFunctionTypeFromReturnExpr(FD, ReturnLoc, RetValExp, AT)) {
if (RetValExp) {
       if (RetValExp) {
         ExprResult ER = ActOnFinishFullExpr(RetValExp, ReturnLoc);
         if (ER.isInvalid())
           return StmtError();
         RetValExp = ER.get();
       }
     }
 
-    Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, nullptr);
+    Result = ReturnStmt::Create(Context, ReturnLoc, RetValExp,
+                                /* NRVOCandidate=*/nullptr);
   } else if (!RetValExp && !HasDependentReturnType) {
     FunctionDecl *FD = getCurFunctionDecl();
 
     unsigned DiagID;
     if (getLangOpts().CPlusPlus11 && FD && FD->isConstexpr()) {
       // C++11 [stmt.return]p2
       DiagID = diag::err_constexpr_return_missing_expr;
       FD->setInvalidDecl();
     } else if (getLangOpts().C99) {
       // C99 6.8.6.4p1 (ext_ since GCC warns)
       DiagID = diag::ext_return_missing_expr;
     } else {
       // C90 6.6.6.4p4
       DiagID = diag::warn_return_missing_expr;
     }
 
     if (FD)
       Diag(ReturnLoc, DiagID) << FD->getIdentifier() << 0/*fn*/;
     else
       Diag(ReturnLoc, DiagID) << getCurMethodDecl()->getDeclName() << 1/*meth*/;
 
-    Result = new (Context) ReturnStmt(ReturnLoc);
+    Result = ReturnStmt::Create(Context, ReturnLoc, /* RetExpr=*/nullptr,
+                                /* NRVOCandidate=*/nullptr);
   } else {
     assert(RetValExp || HasDependentReturnType);
     const VarDecl *NRVOCandidate = nullptr;
 
     QualType RetType = RelatedRetType.isNull() ? FnRetType : RelatedRetType;
 
     // C99 6.8.6.4p3(136): The return statement is not an assignment. The
     // overlap restriction of subclause 6.5.16.1 does not apply to the case of
if (FnRetType->isVoidType()) {
     }
 
     if (RetValExp) {
       ExprResult ER = ActOnFinishFullExpr(RetValExp, ReturnLoc);
       if (ER.isInvalid())
         return StmtError();
       RetValExp = ER.get();
     }
-    Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, NRVOCandidate);
+    Result = ReturnStmt::Create(Context, ReturnLoc, RetValExp, NRVOCandidate);
   }
 
   // If we need to check for the named return value optimization, save the
   // return statement in our scope for later processing.
   if (Result->getNRVOCandidate())
     FunctionScopes.back()->Returns.push_back(Result);
 
   if (FunctionScopes.back()->FirstReturnLoc.isInvalid())

lib/Serialization/ASTReaderStmt.cpp

 
 void ASTStmtReader::VisitBreakStmt(BreakStmt *S) {
   VisitStmt(S);
   S->setBreakLoc(ReadSourceLocation());
 }
 
 void ASTStmtReader::VisitReturnStmt(ReturnStmt *S) {
   VisitStmt(S);
+
+  bool HasNRVOCandidate = Record.readInt();
+
   S->setRetValue(Record.readSubExpr());
-  S->setReturnLoc(ReadSourceLocation());
+  if (HasNRVOCandidate)
     S->setNRVOCandidate(ReadDeclAs<VarDecl>());
+
+  S->setReturnLoc(ReadSourceLocation());
 }
 
 void ASTStmtReader::VisitDeclStmt(DeclStmt *S) {
   VisitStmt(S);
   S->setStartLoc(ReadSourceLocation());
   S->setEndLoc(ReadSourceLocation());
 
   if (Record.size() - Record.getIdx() == 1) {
case STMT_CONTINUE:
       S = new (Context) ContinueStmt(Empty);
       break;
 
     case STMT_BREAK:
       S = new (Context) BreakStmt(Empty);
       break;
 
     case STMT_RETURN:
-      S = new (Context) ReturnStmt(Empty);
+      S = ReturnStmt::CreateEmpty(
+          Context, /* HasNRVOCandidate=*/Record[ASTStmtReader::NumStmtFields]);
       break;
 
     case STMT_DECL:
       S = new (Context) DeclStmt(Empty);
       break;
 
     case STMT_GCCASM:
       S = new (Context) GCCAsmStmt(Empty);

lib/Serialization/ASTWriterStmt.cpp

 void ASTStmtWriter::VisitBreakStmt(BreakStmt *S) {
   VisitStmt(S);
   Record.AddSourceLocation(S->getBreakLoc());
   Code = serialization::STMT_BREAK;
 }
 
 void ASTStmtWriter::VisitReturnStmt(ReturnStmt *S) {
   VisitStmt(S);
+
+  bool HasNRVOCandidate = S->getNRVOCandidate() != nullptr;
+  Record.push_back(HasNRVOCandidate);
+
   Record.AddStmt(S->getRetValue());
-  Record.AddSourceLocation(S->getReturnLoc());
+  if (HasNRVOCandidate)
     Record.AddDeclRef(S->getNRVOCandidate());
+
+  Record.AddSourceLocation(S->getReturnLoc());
   Code = serialization::STMT_RETURN;
 }
 
 void ASTStmtWriter::VisitDeclStmt(DeclStmt *S) {
   VisitStmt(S);
   Record.AddSourceLocation(S->getBeginLoc());
   Record.AddSourceLocation(S->getEndLoc());
   DeclGroupRef DG = S->getDeclGroup();