[Clang] Limit -fintegrated-cc1 to only one TU

[aganea]

Summary

As discussed in D74447, this patch disables integrated-cc1 behavior if there's more than one job to be executed. This is meant to limit memory bloating, given that currently jobs don't clean up after execution (-disable-free is always active in cc1 mode).

I see this behavior as temporary until release 10.0 ships, then we'll reevaluate the situation, see if D74447 makes more sense on the long term.

[aganea]

clang/test/Driver/cc1-spawnprocess.c

9

Out of curiosity: what does -- means to do? (here used along with %s)

[rnk]

lgtm

clang/test/Driver/cc1-spawnprocess.c

9

It indicates that the remaining args should be interpreted as inputs, not flags. We need -- in the test suite on Mac where %s usually expands to /Users/foo/bar.c, which looks like clang-cl /UMYMACRO.

[aganea]

clang/include/clang/Driver/Job.h

87

I forgot to put back the default value(s) here, I'll do it before landing.

clang/tools/driver/cc1_main.cpp

262 ↗

(On Diff #244197)

I'll commit this separately, this is not a requirement for this patch.

[hans]

Cherry-picked to 10.x as 9c9e46d786d0581079e5018e550cbe187a1ec219

clang/include/clang/Driver/Job.h

class Command {
 
   /// The list of program arguments (not including the implicit first
   /// argument, which will be the executable).
   llvm::opt::ArgStringList Arguments;
 
   /// The list of program arguments which are inputs.
   llvm::opt::ArgStringList InputFilenames;
 
-  /// Whether to print the input filenames when executing.
-  bool PrintInputFilenames = false;
-
   /// Response file name, if this command is set to use one, or nullptr
   /// otherwise
   const char *ResponseFile = nullptr;
 
   /// The input file list in case we need to emit a file list instead of a
   /// proper response file
   llvm::opt::ArgStringList InputFileList;
 
class Command {
 
   /// Encodes an array of C strings into a single string separated by whitespace.
   /// This function will also put in quotes arguments that have whitespaces and
   /// will escape the regular backslashes (used in Windows paths) and quotes.
   /// The results are the contents of a response file, written into a raw_ostream.
   void writeResponseFile(raw_ostream &OS) const;
 
 public:
+  /// Whether to print the input filenames when executing.
+  bool PrintInputFilenames = false;
+
+  /// Whether the command will be executed in this process or not.
+  bool InProcess = false;
+
   Command(const Action &Source, const Tool &Creator, const char *Executable,
           const llvm::opt::ArgStringList &Arguments,
           ArrayRef<InputInfo> Inputs);
   // FIXME: This really shouldn't be copyable, but is currently copied in some
   // error handling in Driver::generateCompilationDiagnostics.
   Command(const Command &) = default;
   virtual ~Command() = default;
 
public:
 
   const char *getExecutable() const { return Executable; }
 
   const llvm::opt::ArgStringList &getArguments() const { return Arguments; }
 
   /// Print a command argument, and optionally quote it.
   static void printArg(llvm::raw_ostream &OS, StringRef Arg, bool Quote);
 
-  /// Set whether to print the input filenames when executing.
-  void setPrintInputFilenames(bool P) { PrintInputFilenames = P; }
-
 protected:
   /// Optionally print the filenames to be compiled
   void PrintFileNames() const;
 };
 
 /// Use the CC1 tool callback when available, to avoid creating a new process
 class CC1Command : public Command {
 public:
-  using Command::Command;
+  CC1Command(const Action &Source, const Tool &Creator, const char *Executable,
+             const llvm::opt::ArgStringList &Arguments,
+             ArrayRef<InputInfo> Inputs);
 
   void Print(llvm::raw_ostream &OS, const char *Terminator, bool Quote,
              CrashReportInfo *CrashInfo = nullptr) const override;
 
   int Execute(ArrayRef<Optional<StringRef>> Redirects, std::string *ErrMsg,
               bool *ExecutionFailed) const override;
 
   void setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) override;

clang/lib/Driver/Driver.cpp

for (Action *A : C.getActions()) {
     BuildJobsForAction(C, A, &C.getDefaultToolChain(),
                        /*BoundArch*/ StringRef(),
                        /*AtTopLevel*/ true,
                        /*MultipleArchs*/ ArchNames.size() > 1,
                        /*LinkingOutput*/ LinkingOutput, CachedResults,
                        /*TargetDeviceOffloadKind*/ Action::OFK_None);
   }
 
+  // If we have more than one job, then disable integrated-cc1 for now.
+  if (C.getJobs().size() > 1)
+    for (auto &J : C.getJobs())
+      J.InProcess = false;
+
   // If the user passed -Qunused-arguments or there were errors, don't warn
   // about any unused arguments.
   if (Diags.hasErrorOccurred() ||
       C.getArgs().hasArg(options::OPT_Qunused_arguments))
     return;
 
   // Claim -### here.
   (void)C.getArgs().hasArg(options::OPT__HASH_HASH_HASH);

clang/lib/Driver/Job.cpp

int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
   }
 
   auto Args = llvm::toStringRefArray(Argv.data());
   return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects,
                                    /*secondsToWait*/ 0,
                                    /*memoryLimit*/ 0, ErrMsg, ExecutionFailed);
 }
 
+CC1Command::CC1Command(const Action &Source, const Tool &Creator,
+                       const char *Executable,
+                       const llvm::opt::ArgStringList &Arguments,
+                       ArrayRef<InputInfo> Inputs)
+    : Command(Source, Creator, Executable, Arguments, Inputs) {
+  InProcess = true;
+}
+
 void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
                        CrashReportInfo *CrashInfo) const {
+  if (InProcess)
     OS << " (in-process)\n";
   Command::Print(OS, Terminator, Quote, CrashInfo);
 }
 
-int CC1Command::Execute(ArrayRef<llvm::Optional<StringRef>> /*Redirects*/,
+int CC1Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
                         std::string *ErrMsg, bool *ExecutionFailed) const {
+  // FIXME: Currently, if there're more than one job, we disable
+  // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to
+  // out-of-process execution. See discussion in https://reviews.llvm.org/D74447
+  if (!InProcess)
+    return Command::Execute(Redirects, ErrMsg, ExecutionFailed);
+
   PrintFileNames();
 
   SmallVector<const char *, 128> Argv;
   Argv.push_back(getExecutable());
   Argv.append(getArguments().begin(), getArguments().end());
   Argv.push_back(nullptr);
 
   // This flag simply indicates that the program couldn't start, which isn't

clang/lib/Driver/ToolChains/Clang.cpp

   } else {
     C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
   }
 
   // Make the compile command echo its inputs for /showFilenames.
   if (Output.getType() == types::TY_Object &&
       Args.hasFlag(options::OPT__SLASH_showFilenames,
                    options::OPT__SLASH_showFilenames_, false)) {
-    C.getJobs().getJobs().back()->setPrintInputFilenames(true);
+    C.getJobs().getJobs().back()->PrintInputFilenames = true;
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_pg))
     if (FPKeepKind == CodeGenOptions::FramePointerKind::None)
       D.Diag(diag::err_drv_argument_not_allowed_with) << "-fomit-frame-pointer"
                                                       << A->getAsString(Args);
 
   // Claim some arguments which clang supports automatically.

clang/test/Driver/cc1-spawnprocess.c

-// RUN: %clang -fintegrated-cc1 -### %s 2>&1 | FileCheck %s --check-prefix=YES
-// RUN: %clang -fno-integrated-cc1 -### %s 2>&1 | FileCheck %s --check-prefix=NO
+// RUN: %clang -fintegrated-cc1 -c -### %s 2>&1 | FileCheck %s --check-prefix=YES
+// RUN: %clang -fno-integrated-cc1 -c -### %s 2>&1 | FileCheck %s --check-prefix=NO
 
-// RUN: %clang -fintegrated-cc1 -fno-integrated-cc1 -### %s 2>&1 \
+// RUN: %clang -fintegrated-cc1 -fno-integrated-cc1 -c -### %s 2>&1 \
 // RUN:     | FileCheck %s --check-prefix=NO
-// RUN: %clang -fno-integrated-cc1 -fintegrated-cc1 -### %s 2>&1 \
+// RUN: %clang -fno-integrated-cc1 -fintegrated-cc1 -c -### %s 2>&1 \
 // RUN:     | FileCheck %s --check-prefix=YES
 
-// RUN: %clang_cl -fintegrated-cc1 -### -- %s 2>&1 \
+// RUN: %clang_cl -fintegrated-cc1 -c -### -- %s 2>&1 \
 // RUN:     | FileCheck %s --check-prefix=YES
-// RUN: %clang_cl -fno-integrated-cc1 -### -- %s 2>&1 \
+// RUN: %clang_cl -fno-integrated-cc1 -c -### -- %s 2>&1 \
 // RUN:     | FileCheck %s --check-prefix=NO
 
 // RUN: env CCC_OVERRIDE_OPTIONS=+-fintegrated-cc1 \
-// RUN:     %clang -fintegrated-cc1 -### %s 2>&1 \
+// RUN:     %clang -fintegrated-cc1 -c -### %s 2>&1 \
 // RUN:     | FileCheck %s --check-prefix=YES
 // RUN: env CCC_OVERRIDE_OPTIONS=+-fno-integrated-cc1 \
-// RUN:     %clang -fintegrated-cc1 -### %s 2>&1 \
+// RUN:     %clang -fintegrated-cc1 -c -### %s 2>&1 \
 // RUN:     | FileCheck %s --check-prefix=NO
 
 // YES: (in-process)
 // NO-NOT: (in-process)
+
+// The following tests ensure that only one integrated-cc1 is executed.
+
+// Only one TU, one job, thus integrated-cc1 is enabled.
+// RUN: %clang -fintegrated-cc1 -c %s -### 2>&1 | FileCheck %s --check-prefix=YES
+
+// Only one TU, but we're linking, two jobs, thus integrated-cc1 is disabled.
+// RUN: %clang -fintegrated-cc1 %s -### 2>&1 | FileCheck %s --check-prefix=NO
+
+// RUN: echo 'int main() { return f() + g(); }' > %t1.cpp
+// RUN: echo 'int f() { return 1; }' > %t2.cpp
+// RUN: echo 'int g() { return 2; }' > %t3.cpp
+
+// Three jobs, thus integrated-cc1 is disabled.
+// RUN: %clang -fintegrated-cc1 -c %t1.cpp %t2.cpp %t3.cpp -### 2>&1 | FileCheck %s --check-prefix=NO