[clang-cl] Use the Windows response file tokenizer

[rnk]

Summary

We were still using the Unix response file tokenizer for all driver
modes. This was difficult to get right in the beginning because there is
a circular dependency. The Driver class also can't officially determine
its mode until it can see all possible --driver-mode= flags, and those
flags could come from the response file.

Now we use the Windows parsing algorithm if the program name looks like
clang-cl, or if the --driver-mode=cl flag is present on the main command
line.

[majnemer]

tools/driver/driver.cpp

239–240 ↗

(On Diff #29799)

You could do this in-place on ProgName to save a copy.

[hans]

lgtm!

tools/driver/driver.cpp

400 ↗

(On Diff #29799)

Maybe expand this comment pointing out that we're checking for --driver-mode manually here because it's very early and we don't know the mode yet, like you said in the change description.

cfe/trunk/lib/Driver/Job.cpp

void Command::writeResponseFile(raw_ostream &OS) const {
   // In a file list, we only write the set of inputs to the response file
   if (Creator.getResponseFilesSupport() == Tool::RF_FileList) {
     for (const char *Arg : InputFileList) {
       OS << Arg << '\n';
     }
     return;
   }
 
-  // In regular response files, we send all arguments to the response file
+  // In regular response files, we send all arguments to the response file.
+  // Wrapping all arguments in double quotes ensures that both Unix tools and
+  // Windows tools understand the response file.
   for (const char *Arg : Arguments) {
     OS << '"';
 
     for (; *Arg != '\0'; Arg++) {
       if (*Arg == '\"' || *Arg == '\\') {
         OS << '\\';
       }
       OS << *Arg;

cfe/trunk/test/Driver/cl-response-file.c

+// Don't attempt slash switches on msys bash.
+// REQUIRES: shell-preserves-root
+
+// Test that we use the Windows tokenizer for clang-cl response files. The
+// trailing backslash before the space should be interpreted as a literal
+// backslash. PR23709
+
+
+
+// RUN: echo '/I%S\Inputs\cl-response-file\ /DFOO=2' > %t.rsp
+// RUN: %clang_cl /c -### @%t.rsp -- %s 2>&1 | FileCheck %s
+
+// CHECK: "-D" "FOO=2" "-I" "{{.*}}\\Inputs\\cl-response-file\\"

cfe/trunk/tools/driver/driver.cpp

static const DriverSuffix *FindDriverSuffix(StringRef ProgName) {
   };
 
   for (size_t i = 0; i < llvm::array_lengthof(DriverSuffixes); ++i)
     if (ProgName.endswith(DriverSuffixes[i].Suffix))
       return &DriverSuffixes[i];
   return nullptr;
 }
 
-static void ParseProgName(SmallVectorImpl<const char *> &ArgVector,
-                          std::set<std::string> &SavedStrings) {
+/// Normalize the program name from argv[0] by stripping the file extension if
+/// present and lower-casing the string on Windows.
+static std::string normalizeProgramName(const char *Argv0) {
+  std::string ProgName = llvm::sys::path::stem(Argv0);
+#ifdef LLVM_ON_WIN32
+  // Transform to lowercase for case insensitive file systems.
+  std::transform(ProgName.begin(), ProgName.end(), ProgName.begin(), ::tolower);
+#endif
+  return ProgName;
+}
+
+static const DriverSuffix *parseDriverSuffix(StringRef ProgName) {
   // Try to infer frontend type and default target from the program name by
   // comparing it against DriverSuffixes in order.
 
   // If there is a match, the function tries to identify a target as prefix.
   // E.g. "x86_64-linux-clang" as interpreted as suffix "clang" with target
   // prefix "x86_64-linux". If such a target prefix is found, is gets added via
   // -target as implicit first argument.
-
-  std::string ProgName =llvm::sys::path::stem(ArgVector[0]);
-#ifdef LLVM_ON_WIN32
-  // Transform to lowercase for case insensitive file systems.
-  ProgName = StringRef(ProgName).lower();
-#endif
-
-  StringRef ProgNameRef = ProgName;
-  const DriverSuffix *DS = FindDriverSuffix(ProgNameRef);
+  const DriverSuffix *DS = FindDriverSuffix(ProgName);
 
   if (!DS) {
     // Try again after stripping any trailing version number:
     // clang++3.5 -> clang++
-    ProgNameRef = ProgNameRef.rtrim("0123456789.");
-    DS = FindDriverSuffix(ProgNameRef);
+    ProgName = ProgName.rtrim("0123456789.");
+    DS = FindDriverSuffix(ProgName);
   }
 
   if (!DS) {
     // Try again after stripping trailing -component.
     // clang++-tot -> clang++
-    ProgNameRef = ProgNameRef.slice(0, ProgNameRef.rfind('-'));
-    DS = FindDriverSuffix(ProgNameRef);
-  }
+    ProgName = ProgName.slice(0, ProgName.rfind('-'));
+    DS = FindDriverSuffix(ProgName);
+  }
+   return DS;
 
-  if (DS) {
-    if (const char *Flag = DS->ModeFlag) {
-      // Add Flag to the arguments.
-      auto it = ArgVector.begin();
-      if (it != ArgVector.end())
-        ++it;
-      ArgVector.insert(it, Flag);
-    }
+static void insertArgsFromProgramName(StringRef ProgName,
+                                      const DriverSuffix *DS,
+                                      SmallVectorImpl<const char *> &ArgVector,
+                                      std::set<std::string> &SavedStrings) {
+  if (!DS)
+    return;
+
 
-    StringRef::size_type LastComponent = ProgNameRef.rfind(
-        '-', ProgNameRef.size() - strlen(DS->Suffix));
-    if (LastComponent == StringRef::npos)
-      return;
+  StringRef::size_type LastComponent = ProgName.rfind(
+      '-', ProgName.size() - strlen(DS->Suffix));
 
-    // Infer target from the prefix.
-    StringRef Prefix = ProgNameRef.slice(0, LastComponent);
-    std::string IgnoredError;
-    if (llvm::TargetRegistry::lookupTarget(Prefix, IgnoredError)) {
-      auto it = ArgVector.begin();
-      if (it != ArgVector.end())
-        ++it;
-      const char *arr[] = { "-target", GetStableCStr(SavedStrings, Prefix) };
-      ArgVector.insert(it, std::begin(arr), std::end(arr));
-    }
-  }
-}
+  StringRef Prefix = ProgName.slice(0, LastComponent);
 
 static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
   // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
   TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
   if (TheDriver.CCPrintOptions)
     TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
 
   // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
int main(int argc_, const char **argv_) {
   llvm::SpecificBumpPtrAllocator<char> ArgAllocator;
   std::error_code EC = llvm::sys::Process::GetArgumentVector(
       argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator);
   if (EC) {
     llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n';
     return 1;
   }
 
+  std::string ProgName = normalizeProgramName(argv[0]);
+  const DriverSuffix *DS = parseDriverSuffix(ProgName);
+
   llvm::BumpPtrAllocator A;
   llvm::BumpPtrStringSaver Saver(A);
 
+  // Parse response files using the GNU syntax, unless we're in CL mode. There
+  // are two ways to put clang in CL compatibility mode: argv[0] is either
+  // clang-cl or cl, or --driver-mode=cl is on the command line. The normal
+  // command line parsing can't happen until after response file parsing, so we
+  // have to manually search for a --driver-mode=cl argument the hard way.
+  // Finally, our -cc1 tools don't care which tokenization mode we use because
+  // response files written by clang will tokenize the same way in either mode.
+  llvm::cl::TokenizerCallback Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
+  if ((DS && DS->ModeFlag && strcmp(DS->ModeFlag, "--driver-mode=cl") == 0) ||
+      std::find_if(argv.begin(), argv.end(), [](const char *F) {
+        return F && strcmp(F, "--driver-mode=cl") == 0;
+      }) != argv.end()) {
+    Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
+  }
+
   // Determines whether we want nullptr markers in argv to indicate response
   // files end-of-lines. We only use this for the /LINK driver argument.
   bool MarkEOLs = true;
   if (argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
     MarkEOLs = false;
-  llvm::cl::ExpandResponseFiles(Saver, llvm::cl::TokenizeGNUCommandLine, argv,
-                                MarkEOLs);
+  llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
 
   // Handle -cc1 integrated tools, even if -cc1 was expanded from a response
   // file.
   auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
                                [](const char *A) { return A != nullptr; });
   if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
     // If -cc1 came from a response file, remove the EOL sentinels.
     if (MarkEOLs) {
int main(int argc_, const char **argv_) {
   }
 
   ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
 
   Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
   SetInstallDir(argv, TheDriver);
 
   llvm::InitializeAllTargets();
-  ParseProgName(argv, SavedStrings);
+  insertArgsFromProgramName(ProgName, DS, argv, SavedStrings);
 
   SetBackdoorDriverOutputsFromEnvVars(TheDriver);
 
   std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
   int Res = 0;
   SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
   if (C.get())
     Res = TheDriver.ExecuteCompilation(*C, FailingCommands);