Extend syntax of response files

[sepavloff]

Summary

This change adds two extensions to the syntax of response files, which are
included into command line by constructs @file. These extensions are:

• line continuation using trailing backslash, and
• one line comments started by #.

Processing of response files, made in GCC by libiberty, treats backslash only
as an escape of the next character. It means that using trailing backslash
for line continuation prevents from using files with newline in their names.
Such filenames are rarely used and drop of their support does not look
restrictive.

On Windows backslash is a normal symbol and using it for line split is not
implemented.

Line where the first non-whitespace character is # is considered a
comments and is ignored. This behavior is consistent with sh/ksh/bash on
unix-like systems and PowerShell on Windows. With such extension a file with
name like #file cannot be used at the beginning of line. This problem is
easily solved by using names like ./#file, \#file or by putting file
name in quotes.

[sepavloff]

Is there any opinion about feasibility of this extension?
This patch is a prerequisite for D24933 (Enable configuration files in clang), which is now accepted.
If it is not acceptable for all response files, then it must be implemented for config files only.

[hfinkel]

In D36045#962166, @sepavloff wrote:

Is there any opinion about feasibility of this extension?
This patch is a prerequisite for D24933 (Enable configuration files in clang), which is now accepted.
If it is not acceptable for all response files, then it must be implemented for config files only.

I like the extension, but making our parsing of response files differ from GCC's in this regard might not be an improvements (e.g., the handling of the escaped new lines, and even the comments, could be a breaking change). Can you add a new parameter (e.g., bool ExtendedSyntax = false), and then predicate the new behavior on that? Then set that parameter to true when parsing config files from Clang?

[sepavloff]

In D36045#962255, @hfinkel wrote:

I like the extension, but making our parsing of response files differ from GCC's in this regard might not be an improvements (e.g., the handling of the escaped new lines, and even the comments, could be a breaking change). Can you add a new parameter (e.g., bool ExtendedSyntax = false), and then predicate the new behavior on that? Then set that parameter to true when parsing config files from Clang?

This is not a problem, I will revert the other dependency D24926 to its previous version, where config file was parsed by dedicated function.

lib/Support/CommandLine.cpp

 static bool isWhitespace(char C) { return strchr(" \t\n\r\f\v", C); }
 
 static bool isQuote(char C) { return C == '\"' || C == '\''; }
 
 void cl::TokenizeGNUCommandLine(StringRef Src, StringSaver &Saver,
                                 SmallVectorImpl<const char *> &NewArgv,
                                 bool MarkEOLs) {
   SmallString<128> Token;
+  bool NonWhitespaceSeenInLine = false;
   for (size_t I = 0, E = Src.size(); I != E; ++I) {
     // Consume runs of whitespace.
     if (Token.empty()) {
       while (I != E && isWhitespace(Src[I])) {
         // Mark the end of lines in response files
         if (MarkEOLs && Src[I] == '\n')
           NewArgv.push_back(nullptr);
         ++I;
       }
       if (I == E)
         break;
+      // Skip comment line.
+      if (!NonWhitespaceSeenInLine && Src[I] == '#') {
+        ++I;
+        while (I != E && Src[I] != '\n')
+          ++I;
+        if (I == E)
+          break;
+        continue;
+      }
+      NonWhitespaceSeenInLine = true;
     }
 
     // Backslash escapes the next character.
     if (I + 1 < E && Src[I] == '\\') {
       ++I; // Skip the escape.
+      if (Src[I] == '\n')
+        continue; // Ignore backlash followed by '\n'.
+      if (Src[I] == '\r' && I + 1 < E && Src[I + 1] == '\n') {
+        ++I;
+        continue; // Ignore backlash followed by \r\n.
+      }
       Token.push_back(Src[I]);
       continue;
     }
 
     // Consume a quoted string.
     if (isQuote(Src[I])) {
       char Quote = Src[I++];
       while (I != E && Src[I] != Quote) {
         // Backslash escapes the next character.
         if (Src[I] == '\\' && I + 1 != E)
           ++I;
         Token.push_back(Src[I]);
         ++I;
       }
       if (I == E)
         break;
       continue;
     }
 
     // End the token if this is whitespace.
     if (isWhitespace(Src[I])) {
       if (!Token.empty())
         NewArgv.push_back(Saver.save(StringRef(Token)).data());
       Token.clear();
+      if (Src[I] == '\n')
+        NonWhitespaceSeenInLine = false;
       continue;
     }
 
     // This is a normal character.  Append it.
     Token.push_back(Src[I]);
   }
 
   // Append the last token after hitting EOF with no whitespace.

 void cl::TokenizeWindowsCommandLine(StringRef Src, StringSaver &Saver,
                                     SmallVectorImpl<const char *> &NewArgv,
                                     bool MarkEOLs) {
   SmallString<128> Token;
 
   // This is a small state machine to consume characters until it reaches the
   // end of the source string.
   enum { INIT, UNQUOTED, QUOTED } State = INIT;
+  bool NonWhitespaceSeenInLine = false;
   for (size_t I = 0, E = Src.size(); I != E; ++I) {
     // INIT state indicates that the current input index is at the start of
     // the string or between tokens.
     if (State == INIT) {
       if (isWhitespace(Src[I])) {
         // Mark the end of lines in response files
         if (MarkEOLs && Src[I] == '\n')
           NewArgv.push_back(nullptr);
         continue;
       }
+
+      // Skip comment line.
+      if (!NonWhitespaceSeenInLine && Src[I] == '#') {
+        ++I;
+        while (I != E && Src[I] != '\n')
+          ++I;
+        if (I == E)
+          break;
+        continue;
+      }
+      NonWhitespaceSeenInLine = true;
+
       if (Src[I] == '"') {
         State = QUOTED;
         continue;
       }
       if (Src[I] == '\\') {
         I = parseBackslash(Src, I, Token);
         State = UNQUOTED;
         continue;
       }
       Token.push_back(Src[I]);
       State = UNQUOTED;
       continue;
     }
 
     // UNQUOTED state means that it's reading a token not quoted by double
     // quotes.
     if (State == UNQUOTED) {
       // Whitespace means the end of the token.
       if (isWhitespace(Src[I])) {
         NewArgv.push_back(Saver.save(StringRef(Token)).data());
         Token.clear();
         State = INIT;
+        if (Src[I] == '\n') {
+          NonWhitespaceSeenInLine = false;
           // Mark the end of lines in response files
-        if (MarkEOLs && Src[I] == '\n')
+          if (MarkEOLs)
             NewArgv.push_back(nullptr);
+        }
         continue;
       }
       if (Src[I] == '"') {
         State = QUOTED;
         continue;
       }
       if (Src[I] == '\\') {
         I = parseBackslash(Src, I, Token);

unittests/Support/CommandLineTest.cpp

const char Input[] =
       "foo\"bar\"baz C:\\\\src\\\\foo.cpp \"C:\\src\\foo.cpp\"";
   const char *const Output[] = {
       "foo bar",     "foo bar",   "foo bar",          "foo\\bar",
       "-DFOO=bar()", "foobarbaz", "C:\\src\\foo.cpp", "C:srcfoo.cpp"};
   testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output,
                            array_lengthof(Output));
 }
 
+TEST(CommandLineTest, TokenizeGNUCommandLineBS) {
+  const char *Input = "\\";
+  const char *Output[1] = { "\\" };
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\";
+  Output[0] = "abc\\";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "\\abc";
+  Output[0] = "abc";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\123";
+  Output[0] = "abc123";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\1";
+  Output[0] = "abc1";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\\\123";
+  Output[0] = "abc\\123";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "\\\nabc";
+  Output[0] = "abc";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "\\\r\nabc";
+  Output[0] = "abc";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\\n123";
+  Output[0] = "abc123";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\\r\n123";
+  Output[0] = "abc123";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\\n";
+  Output[0] = "abc";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+
+  Input = "abc\\\r\n";
+  Output[0] = "abc";
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output, 1);
+}
+
 TEST(CommandLineTest, TokenizeWindowsCommandLine) {
   const char Input[] = "a\\b c\\\\d e\\\\\"f g\" h\\\"i j\\\\\\\"k \"lmn\" o pqr "
                       "\"st \\\"u\" \\v";
   const char *const Output[] = { "a\\b", "c\\\\d", "e\\f g", "h\"i", "j\\\"k",
                                  "lmn", "o", "pqr", "st \"u", "\\v" };
   testCommandLineTokenizer(cl::TokenizeWindowsCommandLine, Input, Output,
                            array_lengthof(Output));
 }
 
+TEST(CommandLineTest, TokenizeCommandLineComment) {
+  for (auto Tokenizer : { cl::TokenizeGNUCommandLine ,
+                          cl::TokenizeWindowsCommandLine }) {
+    const char *Input = "# abc\n"
+                        "123";
+    const char *Output[3] = { "123" };
+    testCommandLineTokenizer(Tokenizer, Input, Output, 1);
+
+    Input = "  # abc\n"
+            "123";
+    Output[0] = "123";
+    testCommandLineTokenizer(Tokenizer, Input, Output, 1);
+
+    Input = "123 # abc";
+    Output[0] = "123";
+    Output[1] = "#";
+    Output[2] = "abc";
+    testCommandLineTokenizer(Tokenizer, Input, Output, 3);
+
+    Input = "abc\n"
+            "#123";
+    Output[0] = "abc";
+    testCommandLineTokenizer(Tokenizer, Input, Output, 1);
+
+    Input = "abc def\n"
+            "#123";
+    Output[0] = "abc";
+    Output[1] = "def";
+    testCommandLineTokenizer(Tokenizer, Input, Output, 2);
+  }
+}
+
 TEST(CommandLineTest, AliasesWithArguments) {
   static const size_t ARGC = 3;
   const char *const Inputs[][ARGC] = {
     { "-tool", "-actual=x", "-extra" },
     { "-tool", "-actual", "x" },
     { "-tool", "-alias=x", "-extra" },
     { "-tool", "-alias", "x" }
   };