Use lexing mode based on FormatStyle.Standard.

[alexfh]

Summary

Some valid pre-C++11 constructs change meaning when lexed in C++11
mode, e.g.
#define x(_a) printf("foo"_a);
(example from http://llvm.org/bugs/show_bug.cgi?id=16342). "foo"_a is treated as
a user-defined string literal when parsed in C++11 mode.
In order to deal with this correctly, we need to set lexing mode according to
which standard the code conforms to. We already have a configuration value for
this (FormatStyle.Standard), which seems to be appropriate to use in this case
as well.

[gribozavr]

include/clang/Format/Format.h

222

\param Standard

[alexfh]

\p -> \param

include/clang/Format/Format.h

222

Done.

[alexfh]

Closed by commit rL185149 (authored by @alexfh).

include/clang/Format/Format.h

 /// \brief Reformats the given \p Ranges in \p Code.
 ///
 /// Otherwise identical to the reformat() function consuming a \c Lexer.
 tooling::Replacements reformat(const FormatStyle &Style, StringRef Code,
                                std::vector<tooling::Range> Ranges,
                                StringRef FileName = "<stdin>");
 
 /// \brief Returns the \c LangOpts that the formatter expects you to set.
-LangOptions getFormattingLangOpts();
+///
+/// \p Standard determines lexing mode: LC_Cpp11 and LS_Auto turn on C++11
+/// lexing mode, LS_Cpp03 - C++03 mode.
+LangOptions getFormattingLangOpts(FormatStyle::LanguageStandard Standard =
+                                      FormatStyle::LS_Cpp11);
 
 } // end namespace format
 } // end namespace clang
 
 #endif // LLVM_CLANG_FORMAT_FORMAT_H

lib/Format/Format.cpp

DiagnosticsEngine Diagnostics(
       new DiagnosticOptions);
   SourceManager SourceMgr(Diagnostics, Files);
   llvm::MemoryBuffer *Buf = llvm::MemoryBuffer::getMemBuffer(Code, FileName);
   const clang::FileEntry *Entry =
       Files.getVirtualFile(FileName, Buf->getBufferSize(), 0);
   SourceMgr.overrideFileContents(Entry, Buf);
   FileID ID =
       SourceMgr.createFileID(Entry, SourceLocation(), clang::SrcMgr::C_User);
-  Lexer Lex(ID, SourceMgr.getBuffer(ID), SourceMgr, getFormattingLangOpts());
+  Lexer Lex(ID, SourceMgr.getBuffer(ID), SourceMgr,
+            getFormattingLangOpts(Style.Standard));
   SourceLocation StartOfFile = SourceMgr.getLocForStartOfFile(ID);
   std::vector<CharSourceRange> CharRanges;
   for (unsigned i = 0, e = Ranges.size(); i != e; ++i) {
     SourceLocation Start = StartOfFile.getLocWithOffset(Ranges[i].getOffset());
     SourceLocation End = Start.getLocWithOffset(Ranges[i].getLength());
     CharRanges.push_back(CharSourceRange::getCharRange(Start, End));
   }
   return reformat(Style, Lex, SourceMgr, CharRanges);
 }
 
-LangOptions getFormattingLangOpts() {
+LangOptions getFormattingLangOpts(FormatStyle::LanguageStandard Standard) {
   LangOptions LangOpts;
   LangOpts.CPlusPlus = 1;
-  LangOpts.CPlusPlus11 = 1;
+  LangOpts.CPlusPlus11 = Standard == FormatStyle::LS_Cpp03 ? 0 : 1;
   LangOpts.LineComment = 1;
   LangOpts.Bool = 1;
   LangOpts.ObjC1 = 1;
   LangOpts.ObjC2 = 1;
   return LangOpts;
 }
 
 } // namespace format
 } // namespace clang

tools/clang-format/ClangFormat.cpp

DiagnosticsEngine Diagnostics(
       new DiagnosticOptions);
   SourceManager Sources(Diagnostics, Files);
   OwningPtr<MemoryBuffer> Code;
   if (error_code ec = MemoryBuffer::getFileOrSTDIN(FileName, Code)) {
     llvm::errs() << ec.message() << "\n";
     return true;
   }
   FileID ID = createInMemoryFile(FileName, Code.get(), Sources, Files);
-  Lexer Lex(ID, Sources.getBuffer(ID), Sources, getFormattingLangOpts());
   if (Offsets.empty())
     Offsets.push_back(0);
   if (Offsets.size() != Lengths.size() &&
       !(Offsets.size() == 1 && Lengths.empty())) {
     llvm::errs()
         << "error: number of -offset and -length arguments must match.\n";
     return true;
   }
if (i < Lengths.size()) {
         return true;
       }
       End = Start.getLocWithOffset(Lengths[i]);
     } else {
       End = Sources.getLocForEndOfFile(ID);
     }
     Ranges.push_back(CharSourceRange::getCharRange(Start, End));
   }
-  tooling::Replacements Replaces =
-      reformat(getStyle(Style, FileName), Lex, Sources, Ranges);
+  FormatStyle FormatStyle = getStyle(Style, FileName);
+  Lexer Lex(ID, Sources.getBuffer(ID), Sources,
+            getFormattingLangOpts(FormatStyle.Standard));
+  tooling::Replacements Replaces = reformat(FormatStyle, Lex, Sources, Ranges);
   if (OutputXML) {
     llvm::outs()
         << "<?xml version='1.0'?>\n<replacements xml:space='preserve'>\n";
     for (tooling::Replacements::const_iterator I = Replaces.begin(),
                                                E = Replaces.end();
          I != E; ++I) {
       llvm::outs() << "<replacement "
                    << "offset='" << I->getOffset() << "' "

unittests/Format/FormatTest.cpp

TEST_F(FormatTest, UnderstandsUnaryOperators) {
   verifyFormat("--(a->f());");
   verifyFormat("(a->f())++;");
   verifyFormat("a[42]++;");
   verifyFormat("if (!(a->f())) {\n}");
 
   verifyFormat("a-- > b;");
   verifyFormat("b ? -a : c;");
   verifyFormat("n * sizeof char16;");
-  verifyFormat("n * alignof char16;");
+  verifyFormat("n * alignof char16;", getGoogleStyle());
   verifyFormat("sizeof(char);");
-  verifyFormat("alignof(char);");
+  verifyFormat("alignof(char);", getGoogleStyle());
 
   verifyFormat("return -1;");
   verifyFormat("switch (a) {\n"
                "case -1:\n"
                "  break;\n"
                "}");
   verifyFormat("#define X -1");
   verifyFormat("#define X -kConstant");
TEST_F(FormatTest, FormatsCasts) {
   verifyFormat("void f(int *);");
   verifyFormat("void f(int *) = 0;");
   verifyFormat("void f(SmallVector<int>) {}");
   verifyFormat("void f(SmallVector<int>);");
   verifyFormat("void f(SmallVector<int>) = 0;");
   verifyFormat("void f(int i = (kValue) * kMask) {}");
   verifyFormat("void f(int i = (kA * kB) & kMask) {}");
   verifyFormat("int a = sizeof(int) * b;");
-  verifyFormat("int a = alignof(int) * b;");
+  verifyFormat("int a = alignof(int) * b;", getGoogleStyle());
   verifyFormat("template <> void f<int>(int i) SOME_ANNOTATION;");
   verifyFormat("f(\"%\" SOME_MACRO(ll) \"d\");");
 
   // These are not casts, but at some point were confused with casts.
   verifyFormat("virtual void foo(int *) override;");
   verifyFormat("virtual void foo(char &) const;");
   verifyFormat("virtual void foo(int *a, char *) const;");
   verifyFormat("int a = sizeof(int *) + b;");
-  verifyFormat("int a = alignof(int *) + b;");
+  verifyFormat("int a = alignof(int *) + b;", getGoogleStyle());
 }
 
 TEST_F(FormatTest, FormatsFunctionTypes) {
   verifyFormat("A<bool()> a;");
   verifyFormat("A<SomeType()> a;");
   verifyFormat("A<void (*)(int, std::string)> a;");
   verifyFormat("A<void *(int)>;");
   verifyFormat("void *(*a)(int *, SomeType *);");
TEST_F(FormatTest, FormatObjCMethodExpr) {
   // Unary operators.
   verifyFormat("int a = +[foo bar:baz];");
   verifyFormat("int a = -[foo bar:baz];");
   verifyFormat("int a = ![foo bar:baz];");
   verifyFormat("int a = ~[foo bar:baz];");
   verifyFormat("int a = ++[foo bar:baz];");
   verifyFormat("int a = --[foo bar:baz];");
   verifyFormat("int a = sizeof [foo bar:baz];");
-  verifyFormat("int a = alignof [foo bar:baz];");
+  verifyFormat("int a = alignof [foo bar:baz];", getGoogleStyle());
   verifyFormat("int a = &[foo bar:baz];");
   verifyFormat("int a = *[foo bar:baz];");
   // FIXME: Make casts work, without breaking f()[4].
   //verifyFormat("int a = (int)[foo bar:baz];");
   //verifyFormat("return (int)[foo bar:baz];");
   //verifyFormat("(void)[foo bar:baz];");
   verifyFormat("return (MyType *)[self.tableView cellForRowAtIndexPath:cell];");
 
EXPECT_EQ(
       "#define A \\\n"
       "  \"some \" \\\n"
       "  \"text \" \\\n"
       "  \"other\";",
       format("#define A \"some text other\";", AlignLeft));
 }
 
 TEST_F(FormatTest, SkipsUnknownStringLiterals) {
-  EXPECT_EQ("u8\"unsupported literal\";",
-            format("u8\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+  EXPECT_EQ(
+      "u8\"unsupported literal\";",
+      format("u8\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("u\"unsupported literal\";",
-            format("u\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+            format("u\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("U\"unsupported literal\";",
-            format("U\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+            format("U\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("L\"unsupported literal\";",
-            format("L\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+            format("L\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("R\"x(raw literal)x\";",
-            format("R\"x(raw literal)x\";", getLLVMStyleWithColumns(15)));
+            format("R\"x(raw literal)x\";", getGoogleStyleWithColumns(15)));
+}
+
+TEST_F(FormatTest, DoesNotTryToParseUDLiteralsInPreCpp11Code) {
+  EXPECT_EQ("#define x(_a) printf(\"foo\" _a);",
+            format("#define x(_a) printf(\"foo\"_a);", getLLVMStyle()));
 }
 
 TEST_F(FormatTest, BreakStringLiteralsBeforeUnbreakableTokenSequence) {
   EXPECT_EQ("someFunction(\"aaabbbcccd\"\n"
             "             \"ddeeefff\");",
             format("someFunction(\"aaabbbcccdddeeefff\");",
                    getLLVMStyleWithColumns(25)));
   EXPECT_EQ("someFunction1234567890(\n"
EXPECT_EQ("\"test\"\n"
             "\"\\000000\"\n"
             "\"000001\"",
             format("\"test\\000000000001\"", getLLVMStyleWithColumns(9)));
   EXPECT_EQ("\"test\\000\"\n"
             "\"00000000\"\n"
             "\"1\"",
             format("\"test\\000000000001\"", getLLVMStyleWithColumns(10)));
   EXPECT_EQ("R\"(\\x\\x00)\"\n",
-            format("R\"(\\x\\x00)\"\n", getLLVMStyleWithColumns(7)));
+            format("R\"(\\x\\x00)\"\n", getGoogleStyleWithColumns(7)));
 }
 
 TEST_F(FormatTest, DoNotCreateUnreasonableUnwrappedLines) {
   verifyFormat("void f() {\n"
                "  return g() {}\n"
                "  void h() {}");
   verifyFormat("if (foo)\n"
                "  return { forgot_closing_brace();\n"