[clang] Disable Unicode in asm files

[tinti]

Summary

Clang should not convert tokens to Unicode when preprocessing assembly
files.

Fixes PR25558

[mehdi_amini]

Isn't the patch reversed or do I miss something?

[mehdi_amini]

LGTM, please wait for another pair of eye.

[rengolin]

LGTM, too. Thanks!

[tinti]

r253738

lib/Lex/Lexer.cpp

   // FIXME: What exactly does the StartOfLine bit mean?  There are two
   // possible meanings for the "start" of the line: the first token on the
   // unexpanded line, or the first token on the expanded line.
   IsAtStartOfLine = StartOfLine;
   IsAtPhysicalStartOfLine = StartOfLine;
 }
 
 static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) {
-  if (LangOpts.AsmPreprocessor) {
-    return false;
-  } else if (LangOpts.CPlusPlus11 || LangOpts.C11) {
+  if (LangOpts.CPlusPlus11 || LangOpts.C11) {
     static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
         C11AllowedIDCharRanges);
     return C11AllowedIDChars.contains(C);
   } else if (LangOpts.CPlusPlus) {
     static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars(
         CXX03AllowedIDCharRanges);
     return CXX03AllowedIDChars.contains(C);
   } else {
     static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
         C99AllowedIDCharRanges);
     return C99AllowedIDChars.contains(C);
   }
 }
 
 static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) {
   assert(isAllowedIDChar(C, LangOpts));
-  if (LangOpts.AsmPreprocessor) {
-    return false;
-  } else if (LangOpts.CPlusPlus11 || LangOpts.C11) {
+  if (LangOpts.CPlusPlus11 || LangOpts.C11) {
     static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
         C11DisallowedInitialIDCharRanges);
     return !C11DisallowedInitialIDChars.contains(C);
   } else if (LangOpts.CPlusPlus) {
     return true;
   } else {
     static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
         C99DisallowedInitialIDCharRanges);

test/CodeGen/asm-unicode.S

-// RUN: %clang -S %s -o - | FileCheck %s
-.macro  my_macro, trace=1, uaccess=1
-.if \uaccess
-// CHECK: .if \uaccess
-// CHECK-NOT: .if 곎ss
-// CHECK: my_macro trace=1
-        my_macro trace=1
-.endif
-.endm
-
-foo:
-        my_macro trace=0

test/CodeGen/c-unicode.c

-// RUN: %clang -S %s -o - | FileCheck %s -check-prefix=ALLOWED
-// RUN: not %clang -std=c89 -S %s -o - 2>&1 | FileCheck %s -check-prefix=DENIED
-int \uaccess = 0;
-// ALLOWED: "곎ss":
-// ALLOWED-NOT: "\uaccess":
-// DENIED: warning: universal character names are only valid in C99 or C++; treating as '\' followed by identifier [-Wunicode]
-// DENIED: error: expected identifier or '('