[pseudo] Only expand UCNs for raw_identifiers

[sammccall]

Summary

It turns out clang::expandUCNs only works on tokens that contain valid UCNs
and no other random escapes, and clang only uses it on raw_identifiers.

Currently we can hit an assertion by creating tokens with stray non-valid-UCN
backslashes in them.

Fortunately, expanding UCNs in raw_identifiers is actually all we need.
Most tokens (keywords, punctuation) can't have them. UCNs in literals can be
treated as escape sequences like \n even this isn't the standard's
interpretation. This more or less matches how clang works.
(See https://isocpp.org/files/papers/P2194R0.pdf which points out that the
standard's description of how UCNs work is misaligned with real implementations)

[sammccall]

rename testcase to be more descriptive, add comment

[sammccall]

oops, forgot testcase

[hokein]

clang-tools-extra/pseudo/test/crash/backslashes.c

3

nit: I'd add --print-tokens to make the purpose of this test clearer.

clang-tools-extra/pseudo/include/clang-pseudo/Token.h

enum class LexFlags : uint8_t {
   /// The text() of such tokens will contain the raw trigrah.
   NeedsCleaning = 1 << 1,
 };
 /// A generic lang options suitable for lexing/parsing a langage.
 clang::LangOptions genericLangOpts(
     clang::Language = clang::Language::CXX,
     clang::LangStandard::Kind = clang::LangStandard::lang_unspecified);
 
-/// Derives a token stream by decoding escapes, interpreting raw_identifiers and
-/// splitting the greatergreater token.
+/// Decoding raw tokens written in the source code, returning a derived stream.
 ///
-/// Tokens containing UCNs, escaped newlines, trigraphs etc are decoded and
-/// their backing data is owned by the returned stream.
-/// raw_identifier tokens are assigned specific types (identifier, keyword etc).
+/// - escaped newlines within tokens are removed
+/// - trigraphs are replaced with the characters they encode
+/// - UCNs within raw_identifiers are replaced by the characters they encode
+///   (UCNs within strings, comments etc are not translated)
+/// - raw_identifier tokens are assigned their correct keyword type
+/// - the >> token is split into separate > > tokens
+///   (we use a modified grammar where >> is a nonterminal, not a token)
 ///
 /// The StartsPPLine flag is preserved.
 ///
 /// Formally the identifier correctly happens before preprocessing, while we
 /// should only cook raw_identifiers that survive preprocessing.
 /// However, ignoring the Token::Kind of tokens in directives achieves the same.
 /// (And having cooked token kinds in PP-disabled sections is useful for us).
 TokenStream cook(const TokenStream &, const clang::LangOptions &);
 
 /// Drops comment tokens.
 TokenStream stripComments(const TokenStream &);
 
 } // namespace pseudo
 } // namespace clang
 
 #endif // CLANG_PSEUDO_TOKEN_H

clang-tools-extra/pseudo/lib/Lex.cpp

if (Tok.flag(LexFlags::NeedsCleaning)) {
       const char *Pos = Tok.text().begin();
       while (Pos < Tok.text().end()) {
         unsigned CharSize = 0;
         CleanBuffer.push_back(
             clang::Lexer::getCharAndSizeNoWarn(Pos, CharSize, LangOpts));
         assert(CharSize != 0 && "no progress!");
         Pos += CharSize;
       }
-      // Remove universal character names (UCN).
+      llvm::StringRef Text = CleanBuffer;
       llvm::SmallString<64> UCNBuffer;
+      // A surface reading of the standard suggests UCNs might appear anywhere.
+      // But we need only decode them in raw_identifiers.
+      //  - they cannot appear in punctuation/keyword tokens, because UCNs
+      //    cannot encode basic characters outside of literals [lex.charset]
+      //  - they can appear in literals, but we need not unescape them now.
+      //    We treat them as escape sequences when evaluating the literal.
+      //  - comments are handled similarly to literals
+      // This is good fortune, because expandUCNs requires its input to be a
+      // reasonably valid identifier (e.g. without stray backslashes).
+      if (Tok.Kind == tok::raw_identifier) {
         clang::expandUCNs(UCNBuffer, CleanBuffer);
+        Text = UCNBuffer;
+      }
 
-      llvm::StringRef Text = llvm::StringRef(UCNBuffer).copy(*CleanedStorage);
-      Tok.Data = Text.data();
+      Tok.Data = Text.copy(*CleanedStorage).data();
       Tok.Length = Text.size();
       Tok.Flags &= ~static_cast<decltype(Tok.Flags)>(LexFlags::NeedsCleaning);
     }
 
     if (Tok.Kind == tok::raw_identifier) {
       // Cook raw_identifiers into identifier, keyword, etc.
       Tok.Kind = Identifiers.get(Tok.text()).getTokenID();
     } else if (Tok.Kind == tok::greatergreater) {

clang-tools-extra/pseudo/test/crash/backslashes.c

+// We used to try to interpret these backslashes as UCNs.
+// RUN: clang-pseudo -source=%s -print-tokens
+\
+\ x

clang-tools-extra/pseudo/tool/ClangPseudo.cpp

 #include "clang-pseudo/LRGraph.h"
 #include "clang-pseudo/LRTable.h"
 #include "clang-pseudo/Token.h"
 #include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Signals.h"
 
 using clang::pseudo::Grammar;
 using llvm::cl::desc;
 using llvm::cl::init;
 using llvm::cl::opt;
 
 static opt<std::string>
     Grammar("grammar", desc("Parse and check a BNF grammar file."), init(""));
llvm::errs() << "Error: can't read grammar file '" << Path
                  << "': " << EC.message() << "\n";
     ::exit(1);
   }
   return Text.get()->getBuffer().str();
 }
 
 int main(int argc, char *argv[]) {
   llvm::cl::ParseCommandLineOptions(argc, argv, "");
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
   clang::LangOptions LangOpts = clang::pseudo::genericLangOpts();
   std::string SourceText;
   llvm::Optional<clang::pseudo::TokenStream> RawStream;
   llvm::Optional<clang::pseudo::DirectiveTree> DirectiveStructure;
   llvm::Optional<clang::pseudo::TokenStream> ParseableStream;
   if (Source.getNumOccurrences()) {
     SourceText = readOrDie(Source);