Add PragmaHandler for MSVC pragma execution_character_set

[sigatrev]

Summary

__pragma(execution_character_set(push, "UTF-8")) is used in TraceLoggingProvider.h. This commit
implements a no-op handler for compatability, similar to how the flag -fexec_charset is handled.

[rnk]

Looks good and thorough, but it needs tests.

[sigatrev]

Added tests, and slightly improved associated messaging.

[rnk]

lgtm, thanks! Would you like someone to land this for you?

[sigatrev]

Yeah, that would be great. Thanks!

[thakis]

Does our serialization machinery serialize these correctly automatically? What happens if you compiler a header saying execution_character_set_push, "utf-8") in a header, compile that to a pch, then include that through the pch in a .c file that does pop?

My guess is that this either needs (de)serialization code or a warning when the stack is not empty when a pch is written.

(Examples: https://reviews.llvm.org/rL262493 https://reviews.llvm.org/rL262539 -- the latter one is probably a better example)

[sigatrev]

This implementation doesn't track the push/pop stack, it just verifies the synax is valid and moves on. I modeled it after the PragmaWarningHandler which does the same, and thought it would be fine in this case since the only accepted value is a no-op.

[thakis]

Ah ok, then I agree this doesn't need serialization yet :)

It'd be nice to at least diag that the stack use is valid (also for the warning stack), but it doesn't have to be in this change.

clang-tools-extra/pp-trace/PPCallbacksTracker.h

public:
   void PragmaOpenCLExtension(clang::SourceLocation NameLoc,
                              const clang::IdentifierInfo *Name,
                              clang::SourceLocation StateLoc,
                              unsigned State) override;
   void PragmaWarning(clang::SourceLocation Loc, llvm::StringRef WarningSpec,
                      llvm::ArrayRef<int> Ids) override;
   void PragmaWarningPush(clang::SourceLocation Loc, int Level) override;
   void PragmaWarningPop(clang::SourceLocation Loc) override;
+  void PragmaExecCharsetPush(clang::SourceLocation Loc,
+                             clang::StringRef Str) override;
+  void PragmaExecCharsetPop(clang::SourceLocation Loc) override;
   void MacroExpands(const clang::Token &MacroNameTok,
                     const clang::MacroDefinition &MD, clang::SourceRange Range,
                     const clang::MacroArgs *Args) override;
   void MacroDefined(const clang::Token &MacroNameTok,
                     const clang::MacroDirective *MD) override;
   void MacroUndefined(const clang::Token &MacroNameTok,
                       const clang::MacroDefinition &MD,
                       const clang::MacroDirective *Undef) override;

clang-tools-extra/pp-trace/PPCallbacksTracker.cpp

 }
 
 // Callback invoked when a #pragma warning(pop) directive is read.
 void PPCallbacksTracker::PragmaWarningPop(clang::SourceLocation Loc) {
   beginCallback("PragmaWarningPop");
   appendArgument("Loc", Loc);
 }
 
+// Callback invoked when a #pragma execution_character_set(push) directive
+// is read.
+void PPCallbacksTracker::PragmaExecCharsetPush(clang::SourceLocation Loc,
+                                               clang::StringRef Str) {
+  beginCallback("PragmaExecCharsetPush");
+  appendArgument("Loc", Loc);
+  appendArgument("Charset", Str);
+}
+
+// Callback invoked when a #pragma execution_character_set(pop) directive
+// is read.
+void PPCallbacksTracker::PragmaExecCharsetPop(clang::SourceLocation Loc) {
+  beginCallback("PragmaExecCharsetPop");
+  appendArgument("Loc", Loc);
+}
+
 // Called by Preprocessor::HandleMacroExpandedIdentifier when a
 // macro invocation is found.
 void
 PPCallbacksTracker::MacroExpands(const clang::Token &MacroNameTok,
                                  const clang::MacroDefinition &MacroDefinition,
                                  clang::SourceRange Range,
                                  const clang::MacroArgs *Args) {
   beginCallback("MacroExpands");

clang/include/clang/Basic/DiagnosticLexKinds.td

def warn_pragma_warning_spec_invalid :
   InGroup<UnknownPragmas>;
 def warn_pragma_warning_push_level :
   ExtWarn<"#pragma warning(push, level) requires a level between 0 and 4">,
   InGroup<UnknownPragmas>;
 def warn_pragma_warning_expected_number :
   ExtWarn<"#pragma warning expected a warning number">,
   InGroup<UnknownPragmas>;
 
+// - #pragma execution_character_set(...)
+def warn_pragma_exec_charset_expected :
+  ExtWarn<"#pragma execution_character_set expected '%0'">,
+  InGroup<UnknownPragmas>;
+def warn_pragma_exec_charset_spec_invalid :
+  ExtWarn<"#pragma execution_character_set expected 'push' or 'pop'">,
+  InGroup<UnknownPragmas>;
+def warn_pragma_exec_charset_invalid_token :
+   ExtWarn<"unexpected token in pragma execution_character_set">,
+   InGroup<UnknownPragmas>;
+
 def err__Pragma_malformed : Error<
   "_Pragma takes a parenthesized string literal">;
 def err_pragma_message_malformed : Error<
   "pragma %select{message|warning|error}0 requires parenthesized string">;
 def err_pragma_push_pop_macro_malformed : Error<
    "pragma %0 requires a parenthesized string">;
 def warn_pragma_pop_macro_no_push : Warning<
    "pragma pop_macro could not pop '%0', no matching push_macro">,

clang/include/clang/Lex/PPCallbacks.h

public:
   /// Callback invoked when a \#pragma warning(push) directive is read.
   virtual void PragmaWarningPush(SourceLocation Loc, int Level) {
   }
 
   /// Callback invoked when a \#pragma warning(pop) directive is read.
   virtual void PragmaWarningPop(SourceLocation Loc) {
   }
 
+  /// Callback invoked when a \#pragma execution_character_set(push) directive
+  /// is read.
+  virtual void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) {}
+
+  /// Callback invoked when a \#pragma execution_character_set(pop) directive
+  /// is read.
+  virtual void PragmaExecCharsetPop(SourceLocation Loc) {}
+
   /// Callback invoked when a \#pragma clang assume_nonnull begin directive
   /// is read.
   virtual void PragmaAssumeNonNullBegin(SourceLocation Loc) {}
 
   /// Callback invoked when a \#pragma clang assume_nonnull end directive
   /// is read.
   virtual void PragmaAssumeNonNullEnd(SourceLocation Loc) {}
 
void PragmaWarningPush(SourceLocation Loc, int Level) override {
     Second->PragmaWarningPush(Loc, Level);
   }
 
   void PragmaWarningPop(SourceLocation Loc) override {
     First->PragmaWarningPop(Loc);
     Second->PragmaWarningPop(Loc);
   }
 
+  void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) override {
+    First->PragmaExecCharsetPush(Loc, Str);
+    Second->PragmaExecCharsetPush(Loc, Str);
+  }
+
+  void PragmaExecCharsetPop(SourceLocation Loc) override {
+    First->PragmaExecCharsetPop(Loc);
+    Second->PragmaExecCharsetPop(Loc);
+  }
+
   void PragmaAssumeNonNullBegin(SourceLocation Loc) override {
     First->PragmaAssumeNonNullBegin(Loc);
     Second->PragmaAssumeNonNullBegin(Loc);
   }
 
   void PragmaAssumeNonNullEnd(SourceLocation Loc) override {
     First->PragmaAssumeNonNullEnd(Loc);
     Second->PragmaAssumeNonNullEnd(Loc);

clang/lib/Frontend/PrintPreprocessedOutput.cpp

public:
   void PragmaDiagnosticPush(SourceLocation Loc, StringRef Namespace) override;
   void PragmaDiagnosticPop(SourceLocation Loc, StringRef Namespace) override;
   void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
                         diag::Severity Map, StringRef Str) override;
   void PragmaWarning(SourceLocation Loc, StringRef WarningSpec,
                      ArrayRef<int> Ids) override;
   void PragmaWarningPush(SourceLocation Loc, int Level) override;
   void PragmaWarningPop(SourceLocation Loc) override;
+  void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) override;
+  void PragmaExecCharsetPop(SourceLocation Loc) override;
   void PragmaAssumeNonNullBegin(SourceLocation Loc) override;
   void PragmaAssumeNonNullEnd(SourceLocation Loc) override;
 
   bool HandleFirstTokOnLine(Token &Tok);
 
   /// Move to the line of the provided source location. This will
   /// return true if the output stream required adjustment or if
   /// the requested location is on the first line.

 
 void PrintPPOutputPPCallbacks::PragmaWarningPop(SourceLocation Loc) {
   startNewLineIfNeeded();
   MoveToLine(Loc);
   OS << "#pragma warning(pop)";
   setEmittedDirectiveOnThisLine();
 }
 
+void PrintPPOutputPPCallbacks::PragmaExecCharsetPush(SourceLocation Loc,
+                                                     StringRef Str) {
+  startNewLineIfNeeded();
+  MoveToLine(Loc);
+  OS << "#pragma character_execution_set(push";
+  if (!Str.empty())
+    OS << ", " << Str;
+  OS << ')';
+  setEmittedDirectiveOnThisLine();
+}
+
+void PrintPPOutputPPCallbacks::PragmaExecCharsetPop(SourceLocation Loc) {
+  startNewLineIfNeeded();
+  MoveToLine(Loc);
+  OS << "#pragma character_execution_set(pop)";
+  setEmittedDirectiveOnThisLine();
+}
+
 void PrintPPOutputPPCallbacks::
 PragmaAssumeNonNullBegin(SourceLocation Loc) {
   startNewLineIfNeeded();
   MoveToLine(Loc);
   OS << "#pragma clang assume_nonnull begin";
   setEmittedDirectiveOnThisLine();
 }
 

clang/lib/Lex/Pragma.cpp

void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
     }
 
     PP.Lex(Tok);
     if (Tok.isNot(tok::eod))
       PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma warning";
   }
 };
 
+/// "\#pragma execution_character_set(...)". MSVC supports this pragma only
+/// for "UTF-8". We parse it and ignore it if UTF-8 is provided and warn
+/// otherwise to avoid -Wunknown-pragma warnings.
+struct PragmaExecCharsetHandler : public PragmaHandler {
+  PragmaExecCharsetHandler() : PragmaHandler("execution_character_set") {}
+
+  void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                    Token &Tok) override {
+    // Parse things like:
+    // execution_character_set(push, "UTF-8")
+    // execution_character_set(pop)
+    SourceLocation DiagLoc = Tok.getLocation();
+    PPCallbacks *Callbacks = PP.getPPCallbacks();
+
+    PP.Lex(Tok);
+    if (Tok.isNot(tok::l_paren)) {
+      PP.Diag(Tok, diag::warn_pragma_exec_charset_expected) << "(";
+      return;
+    }
+
+    PP.Lex(Tok);
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+
+    if (II && II->isStr("push")) {
+      // #pragma execution_character_set( push[ , string ] )
+      PP.Lex(Tok);
+      if (Tok.is(tok::comma)) {
+        PP.Lex(Tok);
+
+        std::string ExecCharset;
+        if (!PP.FinishLexStringLiteral(Tok, ExecCharset,
+                                       "pragma execution_character_set",
+                                       /*MacroExpansion=*/false))
+          return;
+
+        // MSVC supports either of these, but nothing else.
+        if (ExecCharset != "UTF-8" && ExecCharset != "utf-8") {
+          PP.Diag(Tok, diag::warn_pragma_exec_charset_spec_invalid);
+          return;
+        }
+      }
+      if (Callbacks)
+        Callbacks->PragmaExecCharsetPush(DiagLoc, "UTF-8");
+    } else if (II && II->isStr("pop")) {
+      // #pragma execution_character_set( pop )
+      PP.Lex(Tok);
+      if (Callbacks)
+        Callbacks->PragmaExecCharsetPop(DiagLoc);
+    } else {
+      PP.Diag(Tok, diag::warn_pragma_exec_charset_invalid_token);
+      return;
+    }
+
+    if (Tok.isNot(tok::r_paren)) {
+      PP.Diag(Tok, diag::warn_pragma_exec_charset_expected) << ")";
+      return;
+    }
+
+    PP.Lex(Tok);
+    if (Tok.isNot(tok::eod))
+      PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma execution_character_set";
+  }
+};
+
 /// PragmaIncludeAliasHandler - "\#pragma include_alias("...")".
 struct PragmaIncludeAliasHandler : public PragmaHandler {
   PragmaIncludeAliasHandler() : PragmaHandler("include_alias") {}
 
   void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                     Token &IncludeAliasTok) override {
     PP.HandlePragmaIncludeAlias(IncludeAliasTok);
   }
void Preprocessor::RegisterBuiltinPragmas() {
 
   // Add region pragmas.
   AddPragmaHandler(new PragmaRegionHandler("region"));
   AddPragmaHandler(new PragmaRegionHandler("endregion"));
 
   // MS extensions.
   if (LangOpts.MicrosoftExt) {
     AddPragmaHandler(new PragmaWarningHandler());
+    AddPragmaHandler(new PragmaExecCharsetHandler());
     AddPragmaHandler(new PragmaIncludeAliasHandler());
     AddPragmaHandler(new PragmaHdrstopHandler());
   }
 
   // Pragmas added by plugins
   for (PragmaHandlerRegistry::iterator it = PragmaHandlerRegistry::begin(),
                                        ie = PragmaHandlerRegistry::end();
        it != ie; ++it) {