[flang] Allow ! and // comments after some preprocessing directives

[klausler]

Summary

Old-style C /*comments*/ are omitted from preprocessor directive
token sequences by the prescanner, but line-ending C++ and Fortran
free-form comments are not since their handling might depend on
the directive. Add code to skip these line-ending comments as
appropriate in place of existing code that just skipped blanks.

[klausler]

Added test.

flang/lib/Parser/preprocessor.cpp

if (++j < tokens && dir.TokenAt(j).size() == 1 &&
           std::make_pair(nameToken, Definition{dir, j, tokens - j}));
     }
   } else if (dirName == "undef") {
     if (nameToken.empty()) {
       prescanner->Say(
           dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
           "# missing or invalid name"_err_en_US);
     } else {
-      j = dir.SkipBlanks(j + 1);
-      if (j != tokens) {
+      if (dir.IsAnythingLeft(++j)) {
         prescanner->Say(dir.GetIntervalProvenanceRange(j, tokens - j),
-            "#undef: excess tokens at end of directive"_err_en_US);
+            "#undef: excess tokens at end of directive"_en_US);
       } else {
         definitions_.erase(nameToken);
       }
     }
   } else if (dirName == "ifdef" || dirName == "ifndef") {
     bool doThen{false};
     if (nameToken.empty()) {
       prescanner->Say(
           dir.GetIntervalProvenanceRange(dirOffset, tokens - dirOffset),
           "#%s: missing name"_err_en_US, dirName);
     } else {
-      j = dir.SkipBlanks(j + 1);
-      if (j != tokens) {
+      if (dir.IsAnythingLeft(++j)) {
         prescanner->Say(dir.GetIntervalProvenanceRange(j, tokens - j),
             "#%s: excess tokens at end of directive"_en_US, dirName);
       }
       doThen = IsNameDefined(nameToken) == (dirName == "ifdef");
     }
     if (doThen) {
       ifStack_.push(CanDeadElseAppear::Yes);
     } else {
       SkipDisabledConditionalCode(dirName, IsElseActive::Yes, prescanner,
           dir.GetTokenProvenance(dirOffset));
     }
   } else if (dirName == "if") {
     if (IsIfPredicateTrue(dir, j, tokens - j, prescanner)) {
       ifStack_.push(CanDeadElseAppear::Yes);
     } else {
       SkipDisabledConditionalCode(dirName, IsElseActive::Yes, prescanner,
           dir.GetTokenProvenanceRange(dirOffset));
     }
   } else if (dirName == "else") {
-    if (j != tokens) {
+    if (dir.IsAnythingLeft(j)) {
       prescanner->Say(dir.GetIntervalProvenanceRange(j, tokens - j),
-          "#else: excess tokens at end of directive"_err_en_US);
+          "#else: excess tokens at end of directive"_en_US);
     } else if (ifStack_.empty()) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#else: not nested within #if, #ifdef, or #ifndef"_err_en_US);
     } else if (ifStack_.top() != CanDeadElseAppear::Yes) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#else: already appeared within this #if, #ifdef, or #ifndef"_err_en_US);
     } else {
       ifStack_.pop();
       SkipDisabledConditionalCode("else", IsElseActive::No, prescanner,
           dir.GetTokenProvenanceRange(dirOffset));
     }
   } else if (dirName == "elif") {
     if (ifStack_.empty()) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#elif: not nested within #if, #ifdef, or #ifndef"_err_en_US);
     } else if (ifStack_.top() != CanDeadElseAppear::Yes) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#elif: #else previously appeared within this #if, #ifdef, or #ifndef"_err_en_US);
     } else {
       ifStack_.pop();
       SkipDisabledConditionalCode("elif", IsElseActive::No, prescanner,
           dir.GetTokenProvenanceRange(dirOffset));
     }
   } else if (dirName == "endif") {
-    if (j != tokens) {
+    if (dir.IsAnythingLeft(j)) {
       prescanner->Say(dir.GetIntervalProvenanceRange(j, tokens - j),
-          "#endif: excess tokens at end of directive"_err_en_US);
+          "#endif: excess tokens at end of directive"_en_US);
     } else if (ifStack_.empty()) {
       prescanner->Say(dir.GetTokenProvenanceRange(dirOffset),
           "#endif: no #if, #ifdef, or #ifndef"_err_en_US);
     } else {
       ifStack_.pop();
     }
   } else if (dirName == "error") {
     prescanner->Say(

flang/lib/Parser/token-sequence.h

public:
   }
   char CharAt(std::size_t j) const { return char_.at(j); }
   CharBlock CurrentOpenToken() const {
     return {&char_[nextStart_], char_.size() - nextStart_};
   }
 
   std::size_t SkipBlanks(std::size_t) const;
 
+  // True if anything remains in the sequence at & after the given offset
+  // except blanks and line-ending C++ and Fortran free-form comments.
+  bool IsAnythingLeft(std::size_t) const;
+
   void PutNextTokenChar(char ch, Provenance provenance) {
     char_.emplace_back(ch);
     provenances_.Put({provenance, 1});
   }
 
   void CloseToken() {
     start_.emplace_back(nextStart_);
     nextStart_ = char_.size();

flang/lib/Parser/token-sequence.cpp

std::size_t TokenSequence::SkipBlanks(std::size_t at) const {
   for (; at < tokens; ++at) {
     if (!TokenAt(at).IsBlank()) {
       return at;
     }
   }
   return tokens; // even if at > tokens
 }
 
+// C-style /*comments*/ are removed from preprocessing directive
+// token sequences by the prescanner, but not C++ or Fortran
+// free-form line-ending comments (//...  and !...) because
+// ignoring them is directive-specific.
+bool TokenSequence::IsAnythingLeft(std::size_t at) const {
+  std::size_t tokens{start_.size()};
+  for (; at < tokens; ++at) {
+    auto tok{TokenAt(at)};
+    const char *end{tok.end()};
+    for (const char *p{tok.begin()}; p < end; ++p) {
+      switch (*p) {
+      case '/':
+        return p + 1 >= end || p[1] != '/';
+      case '!':
+        return false;
+      case ' ':
+        break;
+      default:
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 void TokenSequence::RemoveLastToken() {
   CHECK(!start_.empty());
   CHECK(nextStart_ > start_.back());
   std::size_t bytes{nextStart_ - start_.back()};
   nextStart_ = start_.back();
   start_.pop_back();
   char_.erase(char_.begin() + nextStart_, char_.end());
   provenances_.RemoveLastBytes(bytes);

flang/test/Parser/pp-dir-comments.f90

+! RUN: %f18 -funparse %s 2>&1 | FileCheck %s
+
+#define pmk
+#ifdef pmk // comment
+! CHECK: t1
+real t1
+#endif // comment
+#undef pmk ! comment
+#ifndef pmk ! comment
+! CHECK: t2
+real t2
+#endif // comment
+#if 0 /* C comment */ + 0
+! CHECK-NOT: misinterpreted
+# error misinterpreted #if
+#else // comment
+! CHECK: END PROGRAM
+end
+#endif ! comment