[flang] Fix edge case with Hollerith

[klausler]

Summary

To prevent mistokenization of CHARACTER*2HXY as a Hollerith
literal constant while allowing it in DATA A/2*2HXY/, there's
a little state that tracks whether a / has been seen earlier
in the same statement. But it was being reset on each line,
not statement, so Hollerith in a DATA statement continuation
line was incorrectly tokenized. Fixed.

[PeteSteinfeld]

All builds, tests, and looks good.

flang/lib/Parser/prescan.h

struct LineClassification {
     std::size_t payloadOffset; // byte offset of content
     const char *sentinel; // if it's a compiler directive
   };
 
   void BeginSourceLine(const char *at) {
     at_ = at;
     column_ = 1;
     tabInCurrentLine_ = false;
-    slashInCurrentLine_ = false;
-    preventHollerith_ = false;
-    delimiterNesting_ = 0;
   }
 
   void BeginSourceLineAndAdvance() {
     BeginSourceLine(nextLine_);
     NextLine();
   }
 
+  void BeginStatementAndAdvance() {
+    BeginSourceLineAndAdvance();
+    slashInCurrentStatement_ = false;
+    preventHollerith_ = false;
+    delimiterNesting_ = 0;
+  }
+
   Provenance GetProvenance(const char *sourceChar) const {
     return startProvenance_ + (sourceChar - start_);
   }
 
   ProvenanceRange GetProvenanceRange(
       const char *first, const char *afterLast) const {
     std::size_t bytes = afterLast - first;
     return {startProvenance_ + (first - start_), bytes};
private:
   const char *nextLine_{nullptr}; // next line to process; <= limit_
   const char *directiveSentinel_{nullptr}; // current compiler directive
 
   // This data members are state for processing the source line containing
   // "at_", which goes to up to the newline character before "nextLine_".
   const char *at_{nullptr}; // next character to process; < nextLine_
   int column_{1}; // card image column position of next character
   bool tabInCurrentLine_{false};
-  bool slashInCurrentLine_{false};
-  bool preventHollerith_{false};
+  bool slashInCurrentStatement_{false};
+  bool preventHollerith_{false}; // CHARACTER*4HIMOM not Hollerith
   bool inCharLiteral_{false};
   bool inPreprocessorDirective_{false};
 
   // In some edge cases of compiler directive continuation lines, it
   // is necessary to treat the line break as a space character by
   // setting this flag, which is cleared by EmitChar().
   bool insertASpace_{false};
 

flang/lib/Parser/prescan.cpp

void Prescanner::Statement() {
   case LineClassification::Kind::IncludeDirective:
   case LineClassification::Kind::DefinitionDirective:
   case LineClassification::Kind::PreprocessorDirective:
     preprocessor_.Directive(TokenizePreprocessorDirective(), this);
     return;
   case LineClassification::Kind::CompilerDirective:
     directiveSentinel_ = line.sentinel;
     CHECK(InCompilerDirective());
-    BeginSourceLineAndAdvance();
+    BeginStatementAndAdvance();
     if (inFixedForm_) {
       CHECK(IsFixedFormCommentChar(*at_));
     } else {
       while (*at_ == ' ' || *at_ == '\t') {
         ++at_, ++column_;
       }
       CHECK(*at_ == '!');
     }
if (directiveSentinel_[0] == '$' && directiveSentinel_[1] == '\0') {
       if (*at_ == ' ') {
         EmitChar(tokens, ' ');
         ++at_, ++column_;
       }
       tokens.CloseToken();
     }
     break;
   case LineClassification::Kind::Source:
-    BeginSourceLineAndAdvance();
+    BeginStatementAndAdvance();
     if (inFixedForm_) {
       LabelField(tokens);
     } else if (skipLeadingAmpersand_) {
       skipLeadingAmpersand_ = false;
       const char *p{SkipWhiteSpace(at_)};
       if (p < limit_ && *p == '&') {
         column_ += ++p - at_;
         at_ = p;
if (omitNewline_) {
     cooked_.Put('\n', newlineProvenance);
   }
   directiveSentinel_ = nullptr;
 }
 
 TokenSequence Prescanner::TokenizePreprocessorDirective() {
   CHECK(nextLine_ < limit_ && !inPreprocessorDirective_);
   inPreprocessorDirective_ = true;
-  BeginSourceLineAndAdvance();
+  BeginStatementAndAdvance();
   TokenSequence tokens;
   while (NextToken(tokens)) {
   }
   inPreprocessorDirective_ = false;
   return tokens;
 }
 
 void Prescanner::NextLine() {
if (!inPreprocessorDirective_ && IsDecimalDigit(nch)) {
       EmitCharAndAdvance(tokens, '.'); // variadic macro definition ellipsis
     }
     preventHollerith_ = false;
   } else if (IsLegalInIdentifier(*at_)) {
     do {
     } while (IsLegalInIdentifier(EmitCharAndAdvance(tokens, *at_)));
     if (*at_ == '\'' || *at_ == '"') {
       QuotedCharacterLiteral(tokens, start);
-      preventHollerith_ = false;
-    } else {
-      // Subtle: Don't misrecognize labeled DO statement label as Hollerith
-      // when the loop control variable starts with 'H'.
-      preventHollerith_ = true;
     }
+    preventHollerith_ = false;
   } else if (*at_ == '*') {
     if (EmitCharAndAdvance(tokens, '*') == '*') {
       EmitCharAndAdvance(tokens, '*');
     } else {
       // Subtle ambiguity:
       //  CHARACTER*2H     declares H because *2 is a kind specifier
       //  DATAC/N*2H  /    is repeated Hollerith
-      preventHollerith_ = !slashInCurrentLine_;
+      preventHollerith_ = !slashInCurrentStatement_;
     }
   } else {
     char ch{*at_};
     if (ch == '(' || ch == '[') {
       ++delimiterNesting_;
     } else if ((ch == ')' || ch == ']') && delimiterNesting_ > 0) {
       --delimiterNesting_;
     }
     char nch{EmitCharAndAdvance(tokens, ch)};
     preventHollerith_ = false;
     if ((nch == '=' &&
             (ch == '<' || ch == '>' || ch == '/' || ch == '=' || ch == '!')) ||
         (ch == nch &&
             (ch == '/' || ch == ':' || ch == '*' || ch == '#' || ch == '&' ||
                 ch == '|' || ch == '<' || ch == '>')) ||
         (ch == '=' && nch == '>')) {
       // token comprises two characters
       EmitCharAndAdvance(tokens, nch);
     } else if (ch == '/') {
-      slashInCurrentLine_ = true;
+      slashInCurrentStatement_ = true;
     }
   }
   tokens.CloseToken();
   return true;
 }
 
 bool Prescanner::ExponentAndKind(TokenSequence &tokens) {
   char ed{ToLowerCaseLetter(*at_)};

flang/test/Preprocessing/hollerith.f

+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: character*1hi
+! CHECK: dataa/1*1h /
+! CHECK: datab/1*1h /
+! CHECK: do1h=1,2
+      CHARACTER*1H I
+      CHARACTER*1 A,B
+      INTEGER H
+      DATA A/1*1H /
+      DATA B/
+     +1*1H /
+      DO1H =1,2
+ 1    CONTINUE
+      END