[flang] More clean-up of CookedSource API

[klausler]

Summary

The std::string holding the content of a CookedSource no longer
needs to be exposed in its API after the recent work that allows
the parsing context to hold multiple instances of a CookedSource.
So clean the API. These changes were extracted from some work in
progress that was made easier by the API changes.

[PeteSteinfeld]

All of these changes build, test, and look good.

Note that I'm seeing diffs in f18.cpp when I run clang-format, but they're unrelated to these changes.

flang/include/flang/Parser/parse-state.h

 namespace Fortran::parser {
 
 using common::LanguageFeature;
 
 class ParseState {
 public:
   // TODO: Add a constructor for parsing a normalized module file.
   ParseState(const CookedSource &cooked)
-      : p_{&cooked.data().front()}, limit_{&cooked.data().back() + 1} {}
+      : p_{cooked.AsCharBlock().begin()}, limit_{cooked.AsCharBlock().end()} {}
   ParseState(const ParseState &that)
       : p_{that.p_}, limit_{that.limit_}, context_{that.context_},
         userState_{that.userState_}, inFixedForm_{that.inFixedForm_},
         anyErrorRecovery_{that.anyErrorRecovery_},
         anyConformanceViolation_{that.anyConformanceViolation_},
         deferMessages_{that.deferMessages_},
         anyDeferredMessages_{that.anyDeferredMessages_},
         anyTokenMatched_{that.anyTokenMatched_} {}

flang/include/flang/Parser/provenance.h

public:
   bool IsValid(Provenance at) const { return range_.Contains(at); }
   bool IsValid(ProvenanceRange range) const {
     return range.size() > 0 && range_.Contains(range);
   }
   void EmitMessage(llvm::raw_ostream &, const std::optional<ProvenanceRange> &,
       const std::string &message, bool echoSourceLine = false) const;
   const SourceFile *GetSourceFile(
       Provenance, std::size_t *offset = nullptr) const;
+  const char *GetSource(ProvenanceRange) const;
   std::optional<SourcePosition> GetSourcePosition(Provenance) const;
   std::optional<ProvenanceRange> GetFirstFileProvenance() const;
   std::string GetPath(Provenance) const; // __FILE__
   int GetLineNumber(Provenance) const; // __LINE__
   Provenance CompilerInsertionProvenance(char ch);
   Provenance CompilerInsertionProvenance(const char *, std::size_t);
   ProvenanceRange IntersectionWithSourceFiles(ProvenanceRange) const;
   llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
private:
   Encoding encoding_{Encoding::UTF_8};
 };
 
 // Represents the result of preprocessing and prescanning a single source
 // file (and all its inclusions) or module file.  Parsers operate within
 // single instances of CookedSource.
 class CookedSource {
 public:
-  const std::string &data() const { return data_; }
-
-  bool Contains(const char *p) const {
-    return p >= &data_.front() && p <= &data_.back() + 1;
-  }
-  bool Contains(CharBlock range) const {
-    return !range.empty() && Contains(range.begin()) &&
-        Contains(range.end() - 1);
-  }
-
+  CharBlock AsCharBlock() const { return CharBlock{data_}; }
   std::optional<ProvenanceRange> GetProvenanceRange(CharBlock) const;
   std::optional<CharBlock> GetCharBlock(ProvenanceRange) const;
 
   // The result of a Put() is the offset that the new data
   // will have in the eventually marshaled contiguous buffer.
   std::size_t Put(const char *data, std::size_t bytes) {
     return buffer_.Put(data, bytes);
   }
   std::size_t Put(const std::string &s) { return buffer_.Put(s); }
   std::size_t Put(char ch) { return buffer_.Put(&ch, 1); }
   std::size_t Put(char ch, Provenance p) {
     provenanceMap_.Put(ProvenanceRange{p, 1});
     return buffer_.Put(&ch, 1);
   }
 
   void PutProvenance(Provenance p) { provenanceMap_.Put(ProvenanceRange{p}); }
   void PutProvenance(ProvenanceRange pr) { provenanceMap_.Put(pr); }
   void PutProvenanceMappings(const OffsetToProvenanceMappings &pm) {
     provenanceMap_.Put(pm);
   }
 
   std::size_t BufferedBytes() const;
   void Marshal(AllSources &); // marshals text into one contiguous block
   void CompileProvenanceRangeToOffsetMappings(AllSources &);
-  std::string AcquireData() { return std::move(data_); }
   llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
 
 private:
   CharBuffer buffer_; // before Marshal()
   std::string data_; // all of it, prescanned and preprocessed
   OffsetToProvenanceMappings provenanceMap_;
   ProvenanceRangeToOffsetMappings invertedMap_;
 };
 
 class AllCookedSources {
 public:
   explicit AllCookedSources(AllSources &);
   ~AllCookedSources();
 
   AllSources &allSources() { return allSources_; }
   const AllSources &allSources() const { return allSources_; }
 
   CookedSource &NewCookedSource();
 
   template <typename A> // const char * or CharBlock
   const CookedSource *Find(A x) const {
     for (const auto &c : cooked_) {
-      if (c.Contains(x)) {
+      if (c.AsCharBlock().Contains(x)) {
         return &c;
       }
     }
     return nullptr;
   }
 
   bool IsValid(ProvenanceRange r) const { return allSources_.IsValid(r); }
 

flang/include/flang/Semantics/semantics.h

private:
   std::map<SymbolRef, const IndexVarInfo> activeIndexVars_;
   std::set<SymbolRef> errorSymbols_;
   std::vector<std::string> tempNames_;
 };
 
 class Semantics {
 public:
   explicit Semantics(SemanticsContext &context, parser::Program &program,
-      const parser::CookedSource &cooked, bool debugModuleWriter = false)
+      parser::CharBlock charBlock, bool debugModuleWriter = false)
       : context_{context}, program_{program} {
     context.set_debugModuleWriter(debugModuleWriter);
-    context.globalScope().AddSourceRange(parser::CharBlock{cooked.data()});
+    context.globalScope().AddSourceRange(charBlock);
   }
 
   SemanticsContext &context() const { return context_; }
   bool Perform();
   const Scope &FindScope(const parser::CharBlock &where) const {
     return context_.FindScope(where);
   }
   bool AnyFatalError() const { return context_.AnyFatalError(); }

flang/lib/Lower/OpenACC.cpp

-//===-- OpenMP.cpp -- OpenACC directive lowering --------------------------===//
+//===-- OpenACC.cpp -- OpenACC directive lowering -------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/

flang/lib/Parser/prescan.cpp

if (*p != ' ') {
       return;
     }
   }
   DIE("compiler directive all blank");
 }
 
 void Prescanner::Prescan(ProvenanceRange range) {
   startProvenance_ = range.start();
-  std::size_t offset{0};
-  const SourceFile *source{
-      allSources_.GetSourceFile(startProvenance_, &offset)};
-  CHECK(source);
-  start_ = source->content().data() + offset;
+  start_ = allSources_.GetSource(range);
+  CHECK(start_);
   limit_ = start_ + range.size();
   nextLine_ = start_;
   const bool beganInFixedForm{inFixedForm_};
   if (prescannerNesting_ > maxPrescannerNesting) {
     Say(GetProvenance(start_),
         "too many nested INCLUDE/#include files, possibly circular"_err_en_US);
     return;
   }
-  while (nextLine_ < limit_) {
+  while (!IsAtEnd()) {
     Statement();
   }
   if (inFixedForm_ != beganInFixedForm) {
     std::string dir{"!dir$ "};
     if (beganInFixedForm) {
       dir += "fixed";
     } else {
       dir += "free";
if (omitNewline_) {
     omitNewline_ = false;
   } else {
     cooked_.Put('\n', newlineProvenance);
   }
   directiveSentinel_ = nullptr;
 }
 
 TokenSequence Prescanner::TokenizePreprocessorDirective() {
-  CHECK(nextLine_ < limit_ && !inPreprocessorDirective_);
+  CHECK(!IsAtEnd() && !inPreprocessorDirective_);
   inPreprocessorDirective_ = true;
   BeginStatementAndAdvance();
   TokenSequence tokens;
   while (NextToken(tokens)) {
   }
   inPreprocessorDirective_ = false;
   return tokens;
 }
if (IsCComment(at_)) {
         NextLine();
       } else {
         // Don't emit any messages about unclosed C-style comments, because
         // the sequence /* can appear legally in a FORMAT statement.  There's
         // no ambiguity, since the sequence */ cannot appear legally.
         break;
       }
     } else if (inPreprocessorDirective_ && at_[0] == '\\' && at_ + 2 < limit_ &&
-        at_[1] == '\n' && nextLine_ < limit_) {
+        at_[1] == '\n' && !IsAtEnd()) {
       BeginSourceLineAndAdvance();
     } else {
       break;
     }
   }
 }
 
 void Prescanner::SkipSpaces() {
const char *Prescanner::IsPreprocessorDirectiveLine(const char *start) const {
   return SkipWhiteSpace(p + 1);
 }
 
 bool Prescanner::IsNextLinePreprocessorDirective() const {
   return IsPreprocessorDirectiveLine(nextLine_) != nullptr;
 }
 
 bool Prescanner::SkipCommentLine(bool afterAmpersand) {
-  if (nextLine_ >= limit_) {
+  if (IsAtEnd()) {
     if (afterAmpersand && prescannerNesting_ > 0) {
       // A continuation marker at the end of the last line in an
       // include file inhibits the newline for that line.
       SkipToEndOfLine();
       omitNewline_ = true;
     }
     return false;
   }
if (lineClass.kind == LineClassification::Kind::Comment) {
     skipLeadingAmpersand_ = true;
     return false;
   } else {
     return false;
   }
 }
 
 const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
-  if (nextLine_ >= limit_) {
+  if (IsAtEnd()) {
     return nullptr;
   }
   tabInCurrentLine_ = false;
   char col1{*nextLine_};
   if (InCompilerDirective()) {
     // Must be a continued compiler directive.
     if (!IsFixedFormCommentChar(col1)) {
       return nullptr;
bool Prescanner::FreeFormContinuation() {
   } while (SkipCommentLine(ampersand));
   return false;
 }
 
 // Implicit line continuation allows a preprocessor macro call with
 // arguments to span multiple lines.
 bool Prescanner::IsImplicitContinuation() const {
   return !inPreprocessorDirective_ && !inCharLiteral_ &&
-      delimiterNesting_ > 0 && nextLine_ < limit_ &&
+      delimiterNesting_ > 0 && !IsAtEnd() &&
       ClassifyLine(nextLine_).kind == LineClassification::Kind::Source;
 }
 
 bool Prescanner::Continuation(bool mightNeedFixedFormSpace) {
   if (*at_ == '\n' || *at_ == '&') {
     if (inFixedForm_) {
       return FixedFormContinuation(mightNeedFixedFormSpace);
     } else {

flang/lib/Parser/provenance.cpp

return std::visit(common::visitors{
                             *offset = 0;
                           }
                           return static_cast<const SourceFile *>(nullptr);
                         },
                     },
       origin.u);
 }
 
+const char *AllSources::GetSource(ProvenanceRange range) const {
+  Provenance start{range.start()};
+  const Origin &origin{MapToOrigin(start)};
+  return origin.covers.Contains(range)
+      ? &origin[origin.covers.MemberOffset(start)]
+      : nullptr;
+}
+
 std::optional<SourcePosition> AllSources::GetSourcePosition(
     Provenance prov) const {
   const Origin &origin{MapToOrigin(prov)};
   if (const auto *inc{std::get_if<Inclusion>(&origin.u)}) {
     std::size_t offset{origin.covers.MemberOffset(prov)};
     return inc->source.FindOffsetLineAndColumn(offset);
   } else {
     return std::nullopt;
while (count > 1) {
     }
   }
   CHECK(origin_[low].covers.Contains(at));
   return origin_[low];
 }
 
 std::optional<ProvenanceRange> CookedSource::GetProvenanceRange(
     CharBlock cookedRange) const {
-  if (!Contains(cookedRange)) {
+  if (!AsCharBlock().Contains(cookedRange)) {
     return std::nullopt;
   }
   ProvenanceRange first{provenanceMap_.Map(cookedRange.begin() - &data_[0])};
   if (cookedRange.size() <= first.size()) {
     return first.Prefix(cookedRange.size());
   }
   ProvenanceRange last{provenanceMap_.Map(cookedRange.end() - &data_[0])};
   return {ProvenanceRange{first.start(), last.start() - first.start()}};

flang/tools/f18/f18.cpp

std::string CompileFortran(std::string path, Fortran::parser::Options options,
   auto &parseTree{*parsing.parseTree()};
   if (driver.measureTree) {
     MeasureParseTree(parseTree);
   }
   if (!driver.debugNoSemantics || driver.debugResolveNames ||
       driver.dumpSymbols || driver.dumpUnparseWithSymbols ||
       driver.getDefinition || driver.getSymbolsSources) {
     Fortran::semantics::Semantics semantics{semanticsContext, parseTree,
-        parsing.cooked(), driver.debugModuleWriter};
+        parsing.cooked().AsCharBlock(), driver.debugModuleWriter};
     semantics.Perform();
     semantics.EmitMessages(llvm::errs());
     if (driver.dumpSymbols) {
       semantics.DumpSymbols(llvm::outs());
     }
     if (semantics.AnyFatalError()) {
       llvm::errs() << driver.prefix << "semantic errors in " << path << '\n';
       exitStatus = EXIT_FAILURE;

flang/unittests/Evaluate/intrinsics.cpp

explicit CookedStrings(const std::initializer_list<std::string> &ss) {
     Marshal();
   }
   void Save(const std::string &s) {
     offsets_[s] = cooked_.Put(s);
     cooked_.PutProvenance(allSources_.AddCompilerInsertion(s));
   }
   void Marshal() { cooked_.Marshal(allSources_); }
   parser::CharBlock operator()(const std::string &s) {
-    return {cooked_.data().data() + offsets_[s], s.size()};
+    return {cooked_.AsCharBlock().begin() + offsets_[s], s.size()};
   }
   parser::ContextualMessages Messages(parser::Messages &buffer) {
-    return parser::ContextualMessages{cooked_.data(), &buffer};
+    return parser::ContextualMessages{cooked_.AsCharBlock(), &buffer};
   }
   void Emit(llvm::raw_ostream &o, const parser::Messages &messages) {
     messages.Emit(o, allCookedSources_);
   }
 
 private:
   parser::AllSources allSources_;
   parser::AllCookedSources allCookedSources_{allSources_};