[flang] Runtime must defer formatted/unformatted determination

[klausler]

Summary

What the Fortran standard calls "preconnected" external I/O units
might not be known to be connected to unformatted or formatted files
until the first I/O data transfer statement is executed.
Support this deferred determination by representing the flag as
a tri-state Boolean and adapting its points of use.

[jeanPerier]

LGTM

flang/runtime/connection.h

 enum class Access { Sequential, Direct, Stream };
 
 inline bool IsRecordFile(Access a) { return a != Access::Stream; }
 
 // These characteristics of a connection are immutable after being
 // established in an OPEN statement.
 struct ConnectionAttributes {
   Access access{Access::Sequential}; // ACCESS='SEQUENTIAL', 'DIRECT', 'STREAM'
-  bool isUnformatted{false}; // FORM='UNFORMATTED'
+  std::optional<bool> isUnformatted; // FORM='UNFORMATTED' if true
   bool isUTF8{false}; // ENCODING='UTF-8'
   bool isFixedRecordLength{false}; // RECL= on OPEN
   std::optional<std::int64_t> recordLength; // RECL= or current record
 };
 
 struct ConnectionState : public ConnectionAttributes {
   bool IsAtEOF() const; // true when read has hit EOF or endfile record
   std::size_t RemainingSpaceInRecord() const;

flang/runtime/io-api.cpp

 template <Direction DIR>
 Cookie BeginExternalListIO(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   if (unitNumber == DefaultUnit) {
     unitNumber = DIR == Direction::Input ? 5 : 6;
   }
   ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreateAnonymous(
-      unitNumber, DIR, false /*formatted*/, terminator)};
+      unitNumber, DIR, false /*!unformatted*/, terminator)};
   if (unit.access == Access::Direct) {
     terminator.Crash("List-directed I/O attempted on direct access file");
     return nullptr;
   }
-  if (unit.isUnformatted) {
+  if (!unit.isUnformatted.has_value()) {
+    unit.isUnformatted = false;
+  }
+  if (*unit.isUnformatted) {
     terminator.Crash("List-directed I/O attempted on unformatted file");
     return nullptr;
   }
   IoErrorHandler handler{terminator};
   unit.SetDirection(DIR, handler);
   IoStatementState &io{unit.BeginIoStatement<ExternalListIoStatementState<DIR>>(
       unit, sourceFile, sourceLine)};
   return &io;

 template <Direction DIR>
 Cookie BeginExternalFormattedIO(const char *format, std::size_t formatLength,
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   if (unitNumber == DefaultUnit) {
     unitNumber = DIR == Direction::Input ? 5 : 6;
   }
   ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreateAnonymous(
-      unitNumber, DIR, false /*formatted*/, terminator)};
-  if (unit.isUnformatted) {
+      unitNumber, DIR, false /*!unformatted*/, terminator)};
+  if (!unit.isUnformatted.has_value()) {
+    unit.isUnformatted = false;
+  }
+  if (*unit.isUnformatted) {
     terminator.Crash("Formatted I/O attempted on unformatted file");
     return nullptr;
   }
   IoErrorHandler handler{terminator};
   unit.SetDirection(DIR, handler);
   IoStatementState &io{
       unit.BeginIoStatement<ExternalFormattedIoStatementState<DIR>>(
           unit, format, formatLength, sourceFile, sourceLine)};

 }
 
 template <Direction DIR>
 Cookie BeginUnformattedIO(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreateAnonymous(
       unitNumber, DIR, true /*unformatted*/, terminator)};
-  if (!unit.isUnformatted) {
-    terminator.Crash("Unformatted output attempted on formatted file");
+  if (!unit.isUnformatted.has_value()) {
+    unit.isUnformatted = true;
+  }
+  if (!*unit.isUnformatted) {
+    terminator.Crash("Unformatted I/O attempted on formatted file");
   }
   IoStatementState &io{unit.BeginIoStatement<UnformattedIoStatementState<DIR>>(
       unit, sourceFile, sourceLine)};
   IoErrorHandler handler{terminator};
   unit.SetDirection(DIR, handler);
   if constexpr (DIR == Direction::Output) {
     if (unit.access == Access::Sequential && !unit.isFixedRecordLength) {
       // Create space for (sub)record header to be completed by
Cookie IONAME(BeginBackspace)(
   return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
       unit, ExternalMiscIoStatementState::Backspace, sourceFile, sourceLine);
 }
 
 Cookie IONAME(BeginEndfile)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreateAnonymous(
-      unitNumber, Direction::Output, false /*formatted*/, terminator)};
+      unitNumber, Direction::Output, std::nullopt, terminator)};
   return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
       unit, ExternalMiscIoStatementState::Endfile, sourceFile, sourceLine);
 }
 
 Cookie IONAME(BeginRewind)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
   ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreateAnonymous(
-      unitNumber, Direction::Input, false /*formatted*/, terminator)};
+      unitNumber, Direction::Input, std::nullopt, terminator)};
   return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
       unit, ExternalMiscIoStatementState::Rewind, sourceFile, sourceLine);
 }
 
 Cookie IONAME(BeginInquireUnit)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   if (ExternalFileUnit * unit{ExternalFileUnit::LookUp(unitNumber)}) {
     return &unit->BeginIoStatement<InquireUnitState>(

flang/runtime/io-stmt.cpp

int OpenStatementState::EndIoStatement() {
   if (access_) {
     if (*access_ != unit().access) {
       if (wasExtant_) {
         SignalError("ACCESS= may not be changed on an open unit");
       }
     }
     unit().access = *access_;
   }
-  if (!isUnformatted_) {
-    isUnformatted_ = unit().access != Access::Sequential;
+  if (!unit().isUnformatted) {
+    unit().isUnformatted = isUnformatted_;
   }
-  if (*isUnformatted_ != unit().isUnformatted) {
+  if (isUnformatted_ && *isUnformatted_ != *unit().isUnformatted) {
     if (wasExtant_) {
       SignalError("FORM= may not be changed on an open unit");
     }
     unit().isUnformatted = *isUnformatted_;
   }
   return ExternalIoStatementBase::EndIoStatement();
 }
 
case HashInquiryKeyword("ACCESS"):
     break;
   case HashInquiryKeyword("ACTION"):
     str = unit().mayWrite() ? unit().mayRead() ? "READWRITE" : "WRITE" : "READ";
     break;
   case HashInquiryKeyword("ASYNCHRONOUS"):
     str = unit().mayAsynchronous() ? "YES" : "NO";
     break;
   case HashInquiryKeyword("BLANK"):
-    str = unit().isUnformatted                  ? "UNDEFINED"
+    str = unit().isUnformatted.value_or(true)   ? "UNDEFINED"
         : unit().modes.editingFlags & blankZero ? "ZERO"
                                                 : "NULL";
     break;
   case HashInquiryKeyword("CARRIAGECONTROL"):
     str = "LIST";
     break;
   case HashInquiryKeyword("CONVERT"):
     str = unit().swapEndianness() ? "SWAP" : "NATIVE";
     break;
   case HashInquiryKeyword("DECIMAL"):
-    str = unit().isUnformatted                     ? "UNDEFINED"
+    str = unit().isUnformatted.value_or(true)      ? "UNDEFINED"
         : unit().modes.editingFlags & decimalComma ? "COMMA"
                                                    : "POINT";
     break;
   case HashInquiryKeyword("DELIM"):
-    if (unit().isUnformatted) {
+    if (unit().isUnformatted.value_or(true)) {
       str = "UNDEFINED";
     } else {
       switch (unit().modes.delim) {
       case '\'':
         str = "APOSTROPHE";
         break;
       case '"':
         str = "QUOTE";
         break;
       default:
         str = "NONE";
         break;
       }
     }
     break;
   case HashInquiryKeyword("DIRECT"):
     str = unit().access == Access::Direct ||
             (unit().mayPosition() && unit().isFixedRecordLength)
         ? "YES"
         : "NO";
     break;
   case HashInquiryKeyword("ENCODING"):
-    str = unit().isUnformatted ? "UNDEFINED"
-        : unit().isUTF8        ? "UTF-8"
+    str = unit().isUnformatted.value_or(true) ? "UNDEFINED"
+        : unit().isUTF8                       ? "UTF-8"
                                               : "ASCII";
     break;
   case HashInquiryKeyword("FORM"):
-    str = unit().isUnformatted ? "UNFORMATTED" : "FORMATTED";
+    str = !unit().isUnformatted ? "UNKNOWN"
+        : *unit().isUnformatted ? "UNFORMATTED"
+                                : "FORMATTED";
     break;
   case HashInquiryKeyword("FORMATTED"):
-    str = !unit().isUnformatted ? "YES" : "NO";
+    str = !unit().isUnformatted ? "UNKNOWN"
+        : *unit().isUnformatted ? "NO"
+                                : "YES";
     break;
   case HashInquiryKeyword("NAME"):
     str = unit().path();
     if (!str) {
       return true; // result is undefined
     }
     break;
   case HashInquiryKeyword("PAD"):
-    str = unit().isUnformatted ? "UNDEFINED" : unit().modes.pad ? "YES" : "NO";
+    str = unit().isUnformatted.value_or(true) ? "UNDEFINED"
+        : unit().modes.pad                    ? "YES"
+                                              : "NO";
     break;
   case HashInquiryKeyword("POSITION"):
     if (unit().access == Access::Direct) {
       str = "UNDEFINED";
     } else {
       auto size{unit().knownSize()};
       auto pos{unit().position()};
       if (pos == size.value_or(pos + 1)) {
         str = "APPEND";
       } else if (pos == 0) {
         str = "REWIND";
       } else {
         str = "ASIS"; // processor-dependent & no common behavior
       }
     }
     break;
   case HashInquiryKeyword("READ"):
     str = unit().mayRead() ? "YES" : "NO";
     break;
   case HashInquiryKeyword("READWRITE"):
     str = unit().mayRead() && unit().mayWrite() ? "YES" : "NO";
     break;
   case HashInquiryKeyword("ROUND"):
-    if (unit().isUnformatted) {
+    if (unit().isUnformatted.value_or(true)) {
       str = "UNDEFINED";
     } else {
       switch (unit().modes.round) {
       case decimal::FortranRounding::RoundNearest:
         str = "NEAREST";
         break;
       case decimal::FortranRounding::RoundUp:
         str = "UP";
case HashInquiryKeyword("ROUND"):
     }
     break;
   case HashInquiryKeyword("SEQUENTIAL"):
     // "NO" for Direct, since Sequential would not work if
     // the unit were reopened without RECL=.
     str = unit().access == Access::Sequential ? "YES" : "NO";
     break;
   case HashInquiryKeyword("SIGN"):
-    str = unit().isUnformatted                 ? "UNDEFINED"
+    str = unit().isUnformatted.value_or(true)  ? "UNDEFINED"
         : unit().modes.editingFlags & signPlus ? "PLUS"
                                                : "SUPPRESS";
     break;
   case HashInquiryKeyword("STREAM"):
     str = unit().access == Access::Stream ? "YES" : "NO";
     break;
   case HashInquiryKeyword("WRITE"):
     str = unit().mayWrite() ? "YES" : "NO";
     break;
   case HashInquiryKeyword("UNFORMATTED"):
-    str = unit().isUnformatted ? "YES" : "NO";
+    str = !unit().isUnformatted ? "UNKNOWN"
+        : *unit().isUnformatted ? "YES"
+                                : "NO";
     break;
   }
   if (str) {
     ToFortranDefaultCharacter(result, length, str);
     return true;
   } else {
     BadInquiryKeywordHashCrash(inquiry);
     return false;

flang/runtime/unit.h

public:
   explicit ExternalFileUnit(int unitNumber) : unitNumber_{unitNumber} {}
   int unitNumber() const { return unitNumber_; }
   bool swapEndianness() const { return swapEndianness_; }
 
   static ExternalFileUnit *LookUp(int unit);
   static ExternalFileUnit &LookUpOrCrash(int unit, const Terminator &);
   static ExternalFileUnit &LookUpOrCreate(
       int unit, const Terminator &, bool &wasExtant);
-  static ExternalFileUnit &LookUpOrCreateAnonymous(
-      int unit, Direction, bool isUnformatted, const Terminator &);
+  static ExternalFileUnit &LookUpOrCreateAnonymous(int unit, Direction,
+      std::optional<bool> isUnformatted, const Terminator &);
   static ExternalFileUnit *LookUp(const char *path);
   static ExternalFileUnit &CreateNew(int unit, const Terminator &);
   static ExternalFileUnit *LookUpForClose(int unit);
   static int NewUnit(const Terminator &);
   static void CloseAll(IoErrorHandler &);
   static void FlushAll(IoErrorHandler &);
 
   void OpenUnit(std::optional<OpenStatus>, std::optional<Action>, Position,

flang/runtime/unit.cpp

ExternalFileUnit &ExternalFileUnit::LookUpOrCrash(
   return *file;
 }
 
 ExternalFileUnit &ExternalFileUnit::LookUpOrCreate(
     int unit, const Terminator &terminator, bool &wasExtant) {
   return GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant);
 }
 
-ExternalFileUnit &ExternalFileUnit::LookUpOrCreateAnonymous(
-    int unit, Direction dir, bool isUnformatted, const Terminator &terminator) {
+ExternalFileUnit &ExternalFileUnit::LookUpOrCreateAnonymous(int unit,
+    Direction dir, std::optional<bool> isUnformatted,
+    const Terminator &terminator) {
   bool exists{false};
   ExternalFileUnit &result{
       GetUnitMap().LookUpOrCreate(unit, terminator, exists)};
   if (!exists) {
     IoErrorHandler handler{terminator};
     result.OpenAnonymousUnit(
         dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace,
         Action::ReadWrite, Position::Rewind, Convert::Native, handler);
if (const char *p{FrameNextInput(handler, 1)}) {
     // TODO: UTF-8 decoding; may have to get more bytes in a loop
     return *p;
   }
   return std::nullopt;
 }
 
 const char *ExternalFileUnit::FrameNextInput(
     IoErrorHandler &handler, std::size_t bytes) {
-  RUNTIME_CHECK(handler, !isUnformatted);
+  RUNTIME_CHECK(handler, isUnformatted.has_value() && !*isUnformatted);
   if (static_cast<std::int64_t>(positionInRecord + bytes) <=
       recordLength.value_or(positionInRecord + bytes)) {
     auto at{recordOffsetInFrame_ + positionInRecord};
     auto need{static_cast<std::size_t>(at + bytes)};
     auto got{ReadFrame(frameOffsetInFile_, need, handler)};
     SetSequentialVariableFormattedRecordLength();
     if (got >= need) {
       return Frame() + at;
if (access == Access::Sequential) {
       } else if (isFixedRecordLength) {
         RUNTIME_CHECK(handler, recordLength.has_value());
         auto need{
             static_cast<std::size_t>(recordOffsetInFrame_ + *recordLength)};
         auto got{ReadFrame(frameOffsetInFile_, need, handler)};
         if (got < need) {
           handler.SignalEnd();
         }
-      } else if (isUnformatted) {
+      } else {
+        RUNTIME_CHECK(handler, isUnformatted.has_value());
+        if (isUnformatted.value_or(false)) {
           BeginSequentialVariableUnformattedInputRecord(handler);
         } else { // formatted
           BeginSequentialVariableFormattedInputRecord(handler);
         }
       }
     }
+  }
   RUNTIME_CHECK(handler,
       access != Access::Sequential || recordLength.has_value() ||
           handler.InError());
   return !handler.InError();
 }
 
 void ExternalFileUnit::FinishReadingRecord(IoErrorHandler &handler) {
   RUNTIME_CHECK(handler, direction_ == Direction::Input && beganReadingRecord_);
   beganReadingRecord_ = false;
   if (handler.InError()) {
     // avoid bogus crashes in END/ERR circumstances
   } else if (access == Access::Sequential) {
     RUNTIME_CHECK(handler, recordLength.has_value());
     if (isFixedRecordLength) {
       frameOffsetInFile_ += recordOffsetInFrame_ + *recordLength;
       recordOffsetInFrame_ = 0;
-    } else if (isUnformatted) {
+    } else {
+      RUNTIME_CHECK(handler, isUnformatted.has_value());
+      if (isUnformatted.value_or(false)) {
         // Retain footer in frame for more efficient BACKSPACE
         frameOffsetInFile_ += recordOffsetInFrame_ + *recordLength;
         recordOffsetInFrame_ = sizeof(std::uint32_t);
         recordLength.reset();
       } else { // formatted
         if (Frame()[recordOffsetInFrame_ + *recordLength] == '\r') {
           ++recordOffsetInFrame_;
         }
         recordOffsetInFrame_ += *recordLength + 1;
         RUNTIME_CHECK(handler, Frame()[recordOffsetInFrame_ - 1] == '\n');
         recordLength.reset();
       }
     }
+  }
   ++currentRecordNumber;
   BeginRecord();
 }
 
 bool ExternalFileUnit::AdvanceRecord(IoErrorHandler &handler) {
   if (direction_ == Direction::Input) {
     FinishReadingRecord(handler);
     return BeginReadingRecord(handler);
   } else { // Direction::Output
     bool ok{true};
+    RUNTIME_CHECK(handler, isUnformatted.has_value());
     if (isFixedRecordLength && recordLength) {
       // Pad remainder of fixed length record
       if (furthestPositionInRecord < *recordLength) {
         WriteFrame(
             frameOffsetInFile_, recordOffsetInFrame_ + *recordLength, handler);
         std::memset(Frame() + recordOffsetInFrame_ + furthestPositionInRecord,
-            isUnformatted ? 0 : ' ', *recordLength - furthestPositionInRecord);
+            isUnformatted.value_or(false) ? 0 : ' ',
+            *recordLength - furthestPositionInRecord);
       }
     } else {
       positionInRecord = furthestPositionInRecord;
-      if (isUnformatted) {
+      if (isUnformatted.value_or(false)) {
         // Append the length of a sequential unformatted variable-length record
         // as its footer, then overwrite the reserved first four bytes of the
         // record with its length as its header.  These four bytes were skipped
         // over in BeginUnformattedIO<Output>().
         // TODO: Break very large records up into subrecords with negative
         // headers &/or footers
         std::uint32_t length;
         length = furthestPositionInRecord - sizeof length;
if (access != Access::Sequential) {
     if (endfileRecordNumber && currentRecordNumber > *endfileRecordNumber) {
       // BACKSPACE after ENDFILE
     } else {
       DoImpliedEndfile(handler);
       if (frameOffsetInFile_ + recordOffsetInFrame_ > 0) {
         --currentRecordNumber;
         if (isFixedRecordLength) {
           BackspaceFixedRecord(handler);
-        } else if (isUnformatted) {
+        } else {
+          RUNTIME_CHECK(handler, isUnformatted.has_value());
+          if (isUnformatted.value_or(false)) {
             BackspaceVariableUnformattedRecord(handler);
           } else {
             BackspaceVariableFormattedRecord(handler);
           }
         }
       }
+    }
     BeginRecord();
   }
 }
 
 void ExternalFileUnit::FlushIfTerminal(IoErrorHandler &handler) {
   if (isTerminal()) {
     Flush(handler);
   }