[flang] Extend & fix per-I/O-statement state (ext. I/O work part 7)

[klausler]

Summary

The per-I/O-statement state structures need to support missing
external I/O statements, and some bugs found in testing with
formatted input and record advancement are fixed. The effects
of these changes will not be visible until further patches to
the I/O API handlers are pushed.

flang/runtime/io-stmt.h

 
 template <Direction, typename CHAR = char>
 class InternalFormattedIoStatementState;
 template <Direction, typename CHAR = char> class InternalListIoStatementState;
 template <Direction, typename CHAR = char>
 class ExternalFormattedIoStatementState;
 template <Direction> class ExternalListIoStatementState;
 template <Direction> class UnformattedIoStatementState;
+class ExternalMiscIoStatementState;
 
 // The Cookie type in the I/O API is a pointer (for C) to this class.
 class IoStatementState {
 public:
   template <typename A> explicit IoStatementState(A &x) : u_{x} {}
 
   // These member functions each project themselves into the active alternative.
   // They're used by per-data-item routines in the I/O API (e.g., OutputReal64)
return std::visit(
           }
           return nullptr;
         },
         u_);
   }
 
   bool EmitRepeated(char, std::size_t);
   bool EmitField(const char *, std::size_t length, std::size_t width);
-  void SkipSpaces(std::optional<int> &remaining);
+
+  std::optional<char32_t> SkipSpaces(std::optional<int> &remaining);
   std::optional<char32_t> NextInField(std::optional<int> &remaining);
   std::optional<char32_t> GetNextNonBlank(); // can advance record
 
 private:
   std::variant<std::reference_wrapper<OpenStatementState>,
       std::reference_wrapper<CloseStatementState>,
       std::reference_wrapper<NoopCloseStatementState>,
       std::reference_wrapper<
           InternalFormattedIoStatementState<Direction::Output>>,
       std::reference_wrapper<
           InternalFormattedIoStatementState<Direction::Input>>,
       std::reference_wrapper<InternalListIoStatementState<Direction::Output>>,
       std::reference_wrapper<InternalListIoStatementState<Direction::Input>>,
       std::reference_wrapper<
           ExternalFormattedIoStatementState<Direction::Output>>,
       std::reference_wrapper<
           ExternalFormattedIoStatementState<Direction::Input>>,
       std::reference_wrapper<ExternalListIoStatementState<Direction::Output>>,
       std::reference_wrapper<ExternalListIoStatementState<Direction::Input>>,
       std::reference_wrapper<UnformattedIoStatementState<Direction::Output>>,
-      std::reference_wrapper<UnformattedIoStatementState<Direction::Input>>>
+      std::reference_wrapper<UnformattedIoStatementState<Direction::Input>>,
+      std::reference_wrapper<ExternalMiscIoStatementState>>
       u_;
 };
 
 // Base class for all per-I/O statement state classes.
 // Inherits IoErrorHandler from its base.
 struct IoStatementBase : public DefaultFormatControlCallbacks {
   using DefaultFormatControlCallbacks::DefaultFormatControlCallbacks;
   int EndIoStatement();

 };
 
 template <Direction DIR>
 class ExternalIoStatementState : public ExternalIoStatementBase,
                                  public IoDirectionState<DIR> {
 public:
   using ExternalIoStatementBase::ExternalIoStatementBase;
   int EndIoStatement();
-  bool Emit(const char *, std::size_t chars /* not bytes */);
+  bool Emit(const char *, std::size_t);
   bool Emit(const char16_t *, std::size_t chars /* not bytes */);
   bool Emit(const char32_t *, std::size_t chars /* not bytes */);
   std::optional<char32_t> GetCurrentChar();
   bool AdvanceRecord(int = 1);
   void BackspaceRecord();
   void HandleRelativePosition(std::int64_t);
   void HandleAbsolutePosition(std::int64_t);
 };
public:
   using ExternalIoStatementState<DIR>::ExternalIoStatementState;
   using ListDirectedStatementState<DIR>::GetNextDataEdit;
 };
 
 template <Direction DIR>
 class UnformattedIoStatementState : public ExternalIoStatementState<DIR> {
 public:
   using ExternalIoStatementState<DIR>::ExternalIoStatementState;
+  bool Receive(char *, std::size_t);
   int EndIoStatement();
 };
 
 class OpenStatementState : public ExternalIoStatementBase {
 public:
   OpenStatementState(ExternalFileUnit &unit, bool wasExtant,
       const char *sourceFile = nullptr, int sourceLine = 0)
       : ExternalIoStatementBase{unit, sourceFile, sourceLine}, wasExtant_{
extern template class FormatControl<
     InternalFormattedIoStatementState<Direction::Output>>;
 extern template class FormatControl<
     InternalFormattedIoStatementState<Direction::Input>>;
 extern template class FormatControl<
     ExternalFormattedIoStatementState<Direction::Output>>;
 extern template class FormatControl<
     ExternalFormattedIoStatementState<Direction::Input>>;
 
+class ExternalMiscIoStatementState : public ExternalIoStatementBase {
+public:
+  enum Which { Flush, Backspace, Endfile, Rewind };
+  ExternalMiscIoStatementState(ExternalFileUnit &unit, Which which,
+      const char *sourceFile = nullptr, int sourceLine = 0)
+      : ExternalIoStatementBase{unit, sourceFile, sourceLine}, which_{which} {}
+  int EndIoStatement();
+
+private:
+  Which which_;
+};
+
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_IO_STMT_H_

flang/runtime/io-stmt.cpp

 //===-- runtime/io-stmt.cpp -------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "io-stmt.h"
 #include "connection.h"
 #include "format.h"
 #include "memory.h"
 #include "tools.h"
 #include "unit.h"
 #include <algorithm>
+#include <cstdio>
 #include <cstring>
 #include <limits>
 
 namespace Fortran::runtime::io {
 
 int IoStatementBase::EndIoStatement() { return GetIoStat(); }
 
 std::optional<DataEdit> IoStatementBase::GetNextDataEdit(

 
 int NoopCloseStatementState::EndIoStatement() {
   auto result{IoStatementBase::EndIoStatement()};
   FreeMemory(this);
   return result;
 }
 
 template <Direction DIR> int ExternalIoStatementState<DIR>::EndIoStatement() {
-  if constexpr (DIR == Direction::Output) {
-    if (!unit().nonAdvancing) {
-      unit().AdvanceRecord(*this);
-    }
+  if constexpr (DIR == Direction::Output) {
     unit().FlushIfTerminal(*this);
   }
   return ExternalIoStatementBase::EndIoStatement();
 }
 
 template <Direction DIR>
 bool ExternalIoStatementState<DIR>::Emit(const char *data, std::size_t chars) {
   if constexpr (DIR == Direction::Input) {
bool IoStatementState::AdvanceRecord(int n) {
   return std::visit([=](auto &x) { return x.get().AdvanceRecord(n); }, u_);
 }
 
 void IoStatementState::BackspaceRecord() {
   std::visit([](auto &x) { x.get().BackspaceRecord(); }, u_);
 }
 
 void IoStatementState::HandleRelativePosition(std::int64_t n) {
-  return std::visit([=](auto &x) { x.get().HandleRelativePosition(n); }, u_);
+  std::visit([=](auto &x) { x.get().HandleRelativePosition(n); }, u_);
 }
 
 int IoStatementState::EndIoStatement() {
   return std::visit([](auto &x) { return x.get().EndIoStatement(); }, u_);
 }
 
 ConnectionState &IoStatementState::GetConnectionState() {
   return std::visit(
bool IoStatementState::EmitField(
   if (length > static_cast<std::size_t>(width)) {
     return EmitRepeated('*', width);
   } else {
     return EmitRepeated(' ', static_cast<int>(width - length)) &&
         Emit(p, length);
   }
 }
 
-void IoStatementState::SkipSpaces(std::optional<int> &remaining) {
-  if (!remaining || *remaining > 0) {
-    for (auto ch{GetCurrentChar()}; ch && ch == ' '; ch = GetCurrentChar()) {
+std::optional<char32_t> IoStatementState::SkipSpaces(
+    std::optional<int> &remaining) {
+  while (!remaining || *remaining > 0) {
+    if (auto ch{GetCurrentChar()}) {
+      if (*ch != ' ') {
+        return ch;
+      }
       HandleRelativePosition(1);
-      if (remaining && !--*remaining) {
-        break;
+      if (remaining) {
+        --*remaining;
       }
+    } else {
+      break;
     }
   }
+  return std::nullopt;
 }
 
 std::optional<char32_t> IoStatementState::NextInField(
     std::optional<int> &remaining) {
   if (!remaining) { // list-directed or namelist: check for separators
     if (auto next{GetCurrentChar()}) {
       switch (*next) {
       case ' ':
       case ',':
       case ';':
       case '/':
       case '(':
       case ')':
       case '\'':
       case '"':
       case '*':
+      case '\n': // for stream access
         break;
       default:
         HandleRelativePosition(1);
         return next;
       }
     }
   } else if (*remaining > 0) {
     if (auto next{GetCurrentChar()}) {
       --*remaining;
       HandleRelativePosition(1);
       return next;
     }
     const ConnectionState &connection{GetConnectionState()};
-    if (!connection.IsAtEOF() && connection.recordLength &&
+    if (!connection.IsAtEOF() && connection.isFixedRecordLength &&
+        connection.recordLength &&
         connection.positionInRecord >= *connection.recordLength) {
       if (connection.modes.pad) { // PAD='YES'
         --*remaining;
         return std::optional<char32_t>{' '};
       }
       IoErrorHandler &handler{GetIoErrorHandler()};
       if (connection.nonAdvancing) {
         handler.SignalEor();
ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
   if (!imaginaryPart_ && ch && *ch == '(') {
     realPart_ = true;
     io.HandleRelativePosition(1);
   }
   return edit;
 }
 
 template <Direction DIR>
+bool UnformattedIoStatementState<DIR>::Receive(char *data, std::size_t bytes) {
+  if constexpr (DIR == Direction::Output) {
+    this->Crash(
+        "UnformattedIoStatementState::Receive() called for output statement");
+  }
+  return this->unit().Receive(data, bytes, *this);
+}
+
+template <Direction DIR>
 int UnformattedIoStatementState<DIR>::EndIoStatement() {
-  auto &ext{static_cast<ExternalIoStatementState<DIR> &>(*this)};
-  ExternalFileUnit &unit{ext.unit()};
-  if (unit.access == Access::Sequential && !unit.recordLength.has_value()) {
-    // Overwrite the first four bytes of the record with its length,
-    // and also append the length.  These four bytes were skipped over
-    // in BeginUnformattedOutput().
+  ExternalFileUnit &unit{this->unit()};
+  if constexpr (DIR == Direction::Output) {
+    if (unit.access == Access::Sequential && !unit.isFixedRecordLength) {
+      // Append the length of a sequential unformatted variable-length record
+      // as its footer, then overwrite the reserved first four bytes of the
+      // record with its length as its header.  These four bytes were skipped
+      // over in BeginUnformattedOutput().
       // TODO: Break very large records up into subrecords with negative
       // headers &/or footers
       union {
         std::uint32_t u;
         char c[sizeof u];
       } u;
-    u.u = unit.furthestPositionInRecord - sizeof u.c;
+      u.u = unit.furthestPositionInRecord - sizeof u;
       // TODO: Convert record length to little-endian on big-endian host?
-    if (!(ext.Emit(u.c, sizeof u.c) &&
-            (ext.HandleAbsolutePosition(0), ext.Emit(u.c, sizeof u.c)) &&
-            ext.AdvanceRecord())) {
+      if (!(this->Emit(u.c, sizeof u) &&
+              (this->HandleAbsolutePosition(0), this->Emit(u.c, sizeof u)))) {
         return false;
       }
     }
-  return ext.EndIoStatement();
+  }
+  return ExternalIoStatementState<DIR>::EndIoStatement();
 }
 
 template class InternalIoStatementState<Direction::Output>;
 template class InternalIoStatementState<Direction::Input>;
 template class InternalFormattedIoStatementState<Direction::Output>;
 template class InternalFormattedIoStatementState<Direction::Input>;
 template class InternalListIoStatementState<Direction::Output>;
 template class InternalListIoStatementState<Direction::Input>;
 template class ExternalIoStatementState<Direction::Output>;
 template class ExternalIoStatementState<Direction::Input>;
 template class ExternalFormattedIoStatementState<Direction::Output>;
 template class ExternalFormattedIoStatementState<Direction::Input>;
 template class ExternalListIoStatementState<Direction::Output>;
 template class ExternalListIoStatementState<Direction::Input>;
 template class UnformattedIoStatementState<Direction::Output>;
 template class UnformattedIoStatementState<Direction::Input>;
+
+int ExternalMiscIoStatementState::EndIoStatement() {
+  ExternalFileUnit &ext{unit()};
+  switch (which_) {
+  case Flush:
+    ext.Flush(*this);
+    std::fflush(nullptr); // flushes C stdio output streams (12.9(2))
+    break;
+  case Backspace:
+    ext.BackspaceRecord(*this);
+    break;
+  case Endfile:
+    ext.Endfile(*this);
+    break;
+  case Rewind:
+    ext.Rewind(*this);
+    break;
+  }
+  return ExternalIoStatementBase::EndIoStatement();
+}
+
 } // namespace Fortran::runtime::io

flang/runtime/unit.h

private:
 
   // When an I/O statement is in progress on this unit, holds its state.
   std::variant<std::monostate, OpenStatementState, CloseStatementState,
       ExternalFormattedIoStatementState<Direction::Output>,
       ExternalFormattedIoStatementState<Direction::Input>,
       ExternalListIoStatementState<Direction::Output>,
       ExternalListIoStatementState<Direction::Input>,
       UnformattedIoStatementState<Direction::Output>,
-      UnformattedIoStatementState<Direction::Input>>
+      UnformattedIoStatementState<Direction::Input>,
+      ExternalMiscIoStatementState>
       u_;
 
   // Points to the active alternative (if any) in u_ for use as a Cookie
   std::optional<IoStatementState> io_;
 
   // Subtle: The beginning of the frame can't be allowed to advance
   // during a single list-directed READ due to the possibility of a
   // multi-record CHARACTER value with a "r*" repeat count.  So we
   // manage the frame and the current record therein separately.
   std::int64_t frameOffsetInFile_{0};
   std::size_t recordOffsetInFrame_{0}; // of currentRecordNumber
 };
 
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_IO_UNIT_H_