[flang] Further implementation of external I/O unit operations (part 6)

[klausler]

Summary

Rework initial implementation of external I/O unit operations to
fix problems exposed in unit tests (in a later patch). Add flushing.

flang/runtime/unit-map.h

public:
   }
 
   // To prevent races, the unit is removed from the map if it exists,
   // and put on the closing_ list until DestroyClosed() is called.
   ExternalFileUnit *LookUpForClose(int);
 
   void DestroyClosed(ExternalFileUnit &);
   void CloseAll(IoErrorHandler &);
+  void FlushAll(IoErrorHandler &);
 
 private:
   struct Chain {
     explicit Chain(int n) : unit{n} {}
     ExternalFileUnit unit;
     OwningPtr<Chain> next{nullptr};
   };
 

flang/runtime/unit-map.cpp

while (Chain * p{bucket_[j].get()}) {
       bucket_[j].swap(p->next); // pops p from head of list
       p->unit.CloseUnit(CloseStatus::Keep, handler);
       p->unit.~ExternalFileUnit();
       FreeMemory(p);
     }
   }
 }
 
+void UnitMap::FlushAll(IoErrorHandler &handler) {
+  CriticalSection critical{lock_};
+  for (int j{0}; j < buckets_; ++j) {
+    for (Chain *p{bucket_[j].get()}; p; p = p->next.get()) {
+      p->unit.Flush(handler);
+    }
+  }
+}
+
 ExternalFileUnit &UnitMap::Create(int n, const Terminator &terminator) {
   Chain &chain{*New<Chain>{terminator}(n).release()};
   chain.next.reset(&chain);
   bucket_[Hash(n)].swap(chain.next); // pushes new node as list head
   return chain.unit;
 }
 } // namespace Fortran::runtime::io

flang/runtime/unit.h

public:
 
   static ExternalFileUnit *LookUp(int unit);
   static ExternalFileUnit &LookUpOrCrash(int unit, const Terminator &);
   static ExternalFileUnit &LookUpOrCreate(
       int unit, const Terminator &, bool *wasExtant = nullptr);
   static ExternalFileUnit *LookUpForClose(int unit);
   static int NewUnit(const Terminator &);
   static void CloseAll(IoErrorHandler &);
+  static void FlushAll(IoErrorHandler &);
 
   void OpenUnit(OpenStatus, Position, OwningPtr<char> &&path,
       std::size_t pathLength, IoErrorHandler &);
   void CloseUnit(CloseStatus, IoErrorHandler &);
   void DestroyClosed();
 
+  bool SetDirection(Direction, IoErrorHandler &);
+
   template <typename A, typename... X>
   IoStatementState &BeginIoStatement(X &&... xs) {
     // TODO: Child data transfer statements vs. locking
     lock_.Take(); // dropped in EndIoStatement()
     A &state{u_.emplace<A>(std::forward<X>(xs)...)};
     if constexpr (!std::is_same_v<A, OpenStatementState>) {
       state.mutableModes() = ConnectionState::modes;
     }
     io_.emplace(state);
     return *io_;
   }
 
-  bool Emit(const char *, std::size_t bytes, IoErrorHandler &);
+  bool Emit(const char *, std::size_t, IoErrorHandler &);
+  bool Receive(char *, std::size_t, IoErrorHandler &);
   std::optional<char32_t> GetCurrentChar(IoErrorHandler &);
   void SetLeftTabLimit();
+  void BeginReadingRecord(IoErrorHandler &);
   bool AdvanceRecord(IoErrorHandler &);
   void BackspaceRecord(IoErrorHandler &);
   void FlushIfTerminal(IoErrorHandler &);
+  void Endfile(IoErrorHandler &);
+  void Rewind(IoErrorHandler &);
   void EndIoStatement();
   void SetPosition(std::int64_t pos) {
     frameOffsetInFile_ = pos;
     recordOffsetInFrame_ = 0;
+    BeginRecord();
   }
 
 private:
   static UnitMap &GetUnitMap();
-  void NextSequentialUnformattedInputRecord(IoErrorHandler &);
-  void NextSequentialFormattedInputRecord(IoErrorHandler &);
-  void BackspaceSequentialUnformattedRecord(IoErrorHandler &);
-  void BackspaceSequentialFormattedRecord(IoErrorHandler &);
+  const char *FrameNextInput(IoErrorHandler &, std::size_t);
+  void BeginSequentialVariableUnformattedInputRecord(IoErrorHandler &);
+  void BeginSequentialVariableFormattedInputRecord(IoErrorHandler &);
+  void BackspaceFixedRecord(IoErrorHandler &);
+  void BackspaceVariableUnformattedRecord(IoErrorHandler &);
+  void BackspaceVariableFormattedRecord(IoErrorHandler &);
+  bool SetSequentialVariableFormattedRecordLength();
+  void DoImpliedEndfile(IoErrorHandler &);
+  void DoEndfile(IoErrorHandler &);
 
   int unitNumber_{-1};
-  bool isReading_{false};
+  Direction direction_{Direction::Output};
+  bool impliedEndfile_{false}; // seq. output has taken place
 
   Lock lock_;
 
   // When an I/O statement is in progress on this unit, holds its state.
   std::variant<std::monostate, OpenStatementState, CloseStatementState,
       ExternalFormattedIoStatementState<Direction::Output>,
       ExternalFormattedIoStatementState<Direction::Input>,
       ExternalListIoStatementState<Direction::Output>,
       ExternalListIoStatementState<Direction::Input>,
       UnformattedIoStatementState<Direction::Output>,
       UnformattedIoStatementState<Direction::Input>>
       u_;
 
   // Points to the active alternative (if any) in u_ for use as a Cookie
   std::optional<IoStatementState> io_;
 
   // Subtle: The beginning of the frame can't be allowed to advance
   // during a single list-directed READ due to the possibility of a
-  // multi-record CHARACTER value with a "r*" repeat count.
+  // multi-record CHARACTER value with a "r*" repeat count.  So we
+  // manage the frame and the current record therein separately.
   std::int64_t frameOffsetInFile_{0};
-  std::int64_t recordOffsetInFrame_{0}; // of currentRecordNumber
+  std::size_t recordOffsetInFrame_{0}; // of currentRecordNumber
 };
 
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_IO_UNIT_H_

flang/runtime/unit.cpp

 #include "unit-map.h"
 
 namespace Fortran::runtime::io {
 
 // The per-unit data structures are created on demand so that Fortran I/O
 // should work without a Fortran main program.
 static Lock unitMapLock;
 static UnitMap *unitMap{nullptr};
+static ExternalFileUnit *defaultInput{nullptr};
 static ExternalFileUnit *defaultOutput{nullptr};
 
 void FlushOutputOnCrash(const Terminator &terminator) {
   if (!defaultOutput) {
     return;
   }
   CriticalSection critical{unitMapLock};
   if (defaultOutput) {
if (status == OpenStatus::Old &&
         (!newPath.get() ||
             (path() && pathLength() == newPathLength &&
                 std::memcmp(path(), newPath.get(), newPathLength) == 0))) {
       // OPEN of existing unit, STATUS='OLD', not new FILE=
       newPath.reset();
       return;
     }
     // Otherwise, OPEN on open unit with new FILE= implies CLOSE
+    DoImpliedEndfile(handler);
     Flush(handler);
     Close(CloseStatus::Keep, handler);
   }
   set_path(std::move(newPath), newPathLength);
   Open(status, position, handler);
+  auto totalBytes{knownSize()};
+  if (access == Access::Direct) {
+    if (!isFixedRecordLength || !recordLength) {
+      handler.SignalError(IostatOpenBadRecl,
+          "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known",
+          unitNumber());
+    } else if (*recordLength <= 0) {
+      handler.SignalError(IostatOpenBadRecl,
+          "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid",
+          unitNumber(), static_cast<std::intmax_t>(*recordLength));
+    } else if (!totalBytes) {
+      handler.SignalError(IostatOpenUnknownSize,
+          "OPEN(UNIT=%d,ACCESS='DIRECT'): file size is not known");
+    } else if (*totalBytes % *recordLength != 0) {
+      handler.SignalError(IostatOpenBadAppend,
+          "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an "
+          "even divisor of the file size %jd",
+          unitNumber(), static_cast<std::intmax_t>(*recordLength),
+          static_cast<std::intmax_t>(*totalBytes));
+    }
+  }
+  if (position == Position::Append) {
+    if (*totalBytes && recordLength && *recordLength) {
+      endfileRecordNumber = 1 + (*totalBytes / *recordLength);
+    } else {
+      // Fake it so that we can backspace relative from the end
+      endfileRecordNumber = std::numeric_limits<std::int64_t>::max() - 1;
+    }
+    currentRecordNumber = *endfileRecordNumber;
+  } else {
+    currentRecordNumber = 1;
+  }
 }
 
 void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) {
+  DoImpliedEndfile(handler);
   Flush(handler);
   Close(status, handler);
 }
 
 void ExternalFileUnit::DestroyClosed() {
   GetUnitMap().DestroyClosed(*this); // destroys *this
 }
 
+bool ExternalFileUnit::SetDirection(
+    Direction direction, IoErrorHandler &handler) {
+  if (direction == Direction::Input) {
+    if (mayRead()) {
+      direction_ = Direction::Input;
+      return true;
+    } else {
+      handler.SignalError(IostatReadFromWriteOnly,
+          "READ(UNIT=%d) with ACTION='WRITE'", unitNumber());
+      return false;
+    }
+  } else {
+    if (mayWrite()) {
+      direction_ = Direction::Output;
+      return true;
+    } else {
+      handler.SignalError(IostatWriteToReadOnly,
+          "WRITE(UNIT=%d) with ACTION='READ'", unitNumber());
+      return false;
+    }
+  }
+}
+
 UnitMap &ExternalFileUnit::GetUnitMap() {
   if (unitMap) {
     return *unitMap;
   }
   CriticalSection critical{unitMapLock};
   if (unitMap) {
     return *unitMap;
   }
   Terminator terminator{__FILE__, __LINE__};
+  IoErrorHandler handler{terminator};
   unitMap = New<UnitMap>{terminator}().release();
   ExternalFileUnit &out{ExternalFileUnit::LookUpOrCreate(6, terminator)};
   out.Predefine(1);
   out.set_mayRead(false);
   out.set_mayWrite(true);
   out.set_mayPosition(false);
+  out.SetDirection(Direction::Output, handler);
   defaultOutput = &out;
   ExternalFileUnit &in{ExternalFileUnit::LookUpOrCreate(5, terminator)};
   in.Predefine(0);
   in.set_mayRead(true);
   in.set_mayWrite(false);
   in.set_mayPosition(false);
+  in.SetDirection(Direction::Input, handler);
+  defaultInput = &in;
   // TODO: Set UTF-8 mode from the environment
   return *unitMap;
 }
 
 void ExternalFileUnit::CloseAll(IoErrorHandler &handler) {
   CriticalSection critical{unitMapLock};
   if (unitMap) {
     unitMap->CloseAll(handler);
     FreeMemoryAndNullify(unitMap);
   }
   defaultOutput = nullptr;
 }
 
+void ExternalFileUnit::FlushAll(IoErrorHandler &handler) {
+  CriticalSection critical{unitMapLock};
+  if (unitMap) {
+    unitMap->FlushAll(handler);
+  }
+}
+
 bool ExternalFileUnit::Emit(
     const char *data, std::size_t bytes, IoErrorHandler &handler) {
   auto furthestAfter{std::max(furthestPositionInRecord,
       positionInRecord + static_cast<std::int64_t>(bytes))};
   if (furthestAfter > recordLength.value_or(furthestAfter)) {
-    handler.SignalError(IostatRecordWriteOverrun);
+    handler.SignalError(IostatRecordWriteOverrun,
+        "Attempt to write %zd bytes to position %jd in a fixed-size record of "
+        "%jd bytes",
+        bytes, static_cast<std::intmax_t>(positionInRecord),
+        static_cast<std::intmax_t>(*recordLength));
     return false;
   }
   WriteFrame(frameOffsetInFile_, recordOffsetInFrame_ + furthestAfter, handler);
   if (positionInRecord > furthestPositionInRecord) {
-    std::memset(Frame() + furthestPositionInRecord, ' ',
+    std::memset(Frame() + recordOffsetInFrame_ + furthestPositionInRecord, ' ',
         positionInRecord - furthestPositionInRecord);
   }
-  std::memcpy(Frame() + positionInRecord, data, bytes);
+  std::memcpy(Frame() + recordOffsetInFrame_ + positionInRecord, data, bytes);
   positionInRecord += bytes;
   furthestPositionInRecord = furthestAfter;
   return true;
 }
 
+bool ExternalFileUnit::Receive(
+    char *data, std::size_t bytes, IoErrorHandler &handler) {
+  RUNTIME_CHECK(handler, direction_ == Direction::Input);
+  auto furthestAfter{std::max(furthestPositionInRecord,
+      positionInRecord + static_cast<std::int64_t>(bytes))};
+  if (furthestAfter > recordLength.value_or(furthestAfter)) {
+    handler.SignalError(IostatRecordReadOverrun,
+        "Attempt to read %zd bytes at position %jd in a record of %jd bytes",
+        bytes, static_cast<std::intmax_t>(positionInRecord),
+        static_cast<std::intmax_t>(*recordLength));
+    return false;
+  }
+  auto need{recordOffsetInFrame_ + furthestAfter};
+  auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+  if (got >= need) {
+    std::memcpy(data, Frame() + recordOffsetInFrame_ + positionInRecord, bytes);
+    positionInRecord += bytes;
+    furthestPositionInRecord = furthestAfter;
+    return true;
+  } else {
+    handler.SignalEnd();
+    endfileRecordNumber = currentRecordNumber;
+    return false;
+  }
+}
+
 std::optional<char32_t> ExternalFileUnit::GetCurrentChar(
     IoErrorHandler &handler) {
-  isReading_ = true; // TODO: manage read/write transitions
-  if (isUnformatted) {
-    handler.Crash("GetCurrentChar() called for unformatted input");
-    return std::nullopt;
+  RUNTIME_CHECK(handler, direction_ == Direction::Input);
+  if (const char *p{FrameNextInput(handler, 1)}) {
+    // TODO: UTF-8 decoding; may have to get more bytes in a loop
+    return *p;
   }
-  std::size_t chunk{256}; // for stream input
-  if (recordLength.has_value()) {
-    if (positionInRecord >= *recordLength) {
-      return std::nullopt;
-    }
-    chunk = *recordLength - positionInRecord;
-  }
+
+const char *ExternalFileUnit::FrameNextInput(
+    IoErrorHandler &handler, std::size_t bytes) {
+  RUNTIME_CHECK(handler, !isUnformatted);
+  if (static_cast<std::int64_t>(positionInRecord + bytes) <=
+      recordLength.value_or(positionInRecord + bytes)) {
     auto at{recordOffsetInFrame_ + positionInRecord};
-  std::size_t need{static_cast<std::size_t>(at + 1)};
-  std::size_t want{need + chunk};
-  auto got{ReadFrame(frameOffsetInFile_, want, handler)};
-  if (got <= need) {
-    endfileRecordNumber = currentRecordNumber;
+    auto need{static_cast<std::size_t>(at + bytes)};
+    auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+    SetSequentialVariableFormattedRecordLength();
+    if (got >= need) {
+      return Frame() + at;
+    }
     handler.SignalEnd();
-    return std::nullopt;
+    endfileRecordNumber = currentRecordNumber;
   }
-  const char *p{Frame() + at};
-  if (isUTF8) {
-    // TODO: UTF-8 decoding
-  }
-  return *p;
+  return nullptr;
+
+bool ExternalFileUnit::SetSequentialVariableFormattedRecordLength() {
+  if (recordLength || access != Access::Sequential) {
+    return true;
+  }
+  if (FrameLength() > recordOffsetInFrame_) {
+    const char *record{Frame() + recordOffsetInFrame_};
+    if (const char *nl{reinterpret_cast<const char *>(
+            std::memchr(record, '\n', FrameLength() - recordOffsetInFrame_))}) {
+      recordLength = nl - record;
+      if (*recordLength > 0 && record[*recordLength - 1] == '\r') {
+        --*recordLength;
+      }
+      return true;
+    }
+  }
+  return false;
 }
 
 void ExternalFileUnit::SetLeftTabLimit() {
   leftTabLimit = furthestPositionInRecord;
   positionInRecord = furthestPositionInRecord;
 }
 
+void ExternalFileUnit::BeginReadingRecord(IoErrorHandler &handler) {
+  RUNTIME_CHECK(handler, direction_ == Direction::Input);
+  if (access == Access::Sequential) {
+    if (endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber) {
+      handler.SignalEnd();
+    } else if (isFixedRecordLength) {
+      RUNTIME_CHECK(handler, recordLength.has_value());
+      auto need{static_cast<std::size_t>(recordOffsetInFrame_ + *recordLength)};
+      auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+      if (got < need) {
+        handler.SignalEnd();
+      }
+    } else if (isUnformatted) {
+      BeginSequentialVariableUnformattedInputRecord(handler);
+    } else { // formatted
+      BeginSequentialVariableFormattedInputRecord(handler);
+    }
+  }
+}
+
 bool ExternalFileUnit::AdvanceRecord(IoErrorHandler &handler) {
   bool ok{true};
-  if (isReading_) {
+  if (direction_ == Direction::Input) {
     if (access == Access::Sequential) {
-      if (isUnformatted) {
-        NextSequentialUnformattedInputRecord(handler);
-      } else {
-        NextSequentialFormattedInputRecord(handler);
+      RUNTIME_CHECK(handler, recordLength.has_value());
+      if (isFixedRecordLength) {
+        frameOffsetInFile_ += recordOffsetInFrame_ + *recordLength;
+        recordOffsetInFrame_ = 0;
+      } else if (isUnformatted) {
+        // Retain footer in frame for more efficient BACKSPACE
+        frameOffsetInFile_ += recordOffsetInFrame_ + *recordLength;
+        recordOffsetInFrame_ = sizeof(std::uint32_t);
+        recordLength.reset();
+      } else { // formatted
+        if (Frame()[recordOffsetInFrame_ + *recordLength] == '\r') {
+          ++recordOffsetInFrame_;
+        }
+        recordOffsetInFrame_ += *recordLength + 1;
+        RUNTIME_CHECK(handler, Frame()[recordOffsetInFrame_ - 1] == '\n');
+        recordLength.reset();
       }
     }
-  } else if (!isUnformatted) {
-    if (recordLength.has_value()) {
-      // fill fixed-size record
+  } else { // Direction::Output
+    if (!isUnformatted) {
+      if (isFixedRecordLength && recordLength) {
         if (furthestPositionInRecord < *recordLength) {
           WriteFrame(frameOffsetInFile_, *recordLength, handler);
           std::memset(Frame() + recordOffsetInFrame_ + furthestPositionInRecord,
               ' ', *recordLength - furthestPositionInRecord);
         }
       } else {
         positionInRecord = furthestPositionInRecord;
         ok &= Emit("\n", 1, handler); // TODO: Windows CR+LF
-      frameOffsetInFile_ += recordOffsetInFrame_ + furthestPositionInRecord;
-      recordOffsetInFrame_ = 0;
       }
     }
+    frameOffsetInFile_ +=
+        recordOffsetInFrame_ + recordLength.value_or(furthestPositionInRecord);
+    recordOffsetInFrame_ = 0;
+    impliedEndfile_ = true;
+  }
   ++currentRecordNumber;
-  positionInRecord = 0;
-  furthestPositionInRecord = 0;
-  leftTabLimit.reset();
+  BeginRecord();
   return ok;
 }
 
 void ExternalFileUnit::BackspaceRecord(IoErrorHandler &handler) {
-  if (!isReading_) {
-    handler.Crash("ExternalFileUnit::BackspaceRecord() called during writing");
-    // TODO: create endfile record, &c.
-  }
-  if (access == Access::Sequential) {
-    if (isUnformatted) {
-      BackspaceSequentialUnformattedRecord(handler);
-    } else {
-      BackspaceSequentialFormattedRecord(handler);
-    }
+  if (access != Access::Sequential) {
+    handler.SignalError(IostatBackspaceNonSequential,
+        "BACKSPACE(UNIT=%d) on non-sequential file", unitNumber());
+    DoImpliedEndfile(handler);
+    --currentRecordNumber;
+    BeginRecord();
+    if (isFixedRecordLength) {
+      BackspaceFixedRecord(handler);
+    } else if (isUnformatted) {
+      BackspaceVariableUnformattedRecord(handler);
     } else {
-    // TODO
+      BackspaceVariableFormattedRecord(handler);
+    }
   }
-  positionInRecord = 0;
-  furthestPositionInRecord = 0;
-  leftTabLimit.reset();
 }
 
 void ExternalFileUnit::FlushIfTerminal(IoErrorHandler &handler) {
   if (isTerminal()) {
     Flush(handler);
   }
 }
 
+void ExternalFileUnit::Endfile(IoErrorHandler &handler) {
+  if (access != Access::Sequential) {
+    handler.SignalError(IostatEndfileNonSequential,
+        "ENDFILE(UNIT=%d) on non-sequential file", unitNumber());
+  } else if (!mayWrite()) {
+    handler.SignalError(IostatEndfileUnwritable,
+        "ENDFILE(UNIT=%d) on read-only file", unitNumber());
+  } else {
+    DoEndfile(handler);
+  }
+}
+
+void ExternalFileUnit::Rewind(IoErrorHandler &handler) {
+  if (access == Access::Direct) {
+    handler.SignalError(IostatRewindNonSequential,
+        "REWIND(UNIT=%d) on non-sequential file", unitNumber());
+  } else {
+    DoImpliedEndfile(handler);
+    SetPosition(0);
+    currentRecordNumber = 1;
+    // TODO: reset endfileRecordNumber?
+  }
+}
+
 void ExternalFileUnit::EndIoStatement() {
   frameOffsetInFile_ += recordOffsetInFrame_;
   recordOffsetInFrame_ = 0;
   io_.reset();
   u_.emplace<std::monostate>();
   lock_.Drop();
 }
 
-void ExternalFileUnit::NextSequentialUnformattedInputRecord(
+void ExternalFileUnit::BeginSequentialVariableUnformattedInputRecord(
     IoErrorHandler &handler) {
   std::int32_t header{0}, footer{0};
-  // Retain previous footer (if any) in frame for more efficient BACKSPACE
-  std::size_t retain{sizeof header};
-  if (recordLength) { // not first record - advance to next
-    ++currentRecordNumber;
-    if (endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber) {
-      handler.SignalEnd();
-      return;
-    }
-    frameOffsetInFile_ +=
-        recordOffsetInFrame_ + *recordLength + 2 * sizeof header;
-    recordOffsetInFrame_ = 0;
-  } else {
-    retain = 0;
-  }
-  std::size_t need{retain + sizeof header};
-  std::size_t got{ReadFrame(frameOffsetInFile_ - retain, need, handler)};
+  std::size_t need{recordOffsetInFrame_ + sizeof header};
+  std::size_t got{ReadFrame(frameOffsetInFile_, need, handler)};
   // Try to emit informative errors to help debug corrupted files.
   const char *error{nullptr};
   if (got < need) {
-    if (got == retain) {
+    if (got == recordOffsetInFrame_) {
       handler.SignalEnd();
     } else {
-      error = "Unformatted sequential file input failed at record #%jd (file "
-              "offset %jd): truncated record header";
+      error = "Unformatted variable-length sequential file input failed at "
+              "record #%jd (file offset %jd): truncated record header";
     }
   } else {
-    std::memcpy(&header, Frame() + retain, sizeof header);
-    need = retain + header + 2 * sizeof header;
-    got = ReadFrame(frameOffsetInFile_ - retain,
-        need + sizeof header /* next one */, handler);
+    std::memcpy(&header, Frame() + recordOffsetInFrame_, sizeof header);
+    recordLength = sizeof header + header; // does not include footer
+    need = recordOffsetInFrame_ + *recordLength + sizeof footer;
+    got = ReadFrame(frameOffsetInFile_, need, handler);
     if (got < need) {
-      error = "Unformatted sequential file input failed at record #%jd (file "
-              "offset %jd): hit EOF reading record with length %jd bytes";
+      error = "Unformatted variable-length sequential file input failed at "
+              "record #%jd (file offset %jd): hit EOF reading record with "
+              "length %jd bytes";
     } else {
-      const char *start{Frame() + retain + sizeof header};
-      std::memcpy(&footer, start + header, sizeof footer);
+      std::memcpy(&footer, Frame() + recordOffsetInFrame_ + *recordLength,
+          sizeof footer);
       if (footer != header) {
-        error = "Unformatted sequential file input failed at record #%jd (file "
-                "offset %jd): record header has length %jd that does not match "
-                "record footer (%jd)";
-      } else {
-        recordLength = header;
+        error = "Unformatted variable-length sequential file input failed at "
+                "record #%jd (file offset %jd): record header has length %jd "
+                "that does not match record footer (%jd)";
       }
     }
   }
   if (error) {
     handler.SignalError(error, static_cast<std::intmax_t>(currentRecordNumber),
         static_cast<std::intmax_t>(frameOffsetInFile_),
         static_cast<std::intmax_t>(header), static_cast<std::intmax_t>(footer));
+    // TODO: error recovery
   }
   positionInRecord = sizeof header;
 }
 
-void ExternalFileUnit::NextSequentialFormattedInputRecord(
+void ExternalFileUnit::BeginSequentialVariableFormattedInputRecord(
     IoErrorHandler &handler) {
-  static constexpr std::size_t chunk{256};
-  std::size_t length{0};
-  if (recordLength.has_value()) {
-    // not first record - advance to next
-    ++currentRecordNumber;
-    if (endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber) {
-      handler.SignalEnd();
-      return;
-    }
-    if (Frame()[*recordLength] == '\r') {
-      ++*recordLength;
-    }
-    recordOffsetInFrame_ += *recordLength + 1;
+  if (this == defaultInput && defaultOutput) {
+    defaultOutput->Flush(handler);
   }
-  while (true) {
-    std::size_t got{ReadFrame(
-        frameOffsetInFile_, recordOffsetInFrame_ + length + chunk, handler)};
-    if (got <= recordOffsetInFrame_ + length) {
+  std::size_t length{0};
+  do {
+    std::size_t need{recordOffsetInFrame_ + length + 1};
+    length = ReadFrame(frameOffsetInFile_, need, handler);
+    if (length < need) {
       handler.SignalEnd();
       break;
     }
-    const char *frame{Frame() + recordOffsetInFrame_};
-    if (const char *nl{reinterpret_cast<const char *>(
-            std::memchr(frame + length, '\n', chunk))}) {
-      recordLength = nl - (frame + length) + 1;
-      if (*recordLength > 0 && frame[*recordLength - 1] == '\r') {
-        --*recordLength;
-      }
-      return;
-    }
-    length += got;
+  } while (!SetSequentialVariableFormattedRecordLength());
+
+void ExternalFileUnit::BackspaceFixedRecord(IoErrorHandler &handler) {
+  RUNTIME_CHECK(handler, recordLength.has_value());
+  if (frameOffsetInFile_ < *recordLength) {
+    handler.SignalError(IostatBackspaceAtFirstRecord);
+  } else {
+    frameOffsetInFile_ -= *recordLength;
   }
 }
 
-void ExternalFileUnit::BackspaceSequentialUnformattedRecord(
+void ExternalFileUnit::BackspaceVariableUnformattedRecord(
     IoErrorHandler &handler) {
   std::int32_t header{0}, footer{0};
-  RUNTIME_CHECK(handler, currentRecordNumber > 1);
-  --currentRecordNumber;
-  int overhead{static_cast<int>(2 * sizeof header)};
+  auto headerBytes{static_cast<std::int64_t>(sizeof header)};
+  frameOffsetInFile_ += recordOffsetInFrame_;
+  recordOffsetInFrame_ = 0;
+  if (frameOffsetInFile_ <= headerBytes) {
+    handler.SignalError(IostatBackspaceAtFirstRecord);
+    return;
+  }
   // Error conditions here cause crashes, not file format errors, because the
   // validity of the file structure before the current record will have been
-  // checked informatively in NextSequentialUnformattedInputRecord().
-  RUNTIME_CHECK(handler, frameOffsetInFile_ >= overhead);
+  // checked informatively in NextSequentialVariableUnformattedInputRecord().
   std::size_t got{
-      ReadFrame(frameOffsetInFile_ - sizeof footer, sizeof footer, handler)};
+      ReadFrame(frameOffsetInFile_ - headerBytes, headerBytes, handler)};
   RUNTIME_CHECK(handler, got >= sizeof footer);
   std::memcpy(&footer, Frame(), sizeof footer);
-  RUNTIME_CHECK(handler, frameOffsetInFile_ >= footer + overhead);
-  frameOffsetInFile_ -= footer + 2 * sizeof footer;
-  auto extra{std::max<std::size_t>(sizeof footer, frameOffsetInFile_)};
-  std::size_t want{extra + footer + 2 * sizeof footer};
-  got = ReadFrame(frameOffsetInFile_ - extra, want, handler);
-  RUNTIME_CHECK(handler, got >= want);
-  std::memcpy(&header, Frame() + extra, sizeof header);
-  RUNTIME_CHECK(handler, header == footer);
-  positionInRecord = sizeof header;
   recordLength = footer;
+  RUNTIME_CHECK(handler, frameOffsetInFile_ >= *recordLength + 2 * headerBytes);
+  frameOffsetInFile_ -= *recordLength + 2 * headerBytes;
+  if (frameOffsetInFile_ >= headerBytes) {
+    frameOffsetInFile_ -= headerBytes;
+    recordOffsetInFrame_ = headerBytes;
+  }
+  auto need{static_cast<std::size_t>(
+      recordOffsetInFrame_ + sizeof header + *recordLength)};
+  got = ReadFrame(frameOffsetInFile_, need, handler);
+  RUNTIME_CHECK(handler, got >= need);
+  std::memcpy(&header, Frame() + recordOffsetInFrame_, sizeof header);
+  RUNTIME_CHECK(handler, header == *recordLength);
 }
 
 // There's no portable memrchr(), unfortunately, and strrchr() would
 // fail on a record with a NUL, so we have to do it the hard way.
 static const char *FindLastNewline(const char *str, std::size_t length) {
   for (const char *p{str + length}; p-- > str;) {
     if (*p == '\n') {
       return p;
     }
   }
   return nullptr;
 }
 
-void ExternalFileUnit::BackspaceSequentialFormattedRecord(
+void ExternalFileUnit::BackspaceVariableFormattedRecord(
     IoErrorHandler &handler) {
-  std::int64_t start{frameOffsetInFile_ + recordOffsetInFrame_};
-  --currentRecordNumber;
-  RUNTIME_CHECK(handler, currentRecordNumber > 0);
-  if (currentRecordNumber == 1) {
-    // To simplify the code below, treat a backspace to the first record
-    // as a special case;
-    RUNTIME_CHECK(handler, start > 0);
-    *recordLength = start - 1;
-    frameOffsetInFile_ = 0;
-    recordOffsetInFrame_ = 0;
-    ReadFrame(0, *recordLength + 1, handler);
-  } else {
-    RUNTIME_CHECK(handler, start > 1);
-    std::int64_t at{start - 2}; // byte before previous record's newline
-    while (true) {
+  // File offset of previous record's newline
+  auto prevNL{
+      frameOffsetInFile_ + static_cast<std::int64_t>(recordOffsetInFrame_) - 1};
+  if (prevNL < 0) {
+    handler.SignalError(IostatBackspaceAtFirstRecord);
+    return;
+  }
+    if (frameOffsetInFile_ < prevNL) {
       if (const char *p{
-              FindLastNewline(Frame(), at - frameOffsetInFile_ + 1)}) {
-        // This is the newline that ends the record before the previous one.
+              FindLastNewline(Frame(), prevNL - 1 - frameOffsetInFile_)}) {
         recordOffsetInFrame_ = p - Frame() + 1;
-        *recordLength = start - 1 - (frameOffsetInFile_ + recordOffsetInFrame_);
-        break;
-      }
-      RUNTIME_CHECK(handler, frameOffsetInFile_ > 0);
-      at = frameOffsetInFile_ - 1;
-      if (auto bytesBefore{BytesBufferedBeforeFrame()}) {
-        frameOffsetInFile_ = FrameAt() - bytesBefore;
-      } else {
-        static constexpr int chunk{1024};
-        frameOffsetInFile_ = std::max<std::int64_t>(0, at - chunk);
-      }
-      std::size_t want{static_cast<std::size_t>(start - frameOffsetInFile_)};
-      std::size_t got{ReadFrame(frameOffsetInFile_, want, handler)};
-      RUNTIME_CHECK(handler, got >= want);
-    }
-  }
-  std::size_t want{
-      static_cast<std::size_t>(recordOffsetInFrame_ + *recordLength + 1)};
-  RUNTIME_CHECK(handler, FrameLength() >= want);
+        *recordLength = prevNL - (frameOffsetInFile_ + recordOffsetInFrame_);
+        break;
+      }
+    }
+    if (frameOffsetInFile_ == 0) {
+      recordOffsetInFrame_ = 0;
+      *recordLength = prevNL;
+    frameOffsetInFile_ -= std::min<std::int64_t>(frameOffsetInFile_, 1024);
+    auto need{static_cast<std::size_t>(prevNL + 1 - frameOffsetInFile_)};
+    auto got{ReadFrame(frameOffsetInFile_, need, handler)};
+    RUNTIME_CHECK(handler, got >= need);
+  }
   RUNTIME_CHECK(handler, Frame()[recordOffsetInFrame_ + *recordLength] == '\n');
   if (*recordLength > 0 &&
       Frame()[recordOffsetInFrame_ + *recordLength - 1] == '\r') {
     --*recordLength;
   }
 }
+
+void ExternalFileUnit::DoImpliedEndfile(IoErrorHandler &handler) {
+  if (impliedEndfile_) {
+    impliedEndfile_ = false;
+    if (access == Access::Sequential && mayPosition()) {
+      DoEndfile(handler);
+    }
+  }
+}
+
+void ExternalFileUnit::DoEndfile(IoErrorHandler &handler) {
+  endfileRecordNumber = currentRecordNumber;
+  Truncate(frameOffsetInFile_ + recordOffsetInFrame_, handler);
+  BeginRecord();
+  impliedEndfile_ = false;
+}
 } // namespace Fortran::runtime::io