[flang] Initial UTF-8 support in runtime I/O

[klausler]

Summary

Implements UTF-8 encoding and decoding for external units
with OPEN(ENCODING='UTF-8'). This encoding applies to default
CHARACTER values that are not 7-bit ASCII as well as to
the wide CHARACTER kinds 2 and 4. Basic testing is in place
via direct calls to the runtime I/O APIs, but serious checkout
awaits lowering support of the wide CHARACTER kinds.

[jeanPerier]

The code looks good to me, but something might be wrong since the added test is failing.

flang/unittests/Runtime/ExternalIOTest.cpp

844

This test is failing with the built bots.

[klausler]

flang/unittests/Runtime/ExternalIOTest.cpp

844

Yes, I see; will figure it out.

[klausler]

Fix bugs exposed by testing in other environments

[jeanPerier]

LGTM

flang/runtime/utf.cpp

14

You could use // clang-format off and // clang-format on around UTF8FirstByteTable to prevent anyone from destroying the human readable format of this table when running clang-format.

[klausler]

Silence messages about clang-format for a table that shouldn't be automatically reformatted.

flang/include/flang/Runtime/iostat.h

enum Iostat {
   IostatListIoOnDirectAccessUnit,
   IostatUnformattedChildOnFormattedParent,
   IostatFormattedChildOnUnformattedParent,
   IostatChildInputFromOutputParent,
   IostatChildOutputToInputParent,
   IostatShortRead,
   IostatMissingTerminator,
   IostatBadUnformattedRecord,
+  IostatUTF8Decoding,
 };
 
 const char *IostatErrorString(int);
 
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_IOSTAT_H_

flang/runtime/CMakeLists.txt

add_flang_library(FortranRuntime
   terminator.cpp
   time-intrinsic.cpp
   tools.cpp
   transformational.cpp
   type-code.cpp
   type-info.cpp
   unit.cpp
   unit-map.cpp
+  utf.cpp
 
   LINK_LIBS
   FortranDecimal
 )

flang/runtime/descriptor-io.h

inline bool FormattedCharacterIO(
   SubscriptValue subscripts[maxRank];
   descriptor.GetLowerBounds(subscripts);
   std::size_t length{descriptor.ElementBytes() / sizeof(A)};
   auto *listOutput{io.get_if<ListDirectedStatementState<Direction::Output>>()};
   bool anyInput{false};
   for (std::size_t j{0}; j < numElements; ++j) {
     A *x{&ExtractElement<A>(io, descriptor, subscripts)};
     if (listOutput) {
-      if (!ListDirectedDefaultCharacterOutput(io, *listOutput, x, length)) {
+      if (!ListDirectedCharacterOutput(io, *listOutput, x, length)) {
         return false;
       }
     } else if (auto edit{io.GetNextDataEdit()}) {
       if constexpr (DIR == Direction::Output) {
-        if (!EditDefaultCharacterOutput(io, *edit, x, length)) {
+        if (!EditCharacterOutput(io, *edit, x, length)) {
           return false;
         }
       } else {
         if (edit->descriptor != DataEdit::ListDirectedNullValue) {
-          if (EditDefaultCharacterInput(io, *edit, x, length)) {
+          if (EditCharacterInput(io, *edit, x, length)) {
             anyInput = true;
           } else {
             return anyInput && edit->IsNamelist();
           }
         }
       }
     } else {
       return false;
case TypeCategory::Complex:
             "DescriptorIO: Unimplemented COMPLEX kind (%d) in descriptor",
             kind);
         return false;
       }
     case TypeCategory::Character:
       switch (kind) {
       case 1:
         return FormattedCharacterIO<char, DIR>(io, descriptor);
-      // TODO cases 2, 4
+      case 2:
+        return FormattedCharacterIO<char16_t, DIR>(io, descriptor);
+      case 4:
+        return FormattedCharacterIO<char32_t, DIR>(io, descriptor);
       default:
         handler.Crash(
             "DescriptorIO: Unimplemented CHARACTER kind (%d) in descriptor",
             kind);
         return false;
       }
     case TypeCategory::Logical:
       switch (kind) {

flang/runtime/edit-input.h

 namespace Fortran::runtime::io {
 
 bool EditIntegerInput(IoStatementState &, const DataEdit &, void *, int kind);
 
 template <int KIND>
 bool EditRealInput(IoStatementState &, const DataEdit &, void *);
 
 bool EditLogicalInput(IoStatementState &, const DataEdit &, bool &);
-bool EditDefaultCharacterInput(
-    IoStatementState &, const DataEdit &, char *, std::size_t);
+
+template <typename CHAR>
+bool EditCharacterInput(
+    IoStatementState &, const DataEdit &, CHAR *, std::size_t);
 
 extern template bool EditRealInput<2>(
     IoStatementState &, const DataEdit &, void *);
 extern template bool EditRealInput<3>(
     IoStatementState &, const DataEdit &, void *);
 extern template bool EditRealInput<4>(
     IoStatementState &, const DataEdit &, void *);
 extern template bool EditRealInput<8>(
     IoStatementState &, const DataEdit &, void *);
 extern template bool EditRealInput<10>(
     IoStatementState &, const DataEdit &, void *);
 // TODO: double/double
 extern template bool EditRealInput<16>(
     IoStatementState &, const DataEdit &, void *);
+
+extern template bool EditCharacterInput(
+    IoStatementState &, const DataEdit &, char *, std::size_t);
+extern template bool EditCharacterInput(
+    IoStatementState &, const DataEdit &, char16_t *, std::size_t);
+extern template bool EditCharacterInput(
+    IoStatementState &, const DataEdit &, char32_t *, std::size_t);
+
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_EDIT_INPUT_H_

flang/runtime/edit-input.cpp

 //===-- runtime/edit-input.cpp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "edit-input.h"
 #include "namelist.h"
+#include "utf.h"
 #include "flang/Common/real.h"
 #include "flang/Common/uint128.h"
 #include <algorithm>
 
 namespace Fortran::runtime::io {
 
 static bool EditBOZInput(IoStatementState &io, const DataEdit &edit, void *n,
     int base, int totalBitSize) {

 // Returns true if there's a '-' sign.
 static bool ScanNumericPrefix(IoStatementState &io, const DataEdit &edit,
     std::optional<char32_t> &next, std::optional<int> &remaining) {
   next = io.PrepareInput(edit, remaining);
   bool negative{false};
   if (next) {
     negative = *next == '-';
     if (negative || *next == '+') {
-      io.GotChar();
       io.SkipSpaces(remaining);
       next = io.NextInField(remaining, edit);
     }
   }
   return negative;
 }
 
 bool EditIntegerInput(
case 'I':
     break;
   case 'B':
     return EditBOZInput(io, edit, n, 2, kind << 3);
   case 'O':
     return EditBOZInput(io, edit, n, 8, kind << 3);
   case 'Z':
     return EditBOZInput(io, edit, n, 16, kind << 3);
   case 'A': // legacy extension
-    return EditDefaultCharacterInput(
-        io, edit, reinterpret_cast<char *>(n), kind);
+    return EditCharacterInput(io, edit, reinterpret_cast<char *>(n), kind);
   default:
     io.GetIoErrorHandler().SignalError(IostatErrorInFormat,
         "Data edit descriptor '%c' may not be used with an INTEGER data item",
         edit.descriptor);
     return false;
   }
   std::optional<int> remaining;
   std::optional<char32_t> next;
static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
   }
   // Consume the trailing ')' of a list-directed or NAMELIST complex
   // input value.
   if (edit.descriptor == DataEdit::ListDirectedImaginaryPart) {
     if (next && (*next == ' ' || *next == '\t')) {
       next = io.NextInField(remaining, edit);
     }
     if (!next) { // NextInField fails on separators like ')'
-      next = io.GetCurrentChar();
+      std::size_t byteCount{0};
+      next = io.GetCurrentChar(byteCount);
       if (next && *next == ')') {
-        io.HandleRelativePosition(1);
+        io.HandleRelativePosition(byteCount);
       }
     }
   } else if (remaining) {
     while (next && (*next == ' ' || *next == '\t')) {
       next = io.NextInField(remaining, edit);
     }
     if (next) {
       return 0; // error: unused nonblank character in fixed-width field
return EditBOZInput(
         io, edit, n, 2, common::BitsForBinaryPrecision(binaryPrecision));
   case 'O':
     return EditBOZInput(
         io, edit, n, 8, common::BitsForBinaryPrecision(binaryPrecision));
   case 'Z':
     return EditBOZInput(
         io, edit, n, 16, common::BitsForBinaryPrecision(binaryPrecision));
   case 'A': // legacy extension
-    return EditDefaultCharacterInput(
-        io, edit, reinterpret_cast<char *>(n), KIND);
+    return EditCharacterInput(io, edit, reinterpret_cast<char *>(n), KIND);
   default:
     io.GetIoErrorHandler().SignalError(IostatErrorInFormat,
         "Data edit descriptor '%c' may not be used for REAL input",
         edit.descriptor);
     return false;
   }
 }
 
bool EditLogicalInput(IoStatementState &io, const DataEdit &edit, bool &x) {
   } else if (edit.descriptor == DataEdit::ListDirected) {
     while (io.NextInField(remaining, edit)) { // discard rest of field
     }
   }
   return true;
 }
 
 // See 13.10.3.1 paragraphs 7-9 in Fortran 2018
+template <typename CHAR>
 static bool EditDelimitedCharacterInput(
-    IoStatementState &io, char *x, std::size_t length, char32_t delimiter) {
+    IoStatementState &io, CHAR *x, std::size_t length, char32_t delimiter) {
   bool result{true};
   while (true) {
-    auto ch{io.GetCurrentChar()};
+    std::size_t byteCount{0};
+    auto ch{io.GetCurrentChar(byteCount)};
     if (!ch) {
       if (io.AdvanceRecord()) {
         continue;
       } else {
         result = false; // EOF in character value
         break;
       }
     }
-    io.HandleRelativePosition(1);
+    io.HandleRelativePosition(byteCount);
     if (*ch == delimiter) {
-      auto next{io.GetCurrentChar()};
+      auto next{io.GetCurrentChar(byteCount)};
       if (next && *next == delimiter) {
         // Repeated delimiter: use as character value
-        io.HandleRelativePosition(1);
+        io.HandleRelativePosition(byteCount);
       } else {
         break; // closing delimiter
       }
     }
     if (length > 0) {
       *x++ = *ch;
       --length;
     }
   }
   std::fill_n(x, length, ' ');
   return result;
 }
 
-static bool EditListDirectedDefaultCharacterInput(
-    IoStatementState &io, char *x, std::size_t length, const DataEdit &edit) {
-  auto ch{io.GetCurrentChar()};
+template <typename CHAR>
+static bool EditListDirectedCharacterInput(
+    IoStatementState &io, CHAR *x, std::size_t length, const DataEdit &edit) {
+  std::size_t byteCount{0};
+  auto ch{io.GetCurrentChar(byteCount)};
   if (ch && (*ch == '\'' || *ch == '"')) {
-    io.HandleRelativePosition(1);
+    io.HandleRelativePosition(byteCount);
     return EditDelimitedCharacterInput(io, x, length, *ch);
   }
   if (IsNamelistName(io) || io.GetConnectionState().IsAtEOF()) {
     return false;
   }
   // Undelimited list-directed character input: stop at a value separator
-  // or the end of the current record.
-  std::optional<int> remaining{length};
+  // or the end of the current record.  Subtlety: the "remaining" count
+  // here is a dummy that's used to avoid the interpretation of separators
+  // in NextInField.
+  std::optional<int> remaining{maxUTF8Bytes};
   while (std::optional<char32_t> next{io.NextInField(remaining, edit)}) {
     switch (*next) {
     case ' ':
     case '\t':
     case ',':
     case ';':
     case '/':
       remaining = 0; // value separator: stop
       break;
     default:
       *x++ = *next;
       --length;
+      remaining = maxUTF8Bytes;
     }
   }
   std::fill_n(x, length, ' ');
   return true;
 }
 
-bool EditDefaultCharacterInput(
-    IoStatementState &io, const DataEdit &edit, char *x, std::size_t length) {
+template <typename CHAR>
+bool EditCharacterInput(
+    IoStatementState &io, const DataEdit &edit, CHAR *x, std::size_t length) {
   switch (edit.descriptor) {
   case DataEdit::ListDirected:
-    return EditListDirectedDefaultCharacterInput(io, x, length, edit);
+    return EditListDirectedCharacterInput(io, x, length, edit);
   case 'A':
   case 'G':
     break;
   default:
     io.GetIoErrorHandler().SignalError(IostatErrorInFormat,
         "Data edit descriptor '%c' may not be used with a CHARACTER data item",
         edit.descriptor);
     return false;
   }
-  if (io.GetConnectionState().IsAtEOF()) {
+  const ConnectionState &connection{io.GetConnectionState()};
+  if (connection.IsAtEOF()) {
     return false;
   }
   std::size_t remaining{length};
   if (edit.width && *edit.width > 0) {
     remaining = *edit.width;
   }
   // When the field is wider than the variable, we drop the leading
   // characters.  When the variable is wider than the field, there's
   // trailing padding.
   const char *input{nullptr};
   std::size_t ready{0};
   bool hitEnd{false};
-  if (remaining > length) {
-    // Discard leading bytes.
-    // These bytes don't count towards INQUIRE(IOLENGTH=).
-    std::size_t skip{remaining - length};
-    do {
-      if (ready == 0) {
-        ready = io.GetNextInputBytes(input);
-        if (ready == 0) {
-          hitEnd = true;
-          break;
-        }
-      }
-      std::size_t chunk{std::min<std::size_t>(skip, ready)};
-      io.HandleRelativePosition(chunk);
-      ready -= chunk;
-      input += chunk;
-      skip -= chunk;
-    } while (skip > 0);
-    remaining = length;
-  }
+  // Skip leading bytes.
+  std::size_t skip{remaining > length ? remaining - length : 0};
   // Transfer payload bytes; these do count.
   while (remaining > 0) {
     if (ready == 0) {
       ready = io.GetNextInputBytes(input);
       if (ready == 0) {
         hitEnd = true;
         break;
       }
     }
-    std::size_t chunk{std::min<std::size_t>(remaining, ready)};
+    std::size_t chunk;
+    bool skipping{skip > 0};
+    if (connection.isUTF8) {
+      chunk = MeasureUTF8Bytes(*input);
+      if (skipping) {
+        --skip;
+      } else if (auto ucs{DecodeUTF8(input)}) {
+        *x++ = *ucs;
+        --length;
+      } else if (chunk == 0) {
+        // error recovery: skip bad encoding
+        chunk = 1;
+      }
+      --remaining;
+    } else {
+      if (skipping) {
+        chunk = std::min<std::size_t>(skip, ready);
+        skip -= chunk;
+      } else {
+        chunk = std::min<std::size_t>(remaining, ready);
         std::memcpy(x, input, chunk);
         x += chunk;
+        length -= chunk;
+      }
+      remaining -= chunk;
+    }
     input += chunk;
+    if (!skipping) {
       io.GotChar(chunk);
+    }
     io.HandleRelativePosition(chunk);
     ready -= chunk;
-    remaining -= chunk;
-    length -= chunk;
   }
   // Pad the remainder of the input variable, if any.
-  std::memset(x, ' ', length);
+  std::fill_n(x, length, ' ');
   if (hitEnd) {
     io.CheckForEndOfRecord(); // signal any needed error
   }
   return true;
 }
 
 template bool EditRealInput<2>(IoStatementState &, const DataEdit &, void *);
 template bool EditRealInput<3>(IoStatementState &, const DataEdit &, void *);
 template bool EditRealInput<4>(IoStatementState &, const DataEdit &, void *);
 template bool EditRealInput<8>(IoStatementState &, const DataEdit &, void *);
 template bool EditRealInput<10>(IoStatementState &, const DataEdit &, void *);
 // TODO: double/double
 template bool EditRealInput<16>(IoStatementState &, const DataEdit &, void *);
+
+template bool EditCharacterInput(
+    IoStatementState &, const DataEdit &, char *, std::size_t);
+template bool EditCharacterInput(
+    IoStatementState &, const DataEdit &, char16_t *, std::size_t);
+template bool EditCharacterInput(
+    IoStatementState &, const DataEdit &, char32_t *, std::size_t);
+
 } // namespace Fortran::runtime::io

flang/runtime/edit-output.h

private:
   BinaryFloatingPoint x_;
   char buffer_[BinaryFloatingPoint::maxDecimalConversionDigits +
       EXTRA_DECIMAL_CONVERSION_SPACE];
 };
 
 bool ListDirectedLogicalOutput(
     IoStatementState &, ListDirectedStatementState<Direction::Output> &, bool);
 bool EditLogicalOutput(IoStatementState &, const DataEdit &, bool);
-bool ListDirectedDefaultCharacterOutput(IoStatementState &,
-    ListDirectedStatementState<Direction::Output> &, const char *, std::size_t);
-bool EditDefaultCharacterOutput(
-    IoStatementState &, const DataEdit &, const char *, std::size_t);
+
+template <typename CHAR>
+bool ListDirectedCharacterOutput(IoStatementState &,
+    ListDirectedStatementState<Direction::Output> &, const CHAR *,
+    std::size_t chars);
+extern template bool ListDirectedCharacterOutput(IoStatementState &,
+    ListDirectedStatementState<Direction::Output> &, const char *,
+    std::size_t chars);
+extern template bool ListDirectedCharacterOutput(IoStatementState &,
+    ListDirectedStatementState<Direction::Output> &, const char16_t *,
+    std::size_t chars);
+extern template bool ListDirectedCharacterOutput(IoStatementState &,
+    ListDirectedStatementState<Direction::Output> &, const char32_t *,
+    std::size_t chars);
+
+template <typename CHAR>
+bool EditCharacterOutput(
+    IoStatementState &, const DataEdit &, const CHAR *, std::size_t chars);
+extern template bool EditCharacterOutput(
+    IoStatementState &, const DataEdit &, const char *, std::size_t chars);
+extern template bool EditCharacterOutput(
+    IoStatementState &, const DataEdit &, const char16_t *, std::size_t chars);
+extern template bool EditCharacterOutput(
+    IoStatementState &, const DataEdit &, const char32_t *, std::size_t chars);
 
 extern template bool EditIntegerOutput<1>(
     IoStatementState &, const DataEdit &, std::int8_t);
 extern template bool EditIntegerOutput<2>(
     IoStatementState &, const DataEdit &, std::int16_t);
 extern template bool EditIntegerOutput<4>(
     IoStatementState &, const DataEdit &, std::int32_t);
 extern template bool EditIntegerOutput<8>(

flang/runtime/edit-output.cpp

 //===-- runtime/edit-output.cpp -------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "edit-output.h"
+#include "utf.h"
 #include "flang/Common/uint128.h"
 #include <algorithm>
 
 namespace Fortran::runtime::io {
 
 template <int KIND>
 bool EditIntegerOutput(IoStatementState &io, const DataEdit &edit,
     common::HostSignedIntType<8 * KIND> n) {
case 'O':
     break;
   case 'Z':
     for (; un > 0; un >>= 4) {
       int digit = static_cast<int>(un) & 0xf;
       *--p = digit >= 10 ? 'A' + (digit - 10) : '0' + digit;
     }
     break;
   case 'A': // legacy extension
-    return EditDefaultCharacterOutput(
+    return EditCharacterOutput(
         io, edit, reinterpret_cast<char *>(&n), sizeof n);
   default:
     io.GetIoErrorHandler().Crash(
         "Data edit descriptor '%c' may not be used with an INTEGER data item",
         edit.descriptor);
     return false;
   }
 
template <int KIND> bool RealOutputEditing<KIND>::Edit(const DataEdit &edit) {
   case 'O':
   case 'Z':
     return EditIntegerOutput<KIND>(io_, edit,
         static_cast<common::HostSignedIntType<8 * KIND>>(
             decimal::BinaryFloatingPointNumber<binaryPrecision>{x_}.raw()));
   case 'G':
     return Edit(EditForGOutput(edit));
   case 'A': // legacy extension
-    return EditDefaultCharacterOutput(
+    return EditCharacterOutput(
         io_, edit, reinterpret_cast<char *>(&x_), sizeof x_);
   default:
     if (edit.IsListDirected()) {
       return EditListDirectedOutput(edit);
     }
     io_.GetIoErrorHandler().SignalError(IostatErrorInFormat,
         "Data edit descriptor '%c' may not be used with a REAL data item",
         edit.descriptor);
bool EditLogicalOutput(IoStatementState &io, const DataEdit &edit, bool truth) {
   default:
     io.GetIoErrorHandler().SignalError(IostatErrorInFormat,
         "Data edit descriptor '%c' may not be used with a LOGICAL data item",
         edit.descriptor);
     return false;
   }
 }
 
-bool ListDirectedDefaultCharacterOutput(IoStatementState &io,
-    ListDirectedStatementState<Direction::Output> &list, const char *x,
+template <typename CHAR>
+bool ListDirectedCharacterOutput(IoStatementState &io,
+    ListDirectedStatementState<Direction::Output> &list, const CHAR *x,
     std::size_t length) {
   bool ok{true};
   MutableModes &modes{io.mutableModes()};
   ConnectionState &connection{io.GetConnectionState()};
   if (modes.delim) {
     ok = ok && list.EmitLeadingSpaceOrAdvance(io);
     // Value is delimited with ' or " marks, and interior
     // instances of that character are doubled.
-    auto EmitOne{[&](char ch) {
+    auto EmitOne{[&](CHAR ch) {
       if (connection.NeedAdvance(1)) {
         ok = ok && io.AdvanceRecord();
       }
-      ok = ok && io.Emit(&ch, 1);
+      ok = ok && io.EmitEncoded(&ch, 1);
     }};
     EmitOne(modes.delim);
     for (std::size_t j{0}; j < length; ++j) {
       // Doubled delimiters must be put on the same record
       // in order to be acceptable as list-directed or NAMELIST
       // input; however, this requirement is not always possible
       // when the records have a fixed length, as is the case with
       // internal output.  The standard is silent on what should
       // happen, and no two extant Fortran implementations do
       // the same thing when tested with this case.
       // This runtime splits the doubled delimiters across
       // two records for lack of a better alternative.
-      if (x[j] == modes.delim) {
+      if (x[j] == static_cast<CHAR>(modes.delim)) {
         EmitOne(x[j]);
       }
       EmitOne(x[j]);
     }
     EmitOne(modes.delim);
   } else {
     // Undelimited list-directed output
     ok = ok && list.EmitLeadingSpaceOrAdvance(io, length > 0 ? 1 : 0, true);
     std::size_t put{0};
+    std::size_t oneIfUTF8{connection.isUTF8 ? 1 : length};
     while (ok && put < length) {
-      auto chunk{std::min(length - put, connection.RemainingSpaceInRecord())};
-      ok = ok && io.Emit(x + put, chunk);
+      if (std::size_t chunk{std::min<std::size_t>(
+              std::min<std::size_t>(length - put, oneIfUTF8),
+              connection.RemainingSpaceInRecord())}) {
+        ok = io.EmitEncoded(x + put, chunk);
         put += chunk;
-      if (put < length) {
-        ok = ok && io.AdvanceRecord() && io.Emit(" ", 1);
+      } else {
+        ok = io.AdvanceRecord() && io.Emit(" ", 1);
       }
     }
     list.set_lastWasUndelimitedCharacter(true);
   }
   return ok;
 }
 
-bool EditDefaultCharacterOutput(IoStatementState &io, const DataEdit &edit,
-    const char *x, std::size_t length) {
+template <typename CHAR>
+bool EditCharacterOutput(IoStatementState &io, const DataEdit &edit,
+    const CHAR *x, std::size_t length) {
   switch (edit.descriptor) {
   case 'A':
   case 'G':
     break;
   default:
     io.GetIoErrorHandler().SignalError(IostatErrorInFormat,
         "Data edit descriptor '%c' may not be used with a CHARACTER data item",
         edit.descriptor);
     return false;
   }
   int len{static_cast<int>(length)};
   int width{edit.width.value_or(len)};
   return io.EmitRepeated(' ', std::max(0, width - len)) &&
-      io.Emit(x, std::min(width, len));
+      io.EmitEncoded(x, std::min(width, len));
 }
 
 template bool EditIntegerOutput<1>(
     IoStatementState &, const DataEdit &, std::int8_t);
 template bool EditIntegerOutput<2>(
     IoStatementState &, const DataEdit &, std::int16_t);
 template bool EditIntegerOutput<4>(
     IoStatementState &, const DataEdit &, std::int32_t);
 template bool EditIntegerOutput<8>(
     IoStatementState &, const DataEdit &, std::int64_t);
 template bool EditIntegerOutput<16>(
     IoStatementState &, const DataEdit &, common::int128_t);
 
 template class RealOutputEditing<2>;
 template class RealOutputEditing<3>;
 template class RealOutputEditing<4>;
 template class RealOutputEditing<8>;
 template class RealOutputEditing<10>;
 // TODO: double/double
 template class RealOutputEditing<16>;
+
+template bool ListDirectedCharacterOutput(IoStatementState &,
+    ListDirectedStatementState<Direction::Output> &, const char *,
+    std::size_t chars);
+template bool ListDirectedCharacterOutput(IoStatementState &,
+    ListDirectedStatementState<Direction::Output> &, const char16_t *,
+    std::size_t chars);
+template bool ListDirectedCharacterOutput(IoStatementState &,
+    ListDirectedStatementState<Direction::Output> &, const char32_t *,
+    std::size_t chars);
+
+template bool EditCharacterOutput(
+    IoStatementState &, const DataEdit &, const char *, std::size_t chars);
+template bool EditCharacterOutput(
+    IoStatementState &, const DataEdit &, const char16_t *, std::size_t chars);
+template bool EditCharacterOutput(
+    IoStatementState &, const DataEdit &, const char32_t *, std::size_t chars);
+
 } // namespace Fortran::runtime::io

flang/runtime/environment.h

 #endif
 
 // External unformatted I/O data conversions
 enum class Convert { Unknown, Native, LittleEndian, BigEndian, Swap };
 
 std::optional<Convert> GetConvertFromString(const char *, std::size_t);
 
 struct ExecutionEnvironment {
+  constexpr ExecutionEnvironment(){};
   void Configure(int argc, const char *argv[], const char *envp[]);
   const char *GetEnv(
       const char *name, std::size_t name_length, const Terminator &terminator);
 
-  int argc;
-  const char **argv;
-  const char **envp;
-
-  int listDirectedOutputLineLengthLimit; // FORT_FMT_RECL
-  enum decimal::FortranRounding defaultOutputRoundingMode;
-  Convert conversion; // FORT_CONVERT
-  bool noStopMessage; // NO_STOP_MESSAGE=1 inhibits "Fortran STOP"
+  int argc{0};
+  const char **argv{nullptr};
+  const char **envp{nullptr};
+
+  int listDirectedOutputLineLengthLimit{79}; // FORT_FMT_RECL
+  enum decimal::FortranRounding defaultOutputRoundingMode{
+      decimal::FortranRounding::RoundNearest}; // RP(==PN)
+  Convert conversion{Convert::Unknown}; // FORT_CONVERT
+  bool noStopMessage{false}; // NO_STOP_MESSAGE=1 inhibits "Fortran STOP"
+  bool defaultUTF8{false}; // DEFAULT_UTF8
 };
+
 extern ExecutionEnvironment executionEnvironment;
 } // namespace Fortran::runtime
 
 #endif // FORTRAN_RUNTIME_ENVIRONMENT_H_

flang/runtime/environment.cpp

if (auto *x{std::getenv("NO_STOP_MESSAGE")}) {
     if (n >= 0 && n <= 1 && *end == '\0') {
       noStopMessage = n != 0;
     } else {
       std::fprintf(stderr,
           "Fortran runtime: NO_STOP_MESSAGE=%s is invalid; ignored\n", x);
     }
   }
 
+  if (auto *x{std::getenv("DEFAULT_UTF8")}) {
+    char *end;
+    auto n{std::strtol(x, &end, 10)};
+    if (n >= 0 && n <= 1 && *end == '\0') {
+      defaultUTF8 = n != 0;
+    } else {
+      std::fprintf(
+          stderr, "Fortran runtime: DEFAULT_UTF8=%s is invalid; ignored\n", x);
+    }
+  }
+
   // TODO: Set RP/ROUND='PROCESSOR_DEFINED' from environment
 }
 
 const char *ExecutionEnvironment::GetEnv(
     const char *name, std::size_t name_length, const Terminator &terminator) {
   RUNTIME_CHECK(terminator, name && name_length);
 
   OwningPtr<char> cStyleName{
       SaveDefaultCharacter(name, name_length, terminator)};
   RUNTIME_CHECK(terminator, cStyleName);
 
   return std::getenv(cStyleName.get());
 }
 } // namespace Fortran::runtime

flang/runtime/internal-unit.h

public:
   using Scalar =
       std::conditional_t<DIR == Direction::Input, const char *, char *>;
   InternalDescriptorUnit(Scalar, std::size_t);
   InternalDescriptorUnit(const Descriptor &, const Terminator &);
   void EndIoStatement();
 
   bool Emit(const char *, std::size_t, IoErrorHandler &);
   std::size_t GetNextInputBytes(const char *&, IoErrorHandler &);
-  std::optional<char32_t> GetCurrentChar(IoErrorHandler &);
   bool AdvanceRecord(IoErrorHandler &);
   void BackspaceRecord(IoErrorHandler &);
 
 private:
   Descriptor &descriptor() { return staticDescriptor_.descriptor(); }
   const Descriptor &descriptor() const {
     return staticDescriptor_.descriptor();
   }

flang/runtime/internal-unit.cpp

if constexpr (DIR == Direction::Output) {
     } else {
       p = &record[positionInRecord];
       return *recordLength - positionInRecord;
     }
   }
 }
 
 template <Direction DIR>
-std::optional<char32_t> InternalDescriptorUnit<DIR>::GetCurrentChar(
-    IoErrorHandler &handler) {
-  const char *p{nullptr};
-  std::size_t bytes{GetNextInputBytes(p, handler)};
-  if (bytes == 0) {
-    return std::nullopt;
-  } else {
-    if (isUTF8) {
-      // TODO: UTF-8 decoding
-    }
-    return *p;
-  }
-}
-
-template <Direction DIR>
 bool InternalDescriptorUnit<DIR>::AdvanceRecord(IoErrorHandler &handler) {
   if (currentRecordNumber >= endfileRecordNumber.value_or(0)) {
     handler.SignalEnd();
     return false;
   }
   if constexpr (DIR == Direction::Output) {
     BlankFillOutputRecord();
   }

flang/runtime/io-stmt.h

public:
   void CompleteOperation();
   // Completes an I/O statement and reclaims storage.
   int EndIoStatement();
 
   bool Emit(const char *, std::size_t, std::size_t elementBytes);
   bool Emit(const char *, std::size_t);
   bool Emit(const char16_t *, std::size_t chars);
   bool Emit(const char32_t *, std::size_t chars);
+  template <typename CHAR> bool EmitEncoded(const CHAR *, std::size_t);
   bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
   std::size_t GetNextInputBytes(const char *&);
   bool AdvanceRecord(int = 1);
   void BackspaceRecord();
   void HandleRelativePosition(std::int64_t);
   void HandleAbsolutePosition(std::int64_t); // for r* in list I/O
   std::optional<DataEdit> GetNextDataEdit(int maxRepeat = 1);
   ExternalFileUnit *GetExternalFileUnit() const; // null if internal unit
return std::visit(
             return &x.get();
           }
           return nullptr;
         },
         u_);
   }
 
   // Vacant after the end of the current record
-  std::optional<char32_t> GetCurrentChar() {
-    const char *p{nullptr};
-    std::size_t bytes{GetNextInputBytes(p)};
-    if (bytes == 0) {
-      return std::nullopt;
-    } else {
-      // TODO: UTF-8 decoding; may have to get more bytes in a loop
-      return *p;
-    }
-  }
+  std::optional<char32_t> GetCurrentChar(std::size_t &byteCount);
 
   bool EmitRepeated(char, std::size_t);
   bool EmitField(const char *, std::size_t length, std::size_t width);
 
   // For fixed-width fields, initialize the number of remaining characters.
   // Skip over leading blanks, then return the first non-blank character (if
   // any).
   std::optional<char32_t> PrepareInput(
       const DataEdit &edit, std::optional<int> &remaining) {
     remaining.reset();
     if (edit.descriptor == DataEdit::ListDirected) {
-      GetNextNonBlank();
+      std::size_t byteCount{0};
+      GetNextNonBlank(byteCount);
     } else {
       if (edit.width.value_or(0) > 0) {
         remaining = *edit.width;
       }
       SkipSpaces(remaining);
     }
     return NextInField(remaining, edit);
   }
 
   std::optional<char32_t> SkipSpaces(std::optional<int> &remaining) {
     while (!remaining || *remaining > 0) {
-      if (auto ch{GetCurrentChar()}) {
+      std::size_t byteCount{0};
+      if (auto ch{GetCurrentChar(byteCount)}) {
         if (*ch != ' ' && *ch != '\t') {
           return ch;
         }
-        HandleRelativePosition(1);
         if (remaining) {
-          GotChar();
-          --*remaining;
+          if (static_cast<std::size_t>(*remaining) < byteCount) {
+            break;
+          }
+          GotChar(byteCount);
+          *remaining -= byteCount;
         }
+        HandleRelativePosition(byteCount);
       } else {
         break;
       }
     }
     return std::nullopt;
   }
 
   // Acquires the next input character, respecting any applicable field width
   // or separator character.
   std::optional<char32_t> NextInField(
       std::optional<int> &remaining, const DataEdit &);
 
   // Detect and signal any end-of-record condition after input.
   // Returns true if at EOR and remaining input should be padded with blanks.
   bool CheckForEndOfRecord();
 
   // Skips spaces, advances records, and ignores NAMELIST comments
-  std::optional<char32_t> GetNextNonBlank() {
-    auto ch{GetCurrentChar()};
+  std::optional<char32_t> GetNextNonBlank(std::size_t &byteCount) {
+    auto ch{GetCurrentChar(byteCount)};
     bool inNamelist{mutableModes().inNamelist};
     while (!ch || *ch == ' ' || *ch == '\t' || (inNamelist && *ch == '!')) {
       if (ch && (*ch == ' ' || *ch == '\t')) {
-        HandleRelativePosition(1);
+        HandleRelativePosition(byteCount);
       } else if (!AdvanceRecord()) {
         return std::nullopt;
       }
-      ch = GetCurrentChar();
+      ch = GetCurrentChar(byteCount);
     }
     return ch;
   }
 
   template <Direction D> void CheckFormattedStmtType(const char *name) {
     if (!get_if<FormattedIoStatementState<D>>()) {
       GetIoErrorHandler().Crash(
           "%s called for I/O statement that is not formatted %s", name,
public:
   int EndIoStatement();
   ConnectionState &GetConnectionState() { return connection_; }
   MutableModes &mutableModes() { return connection_.modes; }
 
 private:
   ConnectionState connection_;
 };
 
+extern template bool IoStatementState::EmitEncoded<char>(
+    const char *, std::size_t);
+extern template bool IoStatementState::EmitEncoded<char16_t>(
+    const char16_t *, std::size_t);
+extern template bool IoStatementState::EmitEncoded<char32_t>(
+    const char32_t *, std::size_t);
+
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_IO_STMT_H_

flang/runtime/io-stmt.cpp

 //===-- runtime/io-stmt.cpp -----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "io-stmt.h"
 #include "connection.h"
 #include "format.h"
 #include "tools.h"
 #include "unit.h"
+#include "utf.h"
 #include "flang/Runtime/memory.h"
 #include <algorithm>
 #include <cstdio>
 #include <cstring>
 #include <limits>
+#include <type_traits>
 
 namespace Fortran::runtime::io {
 
 bool IoStatementBase::Emit(const char *, std::size_t, std::size_t) {
   return false;
 }
 
 bool IoStatementBase::Emit(const char *, std::size_t) { return false; }

 
 template <Direction DIR>
 bool ExternalIoStatementState<DIR>::Emit(
     const char16_t *data, std::size_t chars) {
   if constexpr (DIR == Direction::Input) {
     Crash(
         "ExternalIoStatementState::Emit(char16_t) called for input statement");
   }
-  // TODO: UTF-8 encoding
   return unit().Emit(reinterpret_cast<const char *>(data), chars * sizeof *data,
       sizeof *data, *this);
 }
 
 template <Direction DIR>
 bool ExternalIoStatementState<DIR>::Emit(
     const char32_t *data, std::size_t chars) {
   if constexpr (DIR == Direction::Input) {
     Crash(
         "ExternalIoStatementState::Emit(char32_t) called for input statement");
   }
-  // TODO: UTF-8 encoding
   return unit().Emit(reinterpret_cast<const char *>(data), chars * sizeof *data,
       sizeof *data, *this);
 }
 
 template <Direction DIR>
 std::size_t ExternalIoStatementState<DIR>::GetNextInputBytes(const char *&p) {
   return unit().GetNextInputBytes(p, *this);
 }

 bool IoStatementState::Emit(const char16_t *data, std::size_t chars) {
   return std::visit([=](auto &x) { return x.get().Emit(data, chars); }, u_);
 }
 
 bool IoStatementState::Emit(const char32_t *data, std::size_t chars) {
   return std::visit([=](auto &x) { return x.get().Emit(data, chars); }, u_);
 }
 
+template <typename CHAR>
+bool IoStatementState::EmitEncoded(const CHAR *data0, std::size_t chars) {
+  // Don't allow sign extension
+  using UnsignedChar = std::make_unsigned_t<CHAR>;
+  const UnsignedChar *data{reinterpret_cast<const UnsignedChar *>(data0)};
+  if (GetConnectionState().isUTF8) {
+    char buffer[256];
+    std::size_t at{0};
+    while (chars-- > 0) {
+      auto len{EncodeUTF8(buffer + at, *data++)};
+      at += len;
+      if (at + maxUTF8Bytes > sizeof buffer) {
+        if (!Emit(buffer, at)) {
+          return false;
+        }
+        at = 0;
+      }
+    }
+    return at == 0 || Emit(buffer, at);
+  } else {
+    return Emit(data0, chars);
+  }
+}
+
 bool IoStatementState::Receive(
     char *data, std::size_t n, std::size_t elementBytes) {
   return std::visit(
       [=](auto &x) { return x.get().Receive(data, n, elementBytes); }, u_);
 }
 
 std::size_t IoStatementState::GetNextInputBytes(const char *&p) {
   return std::visit([&](auto &x) { return x.get().GetNextInputBytes(p); }, u_);
return std::visit(
       },
       u_);
 }
 
 ExternalFileUnit *IoStatementState::GetExternalFileUnit() const {
   return std::visit([](auto &x) { return x.get().GetExternalFileUnit(); }, u_);
 }
 
+std::optional<char32_t> IoStatementState::GetCurrentChar(
+    std::size_t &byteCount) {
+  const char *p{nullptr};
+  std::size_t bytes{GetNextInputBytes(p)};
+  if (bytes == 0) {
+    byteCount = 0;
+    return std::nullopt;
+  } else {
+    if (GetConnectionState().isUTF8) {
+      std::size_t length{MeasureUTF8Bytes(*p)};
+      if (length <= bytes) {
+        if (auto result{DecodeUTF8(p)}) {
+          byteCount = length;
+          return result;
+        }
+      }
+      GetIoErrorHandler().SignalError(IostatUTF8Decoding);
+      // Error recovery: return the next byte
+    }
+    byteCount = 1;
+    return *p;
+  }
+}
+
 bool IoStatementState::EmitRepeated(char ch, std::size_t n) {
   return std::visit(
       [=](auto &x) {
         for (std::size_t j{0}; j < n; ++j) {
           if (!x.get().Emit(&ch, 1)) {
             return false;
           }
         }
bool IoStatementState::EmitField(
   } else {
     return EmitRepeated(' ', static_cast<int>(width - length)) &&
         Emit(p, length);
   }
 }
 
 std::optional<char32_t> IoStatementState::NextInField(
     std::optional<int> &remaining, const DataEdit &edit) {
+  std::size_t byteCount{0};
   if (!remaining) { // Stream, list-directed, or NAMELIST
-    if (auto next{GetCurrentChar()}) {
+    if (auto next{GetCurrentChar(byteCount)}) {
       if (edit.IsListDirected()) {
         // list-directed or NAMELIST: check for separators
         switch (*next) {
         case ' ':
         case '\t':
         case ';':
         case '/':
         case '(':
         case ')':
         case '\'':
         case '"':
         case '*':
         case '\n': // for stream access
           return std::nullopt;
         case ',':
           if (edit.modes.editingFlags & decimalComma) {
             break;
           } else {
             return std::nullopt;
           }
         default:
           break;
         }
       }
-      HandleRelativePosition(1);
-      GotChar();
+      HandleRelativePosition(byteCount);
+      GotChar(byteCount);
       return next;
     }
   } else if (*remaining > 0) {
-    if (auto next{GetCurrentChar()}) {
-      --*remaining;
-      HandleRelativePosition(1);
-      GotChar();
+    if (auto next{GetCurrentChar(byteCount)}) {
+      if (byteCount > static_cast<std::size_t>(*remaining)) {
+        return std::nullopt;
+      }
+      *remaining -= byteCount;
+      HandleRelativePosition(byteCount);
+      GotChar(byteCount);
       return next;
     }
     if (CheckForEndOfRecord()) { // do padding
       --*remaining;
       return std::optional<char32_t>{' '};
     }
   }
   return std::nullopt;
ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
   if (hitSlash_) { // everything after '/' is nullified
     edit.descriptor = DataEdit::ListDirectedNullValue;
     return edit;
   }
   char32_t comma{','};
   if (edit.modes.editingFlags & decimalComma) {
     comma = ';';
   }
+  std::size_t byteCount{0};
   if (remaining_ > 0 && !realPart_) { // "r*c" repetition in progress
     RUNTIME_CHECK(io.GetIoErrorHandler(), repeatPosition_.has_value());
     repeatPosition_.reset(); // restores the saved position
     if (!imaginaryPart_) {
       edit.repeat = std::min<int>(remaining_, maxRepeat);
-      auto ch{io.GetCurrentChar()};
+      auto ch{io.GetCurrentChar(byteCount)};
       if (!ch || *ch == ' ' || *ch == '\t' || *ch == comma) {
         // "r*" repeated null
         edit.descriptor = DataEdit::ListDirectedNullValue;
       }
     }
     remaining_ -= edit.repeat;
     if (remaining_ > 0) {
       repeatPosition_.emplace(io);
     }
     return edit;
   }
   // Skip separators, handle a "r*c" repeat count; see 13.10.2 in Fortran 2018
   if (imaginaryPart_) {
     imaginaryPart_ = false;
   } else if (realPart_) {
     realPart_ = false;
     imaginaryPart_ = true;
     edit.descriptor = DataEdit::ListDirectedImaginaryPart;
   }
-  auto ch{io.GetNextNonBlank()};
+  auto ch{io.GetNextNonBlank(byteCount)};
   if (ch && *ch == comma && eatComma_) {
     // Consume comma & whitespace after previous item.
     // This includes the comma between real and imaginary components
     // in list-directed/NAMELIST complex input.
     // (When DECIMAL='COMMA', the comma is actually a semicolon.)
-    io.HandleRelativePosition(1);
-    ch = io.GetNextNonBlank();
+    io.HandleRelativePosition(byteCount);
+    ch = io.GetNextNonBlank(byteCount);
   }
   eatComma_ = true;
   if (!ch) {
     return std::nullopt;
   }
   if (*ch == '/') {
     hitSlash_ = true;
     edit.descriptor = DataEdit::ListDirectedNullValue;
if (*ch >= '0' && *ch <= '9') { // look for "r*" repetition count
     int r{0};
     do {
       static auto constexpr clamp{(std::numeric_limits<int>::max() - '9') / 10};
       if (r >= clamp) {
         r = 0;
         break;
       }
       r = 10 * r + (*ch - '0');
-      io.HandleRelativePosition(1);
-      ch = io.GetCurrentChar();
+      io.HandleRelativePosition(byteCount);
+      ch = io.GetCurrentChar(byteCount);
     } while (ch && *ch >= '0' && *ch <= '9');
     if (r > 0 && ch && *ch == '*') { // subtle: r must be nonzero
-      io.HandleRelativePosition(1);
-      ch = io.GetCurrentChar();
+      io.HandleRelativePosition(byteCount);
+      ch = io.GetCurrentChar(byteCount);
       if (ch && *ch == '/') { // r*/
         hitSlash_ = true;
         edit.descriptor = DataEdit::ListDirectedNullValue;
         return edit;
       }
       if (!ch || *ch == ' ' || *ch == '\t' || *ch == comma) { // "r*" null
         edit.descriptor = DataEdit::ListDirectedNullValue;
       }
       edit.repeat = std::min<int>(r, maxRepeat);
       remaining_ = r - edit.repeat;
       if (remaining_ > 0) {
         repeatPosition_.emplace(io);
       }
     } else { // not a repetition count, just an integer value; rewind
       connection.positionInRecord = start;
     }
   }
   if (!imaginaryPart_ && ch && *ch == '(') {
     realPart_ = true;
-    io.HandleRelativePosition(1);
+    io.HandleRelativePosition(byteCount);
     edit.descriptor = DataEdit::ListDirectedRealPart;
   }
   return edit;
 }
 
 template <Direction DIR>
 bool ExternalUnformattedIoStatementState<DIR>::Receive(
     char *data, std::size_t bytes, std::size_t elementBytes) {
bool InquireIOLengthState::Emit(const char32_t *p, std::size_t n) {
   return true;
 }
 
 int ErroneousIoStatementState::EndIoStatement() {
   SignalPendingError();
   return IoStatementBase::EndIoStatement();
 }
 
+template bool IoStatementState::EmitEncoded<char>(const char *, std::size_t);
+template bool IoStatementState::EmitEncoded<char16_t>(
+    const char16_t *, std::size_t);
+template bool IoStatementState::EmitEncoded<char32_t>(
+    const char32_t *, std::size_t);
+
 } // namespace Fortran::runtime::io

flang/runtime/iostat.cpp

const char *IostatErrorString(int iostat) {
   case IostatChildOutputToInputParent:
     return "Child output to input parent unit";
   case IostatShortRead:
     return "Read from external unit returned insufficient data";
   case IostatMissingTerminator:
     return "Sequential record missing its terminator";
   case IostatBadUnformattedRecord:
     return "Erroneous unformatted sequential file record structure";
+  case IostatUTF8Decoding:
+    return "UTF-8 decoding error";
   default:
     return nullptr;
   }
 }
 
 } // namespace Fortran::runtime::io

flang/runtime/namelist.cpp

 }
 
 static constexpr char NormalizeIdChar(char32_t ch) {
   return static_cast<char>(ch >= 'A' && ch <= 'Z' ? ch - 'A' + 'a' : ch);
 }
 
 static bool GetLowerCaseName(
     IoStatementState &io, char buffer[], std::size_t maxLength) {
-  if (auto ch{io.GetNextNonBlank()}) {
+  std::size_t byteLength{0};
+  if (auto ch{io.GetNextNonBlank(byteLength)}) {
     if (IsLegalIdStart(*ch)) {
       std::size_t j{0};
       do {
         buffer[j] = NormalizeIdChar(*ch);
-        io.HandleRelativePosition(1);
-        ch = io.GetCurrentChar();
+        io.HandleRelativePosition(byteLength);
+        ch = io.GetCurrentChar(byteLength);
       } while (++j < maxLength && ch && IsLegalIdChar(*ch));
       buffer[j++] = '\0';
       if (j <= maxLength) {
         return true;
       }
       io.GetIoErrorHandler().SignalError(
           "Identifier '%s...' in NAMELIST input group is too long", buffer);
     }
   }
   return false;
 }
 
 static std::optional<SubscriptValue> GetSubscriptValue(IoStatementState &io) {
   std::optional<SubscriptValue> value;
-  std::optional<char32_t> ch{io.GetCurrentChar()};
+  std::size_t byteCount{0};
+  std::optional<char32_t> ch{io.GetCurrentChar(byteCount)};
   bool negate{ch && *ch == '-'};
   if ((ch && *ch == '+') || negate) {
-    io.HandleRelativePosition(1);
-    ch = io.GetCurrentChar();
+    io.HandleRelativePosition(byteCount);
+    ch = io.GetCurrentChar(byteCount);
   }
   bool overflow{false};
   while (ch && *ch >= '0' && *ch <= '9') {
     SubscriptValue was{value.value_or(0)};
     overflow |= was >= std::numeric_limits<SubscriptValue>::max() / 10;
     value = 10 * was + *ch - '0';
-    io.HandleRelativePosition(1);
-    ch = io.GetCurrentChar();
+    io.HandleRelativePosition(byteCount);
+    ch = io.GetCurrentChar(byteCount);
   }
   if (overflow) {
     io.GetIoErrorHandler().SignalError(
         "NAMELIST input subscript value overflow");
     return std::nullopt;
   }
   if (negate) {
     if (value) {
       return -*value;
     } else {
-      io.HandleRelativePosition(-1); // give back '-' with no digits
+      io.HandleRelativePosition(-byteCount); // give back '-' with no digits
     }
   }
   return value;
 }
 
 static bool HandleSubscripts(IoStatementState &io, Descriptor &desc,
     const Descriptor &source, const char *name) {
   IoErrorHandler &handler{io.GetIoErrorHandler()};
   io.HandleRelativePosition(1); // skip '('
   // Allow for blanks in subscripts; they're nonstandard, but not
   // ambiguous within the parentheses.
   SubscriptValue lower[maxRank], upper[maxRank], stride[maxRank];
   int j{0};
   std::size_t contiguousStride{source.ElementBytes()};
   bool ok{true};
-  std::optional<char32_t> ch{io.GetNextNonBlank()};
+  std::size_t byteCount{0};
+  std::optional<char32_t> ch{io.GetNextNonBlank(byteCount)};
   char32_t comma{GetComma(io)};
   for (; ch && *ch != ')'; ++j) {
     SubscriptValue dimLower{0}, dimUpper{0}, dimStride{0};
     if (j < maxRank && j < source.rank()) {
       const Dimension &dim{source.GetDimension(j)};
       dimLower = dim.LowerBound();
       dimUpper = dim.UpperBound();
       dimStride =
if (auto low{GetSubscriptValue(io)}) {
               static_cast<std::intmax_t>(*low),
               static_cast<std::intmax_t>(dimLower),
               static_cast<std::intmax_t>(dimUpper), name, j + 1);
           ok = false;
         }
       } else {
         dimLower = *low;
       }
-      ch = io.GetNextNonBlank();
+      ch = io.GetNextNonBlank(byteCount);
     }
     if (ch && *ch == ':') {
-      io.HandleRelativePosition(1);
-      ch = io.GetNextNonBlank();
+      io.HandleRelativePosition(byteCount);
+      ch = io.GetNextNonBlank(byteCount);
       if (auto high{GetSubscriptValue(io)}) {
         if (*high > dimUpper) {
           if (ok) {
             handler.SignalError(
                 "Subscript triplet upper bound %jd out of range (>%jd) in "
                 "NAMELIST group item '%s' dimension %d",
                 static_cast<std::intmax_t>(*high),
                 static_cast<std::intmax_t>(dimUpper), name, j + 1);
             ok = false;
           }
         } else {
           dimUpper = *high;
         }
-        ch = io.GetNextNonBlank();
+        ch = io.GetNextNonBlank(byteCount);
       }
       if (ch && *ch == ':') {
-        io.HandleRelativePosition(1);
-        ch = io.GetNextNonBlank();
+        io.HandleRelativePosition(byteCount);
+        ch = io.GetNextNonBlank(byteCount);
         if (auto str{GetSubscriptValue(io)}) {
           dimStride = *str;
-          ch = io.GetNextNonBlank();
+          ch = io.GetNextNonBlank(byteCount);
         }
       }
     } else { // scalar
       dimUpper = dimLower;
       dimStride = 0;
     }
     if (ch && *ch == comma) {
-      io.HandleRelativePosition(1);
-      ch = io.GetNextNonBlank();
+      io.HandleRelativePosition(byteCount);
+      ch = io.GetNextNonBlank(byteCount);
     }
     if (ok) {
       lower[j] = dimLower;
       upper[j] = dimUpper;
       stride[j] = dimStride;
     }
   }
   if (ok) {
     if (ch && *ch == ')') {
-      io.HandleRelativePosition(1);
+      io.HandleRelativePosition(byteCount);
       if (desc.EstablishPointerSection(source, lower, upper, stride)) {
         return true;
       } else {
         handler.SignalError(
             "Bad subscripts for NAMELIST input group item '%s'", name);
       }
     } else {
       handler.SignalError(
if (!pair || pair->first != TypeCategory::Character) {
     return false;
   }
   int kind{pair->second};
   SubscriptValue chars{static_cast<SubscriptValue>(desc.ElementBytes()) / kind};
   // Allow for blanks in substring bounds; they're nonstandard, but not
   // ambiguous within the parentheses.
   io.HandleRelativePosition(1); // skip '('
   std::optional<SubscriptValue> lower, upper;
-  std::optional<char32_t> ch{io.GetNextNonBlank()};
+  std::size_t byteCount{0};
+  std::optional<char32_t> ch{io.GetNextNonBlank(byteCount)};
   if (ch) {
     if (*ch == ':') {
       lower = 1;
     } else {
       lower = GetSubscriptValue(io);
-      ch = io.GetNextNonBlank();
+      ch = io.GetNextNonBlank(byteCount);
     }
   }
   if (ch && ch == ':') {
-    io.HandleRelativePosition(1);
-    ch = io.GetNextNonBlank();
+    io.HandleRelativePosition(byteCount);
+    ch = io.GetNextNonBlank(byteCount);
     if (ch) {
       if (*ch == ')') {
         upper = chars;
       } else {
         upper = GetSubscriptValue(io);
-        ch = io.GetNextNonBlank();
+        ch = io.GetNextNonBlank(byteCount);
       }
     }
   }
   if (ch && *ch == ')') {
-    io.HandleRelativePosition(1);
+    io.HandleRelativePosition(byteCount);
     if (lower && upper) {
       if (*lower > *upper) {
         // An empty substring, whatever the values are
         desc.raw().elem_len = 0;
         return true;
       }
       if (*lower >= 1 || *upper <= chars) {
         // Offset the base address & adjust the element byte length
handler.SignalError("NAMELIST component reference of input group item %s "
                         "has no name after '%'",
         name);
   }
   return false;
 }
 
 // Advance to the terminal '/' of a namelist group.
 static void SkipNamelistGroup(IoStatementState &io) {
-  while (auto ch{io.GetNextNonBlank()}) {
-    io.HandleRelativePosition(1);
+  std::size_t byteCount{0};
+  while (auto ch{io.GetNextNonBlank(byteCount)}) {
+    io.HandleRelativePosition(byteCount);
     if (*ch == '/') {
       break;
     } else if (*ch == '\'' || *ch == '"') {
       // Skip quoted character literal
       char32_t quote{*ch};
       while (true) {
-        if ((ch = io.GetCurrentChar())) {
-          io.HandleRelativePosition(1);
+        if ((ch = io.GetCurrentChar(byteCount))) {
+          io.HandleRelativePosition(byteCount);
           if (*ch == quote) {
             break;
           }
         } else if (!io.AdvanceRecord()) {
           return;
         }
       }
     }
   }
 }
 
 bool IONAME(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
   IoStatementState &io{*cookie};
   io.CheckFormattedStmtType<Direction::Input>("InputNamelist");
   io.mutableModes().inNamelist = true;
   IoErrorHandler &handler{io.GetIoErrorHandler()};
   auto *listInput{io.get_if<ListDirectedStatementState<Direction::Input>>()};
   RUNTIME_CHECK(handler, listInput != nullptr);
   // Find this namelist group's header in the input
   io.BeginReadingRecord();
   std::optional<char32_t> next;
   char name[nameBufferSize];
   RUNTIME_CHECK(handler, group.groupName != nullptr);
   char32_t comma{GetComma(io)};
+  std::size_t byteCount{0};
   while (true) {
-    next = io.GetNextNonBlank();
+    next = io.GetNextNonBlank(byteCount);
     while (next && *next != '&') {
       // Extension: comment lines without ! before namelist groups
       if (!io.AdvanceRecord()) {
         next.reset();
       } else {
-        next = io.GetNextNonBlank();
+        next = io.GetNextNonBlank(byteCount);
       }
     }
     if (!next || *next != '&') {
       handler.SignalError(
           "NAMELIST input group does not begin with '&' (at '%lc')", *next);
       return false;
     }
-    io.HandleRelativePosition(1);
+    io.HandleRelativePosition(byteCount);
     if (!GetLowerCaseName(io, name, sizeof name)) {
       handler.SignalError("NAMELIST input group has no name");
       return false;
     }
     if (std::strcmp(group.groupName, name) == 0) {
       break; // found it
     }
     SkipNamelistGroup(io);
   }
   // Read the group's items
   while (true) {
-    next = io.GetNextNonBlank();
+    next = io.GetNextNonBlank(byteCount);
     if (!next || *next == '/') {
       break;
     }
     if (!GetLowerCaseName(io, name, sizeof name)) {
       handler.SignalError(
           "NAMELIST input group '%s' was not terminated at '%c'",
           group.groupName, static_cast<char>(*next));
       return false;
if (itemIndex >= group.items) {
       return false;
     }
     // Handle indexing and components, if any.  No spaces are allowed.
     // A copy of the descriptor is made if necessary.
     const Descriptor &itemDescriptor{group.item[itemIndex].descriptor};
     const Descriptor *useDescriptor{&itemDescriptor};
     StaticDescriptor<maxRank, true, 16> staticDesc[2];
     int whichStaticDesc{0};
-    next = io.GetCurrentChar();
+    next = io.GetCurrentChar(byteCount);
     bool hadSubscripts{false};
     bool hadSubstring{false};
     if (next && (*next == '(' || *next == '%')) {
       do {
         Descriptor &mutableDescriptor{staticDesc[whichStaticDesc].descriptor()};
         whichStaticDesc ^= 1;
         if (*next == '(') {
           if (!hadSubstring && (hadSubscripts || useDescriptor->rank() == 0)) {
if (next && (*next == '(' || *next == '%')) {
         } else {
           if (!HandleComponent(io, mutableDescriptor, *useDescriptor, name)) {
             return false;
           }
           hadSubscripts = false;
           hadSubstring = false;
         }
         useDescriptor = &mutableDescriptor;
-        next = io.GetCurrentChar();
+        next = io.GetCurrentChar(byteCount);
       } while (next && (*next == '(' || *next == '%'));
     }
     // Skip the '='
-    next = io.GetNextNonBlank();
+    next = io.GetNextNonBlank(byteCount);
     if (!next || *next != '=') {
       handler.SignalError("No '=' found after item '%s' in NAMELIST group '%s'",
           name, group.groupName);
       return false;
     }
-    io.HandleRelativePosition(1);
+    io.HandleRelativePosition(byteCount);
     // Read the values into the descriptor.  An array can be short.
     listInput->ResetForNextNamelistItem();
     if (!descr::DescriptorIO<Direction::Input>(io, *useDescriptor)) {
       return false;
     }
-    next = io.GetNextNonBlank();
+    next = io.GetNextNonBlank(byteCount);
     if (next && *next == comma) {
-      io.HandleRelativePosition(1);
+      io.HandleRelativePosition(byteCount);
     }
   }
   if (!next || *next != '/') {
     handler.SignalError(
         "No '/' found after NAMELIST group '%s'", group.groupName);
     return false;
   }
   io.HandleRelativePosition(1);
   return true;
 }
 
 bool IsNamelistName(IoStatementState &io) {
   if (io.get_if<ListDirectedStatementState<Direction::Input>>()) {
     if (io.mutableModes().inNamelist) {
       SavedPosition savedPosition{io};
-      if (auto ch{io.GetNextNonBlank()}) {
+      std::size_t byteCount{0};
+      if (auto ch{io.GetNextNonBlank(byteCount)}) {
         if (IsLegalIdStart(*ch)) {
           do {
-            io.HandleRelativePosition(1);
-            ch = io.GetCurrentChar();
+            io.HandleRelativePosition(byteCount);
+            ch = io.GetCurrentChar(byteCount);
           } while (ch && IsLegalIdChar(*ch));
-          ch = io.GetNextNonBlank();
+          ch = io.GetNextNonBlank(byteCount);
           // TODO: how to deal with NaN(...) ambiguity?
           return ch && (*ch == '=' || *ch == '(' || *ch == '%');
         }
       }
     }
   }
   return false;
 }
 
 } // namespace Fortran::runtime::io

flang/runtime/unit.h

 //===-- runtime/unit.h ------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 // Fortran external I/O units
 
 #ifndef FORTRAN_RUNTIME_IO_UNIT_H_
 #define FORTRAN_RUNTIME_IO_UNIT_H_
 
 #include "buffer.h"
 #include "connection.h"
+#include "environment.h"
 #include "file.h"
 #include "format.h"
 #include "io-error.h"
 #include "io-stmt.h"
 #include "lock.h"
 #include "terminator.h"
 #include "flang/Runtime/memory.h"
 #include <cstdlib>
 #include <cstring>
 #include <optional>
 #include <variant>
 
 namespace Fortran::runtime::io {
 
 class UnitMap;
 class ChildIo;
 
 class ExternalFileUnit : public ConnectionState,
                          public OpenFile,
                          public FileFrame<ExternalFileUnit> {
 public:
-  explicit ExternalFileUnit(int unitNumber) : unitNumber_{unitNumber} {}
+  explicit ExternalFileUnit(int unitNumber) : unitNumber_{unitNumber} {
+    isUTF8 = executionEnvironment.defaultUTF8;
+  }
   ~ExternalFileUnit() {}
 
   int unitNumber() const { return unitNumber_; }
   bool swapEndianness() const { return swapEndianness_; }
   bool createdForInternalChildIo() const { return createdForInternalChildIo_; }
 
   static ExternalFileUnit *LookUp(int unit);
   static ExternalFileUnit &LookUpOrCrash(int unit, const Terminator &);
IoStatementState &BeginIoStatement(X &&...xs) {
     io_.emplace(state);
     return *io_;
   }
 
   bool Emit(
       const char *, std::size_t, std::size_t elementBytes, IoErrorHandler &);
   bool Receive(char *, std::size_t, std::size_t elementBytes, IoErrorHandler &);
   std::size_t GetNextInputBytes(const char *&, IoErrorHandler &);
-  std::optional<char32_t> GetCurrentChar(IoErrorHandler &);
   void SetLeftTabLimit();
   bool BeginReadingRecord(IoErrorHandler &);
   void FinishReadingRecord(IoErrorHandler &);
   bool AdvanceRecord(IoErrorHandler &);
   void BackspaceRecord(IoErrorHandler &);
   void FlushOutput(IoErrorHandler &);
   void FlushIfTerminal(IoErrorHandler &);
   void Endfile(IoErrorHandler &);

flang/runtime/unit.cpp

 //===-- runtime/unit.cpp --------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "unit.h"
-#include "environment.h"
 #include "io-error.h"
 #include "lock.h"
 #include "unit-map.h"
 #include <cstdio>
 #include <limits>
 #include <utility>
 
 namespace Fortran::runtime::io {
UnitMap &ExternalFileUnit::GetUnitMap() {
 
   ExternalFileUnit &error{newUnitMap->LookUpOrCreate(0, terminator, wasExtant)};
   RUNTIME_CHECK(terminator, !wasExtant);
   error.Predefine(2);
   handler.SignalError(error.SetDirection(Direction::Output));
   error.isUnformatted = false;
   errorOutput = &error;
 
-  // TODO: Set UTF-8 mode from the environment
   unitMap = newUnitMap;
   return *unitMap;
 }
 
 void ExternalFileUnit::CloseAll(IoErrorHandler &handler) {
   CriticalSection critical{unitMapLock};
   if (unitMap) {
     unitMap->CloseAll(handler);
if (positionInRecord < *recl) {
       p = nullptr;
       return 0;
     }
   }
   p = FrameNextInput(handler, length);
   return p ? length : 0;
 }
 
-std::optional<char32_t> ExternalFileUnit::GetCurrentChar(
-    IoErrorHandler &handler) {
-  const char *p{nullptr};
-  std::size_t bytes{GetNextInputBytes(p, handler)};
-  if (bytes == 0) {
-    return std::nullopt;
-  } else {
-    // TODO: UTF-8 decoding; may have to get more bytes in a loop
-    return *p;
-  }
-}
-
 const char *ExternalFileUnit::FrameNextInput(
     IoErrorHandler &handler, std::size_t bytes) {
   RUNTIME_CHECK(handler, isUnformatted.has_value() && !*isUnformatted);
   if (static_cast<std::int64_t>(positionInRecord + bytes) <=
       recordLength.value_or(positionInRecord + bytes)) {
     auto at{recordOffsetInFrame_ + positionInRecord};
     auto need{static_cast<std::size_t>(at + bytes)};
     auto got{ReadFrame(frameOffsetInFile_, need, handler)};

flang/runtime/utf.h

+//===-- runtime/utf.h -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UTF-8 is the variant-width standard encoding of Unicode (ISO 10646)
+// code points.
+//
+// 7-bit values in [00 .. 7F] represent themselves as single bytes, so true
+// 7-bit ASCII is also valid UTF-8.
+//
+// Larger values are encoded with a start byte in [C0 .. FE] that carries
+// the length of the encoding and some of the upper bits of the value, followed
+// by one or more bytes in the range [80 .. BF].
+//
+// Specifically, the first byte holds two or more uppermost set bits,
+// a zero bit, and some payload; the second and later bytes each start with
+// their uppermost bit set, the next bit clear, and six bits of payload.
+// Payload parcels are in big-endian order.  All bytes must be present in a
+// valid sequence; i.e., low-order sezo bits must be explicit.  UTF-8 is
+// self-synchronizing on input as any byte value cannot be both a valid
+// first byte or trailing byte.
+//
+// 0xxxxxxx - 7 bit ASCII
+// 110xxxxx 10xxxxxx - 11-bit value
+// 1110xxxx 10xxxxxx 10xxxxxx - 16-bit value
+// 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - 21-bit value
+// 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - 26-bit value
+// 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - 31-bit value
+// 11111110 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - 36-bit value
+//
+// Canonical UTF-8 sequences should be minimal, and our output is so, but
+// we do not reject non-minimal sequences on input.  Unicode only defines
+// code points up to 0x10FFFF, so 21-bit (4-byte) UTF-8 is the actual
+// standard maximum.  However, we support extended forms up to 32 bits so that
+// CHARACTER(KIND=4) can be abused to hold arbitrary 32-bit data.
+
+#ifndef FORTRAN_RUNTIME_UTF_H_
+#define FORTRAN_RUNTIME_UTF_H_
+
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+
+namespace Fortran::runtime {
+
+// Derive the length of a UTF-8 character encoding from its first byte.
+// A zero result signifies an invalid encoding.
+extern const std::uint8_t UTF8FirstByteTable[256];
+static inline std::size_t MeasureUTF8Bytes(char first) {
+  return UTF8FirstByteTable[static_cast<std::uint8_t>(first)];
+}
+
+static constexpr std::size_t maxUTF8Bytes{7};
+
+// Ensure that all bytes are present in sequence in the input buffer
+// before calling; use MeasureUTF8Bytes(first byte) to count them.
+std::optional<char32_t> DecodeUTF8(const char *);
+
+// Ensure that at least maxUTF8Bytes remain in the output
+// buffer before calling.
+std::size_t EncodeUTF8(char *, char32_t);
+
+} // namespace Fortran::runtime
+#endif // FORTRAN_RUNTIME_UTF_H_

flang/runtime/utf.cpp

+//===-- runtime/utf.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "utf.h"
+
+namespace Fortran::runtime {
+
+// clang-format off
+const std::uint8_t UTF8FirstByteTable[256]{
+  /* 00 - 7F:  7 bit payload in single byte */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  /* 80 - BF: invalid first byte, valid later byte */
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  /* C0 - DF: 11 bit payload */
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  /* E0 - EF: 16 bit payload */
+    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+  /* F0 - F7: 21 bit payload */ 4, 4, 4, 4, 4, 4, 4, 4,
+  /* F8 - FB: 26 bit payload */ 5, 5, 5, 5,
+  /* FC - FD: 31 bit payload */ 6, 6,
+  /* FE:      32 bit payload */ 7,
+  /* FF:      invalid */ 0
+};
+// clang-format on
+
+// Non-minimal encodings are accepted.
+std::optional<char32_t> DecodeUTF8(const char *p0) {
+  const std::uint8_t *p{reinterpret_cast<const std::uint8_t *>(p0)};
+  std::size_t bytes{MeasureUTF8Bytes(*p0)};
+  if (bytes == 1) {
+    return char32_t{*p};
+  } else if (bytes > 1) {
+    std::uint64_t result{char32_t{*p} & (0x7f >> bytes)};
+    for (std::size_t j{1}; j < bytes; ++j) {
+      std::uint8_t next{p[j]};
+      if (next < 0x80 || next > 0xbf) {
+        return std::nullopt;
+      }
+      result = (result << 6) | (next & 0x3f);
+    }
+    if (result <= 0xffffffff) {
+      return static_cast<char32_t>(result);
+    }
+  }
+  return std::nullopt;
+}
+
+std::size_t EncodeUTF8(char *p0, char32_t ucs) {
+  std::uint8_t *p{reinterpret_cast<std::uint8_t *>(p0)};
+  if (ucs <= 0x7f) {
+    p[0] = ucs;
+    return 1;
+  } else if (ucs <= 0x7ff) {
+    p[0] = 0xc0 | (ucs >> 6);
+    p[1] = 0x80 | (ucs & 0x3f);
+    return 2;
+  } else if (ucs <= 0xffff) {
+    p[0] = 0xe0 | (ucs >> 12);
+    p[1] = 0x80 | ((ucs >> 6) & 0x3f);
+    p[2] = 0x80 | (ucs & 0x3f);
+    return 3;
+  } else if (ucs <= 0x1fffff) {
+    p[0] = 0xf0 | (ucs >> 18);
+    p[1] = 0x80 | ((ucs >> 12) & 0x3f);
+    p[2] = 0x80 | ((ucs >> 6) & 0x3f);
+    p[3] = 0x80 | (ucs & 0x3f);
+    return 4;
+  } else if (ucs <= 0x3ffffff) {
+    p[0] = 0xf8 | (ucs >> 24);
+    p[1] = 0x80 | ((ucs >> 18) & 0x3f);
+    p[2] = 0x80 | ((ucs >> 12) & 0x3f);
+    p[3] = 0x80 | ((ucs >> 6) & 0x3f);
+    p[4] = 0x80 | (ucs & 0x3f);
+    return 5;
+  } else if (ucs <= 0x7ffffff) {
+    p[0] = 0xf8 | (ucs >> 30);
+    p[1] = 0x80 | ((ucs >> 24) & 0x3f);
+    p[2] = 0x80 | ((ucs >> 18) & 0x3f);
+    p[3] = 0x80 | ((ucs >> 12) & 0x3f);
+    p[4] = 0x80 | ((ucs >> 6) & 0x3f);
+    p[5] = 0x80 | (ucs & 0x3f);
+    return 6;
+  } else {
+    p[0] = 0xfe;
+    p[1] = 0x80 | ((ucs >> 30) & 0x3f);
+    p[2] = 0x80 | ((ucs >> 24) & 0x3f);
+    p[3] = 0x80 | ((ucs >> 18) & 0x3f);
+    p[4] = 0x80 | ((ucs >> 12) & 0x3f);
+    p[5] = 0x80 | ((ucs >> 6) & 0x3f);
+    p[6] = 0x80 | (ucs & 0x3f);
+    return 7;
+  }
+}
+
+} // namespace Fortran::runtime

flang/unittests/Runtime/ExternalIOTest.cpp

ASSERT_TRUE(
         IONAME(InputAscii)(io, inputItem.item.data(), inputItem.item.length()))
         << "InputAscii() " << j;
     ASSERT_EQ(IONAME(EndIoStatement)(io), inputItem.expectedIoStat)
         << "EndIoStatement() for Read " << j;
     ASSERT_EQ(inputItem.item, inputItem.expectedItemValue)
         << "Input-item value after non advancing read " << j;
     j++;
   }
+  // CLOSE(UNIT=unit)
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for Close";
 }
 
 TEST(ExternalIOTests, TestWriteAfterNonAvancingInput) {
   // OPEN(NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
   //   FORM='FORMATTED',STATUS='SCRATCH')
   auto *io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
   ASSERT_TRUE(IONAME(SetAccess)(io, "SEQUENTIAL", 10))
       << "SetAccess(SEQUENTIAL)";
TEST(ExternalIOTests, TestWriteAfterNonAvancingInput) {
   io = IONAME(BeginExternalFormattedInput)(
       fmt.data(), fmt.length(), unit, __FILE__, __LINE__);
   IONAME(EnableHandlers)(io, true, false, false, false, false);
   ASSERT_TRUE(
       IONAME(InputAscii)(io, resultRecord.data(), resultRecord.length()))
       << "InputAscii() ";
   ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
       << "EndIoStatement() for Read ";
-
   ASSERT_EQ(resultRecord, expectedRecord)
       << "Record after non advancing read followed by write";
+  // CLOSE(UNIT=unit)
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for Close";
 }
 
 TEST(ExternalIOTests, TestWriteAfterEndfile) {
   // OPEN(NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
   //   FORM='FORMATTED',STATUS='SCRATCH')
   auto *io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
   ASSERT_TRUE(IONAME(SetAccess)(io, "SEQUENTIAL", 10))
       << "SetAccess(SEQUENTIAL)";
TEST(ExternalIOTests, TestWriteAfterEndfile) {
   IONAME(EnableHandlers)(io, false, false, true /*END=*/);
   ASSERT_TRUE(IONAME(InputInteger)(io, j)) << "InputInteger(j)";
   ASSERT_EQ(j, 1234) << "READ(j)";
   ASSERT_TRUE(IONAME(InputInteger)(io, k)) << "InputInteger(k)";
   ASSERT_EQ(k, 3456) << "READ(k)";
   ASSERT_FALSE(IONAME(InputInteger)(io, eof)) << "InputInteger(eof)";
   ASSERT_EQ(eof, -1) << "READ(eof)";
   ASSERT_EQ(IONAME(EndIoStatement)(io), IostatEnd) << "EndIoStatement for READ";
+  // CLOSE(UNIT=unit)
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for Close";
+}
+
+TEST(ExternalIOTests, TestUTF8Encoding) {
+  // OPEN(FILE="utf8test",NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
+  //   FORM='FORMATTED',STATUS='REPLACE',ENCODING='UTF-8')
+  auto *io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  ASSERT_TRUE(IONAME(SetAccess)(io, "SEQUENTIAL", 10))
+      << "SetAccess(SEQUENTIAL)";
+  ASSERT_TRUE(IONAME(SetAction)(io, "READWRITE", 9)) << "SetAction(READWRITE)";
+  ASSERT_TRUE(IONAME(SetFile)(io, "utf8test", 8)) << "SetFile(utf8test)";
+  ASSERT_TRUE(IONAME(SetForm)(io, "FORMATTED", 9)) << "SetForm(FORMATTED)";
+  ASSERT_TRUE(IONAME(SetStatus)(io, "REPLACE", 7)) << "SetStatus(REPLACE)";
+  ASSERT_TRUE(IONAME(SetEncoding)(io, "UTF-8", 5)) << "SetEncoding(UTF-8)";
+  int unit{-1};
+  ASSERT_TRUE(IONAME(GetNewUnit)(io, unit)) << "GetNewUnit()";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for first OPEN";
+  char buffer[12];
+  std::memcpy(buffer,
+      "abc\x80\xff"
+      "de\0\0\0\0\0",
+      12);
+  // WRITE(unit, *) buffer
+  io = IONAME(BeginExternalListOutput)(unit, __FILE__, __LINE__);
+  StaticDescriptor<0> staticDescriptor;
+  Descriptor &desc{staticDescriptor.descriptor()};
+  desc.Establish(TypeCode{CFI_type_char}, 7, buffer, 0);
+  desc.Check();
+  ASSERT_TRUE(IONAME(OutputDescriptor)(io, desc));
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for WRITE";
+  // REWIND(unit)
+  io = IONAME(BeginRewind)(unit, __FILE__, __LINE__);
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement for REWIND";
+  // READ(unit, *) buffer
+  desc.Establish(TypeCode(CFI_type_char), sizeof buffer, buffer, 0);
+  desc.Check();
+  io = IONAME(BeginExternalListInput)(unit, __FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(InputDescriptor)(io, desc));
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for first READ";
+  ASSERT_EQ(std::memcmp(buffer,
+                "abc\x80\xff"
+                "de     ",
+                12),
+      0);
+  // CLOSE(UNIT=unit,STATUS='KEEP')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(SetStatus)(io, "KEEP", 4)) << "SetStatus(KEEP)";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for first CLOSE";
+  // OPEN(FILE="utf8test",NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
+  //   FORM='FORMATTED',STATUS='OLD')
+  io = IONAME(BeginOpenNewUnit)(__FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(SetAccess)(io, "SEQUENTIAL", 10))
+      << "SetAccess(SEQUENTIAL)";
+  ASSERT_TRUE(IONAME(SetAction)(io, "READWRITE", 9)) << "SetAction(READWRITE)";
+  ASSERT_TRUE(IONAME(SetFile)(io, "utf8test", 8)) << "SetFile(utf8test)";
+  ASSERT_TRUE(IONAME(SetForm)(io, "FORMATTED", 9)) << "SetForm(FORMATTED)";
+  ASSERT_TRUE(IONAME(SetStatus)(io, "OLD", 3)) << "SetStatus(OLD)";
+  ASSERT_TRUE(IONAME(GetNewUnit)(io, unit)) << "GetNewUnit()";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for second OPEN";
+  // READ(unit, *) buffer
+  io = IONAME(BeginExternalListInput)(unit, __FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(InputDescriptor)(io, desc));
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for second READ";
+  ASSERT_EQ(std::memcmp(buffer,
+                "abc\xc2\x80\xc3\xbf"
+                "de   ",
+                12),
+      0);
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(SetStatus)(io, "DELETE", 6)) << "SetStatus(DELETE)";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for second CLOSE";
+}
+
+TEST(ExternalIOTests, TestUCS) {
+  // OPEN(FILE="ucstest',NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
+  //   FORM='FORMATTED',STATUS='REPLACE',ENCODING='UTF-8')
+  auto *io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  ASSERT_TRUE(IONAME(SetAccess)(io, "SEQUENTIAL", 10))
+      << "SetAccess(SEQUENTIAL)";
+  ASSERT_TRUE(IONAME(SetAction)(io, "READWRITE", 9)) << "SetAction(READWRITE)";
+  ASSERT_TRUE(IONAME(SetFile)(io, "ucstest", 7)) << "SetAction(ucstest)";
+  ASSERT_TRUE(IONAME(SetForm)(io, "FORMATTED", 9)) << "SetForm(FORMATTED)";
+  ASSERT_TRUE(IONAME(SetStatus)(io, "REPLACE", 7)) << "SetStatus(REPLACE)";
+  ASSERT_TRUE(IONAME(SetEncoding)(io, "UTF-8", 5)) << "SetEncoding(UTF-8)";
+  int unit{-1};
+  ASSERT_TRUE(IONAME(GetNewUnit)(io, unit)) << "GetNewUnit()";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for first OPEN";
+  char32_t wbuffer[8]{U"abc\u0080\uffff"
+                      "de"};
+  // WRITE(unit, *) wbuffec
+  io = IONAME(BeginExternalListOutput)(unit, __FILE__, __LINE__);
+  StaticDescriptor<0> staticDescriptor;
+  Descriptor &desc{staticDescriptor.descriptor()};
+  desc.Establish(TypeCode{CFI_type_char32_t}, sizeof wbuffer - sizeof(char32_t),
+      wbuffer, 0);
+  desc.Check();
+  ASSERT_TRUE(IONAME(OutputDescriptor)(io, desc));
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for WRITE";
+  // REWIND(unit)
+  io = IONAME(BeginRewind)(unit, __FILE__, __LINE__);
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement for REWIND";
+  // READ(unit, *) buffer
+  io = IONAME(BeginExternalListInput)(unit, __FILE__, __LINE__);
+  desc.Establish(TypeCode{CFI_type_char32_t}, sizeof wbuffer, wbuffer, 0);
+  desc.Check();
+  ASSERT_TRUE(IONAME(InputDescriptor)(io, desc));
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for first READ";
+  char dump[80];
+  dump[0] = '\0';
+  for (int j{0}; j < 8; ++j) {
+    std::size_t dumpLen{std::strlen(dump)};
+    std::snprintf(
+        dump + dumpLen, sizeof dump - dumpLen, " %x", (unsigned)wbuffer[j]);
+  }
+  EXPECT_EQ(wbuffer[0], U'a') << dump;
+  EXPECT_EQ(wbuffer[1], U'b') << dump;
+  EXPECT_EQ(wbuffer[2], U'c') << dump;
+  EXPECT_EQ(wbuffer[3], U'\u0080') << dump;
+  EXPECT_EQ(wbuffer[4], U'\uffff') << dump;
+  EXPECT_EQ(wbuffer[5], U'd') << dump;
+  EXPECT_EQ(wbuffer[6], U'e') << dump;
+  EXPECT_EQ(wbuffer[7], U' ') << dump;
+  // CLOSE(UNIT=unit,STATUS='KEEP')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(SetStatus)(io, "KEEP", 4)) << "SetStatus(KEEP)";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for first CLOSE";
+  // OPEN(FILE="ucstest",NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
+  //   FORM='FORMATTED',STATUS='OLD')
+  io = IONAME(BeginOpenNewUnit)(__FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(SetAccess)(io, "SEQUENTIAL", 10))
+      << "SetAccess(SEQUENTIAL)";
+  ASSERT_TRUE(IONAME(SetAction)(io, "READWRITE", 9)) << "SetAction(READWRITE)";
+  ASSERT_TRUE(IONAME(SetFile)(io, "ucstest", 7)) << "SetFile(ucstest)";
+  ASSERT_TRUE(IONAME(SetForm)(io, "FORMATTED", 9)) << "SetForm(FORMATTED)";
+  ASSERT_TRUE(IONAME(SetStatus)(io, "OLD", 3)) << "SetStatus(OLD)";
+  ASSERT_TRUE(IONAME(GetNewUnit)(io, unit)) << "GetNewUnit()";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for second OPEN";
+  char buffer[12];
+  // READ(unit, *) buffer
+  io = IONAME(BeginExternalListInput)(unit, __FILE__, __LINE__);
+  desc.Establish(TypeCode{CFI_type_char}, sizeof buffer, buffer, 0);
+  desc.Check();
+  ASSERT_TRUE(IONAME(InputDescriptor)(io, desc));
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for second READ";
+  dump[0] = '\0';
+  for (int j{0}; j < 12; ++j) {
+    std::size_t dumpLen{std::strlen(dump)};
+    std::snprintf(dump + dumpLen, sizeof dump - dumpLen, " %x",
+        (unsigned)(unsigned char)buffer[j]);
+  }
+  EXPECT_EQ(std::memcmp(buffer,
+                "abc\xc2\x80\xef\xbf\xbf"
+                "de  ",
+                12),
+      0)
+      << dump;
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  ASSERT_TRUE(IONAME(SetStatus)(io, "DELETE", 6)) << "SetStatus(DELETE)";
+  ASSERT_EQ(IONAME(EndIoStatement)(io), IostatOk)
+      << "EndIoStatement() for second CLOSE";
 }