[SystemZ][z/OS] Add the functions needed for handling EBCDIC I/O

[abhina.sreeskantharajan]

Summary

This patch adds the basic functions needed for controlling auto conversion on z/OS.
Auto conversion is enabled on untagged input file to ASCII by making the assumption that all untagged files are EBCDIC encoded. Output files are auto converted to EBCDIC IBM-1047.
This change also enables conversion for stdin/stdout/stderr.

For more information on how fcntl controls codepage https://www.ibm.com/docs/en/zos/2.4.0?topic=descriptions-fcntl-bpx1fct-bpx4fct-control-open-file-descriptors

[anirudhp]

Looks good to me. Minor nits noted inline. It would be nice if we had a bit more testing coverage.

llvm/lib/Support/AutoConvert.cpp

63

minor nit: #endif __MVS__

[abhina.sreeskantharajan]

Address comments, fix bug in lit test

[abhina.sreeskantharajan]

rebase + removing call to enableAutoConversion in getSTDIN anymore because changeStdinToBinary is no longer called because we are passing the OF_Text flag now

[SeanP]

llvm/lib/Support/Unix/Path.inc

1011

Some places use #ifdef and others #if defined(__MVS__). We should be consistent.

llvm/test/Support/encoding.ll

7

minor nit. Rather than using the short names 819 & 1047 we should use the full names iso8859-1 & ibm-1047.

[abhina.sreeskantharajan]

Changes in this new diff:

• changed 819,1047 to full names in encoding.ll
• use #ifdef MVS for consistency

[abhina.sreeskantharajan]

llvm/lib/Support/Unix/Path.inc

1011

Thanks Sean. There's seems to be mixed use in this file, so I just updated all uses of MVS in this patch to be #ifdef for consistency.

[abhina.sreeskantharajan]

Remove unnecessary include

[abhina.sreeskantharajan]

Hi Sean @SeanP, are there any more comments on this patch?

llvm/include/llvm/Support/AutoConvert.h

+//===- AutoConvert.h - Auto conversion between ASCII/EBCDIC -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains functions used for auto conversion between
+// ASCII/EBCDIC codepages specific to z/OS.
+//
+//===----------------------------------------------------------------------===//i
+
+#ifndef LLVM_SUPPORT_AUTOCONVERT_H
+#define LLVM_SUPPORT_AUTOCONVERT_H
+
+#ifdef __MVS__
+#define CCSID_IBM_1047 1047
+#define CCSID_UTF_8 1208
+#include <system_error>
+
+namespace llvm {
+
+/// \brief Disable the z/OS enhanced ASCII auto-conversion for the file
+/// descriptor.
+std::error_code disableAutoConversion(int FD);
+
+/// \brief Query the z/OS enhanced ASCII auto-conversion status of a file
+/// descriptor and force the conversion if the file is not tagged with a
+/// codepage.
+std::error_code enableAutoConversion(int FD);
+
+/// \brief Set the tag information for a file descriptor.
+std::error_code setFileTag(int FD, int CCSID, bool Text);
+
+} // namespace llvm
+
+#endif // __MVS__
+
+#endif // LLVM_SUPPORT_AUTOCONVERT_H

llvm/lib/Support/AutoConvert.cpp

+//===- AutoConvert.cpp - Auto conversion between ASCII/EBCDIC -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains functions used for auto conversion between
+// ASCII/EBCDIC codepages specific to z/OS.
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __MVS__
+
+#include "llvm/Support/AutoConvert.h"
+#include "llvm/Support/raw_ostream.h"
+#include <fcntl.h>
+#include <sys/stat.h>
+
+std::error_code llvm::disableAutoConversion(int FD) {
+  static const struct f_cnvrt Convert = {
+      SETCVTOFF,        // cvtcmd
+      0,                // pccsid
+      (short)FT_BINARY, // fccsid
+  };
+  return fcntl(FD, F_CONTROL_CVT, &Convert);
+}
+
+std::error_code llvm::enableAutoConversion(int FD) {
+  struct f_cnvrt Query = {
+      QUERYCVT, // cvtcmd
+      0,        // pccsid
+      0,        // fccsid
+  };
+
+  if (fcntl(FD, F_CONTROL_CVT, &Query) == -1)
+    return -1;
+
+  Query.cvtcmd = SETCVTALL;
+  Query.pccsid =
+      (FD == STDIN_FILENO || FD == STDOUT_FILENO || FD == STDERR_FILENO)
+          ? 0
+          : CCSID_UTF_8;
+  // Assume untagged files to be IBM-1047 encoded.
+  Query.fccsid = (Query.fccsid == FT_UNTAGGED) ? CCSID_IBM_1047 : Query.fccsid;
+  return fcntl(FD, F_CONTROL_CVT, &Query);
+}
+
+std::error_code llvm::setFileTag(int FD, int CCSID, bool Text) {
+  assert((!Text || (CCSID != FT_UNTAGGED && CCSID != FT_BINARY)) &&
+         "FT_UNTAGGED and FT_BINARY are not allowed for text files");
+  struct file_tag Tag;
+  Tag.ft_ccsid = CCSID;
+  Tag.ft_txtflag = Text;
+  Tag.ft_deferred = 0;
+  Tag.ft_rsvflags = 0;
+
+  return fcntl(FD, F_SETTAG, &Tag);
+}
+
+#endif // __MVS__

llvm/lib/Support/CMakeLists.txt

add_llvm_component_library(LLVMSupport
   APFixedPoint.cpp
   APFloat.cpp
   APInt.cpp
   APSInt.cpp
   ARMBuildAttrs.cpp
   ARMAttributeParser.cpp
   ARMWinEH.cpp
   Allocator.cpp
+  AutoConvert.cpp
   BinaryStreamError.cpp
   BinaryStreamReader.cpp
   BinaryStreamRef.cpp
   BinaryStreamWriter.cpp
   BlockFrequency.cpp
   BranchProbability.cpp
   BuryPointer.cpp
   CachePruning.cpp

llvm/lib/Support/MemoryBuffer.cpp

 //===--- MemoryBuffer.cpp - Memory Buffer implementation ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file implements the MemoryBuffer interface.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/AutoConvert.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/SmallVectorMemoryBuffer.h"
if (shouldUseMmap(FD, FileSize, MapSize, Offset, RequiresNullTerminator,
     std::error_code EC;
     std::unique_ptr<MB> Result(
         new (NamedBufferAlloc(Filename)) MemoryBufferMMapFile<MB>(
             RequiresNullTerminator, FD, MapSize, Offset, EC));
     if (!EC)
       return std::move(Result);
   }
 
+#if defined(__MVS__)
+  // Set codepage auto-conversion for z/OS.
+  if (auto EC = llvm::enableAutoConversion(FD))
+    return EC;
+#endif
+
   auto Buf = WritableMemoryBuffer::getNewUninitMemBuffer(MapSize, Filename);
   if (!Buf) {
     // Failed to create a buffer. The only way it can fail is if
     // new(std::nothrow) returns 0.
     return make_error_code(errc::not_enough_memory);
   }
 
   // Read until EOF, zero-initialize the rest.

 }
 
 ErrorOr<std::unique_ptr<MemoryBuffer>> MemoryBuffer::getSTDIN() {
   // Read in all of the data from stdin, we cannot mmap stdin.
   //
   // FIXME: That isn't necessarily true, we should try to mmap stdin and
   // fallback if it fails.
   sys::ChangeStdinToBinary();
+#if defined(__MVS__)
+  // Set codepage auto-conversion for z/OS.
+  // Under MVS the conversion from EBCDIC is turned on if there is no
+  // code page associated with stdin. The call to change the stream to
+  // binary is a no-op and does not change the attributes of the stream.
+  if (auto EC = llvm::enableAutoConversion(sys::fs::getStdinHandle())
+    return EC;
+#endif
 
   return getMemoryBufferForStream(sys::fs::getStdinHandle(), "<stdin>");
 }
 
 ErrorOr<std::unique_ptr<MemoryBuffer>>
 MemoryBuffer::getFileAsStream(const Twine &Filename) {
   Expected<sys::fs::file_t> FDOrErr =
       sys::fs::openNativeFileForRead(Filename, sys::fs::OF_None);

llvm/lib/Support/Unix/Path.inc

 #include <sys/auxv.h>
 #else
 #include <machine/elf.h>
 extern char **environ;
 #endif
 #elif defined(__DragonFly__)
 #include <sys/mount.h>
 #elif defined(__MVS__)
+#include "llvm/Support/AutoConvert.h"
 #include <sys/ps.h>
 #endif
 
 // Both stdio.h and cstdio are included via different paths and
 // stdcxx's cstdio doesn't include stdio.h, so it doesn't #undef the macros
 // either.
 #undef ferror
 #undef feof
if (Disp == CD_CreateNew) {
     Result |= O_CREAT; // Create if it doesn't exist.
     Result |= O_TRUNC; // Truncate if it does.
   } else if (Disp == CD_OpenAlways) {
     Result |= O_CREAT; // Create if it doesn't exist.
   } else if (Disp == CD_OpenExisting) {
     // Nothing special, just don't add O_CREAT and we get these semantics.
   }
 
+// Using append mode with z/OS UTF-8 auto-conversion results in EINVAL when
+// calling write(). Instead we need to use lseek() to set offset to EOF after
+// open().
+#ifndef __MVS__
   if (Flags & OF_Append)
     Result |= O_APPEND;
+#endif
 
 #ifdef O_CLOEXEC
   if (!(Flags & OF_ChildInherit))
     Result |= O_CLOEXEC;
 #endif
 
   return Result;
 }
if ((ResultFD = sys::RetryAfterSignal(-1, Open)) < 0)
     return std::error_code(errno, std::generic_category());
 #ifndef O_CLOEXEC
   if (!(Flags & OF_ChildInherit)) {
     int r = fcntl(ResultFD, F_SETFD, FD_CLOEXEC);
     (void)r;
     assert(r == 0 && "fcntl(F_SETFD, FD_CLOEXEC) failed");
   }
 #endif
+
+#ifdef __MVS__
+  /* Reason about auto-conversion and file tags. Setting the file tag only
+   * applies if file is opened in write mode:
+   *
+   * Text file:
+   *                  File exists       File created
+   * CD_CreateNew     n/a               conv: on
+   *                                    tag: set 1047
+   * CD_CreateAlways  conv: auto        conv: on
+   *                  tag: auto 1047    tag: set 1047
+   * CD_OpenAlways    conv: auto        conv: on
+   *                  tag: auto 1047    tag: set 1047
+   * CD_OpenExisting  conv: auto        n/a
+   *                  tag: unchanged
+   *
+   * Binary file:
+   *                  File exists       File created
+   * CD_CreateNew     n/a               conv: off
+   *                                    tag: set binary
+   * CD_CreateAlways  conv: off         conv: off
+   *                  tag: auto binary  tag: set binary
+   * CD_OpenAlways    conv: off         conv: off
+   *                  tag: auto binary  tag: set binary
+   * CD_OpenExisting  conv: off         n/a
+   *                  tag: unchanged
+   *
+   * Actions:
+   *   conv: off        -> auto-conversion is turned off
+   *   conv: on         -> auto-conversion is turned on
+   *   conv: auto       -> auto-conversion is turned on if the file is untagged
+   *   tag: set 1047    -> set the file tag to text encoded in 1047
+   *   tag: set binary  -> set the file tag to binary
+   *   tag: auto 1047   -> set file tag to 1047 if not set
+   *   tag: auto binary -> set file tag to binary if not set
+   *   tag: unchanged   -> do not care about the file tag
+   *
+   * It is not possible to distinguish between the cases "file exists" and
+   * "file created". In the latter case, the file tag is not set and the file
+   * size is zero. The decision table boils down to:
+   *
+   * the file tag is set if
+   *   - the file is opened for writing
+   *   - the create disposition is not equal to CD_OpenExisting
+   *   - the file tag is not set
+   *   - the file size is zero
+   *
+   * This only applies if the file is a regular file. E.g. enabling
+   * auto-conversion for reading from /dev/null results in error EINVAL when
+   * calling read().
+   *
+   * Using append mode with z/OS UTF-8 auto-conversion results in EINVAL when
+   * calling write(). Instead we need to use lseek() to set offset to EOF after
+   * open().
+   */
+  if ((Flags & OF_Append) && lseek(ResultFD, 0, SEEK_END) == -1)
+    return std::error_code(errno, std::generic_category());
+  struct stat Stat;
+  if (fstat(ResultFD, &Stat) == -1)
+    return std::error_code(errno, std::generic_category());
+  if (S_ISREG(Stat.st_mode)) {
+    bool DoSetTag = (Access & FA_Write) && (Disp != CD_OpenExisting) &&
+                    !Stat.st_tag.ft_txtflag && !Stat.st_tag.ft_ccsid &&
+                    Stat.st_size == 0;
+    if (Flags & OF_Text) {
+      if (auto EC = llvm::enableAutoConversion(ResultFD))
+        return EC;
+      if (DoSetTag) {
+        if (auto EC = llvm::setFileTag(ResultFD, CCSID_IBM_1047, true))
+          return EC;
+      }
+    } else {
+      if (auto EC = llvm::disableAutoConversion(ResultFD))
+        return EC;
+      if (DoSetTag) {
+        if (auto EC = llvm::setFileTag(ResultFD, FT_BINARY, false))
+          return EC;
+      }
+    }
+  }
+#endif
+
   return std::error_code();
 }
 
 Expected<int> openNativeFile(const Twine &Name, CreationDisposition Disp,
                              FileAccess Access, OpenFlags Flags,
                              unsigned Mode) {
 
   int FD;

llvm/test/Support/encoding.ll

+; Checks if llc can deal with different char encodings.
+; This is only required for z/OS.
+;
+; UNSUPPORTED: !s390x-none-zos
+;
+; RUN: cat %s >%t && chtag -tc 819 %t && llc %t -o - >/dev/null
+; RUN: iconv -f 819 -t 1047 <%s >%t && chtag -tc 1047 %t && llc %t -o - >/dev/null
+; RUN: iconv -f 819 -t 1047 <%s >%t && chtag -r %t && llc %t -o - >/dev/null
+
+@g_105 = external dso_local global i8, align 2