[SystemZ][z/OS] Create utility functions for converting between EBCDIC and UTF-8.

[Everybody0523]

Summary

This patch adds some utility functions for converting between EBCDIC and UTF-8.

This functionality was originally intended to be packed alongside general character set conversion, but based on comments on an RFC it seems more prudent to separate the two tasks.

This functionality will be used in the SystemZ back-end for z/OS as well as GOFF Object File read/write support.

[efriedma]

llvm/include/llvm/Support/ConvertEBCDIC.h

7

Probably should indicate in the header somewhere precisely what encoding this uses; there are a number of different "EBCDIC" encodings (although probably most of them are irrelevant these days).

llvm/lib/Support/ConvertEBCDIC.cpp

110

Out-of-date comment?

[Everybody0523]

Correct comments.

[efriedma]

llvm/lib/Support/ConvertEBCDIC.cpp

100

Maybe convertToUTF8 should return void, instead of an std::error_code, because it can't fail. (The only reason I can think of to keep it is consistency with conversions which can fail. But that doesn't seem compelling given an explicit entry point like this.)

[Everybody0523]

llvm/lib/Support/ConvertEBCDIC.cpp

100

FWIW I actually did personally prefer the consistency with the reverse direction (which can fail), but I'm open to changing it.

[Everybody0523]

Make convertToUTF8 return void since it cannot fail.

[efriedma]

LGTM

[cor3ntin]

llvm/lib/Support/ConvertEBCDIC.cpp

76

I think I'd prefer to assert Result is empty
also, I think we should do something like Result.reserve(Source.size()) to minimize allocations in the common case

107

Same comment as above

[Everybody0523]

Require Result to be an empty SmallVector.

llvm/include/llvm/Support/ConvertEBCDIC.h

+//===--- ConvertEBCDIC.h - UTF8/EBCDIC CharSet Conversion -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides utility functions for converting between EBCDIC-1047 and
+/// UTF-8.
+///
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include <system_error>
+
+namespace llvm {
+namespace ConverterEBCDIC {
+std::error_code convertToEBCDIC(StringRef Source,
+                                SmallVectorImpl<char> &Result);
+
+std::error_code convertToUTF8(StringRef Source, SmallVectorImpl<char> &Result);
+
+} // namespace ConverterEBCDIC
+} // namespace llvm

llvm/lib/Support/CMakeLists.txt

add_llvm_component_library(LLVMSupport
   circular_raw_ostream.cpp
   Chrono.cpp
   COM.cpp
   CodeGenCoverage.cpp
   CommandLine.cpp
   Compression.cpp
   CRC.cpp
   ConvertUTF.cpp
+  ConvertEBCDIC.cpp
   ConvertUTFWrapper.cpp
   CrashRecoveryContext.cpp
   CSKYAttributes.cpp
   CSKYAttributeParser.cpp
   DataExtractor.cpp
   Debug.cpp
   DebugCounter.cpp
   DeltaAlgorithm.cpp

llvm/lib/Support/ConvertEBCDIC.cpp

+//===--- ConvertEBCDIC.cpp - UTF8/EBCDIC CharSet Conversion -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides utility functions for converting between EBCDIC-1047 and
+/// UTF-8.
+///
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ConvertEBCDIC.h"
+
+using namespace llvm;
+
+static const unsigned char ISO88591ToIBM1047[256] = {
+    0x00, 0x01, 0x02, 0x03, 0x37, 0x2d, 0x2e, 0x2f, 0x16, 0x05, 0x15, 0x0b,
+    0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x3c, 0x3d, 0x32, 0x26,
+    0x18, 0x19, 0x3f, 0x27, 0x1c, 0x1d, 0x1e, 0x1f, 0x40, 0x5a, 0x7f, 0x7b,
+    0x5b, 0x6c, 0x50, 0x7d, 0x4d, 0x5d, 0x5c, 0x4e, 0x6b, 0x60, 0x4b, 0x61,
+    0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0x7a, 0x5e,
+    0x4c, 0x7e, 0x6e, 0x6f, 0x7c, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+    0xc8, 0xc9, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xe2,
+    0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xad, 0xe0, 0xbd, 0x5f, 0x6d,
+    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92,
+    0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6,
+    0xa7, 0xa8, 0xa9, 0xc0, 0x4f, 0xd0, 0xa1, 0x07, 0x20, 0x21, 0x22, 0x23,
+    0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x09, 0x0a, 0x1b,
+    0x30, 0x31, 0x1a, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3a, 0x3b,
+    0x04, 0x14, 0x3e, 0xff, 0x41, 0xaa, 0x4a, 0xb1, 0x9f, 0xb2, 0x6a, 0xb5,
+    0xbb, 0xb4, 0x9a, 0x8a, 0xb0, 0xca, 0xaf, 0xbc, 0x90, 0x8f, 0xea, 0xfa,
+    0xbe, 0xa0, 0xb6, 0xb3, 0x9d, 0xda, 0x9b, 0x8b, 0xb7, 0xb8, 0xb9, 0xab,
+    0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9e, 0x68, 0x74, 0x71, 0x72, 0x73,
+    0x78, 0x75, 0x76, 0x77, 0xac, 0x69, 0xed, 0xee, 0xeb, 0xef, 0xec, 0xbf,
+    0x80, 0xfd, 0xfe, 0xfb, 0xfc, 0xba, 0xae, 0x59, 0x44, 0x45, 0x42, 0x46,
+    0x43, 0x47, 0x9c, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57,
+    0x8c, 0x49, 0xcd, 0xce, 0xcb, 0xcf, 0xcc, 0xe1, 0x70, 0xdd, 0xde, 0xdb,
+    0xdc, 0x8d, 0x8e, 0xdf};
+
+static const unsigned char IBM1047ToISO88591[256] = {
+    0x00, 0x01, 0x02, 0x03, 0x9c, 0x09, 0x86, 0x7f, 0x97, 0x8d, 0x8e, 0x0b,
+    0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x9d, 0x0a, 0x08, 0x87,
+    0x18, 0x19, 0x92, 0x8f, 0x1c, 0x1d, 0x1e, 0x1f, 0x80, 0x81, 0x82, 0x83,
+    0x84, 0x85, 0x17, 0x1b, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x05, 0x06, 0x07,
+    0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9a, 0x9b,
+    0x14, 0x15, 0x9e, 0x1a, 0x20, 0xa0, 0xe2, 0xe4, 0xe0, 0xe1, 0xe3, 0xe5,
+    0xe7, 0xf1, 0xa2, 0x2e, 0x3c, 0x28, 0x2b, 0x7c, 0x26, 0xe9, 0xea, 0xeb,
+    0xe8, 0xed, 0xee, 0xef, 0xec, 0xdf, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x5e,
+    0x2d, 0x2f, 0xc2, 0xc4, 0xc0, 0xc1, 0xc3, 0xc5, 0xc7, 0xd1, 0xa6, 0x2c,
+    0x25, 0x5f, 0x3e, 0x3f, 0xf8, 0xc9, 0xca, 0xcb, 0xc8, 0xcd, 0xce, 0xcf,
+    0xcc, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22, 0xd8, 0x61, 0x62, 0x63,
+    0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xab, 0xbb, 0xf0, 0xfd, 0xfe, 0xb1,
+    0xb0, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xaa, 0xba,
+    0xe6, 0xb8, 0xc6, 0xa4, 0xb5, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+    0x79, 0x7a, 0xa1, 0xbf, 0xd0, 0x5b, 0xde, 0xae, 0xac, 0xa3, 0xa5, 0xb7,
+    0xa9, 0xa7, 0xb6, 0xbc, 0xbd, 0xbe, 0xdd, 0xa8, 0xaf, 0x5d, 0xb4, 0xd7,
+    0x7b, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xad, 0xf4,
+    0xf6, 0xf2, 0xf3, 0xf5, 0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50,
+    0x51, 0x52, 0xb9, 0xfb, 0xfc, 0xf9, 0xfa, 0xff, 0x5c, 0xf7, 0x53, 0x54,
+    0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0xb2, 0xd4, 0xd6, 0xd2, 0xd3, 0xd5,
+    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xb3, 0xdb,
+    0xdc, 0xd9, 0xda, 0x9f};
+
+std::error_code
+ConverterEBCDIC::convertToEBCDIC(StringRef Source,
+                                 SmallVectorImpl<char> &Result) {
+  const unsigned char *Table = ISO88591ToIBM1047;
+  const unsigned char *Ptr =
+      reinterpret_cast<const unsigned char *>(Source.data());
+  size_t Length = Source.size();
+  Result.clear();
+  while (Length--) {
+    unsigned char Ch = *Ptr++;
+    // Handle UTF-8 2-byte-sequences in input.
+    if (Ch >= 128) {
+      // Only two-byte sequences can be decoded.
+      if (Ch != 0xc2 && Ch != 0xc3)
+        return std::make_error_code(std::errc::illegal_byte_sequence);
+      // Is buffer truncated?
+      if (!Length)
+        return std::make_error_code(std::errc::invalid_argument);
+      unsigned char Ch2 = *Ptr++;
+      // Is second byte well-formed?
+      if ((Ch2 & 0xc0) != 0x80)
+        return std::make_error_code(std::errc::illegal_byte_sequence);
+      Ch = Ch2 | (Ch << 6);
+      Length--;
+    }
+    // Translate the character.
+    Ch = Table[Ch];
+    Result.push_back(static_cast<char>(Ch));
+  }
+  return std::error_code();
+}
+
+std::error_code ConverterEBCDIC::convertToUTF8(StringRef Source,
+                                               SmallVectorImpl<char> &Result) {
+  const unsigned char *Table = IBM1047ToISO88591;
+  const unsigned char *Ptr =
+      reinterpret_cast<const unsigned char *>(Source.data());
+  size_t Length = Source.size();
+  Result.clear();
+  while (Length--) {
+    unsigned char Ch = *Ptr++;
+    // Translate the character.
+    Ch = Table[Ch];
+    // Handle UTF-8 2-byte-sequences in output.
+    if (Ch >= 128) {
+      // First byte prefixed with either 0xc2 or 0xc3.
+      Result.push_back(static_cast<char>(0xc0 | (Ch >> 6)));
+      // Second byte is either the same as the ASCII byte or ASCII byte -64.
+      Ch = Ch & 0xbf;
+    }
+    Result.push_back(static_cast<char>(Ch));
+  }
+  return std::error_code();
+}

llvm/unittests/Support/CMakeLists.txt

add_llvm_unittest(SupportTests
   BranchProbabilityTest.cpp
   CachePruningTest.cpp
   CrashRecoveryTest.cpp
   Casting.cpp
   CheckedArithmeticTest.cpp
   Chrono.cpp
   CommandLineTest.cpp
   CompressionTest.cpp
+  ConvertEBCDICTest.cpp
   ConvertUTFTest.cpp
   CRCTest.cpp
   CSKYAttributeParserTest.cpp
   DataExtractorTest.cpp
   DebugCounterTest.cpp
   DebugTest.cpp
   DivisionByConstantTest.cpp
   DJBTest.cpp

llvm/unittests/Support/ConvertEBCDICTest.cpp

+//===- unittests/Support/ConvertEBCDICTest.cpp - EBCDIC/UTF8 conversion tests -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--------------------------------------------------------------------------===//
+
+#include "llvm/Support/ConvertEBCDIC.h"
+#include "llvm/ADT/SmallString.h"
+#include "gtest/gtest.h"
+using namespace llvm;
+
+namespace {
+
+// String "Hello World!"
+static const char HelloA[] =
+    "\x48\x65\x6C\x6C\x6F\x20\x57\x6F\x72\x6C\x64\x21\x0a";
+static const char HelloE[] =
+    "\xC8\x85\x93\x93\x96\x40\xE6\x96\x99\x93\x84\x5A\x15";
+
+// String "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+static const char ABCStrA[] =
+    "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52"
+    "\x53\x54\x55\x56\x57\x58\x59\x5A\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A"
+    "\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7A";
+static const char ABCStrE[] =
+    "\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9"
+    "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\x81\x82\x83\x84\x85\x86\x87\x88\x89\x91"
+    "\x92\x93\x94\x95\x96\x97\x98\x99\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9";
+
+// String "¡¢£AÄÅÆEÈÉÊaàáâãäeèéêë"
+static const char AccentUTF[] =
+    "\xc2\xa1\xc2\xa2\xc2\xa3\x41\xc3\x84\xc3\x85\xc3\x86\x45\xc3\x88\xc3\x89"
+    "\xc3\x8a\x61\xc3\xa0\xc3\xa1\xc3\xa2\xc3\xa3\xc3\xa4\x65\xc3\xa8\xc3\xa9"
+    "\xc3\xaa\xc3\xab";
+static const char AccentE[] = "\xaa\x4a\xb1\xc1\x63\x67\x9e\xc5\x74\x71\x72"
+                              "\x81\x44\x45\x42\x46\x43\x85\x54\x51\x52\x53";
+
+// String with Cyrillic character ya.
+static const char CyrillicUTF[] = "\xd0\xaf";
+
+TEST(CharSet, FromUTF8) {
+  // Hello string.
+  StringRef Src(HelloA);
+  SmallString<64> Dst;
+
+  std::error_code EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(HelloE, static_cast<std::string>(Dst).c_str());
+
+  // ABC string.
+  Src = ABCStrA;
+  EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(ABCStrE, static_cast<std::string>(Dst).c_str());
+
+  // Accent string.
+  Src = AccentUTF;
+  EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(AccentE, static_cast<std::string>(Dst).c_str());
+
+  // Cyrillic string. Results in error because not representable in 1047.
+  Src = CyrillicUTF;
+  EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_EQ(EC, std::errc::illegal_byte_sequence);
+}
+
+TEST(CharSet, ToUTF8) {
+  // Hello string.
+  StringRef Src(HelloE);
+  SmallString<64> Dst;
+
+  std::error_code EC = ConverterEBCDIC::convertToUTF8(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(HelloA, static_cast<std::string>(Dst).c_str());
+
+  // ABC string.
+  Src = ABCStrE;
+  EC = ConverterEBCDIC::convertToUTF8(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(ABCStrA, static_cast<std::string>(Dst).c_str());
+
+  // Accent string.
+  Src = AccentE;
+  EC = ConverterEBCDIC::convertToUTF8(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(AccentUTF, static_cast<std::string>(Dst).c_str());
+}
+
+} // namespace