[libc] add basic wide char functions

[michaelrj]

Summary

This patch adds the wchar header, as well as the functions to convert to
and from wide chars. The header also sets up the definitions for wint
and wchar.

[lntue]

libc/include/llvm-libc-macros/wchar-macros.h

1

wchar.h ?

libc/src/__support/wctype_utils.h

13

nit: space after //

[sivachandra]

Except of the definition of the new types, the rest LGTM. We should of course formally define our position on the locales that we care about.

libc/include/wchar.h.def

15

Each of these types should have their own type header (we already have one for size_t). They will get into the final header file if you add them to api.td.

[michaelrj]

clean up type definitions to use the API system.

I agree and will try to make a document describing our plan for locale support as a followup patch.

libc/config/linux/x86_64/entrypoints.txt

set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.rmdir
     libc.src.unistd.symlink
     libc.src.unistd.symlinkat
     libc.src.unistd.sysconf
     libc.src.unistd.truncate
     libc.src.unistd.unlink
     libc.src.unistd.unlinkat
     libc.src.unistd.write
+
+    # wchar.h entrypoints
+    libc.src.wchar.wctob
 )
 
 set(TARGET_LIBM_ENTRYPOINTS
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
     libc.src.fenv.fedisableexcept
     libc.src.fenv.feenableexcept
     libc.src.fenv.fegetenv

libc/config/linux/x86_64/headers.txt

set(TARGET_PUBLIC_HEADERS
     libc.include.setjmp
     libc.include.stdio
     libc.include.stdlib
     libc.include.string
     libc.include.termios
     libc.include.threads
     libc.include.time
     libc.include.unistd
+    libc.include.wchar
 
     libc.include.arpa_inet
 
     libc.include.sys_auxv
     libc.include.sys_ioctl
     libc.include.sys_mman
     libc.include.sys_prctl
     libc.include.sys_random

libc/include/CMakeLists.txt

DEPENDS
     .llvm-libc-macros.termios_macros
     .llvm-libc-types.cc_t
     .llvm-libc-types.pid_t
     .llvm-libc-types.speed_t
     .llvm-libc-types.struct_termios
     .llvm-libc-types.tcflag_t
 )
 
+add_gen_header(
+  wchar
+  DEF_FILE wchar.h.def
+  GEN_HDR wchar.h
+  DEPENDS
+    .llvm_libc_common_h
+    .llvm-libc-macros.wchar_macros
+)
+
 if(NOT LLVM_LIBC_FULL_BUILD)
   # We don't install headers in non-fullbuild mode.
   return()
 endif()
 
 function(get_all_install_header_targets out_var)
   set(all_deps ${ARGN})
   foreach(target IN LISTS ARGN)

libc/include/llvm-libc-macros/CMakeLists.txt

 
 add_header(
   unistd_macros
   HDR
     unistd-macros.h
   DEPENDS
     .linux.unistd_macros
 )
+
+add_header(
+  wchar_macros
+  HDR
+    wchar-macros.h
+)

libc/include/llvm-libc-macros/wchar-macros.h

+//===-- Macros defined in stdio.h header file -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_MACROS_WCHAR_MACROS_H
+#define __LLVM_LIBC_MACROS_WCHAR_MACROS_H
+
+#ifndef WEOF
+#define WEOF 0xffffffffu
+#endif
+
+#endif // __LLVM_LIBC_MACROS_WCHAR_MACROS_H

libc/include/wchar.h.def

+//===-- C standard library header wchar.h ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_WCHAR_H
+#define LLVM_LIBC_WCHAR_H
+
+#define __need_size_t
+#define __need_wint_t
+#define __need_wchar_t
+#include <stddef.h>
+
+#include <__llvm-libc-common.h>
+#include <llvm-libc-macros/wchar-macros.h>
+
+%%public_api()
+
+#endif // LLVM_LIBC_WCHAR_H

libc/spec/spec.td

 def VoidPtrPtr : PtrType<VoidPtr>;
 def RestrictedVoidPtrPtr : RestrictedPtrType<VoidPtr>;
 def ConstVoidPtr : ConstType<VoidPtr>;
 
 def SizeTType : NamedType<"size_t">;
 def SizeTPtr : PtrType<SizeTType>;
 def RestrictedSizeTPtr : RestrictedPtrType<SizeTType>;
 
+def WCharType : NamedType<"wchar_t">;
+def WIntType : NamedType<"wint_t">;
+def MBStateType : NamedType<"mbstate_t">;
+
 def LongDoublePtr : PtrType<LongDoubleType>;
 
 def IntMaxTType : NamedType<"intmax_t">;
 def UIntMaxTType : NamedType<"uintmax_t">;
 
 def UInt16Type : NamedType<"uint16_t">;
 def UInt32Type : NamedType<"uint32_t">;
 

libc/spec/stdc.td

HeaderSpec SetJmp = HeaderSpec<
           FunctionSpec<
               "longjmp",
               RetValSpec<VoidType>,
               [ArgSpec<JmpBuf>, ArgSpec<IntType>]
           >,
       ]
   >;
 
+  HeaderSpec WChar = HeaderSpec<
+      "wchar.h",
+      [ // Macros
+        Macro<"WEOF">,
+      ], 
+      [ //Types
+        SizeTType,
+        WIntType,
+        WCharType,
+        MBStateType,
+        StructTmType,
+      ],
+      [], // Enumerations
+      [
+          FunctionSpec<
+              "wctob",
+              RetValSpec<IntType>,
+              [ArgSpec<WIntType>]
+          >,
+      ]
+  >;
+
+
   let Headers = [
     Assert,
     CType,
     Errno,
     Fenv,
     Math,
     String,
     StdIO,
     StdLib,
     IntTypes,
     SetJmp,
     Signal,
     Threads,
     Time,
+    WChar,
   ];
 }

libc/src/CMakeLists.txt

 add_subdirectory(__support)
 
 add_subdirectory(ctype)
 add_subdirectory(errno)
 add_subdirectory(fenv)
 add_subdirectory(inttypes)
 add_subdirectory(math)
 add_subdirectory(string)
 add_subdirectory(stdlib)
 add_subdirectory(stdio)
+add_subdirectory(wchar)
 
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(dirent)
   add_subdirectory(fcntl)
   add_subdirectory(pthread)
   add_subdirectory(sched)
   add_subdirectory(sys)
   add_subdirectory(termios)

libc/src/__support/CMakeLists.txt

 
 add_header_library(
   ctype_utils
   HDRS
     ctype_utils.h
 )
 
 add_header_library(
+  wctype_utils
+  HDRS
+    wctype_utils.h
+)
+
+add_header_library(
   str_to_num_result
   HDRS
     str_to_num_result.h
 )
 
 add_header_library(
   str_to_integer
   HDRS

libc/src/__support/wctype_utils.h

+//===-- Collection of utils for implementing wide char functions --*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SUPPORT_WCTYPE_UTILS_H
+#define LLVM_LIBC_SRC_SUPPORT_WCTYPE_UTILS_H
+
+#include "src/__support/CPP/optional.h"
+#include "src/__support/macros/attributes.h" //LIBC_INLINE
+
+#define __need_wint_t
+#define __need_wchar_t
+#include <stddef.h> // needed for wint_t and wchar_t
+
+namespace __llvm_libc {
+namespace internal {
+
+// ------------------------------------------------------
+// Rationale: Since these classification functions are
+// called in other functions, we will avoid the overhead
+// of a function call by inlining them.
+// ------------------------------------------------------
+
+LIBC_INLINE cpp::optional<int> wctob(wint_t c) {
+  // This needs to be translated to EOF at the callsite. This is to avoid
+  // including stdio.h in this file.
+  if (c > 127 || c < 0)
+    return cpp::nullopt;
+  return static_cast<int>(c);
+}
+
+LIBC_INLINE cpp::optional<wint_t> btowc(int c) {
+  if (c > 127 || c < 0)
+    return cpp::nullopt;
+  return static_cast<wint_t>(c);
+}
+
+} // namespace internal
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_SUPPORT_WCTYPE_UTILS_H

libc/src/wchar/CMakeLists.txt

+
+add_entrypoint_object(
+  wctob
+  SRCS
+    wctob.cpp
+  HDRS
+    wctob.h
+  DEPENDS 
+    libc.src.__support.wctype_utils
+)

libc/src/wchar/btowc.h

+//===-- Implementation header for btowc -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_BTOWC_H
+#define LLVM_LIBC_SRC_WCHAR_BTOWC_H
+
+#include <wchar.h>
+
+namespace __llvm_libc {
+
+wint_t btowc(int c);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_WCHAR_BTOWC_H

libc/src/wchar/btowc.cpp

+//===-- Implementation of btowc -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/btowc.h"
+#include "src/__support/common.h"
+#include "src/__support/wctype_utils.h"
+
+#include <stdio.h> // for EOF.
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(int, btowc, (wint_t c)) {
+  auto result = internal::btowc(c);
+  if (result.has_value()) {
+    return result.value();
+  } else {
+    return WEOF;
+  }
+}
+
+} // namespace __llvm_libc

libc/src/wchar/wctob.h

+//===-- Implementation header for wctob -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCTOB_H
+#define LLVM_LIBC_SRC_WCHAR_WCTOB_H
+
+#include <wchar.h>
+
+namespace __llvm_libc {
+
+int wctob(wint_t c);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCTOB_H

libc/src/wchar/wctob.cpp

+//===-- Implementation of wctob -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wctob.h"
+#include "src/__support/common.h"
+#include "src/__support/wctype_utils.h"
+
+#include <stdio.h> // for EOF.
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(int, wctob, (wint_t c)) {
+  auto result = internal::wctob(c);
+  if (result.has_value()) {
+    return result.value();
+  } else {
+    return EOF;
+  }
+}
+
+} // namespace __llvm_libc

libc/test/src/CMakeLists.txt

 add_subdirectory(ctype)
 add_subdirectory(errno)
 add_subdirectory(fenv)
 add_subdirectory(math)
 add_subdirectory(string)
 add_subdirectory(stdlib)
 add_subdirectory(inttypes)
 add_subdirectory(stdio)
+add_subdirectory(wchar)
 
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(fcntl)
   add_subdirectory(sched)
   add_subdirectory(sys)
   add_subdirectory(termios)
   add_subdirectory(unistd)
 endif()

libc/test/src/wchar/CMakeLists.txt

+add_libc_testsuite(libc_wchar_unittests)
+
+add_libc_unittest(
+  btowc_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    btowc_test.cpp
+  DEPENDS
+    libc.src.wchar.btowc
+)
+
+add_libc_unittest(
+  wctob_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wctob_test.cpp
+  DEPENDS
+    libc.src.wchar.wctob
+)

libc/test/src/wchar/btowc_test.cpp

+//===-- Unittests for btowc ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <wchar.h> //for WEOF
+
+#include "src/wchar/btowc.h"
+
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcBtowc, DefaultLocale) {
+  // Loops through all characters, verifying that ascii returns itself and
+  // everything else returns WEOF.
+  for (int c = 0; c < 255; ++c) {
+    if (c < 128)
+      EXPECT_EQ(__llvm_libc::btowc(c), static_cast<wint_t>(c));
+    else
+      EXPECT_EQ(__llvm_libc::btowc(c), WEOF);
+  }
+}

libc/test/src/wchar/wctob_test.cpp

+//===-- Unittests for wctob ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <stdio.h> //for EOF
+
+#include "src/wchar/wctob.h"
+
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWctob, DefaultLocale) {
+  // Loops through a subset of the wide characters, verifying that ascii returns
+  // itself and everything else returns EOF.
+  for (wint_t c = 0; c < 32767; ++c) {
+    if (c < 128)
+      EXPECT_EQ(__llvm_libc::wctob(c), static_cast<int>(c));
+    else
+      EXPECT_EQ(__llvm_libc::wctob(c), EOF);
+  }
+}