[libc] add basic wide char functions

[michaelrj]

Summary

This patch adds the wchar header, as well as the functions to convert to
and from wide chars. The header also sets up the definitions for wint
and wchar.

[lntue]

libc/include/llvm-libc-macros/wchar-macros.h

2

wchar.h ?

libc/src/__support/wctype_utils.h

14

nit: space after //

[sivachandra]

Except of the definition of the new types, the rest LGTM. We should of course formally define our position on the locales that we care about.

libc/include/wchar.h.def

16

Each of these types should have their own type header (we already have one for size_t). They will get into the final header file if you add them to api.td.

[michaelrj]

clean up type definitions to use the API system.

I agree and will try to make a document describing our plan for locale support as a followup patch.

libc/config/linux/api.td

def DirentAPI : PublicAPI<"dirent.h"> {
   ];
 }
 
 def UniStdAPI : PublicAPI<"unistd.h"> {
   let Types = ["__exec_argv_t", "__exec_envp_t", "off_t", "pid_t", "size_t",
                "ssize_t", "uid_t", "__getoptargv_t"];
 }
 
+def WCharAPI : PublicAPI<"wchar.h"> {
+  let Types = [
+    "wchar_t",
+    "wint_t",
+    "size_t",
+  ];
+}
+
 def SysRandomAPI : PublicAPI<"sys/random.h"> {
   let Types = ["size_t", "ssize_t"];
 }
 
 def SysSelectAPI : PublicAPI<"sys/select.h"> {
   let Types = ["fd_set", "sigset_t", "suseconds_t", "time_t", "struct timespec",
                "struct timeval"];
 }

libc/config/linux/x86_64/entrypoints.txt

set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.rmdir
     libc.src.unistd.symlink
     libc.src.unistd.symlinkat
     libc.src.unistd.sysconf
     libc.src.unistd.truncate
     libc.src.unistd.unlink
     libc.src.unistd.unlinkat
     libc.src.unistd.write
+
+    # wchar.h entrypoints
+    libc.src.wchar.wctob
 )
 
 set(TARGET_LIBM_ENTRYPOINTS
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
     libc.src.fenv.fedisableexcept
     libc.src.fenv.feenableexcept
     libc.src.fenv.fegetenv

libc/config/linux/x86_64/headers.txt

set(TARGET_PUBLIC_HEADERS
     libc.include.stdio
     libc.include.stdlib
     libc.include.string
     libc.include.strings
     libc.include.termios
     libc.include.threads
     libc.include.time
     libc.include.unistd
+    libc.include.wchar
 
     libc.include.arpa_inet
 
     libc.include.sys_auxv
     libc.include.sys_ioctl
     libc.include.sys_mman
     libc.include.sys_prctl
     libc.include.sys_random

libc/include/CMakeLists.txt

DEPENDS
     .llvm-libc-macros.termios_macros
     .llvm-libc-types.cc_t
     .llvm-libc-types.pid_t
     .llvm-libc-types.speed_t
     .llvm-libc-types.struct_termios
     .llvm-libc-types.tcflag_t
 )
 
+add_gen_header(
+  wchar
+  DEF_FILE wchar.h.def
+  GEN_HDR wchar.h
+  DEPENDS
+    .llvm_libc_common_h
+    .llvm-libc-macros.wchar_macros
+)
+
 if(NOT LLVM_LIBC_FULL_BUILD)
   # We don't install headers in non-fullbuild mode.
   return()
 endif()
 
 function(get_all_install_header_targets out_var)
   set(all_deps ${ARGN})
   foreach(target IN LISTS ARGN)

libc/include/llvm-libc-macros/CMakeLists.txt

 
 add_header(
   unistd_macros
   HDR
     unistd-macros.h
   DEPENDS
     .linux.unistd_macros
 )
+
+add_header(
+  wchar_macros
+  HDR
+    wchar-macros.h
+)

libc/include/llvm-libc-macros/wchar-macros.h

+//===-- Macros defined in wchar.h header file -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_MACROS_WCHAR_MACROS_H
+#define __LLVM_LIBC_MACROS_WCHAR_MACROS_H
+
+#ifndef WEOF
+#define WEOF 0xffffffffu
+#endif
+
+#endif // __LLVM_LIBC_MACROS_WCHAR_MACROS_H

libc/include/llvm-libc-types/CMakeLists.txt

 add_header(thrd_t HDR thrd_t.h DEPENDS .__thread_type)
 add_header(tss_t HDR tss_t.h)
 add_header(tss_dtor_t HDR tss_dtor_t.h)
 add_header(__atexithandler_t HDR __atexithandler_t.h)
 add_header(speed_t HDR speed_t.h)
 add_header(tcflag_t HDR tcflag_t.h)
 add_header(struct_termios HDR struct_termios.h DEPENDS .cc_t .speed_t .tcflag_t)
 add_header(__getoptargv_t HDR __getoptargv_t.h)
+add_header(wchar_t HDR wchar_t.h)
+add_header(wint_t HDR wint_t.h)

libc/include/llvm-libc-types/size_t.h

 //===-- Definition of size_t types ----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef __LLVM_LIBC_TYPES_SIZE_T_H__
 #define __LLVM_LIBC_TYPES_SIZE_T_H__
 
 // Since __need_size_t is defined, we get the definition of size_t from the
 // standalone C header stddef.h. Also, because __need_size_t is defined,
-// including stddef.h will pull only the type size_t and nothing else.a
+// including stddef.h will pull only the type size_t and nothing else.
 #define __need_size_t
 #include <stddef.h>
+#undef __need_size_t
 
 #endif // __LLVM_LIBC_TYPES_SIZE_T_H__

libc/include/llvm-libc-types/wchar_t.h

+//===-- Definition of wchar_t types ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_WCHAR_T_H__
+#define __LLVM_LIBC_TYPES_WCHAR_T_H__
+
+// Since __need_wchar_t is defined, we get the definition of wchar_t from the
+// standalone C header stddef.h. Also, because __need_wchar_t is defined,
+// including stddef.h will pull only the type wchar_t and nothing else.
+#define __need_wchar_t
+#include <stddef.h>
+#undef __need_wchar_t
+
+#endif // __LLVM_LIBC_TYPES_WCHAR_T_H__

libc/include/llvm-libc-types/wint_t.h

+//===-- Definition of wint_t types ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_WINT_T_H__
+#define __LLVM_LIBC_TYPES_WINT_T_H__
+
+// Since __need_wint_t is defined, we get the definition of wint_t from the
+// standalone C header stddef.h. Also, because __need_wint_t is defined,
+// including stddef.h will pull only the type wint_t and nothing else.
+#define __need_wint_t
+#include <stddef.h>
+#undef __need_wint_t
+
+#endif // __LLVM_LIBC_TYPES_WINT_T_H__

libc/include/wchar.h.def

+//===-- C standard library header wchar.h ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_WCHAR_H
+#define LLVM_LIBC_WCHAR_H
+
+#include <__llvm-libc-common.h>
+#include <llvm-libc-macros/wchar-macros.h>
+
+%%public_api()
+
+#endif // LLVM_LIBC_WCHAR_H

libc/spec/spec.td

 def VoidPtrPtr : PtrType<VoidPtr>;
 def RestrictedVoidPtrPtr : RestrictedPtrType<VoidPtr>;
 def ConstVoidPtr : ConstType<VoidPtr>;
 
 def SizeTType : NamedType<"size_t">;
 def SizeTPtr : PtrType<SizeTType>;
 def RestrictedSizeTPtr : RestrictedPtrType<SizeTType>;
 
+def WCharType : NamedType<"wchar_t">;
+def WIntType : NamedType<"wint_t">;
+def MBStateType : NamedType<"mbstate_t">;
+
 def LongDoublePtr : PtrType<LongDoubleType>;
 
 def IntMaxTType : NamedType<"intmax_t">;
 def UIntMaxTType : NamedType<"uintmax_t">;
 
 def UInt16Type : NamedType<"uint16_t">;
 def UInt32Type : NamedType<"uint32_t">;
 

libc/spec/stdc.td

HeaderSpec SetJmp = HeaderSpec<
           FunctionSpec<
               "longjmp",
               RetValSpec<VoidType>,
               [ArgSpec<JmpBuf>, ArgSpec<IntType>]
           >,
       ]
   >;
 
+  HeaderSpec WChar = HeaderSpec<
+      "wchar.h",
+      [ // Macros
+        Macro<"WEOF">,
+      ], 
+      [ //Types
+        SizeTType,
+        WIntType,
+        WCharType,
+        MBStateType,
+        StructTmType,
+      ],
+      [], // Enumerations
+      [
+          FunctionSpec<
+              "wctob",
+              RetValSpec<IntType>,
+              [ArgSpec<WIntType>]
+          >,
+      ]
+  >;
+
+
   let Headers = [
     Assert,
     CType,
     Errno,
     Fenv,
     Math,
     String,
     StdIO,
     StdLib,
     IntTypes,
     SetJmp,
     Signal,
     Threads,
     Time,
+    WChar,
   ];
 }

libc/src/CMakeLists.txt

 add_subdirectory(__support)
 
 add_subdirectory(ctype)
 add_subdirectory(errno)
 add_subdirectory(fenv)
 add_subdirectory(inttypes)
 add_subdirectory(math)
 add_subdirectory(string)
 add_subdirectory(stdlib)
 add_subdirectory(stdio)
+add_subdirectory(wchar)
 
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(dirent)
   add_subdirectory(fcntl)
   add_subdirectory(pthread)
   add_subdirectory(sched)
   add_subdirectory(sys)
   add_subdirectory(termios)

libc/src/__support/CMakeLists.txt

 
 add_header_library(
   ctype_utils
   HDRS
     ctype_utils.h
 )
 
 add_header_library(
+  wctype_utils
+  HDRS
+    wctype_utils.h
+)
+
+add_header_library(
   str_to_num_result
   HDRS
     str_to_num_result.h
 )
 
 add_header_library(
   str_to_integer
   HDRS

libc/src/__support/wctype_utils.h

+//===-- Collection of utils for implementing wide char functions --*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SUPPORT_WCTYPE_UTILS_H
+#define LLVM_LIBC_SRC_SUPPORT_WCTYPE_UTILS_H
+
+#include "src/__support/CPP/optional.h"
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+
+#define __need_wint_t
+#define __need_wchar_t
+#include <stddef.h> // needed for wint_t and wchar_t
+
+namespace __llvm_libc {
+namespace internal {
+
+// ------------------------------------------------------
+// Rationale: Since these classification functions are
+// called in other functions, we will avoid the overhead
+// of a function call by inlining them.
+// ------------------------------------------------------
+
+LIBC_INLINE cpp::optional<int> wctob(wint_t c) {
+  // This needs to be translated to EOF at the callsite. This is to avoid
+  // including stdio.h in this file.
+  if (c > 127 || c < 0)
+    return cpp::nullopt;
+  return static_cast<int>(c);
+}
+
+LIBC_INLINE cpp::optional<wint_t> btowc(int c) {
+  if (c > 127 || c < 0)
+    return cpp::nullopt;
+  return static_cast<wint_t>(c);
+}
+
+} // namespace internal
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_SUPPORT_WCTYPE_UTILS_H

libc/src/wchar/CMakeLists.txt

+
+add_entrypoint_object(
+  wctob
+  SRCS
+    wctob.cpp
+  HDRS
+    wctob.h
+  DEPENDS 
+    libc.src.__support.wctype_utils
+)

libc/src/wchar/btowc.h

+//===-- Implementation header for btowc -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_BTOWC_H
+#define LLVM_LIBC_SRC_WCHAR_BTOWC_H
+
+#include <wchar.h>
+
+namespace __llvm_libc {
+
+wint_t btowc(int c);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_WCHAR_BTOWC_H

libc/src/wchar/btowc.cpp

+//===-- Implementation of btowc -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/btowc.h"
+#include "src/__support/common.h"
+#include "src/__support/wctype_utils.h"
+
+#include <stdio.h> // for EOF.
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(int, btowc, (wint_t c)) {
+  auto result = internal::btowc(c);
+  if (result.has_value()) {
+    return result.value();
+  } else {
+    return WEOF;
+  }
+}
+
+} // namespace __llvm_libc

libc/src/wchar/wctob.h

+//===-- Implementation header for wctob -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCTOB_H
+#define LLVM_LIBC_SRC_WCHAR_WCTOB_H
+
+#include <wchar.h>
+
+namespace __llvm_libc {
+
+int wctob(wint_t c);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCTOB_H

libc/src/wchar/wctob.cpp

+//===-- Implementation of wctob -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wctob.h"
+#include "src/__support/common.h"
+#include "src/__support/wctype_utils.h"
+
+#include <stdio.h> // for EOF.
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(int, wctob, (wint_t c)) {
+  auto result = internal::wctob(c);
+  if (result.has_value()) {
+    return result.value();
+  } else {
+    return EOF;
+  }
+}
+
+} // namespace __llvm_libc

libc/test/src/CMakeLists.txt

 add_subdirectory(ctype)
 add_subdirectory(errno)
 add_subdirectory(fenv)
 add_subdirectory(math)
 add_subdirectory(string)
 add_subdirectory(stdlib)
 add_subdirectory(inttypes)
 add_subdirectory(stdio)
+add_subdirectory(wchar)
 
 if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(fcntl)
   add_subdirectory(sched)
   add_subdirectory(sys)
   add_subdirectory(termios)
   add_subdirectory(unistd)
 endif()

libc/test/src/wchar/CMakeLists.txt

+add_libc_testsuite(libc_wchar_unittests)
+
+add_libc_unittest(
+  btowc_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    btowc_test.cpp
+  DEPENDS
+    libc.src.wchar.btowc
+)
+
+add_libc_unittest(
+  wctob_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wctob_test.cpp
+  DEPENDS
+    libc.src.wchar.wctob
+)

libc/test/src/wchar/btowc_test.cpp

+//===-- Unittests for btowc ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <wchar.h> //for WEOF
+
+#include "src/wchar/btowc.h"
+
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcBtowc, DefaultLocale) {
+  // Loops through all characters, verifying that ascii returns itself and
+  // everything else returns WEOF.
+  for (int c = 0; c < 255; ++c) {
+    if (c < 128)
+      EXPECT_EQ(__llvm_libc::btowc(c), static_cast<wint_t>(c));
+    else
+      EXPECT_EQ(__llvm_libc::btowc(c), WEOF);
+  }
+}

libc/test/src/wchar/wctob_test.cpp

+//===-- Unittests for wctob ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <stdio.h> //for EOF
+
+#include "src/wchar/wctob.h"
+
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWctob, DefaultLocale) {
+  // Loops through a subset of the wide characters, verifying that ascii returns
+  // itself and everything else returns EOF.
+  for (wint_t c = 0; c < 32767; ++c) {
+    if (c < 128)
+      EXPECT_EQ(__llvm_libc::wctob(c), static_cast<int>(c));
+    else
+      EXPECT_EQ(__llvm_libc::wctob(c), EOF);
+  }
+}