Added l16/l32 length modifiers for char16_t/char32_t

[MarcusJohnson91]

Summary

Split from D103426

[MarcusJohnson91]

Clang-formatted the diff

[MarcusJohnson91]

Added a couple tests

[MarcusJohnson91]

Rebased on Main

[MarcusJohnson91]

Clang-FormatDiff.py; Still waiting for the UTFConvert patch to land first.

clang/include/clang/AST/ASTContext.h

public:
   /// __wchar_t as a Microsoft extension).
   QualType getWCharType() const { return WCharTy; }
 
   /// Return the type of wide characters. In C++, this returns the
   /// unique wchar_t type. In C99, this returns a type compatible with the type
   /// defined in <stddef.h> as defined by the target.
   QualType getWideCharType() const { return WideCharTy; }
 
+  /// Return the type of char16 characters. In C++, this returns the
+  /// unique char16_t type. In C11, this returns a type compatible with the type
+  /// defined in <uchar.h> as defined by the target.
+  QualType getChar16Type() const { return Char16Ty; }
+
+  /// Return the type of char32 characters. In C++, this returns the
+  /// unique char32_t type. In C11, this returns a type compatible with the type
+  /// defined in <uchar.h> as defined by the target.
+  QualType getChar32Type() const { return Char32Ty; }
+
   /// Return the type of "signed wchar_t".
   ///
   /// Used when in C++, as a GCC extension.
   QualType getSignedWCharType() const;
 
   /// Return the type of "unsigned wchar_t".
   ///
   /// Used when in C++, as a GCC extension.

clang/include/clang/AST/FormatString.h

private:
   bool flag;
 };
 
 /// Represents the length modifier in a format string in scanf/printf.
 class LengthModifier {
 public:
   enum Kind {
     None,
-    AsChar,       // 'hh'
-    AsShort,      // 'h'
-    AsShortLong,  // 'hl' (OpenCL float/int vector element)
-    AsLong,       // 'l'
-    AsLongLong,   // 'll'
-    AsQuad,       // 'q' (BSD, deprecated, for 64-bit integer types)
-    AsIntMax,     // 'j'
-    AsSizeT,      // 'z'
-    AsPtrDiff,    // 't'
-    AsInt32,      // 'I32' (MSVCRT, like __int32)
-    AsInt3264,    // 'I'   (MSVCRT, like __int3264 from MIDL)
-    AsInt64,      // 'I64' (MSVCRT, like __int64)
-    AsLongDouble, // 'L'
-    AsAllocate,   // for '%as', GNU extension to C90 scanf
-    AsMAllocate,  // for '%ms', GNU extension to scanf
-    AsWide,       // 'w' (MSVCRT, like l but only for c, C, s, S, or Z
+    AsChar,             // 'hh'
+    AsShort,            // 'h'
+    AsShortLong,        // 'hl' (OpenCL float/int vector element)
+    AsLong,             // 'l'
+    AsLongLong,         // 'll'
+    AsQuad,             // 'q' (BSD, deprecated, for 64-bit integer types)
+    AsIntMax,           // 'j'
+    AsSizeT,            // 'z'
+    AsPtrDiff,          // 't'
+    AsInt32,            // 'I32' (MSVCRT, like __int32)
+    AsInt3264,          // 'I'   (MSVCRT, like __int3264 from MIDL)
+    AsInt64,            // 'I64' (MSVCRT, like __int64)
+    AsLongDouble,       // 'L'
+    AsAllocate,         // for '%as', GNU extension to C90 scanf
+    AsMAllocate,        // for '%ms', GNU extension to scanf
+    AsUTF16,            // for '%l16(c|s)', Clang extension
+    AsUTF32,            // for '%l32(c|s)', Clang extension
+    AsWide,             // 'w' (MSVCRT, like l but only for c, C, s, S, or Z
     AsWideChar = AsLong // for '%ls', only makes sense for printf
   };
 
   LengthModifier()
     : Position(nullptr), kind(None) {}
   LengthModifier(const char *pos, Kind k)
     : Position(pos), kind(k) {}
 
protected:
   bool IsPrintf;
   const char *Position;
   const char *EndScanList;
   Kind kind;
 };
 
 class ArgType {
 public:
-  enum Kind { UnknownTy, InvalidTy, SpecificTy, ObjCPointerTy, CPointerTy,
-              AnyCharTy, CStrTy, WCStrTy, WIntTy };
+  enum Kind {
+    UnknownTy,
+    InvalidTy,
+    SpecificTy,
+    ObjCPointerTy,
+    CPointerTy,
+    AnyCharTy,
+    CStrTy,
+    WCStrTy,
+    WIntTy,
+    Char16Ty,
+    Char32Ty
+  };
 
   /// How well a given conversion specifier matches its argument.
   enum MatchKind {
     /// The conversion specifier and the argument types are incompatible. For
     /// instance, "%d" and float.
     NoMatch = 0,
     /// The conversion specifier and the argument type are compatible. For
     /// instance, "%d" and _Bool.

clang/lib/AST/FormatString.cpp

case 'h':
         ++I;
         lmKind = LengthModifier::AsShortLong;
       } else {
         lmKind = LengthModifier::AsShort;
       }
       break;
     case 'l':
       ++I;
+      if (I != E && I + 1 != E && I + 2 != E) {
+        if (*I + 1 == '1' && *I + 2 == '6') {
+          I += 2;
+          lmKind = LengthModifier::AsUTF16;
+        } else if (*I + 1 == '3' && *I + 2 == '2') {
+          I += 2;
+          lmKind = LengthModifier::AsUTF32;
+        }
+      }
       if (I != E && *I == 'l') {
         ++I;
         lmKind = LengthModifier::AsLongLong;
       } else {
         lmKind = LengthModifier::AsLong;
       }
       break;
     case 'j': lmKind = LengthModifier::AsIntMax;     ++I; break;
case AnyCharTy: {
       if (const BuiltinType *BT = argTy->getAs<BuiltinType>())
         switch (BT->getKind()) {
           default:
             break;
           case BuiltinType::Char_S:
           case BuiltinType::SChar:
           case BuiltinType::UChar:
           case BuiltinType::Char_U:
+          case BuiltinType::Char8:
+          case BuiltinType::Char16:
+          case BuiltinType::Char32:
           case BuiltinType::Bool:
             return Match;
         }
       return NoMatch;
     }
 
     case SpecificTy: {
       if (const EnumType *ETy = argTy->getAs<EnumType>()) {
case SpecificTy:
       Res = T;
       break;
     case CStrTy:
       Res = C.getPointerType(C.CharTy);
       break;
     case WCStrTy:
       Res = C.getPointerType(C.getWideCharType());
       break;
+    case Char16Ty:
+      Res = C.getPointerType(C.getChar16Type());
+      break;
+    case Char32Ty:
+      Res = C.getPointerType(C.getChar32Type());
+      break;
     case ObjCPointerTy:
       Res = C.ObjCBuiltinIdTy;
       break;
     case CPointerTy:
       Res = C.VoidPtrTy;
       break;
     case WIntTy: {
       Res = C.getWIntType();
analyze_format_string::LengthModifier::toString() const {
   case AsLongDouble:
     return "L";
   case AsAllocate:
     return "a";
   case AsMAllocate:
     return "m";
   case AsWide:
     return "w";
+  case AsUTF16:
+    return "l16";
+  case AsUTF32:
+    return "l32";
   case None:
     return "";
   }
   return nullptr;
 }
 
 //===----------------------------------------------------------------------===//
 // Methods on ConversionSpecifier.
case LengthModifier::AsInt64:
         case ConversionSpecifier::oArg:
         case ConversionSpecifier::uArg:
         case ConversionSpecifier::xArg:
         case ConversionSpecifier::XArg:
           return Target.getTriple().isOSMSVCRT();
         default:
           return false;
       }
+    case LengthModifier::AsUTF16:
+    case LengthModifier::AsUTF32:
+      switch (CS.getKind()) {
+      case ConversionSpecifier::cArg:
+      case ConversionSpecifier::CArg:
+      case ConversionSpecifier::sArg:
+      case ConversionSpecifier::SArg:
+        return true;
+      default:
+        return false;
+      }
     case LengthModifier::AsWide:
       switch (CS.getKind()) {
         case ConversionSpecifier::cArg:
         case ConversionSpecifier::CArg:
         case ConversionSpecifier::sArg:
         case ConversionSpecifier::SArg:
         case ConversionSpecifier::ZArg:
           return Target.getTriple().isOSMSVCRT();
switch (LM.getKind()) {
     case LengthModifier::AsChar:
     case LengthModifier::AsShort:
     case LengthModifier::AsLong:
     case LengthModifier::AsLongLong:
     case LengthModifier::AsIntMax:
     case LengthModifier::AsSizeT:
     case LengthModifier::AsPtrDiff:
     case LengthModifier::AsLongDouble:
+    case LengthModifier::AsUTF16:
+    case LengthModifier::AsUTF32:
       return true;
     case LengthModifier::AsAllocate:
     case LengthModifier::AsMAllocate:
     case LengthModifier::AsQuad:
     case LengthModifier::AsInt32:
     case LengthModifier::AsInt3264:
     case LengthModifier::AsInt64:
     case LengthModifier::AsWide:
if (Identifier->getName() == "size_t") {
       LM.setKind(LengthModifier::AsIntMax);
       return true;
     } else if (Identifier->getName() == "uintmax_t") {
       LM.setKind(LengthModifier::AsIntMax);
       return true;
     } else if (Identifier->getName() == "ptrdiff_t") {
       LM.setKind(LengthModifier::AsPtrDiff);
       return true;
+    } else if (Identifier->getName() == "char16_t") {
+      LM.setKind(LengthModifier::AsUTF16);
+      return true;
+    } else if (Identifier->getName() == "char32_t") {
+      LM.setKind(LengthModifier::AsUTF32);
+      return true;
     }
 
     QualType T = Typedef->getUnderlyingType();
     if (!isa<TypedefType>(T))
       break;
 
     Typedef = cast<TypedefType>(T)->getDecl();
   }
   return false;
 }

clang/lib/AST/PrintfFormatString.cpp

switch (LM.getKind()) {
                    ? ArgType(Ctx.LongLongTy, "__int64")
                    : ArgType(Ctx.IntTy, "__int32");
       case LengthModifier::AsPtrDiff:
         return ArgType::makePtrdiffT(
             ArgType(Ctx.getPointerDiffType(), "ptrdiff_t"));
       case LengthModifier::AsAllocate:
       case LengthModifier::AsMAllocate:
       case LengthModifier::AsWide:
+      case LengthModifier::AsUTF16:
+      case LengthModifier::AsUTF32:
         return ArgType::Invalid();
     }
 
   if (CS.isUIntArg())
     switch (LM.getKind()) {
       case LengthModifier::AsLongDouble:
         // GNU extension.
         return Ctx.UnsignedLongLongTy;
switch (LM.getKind()) {
                    ? ArgType(Ctx.UnsignedLongLongTy, "unsigned __int64")
                    : ArgType(Ctx.UnsignedIntTy, "unsigned __int32");
       case LengthModifier::AsPtrDiff:
         return ArgType::makePtrdiffT(
             ArgType(Ctx.getUnsignedPointerDiffType(), "unsigned ptrdiff_t"));
       case LengthModifier::AsAllocate:
       case LengthModifier::AsMAllocate:
       case LengthModifier::AsWide:
+      case LengthModifier::AsUTF16:
+      case LengthModifier::AsUTF32:
         return ArgType::Invalid();
     }
 
   if (CS.isDoubleArg()) {
     if (!VectorNumElts.isInvalid()) {
       switch (LM.getKind()) {
       case LengthModifier::AsShort:
         return Ctx.HalfTy;
switch (LM.getKind()) {
       case LengthModifier::AsLongDouble:
         return ArgType(); // FIXME: Is this a known extension?
       case LengthModifier::AsAllocate:
       case LengthModifier::AsMAllocate:
       case LengthModifier::AsInt32:
       case LengthModifier::AsInt3264:
       case LengthModifier::AsInt64:
       case LengthModifier::AsWide:
+      case LengthModifier::AsUTF16:
+      case LengthModifier::AsUTF32:
         return ArgType::Invalid();
       case LengthModifier::AsShortLong:
         llvm_unreachable("only used for OpenCL which doesn not handle nArg");
     }
   }
 
   switch (CS.getKind()) {
     case ConversionSpecifier::sArg:

clang/lib/AST/ScanfFormatString.cpp

case ConversionSpecifier::iArg:
         case LengthModifier::AsLongDouble:
           // GNU extension.
           return ArgType::PtrTo(Ctx.LongLongTy);
         case LengthModifier::AsAllocate:
         case LengthModifier::AsMAllocate:
         case LengthModifier::AsInt32:
         case LengthModifier::AsInt3264:
         case LengthModifier::AsWide:
+        case LengthModifier::AsUTF16:
+        case LengthModifier::AsUTF32:
         case LengthModifier::AsShortLong:
           return ArgType::Invalid();
       }
       llvm_unreachable("Unsupported LengthModifier Type");
 
     // Unsigned int.
     case ConversionSpecifier::oArg:
     case ConversionSpecifier::OArg:
case ConversionSpecifier::XArg:
         case LengthModifier::AsLongDouble:
           // GNU extension.
           return ArgType::PtrTo(Ctx.UnsignedLongLongTy);
         case LengthModifier::AsAllocate:
         case LengthModifier::AsMAllocate:
         case LengthModifier::AsInt32:
         case LengthModifier::AsInt3264:
         case LengthModifier::AsWide:
+        case LengthModifier::AsUTF16:
+        case LengthModifier::AsUTF32:
         case LengthModifier::AsShortLong:
           return ArgType::Invalid();
       }
       llvm_unreachable("Unsupported LengthModifier Type");
 
     // Float.
     case ConversionSpecifier::aArg:
     case ConversionSpecifier::AArg:
switch(CS.getKind()) {
     case ConversionSpecifier::sArg:
     case ConversionSpecifier::ScanListArg:
       switch (LM.getKind()) {
         case LengthModifier::None:
           return ArgType::PtrTo(ArgType::AnyCharTy);
         case LengthModifier::AsLong:
         case LengthModifier::AsWide:
           return ArgType::PtrTo(ArgType(Ctx.getWideCharType(), "wchar_t"));
+        case LengthModifier::AsUTF16:
+          return ArgType::PtrTo(ArgType(Ctx.getChar16Type(), "char16_t"));
+        case LengthModifier::AsUTF32:
+          return ArgType::PtrTo(ArgType(Ctx.getChar32Type(), "char32_t"));
         case LengthModifier::AsAllocate:
         case LengthModifier::AsMAllocate:
           return ArgType::PtrTo(ArgType::CStrTy);
         case LengthModifier::AsShort:
           if (Ctx.getTargetInfo().getTriple().isOSMSVCRT())
             return ArgType::PtrTo(ArgType::AnyCharTy);
           LLVM_FALLTHROUGH;
         default:
           return ArgType::Invalid();
       }
     case ConversionSpecifier::CArg:
     case ConversionSpecifier::SArg:
       // FIXME: Mac OS X specific?
       switch (LM.getKind()) {
         case LengthModifier::None:
         case LengthModifier::AsWide:
           return ArgType::PtrTo(ArgType(Ctx.getWideCharType(), "wchar_t"));
+        case LengthModifier::AsUTF16:
+          return ArgType::PtrTo(ArgType(Ctx.getChar16Type(), "char16_t"));
+        case LengthModifier::AsUTF32:
+          return ArgType::PtrTo(ArgType(Ctx.getChar32Type(), "char32_t"));
         case LengthModifier::AsAllocate:
         case LengthModifier::AsMAllocate:
           return ArgType::PtrTo(ArgType(ArgType::WCStrTy, "wchar_t *"));
         case LengthModifier::AsShort:
           if (Ctx.getTargetInfo().getTriple().isOSMSVCRT())
             return ArgType::PtrTo(ArgType::AnyCharTy);
           LLVM_FALLTHROUGH;
         default:
case ConversionSpecifier::nArg:
           return ArgType::PtrTo(ArgType(Ctx.getPointerDiffType(), "ptrdiff_t"));
         case LengthModifier::AsLongDouble:
           return ArgType(); // FIXME: Is this a known extension?
         case LengthModifier::AsAllocate:
         case LengthModifier::AsMAllocate:
         case LengthModifier::AsInt32:
         case LengthModifier::AsInt3264:
         case LengthModifier::AsWide:
+        case LengthModifier::AsUTF16:
+        case LengthModifier::AsUTF32:
         case LengthModifier::AsShortLong:
           return ArgType::Invalid();
         }
 
     default:
       break;
   }
 
case BuiltinType::ULongLong:
       LM.setKind(LengthModifier::AsLongLong);
       break;
 
     // L
     case BuiltinType::LongDouble:
       LM.setKind(LengthModifier::AsLongDouble);
       break;
 
+    case BuiltinType::Char16:
+      LM.setKind(LengthModifier::AsUTF16);
+      break;
+    case BuiltinType::Char32:
+      LM.setKind(LengthModifier::AsUTF32);
+
     // Don't know.
     default:
       return false;
   }
 
   // Handle size_t, ptrdiff_t, etc. that have dedicated length modifiers in C99.
   if (isa<TypedefType>(PT) && (LangOpt.C99 || LangOpt.CPlusPlus11))
     namedTypeToLengthModifier(PT, LM);

clang/test/Sema/format-strings-int-typedefs.c

 // RUN: %clang_cc1 -triple i386-apple-darwin9 -fsyntax-only -verify %s
 
 int printf(char const *, ...);
 int scanf(char const *, ...);
 
 void test(void) {
   printf("%jd", 42.0); // expected-warning {{format specifies type 'intmax_t' (aka 'long long')}}
   printf("%ju", 42.0); // expected-warning {{format specifies type 'uintmax_t' (aka 'unsigned long long')}}
   printf("%zu", 42.0); // expected-warning {{format specifies type 'size_t' (aka 'unsigned long')}}
   printf("%td", 42.0); // expected-warning {{format specifies type 'ptrdiff_t' (aka 'int')}}
   printf("%lc", 42.0); // expected-warning {{format specifies type 'wint_t' (aka 'int')}}
   printf("%ls", 42.0); // expected-warning {{format specifies type 'wchar_t *' (aka 'int *')}}
+  printf("%l16c", 42.0); // expected-warning {{format specifies type 'char16_t' (aka 'int')}}
+  printf("%l16s", 42.0); // expected-warning {{format specifies type 'char16_t *' (aka 'int *')}}
+  printf("%l32c", 42.0); // expected-warning {{format specifies type 'char32_t' (aka 'int')}}
+  printf("%l32s", 42.0); // expected-warning {{format specifies type 'char32_t *' (aka 'int *')}}
   printf("%S", 42.0);  // expected-warning {{format specifies type 'wchar_t *' (aka 'int *')}}
   printf("%C", 42.0);  // expected-warning {{format specifies type 'wchar_t' (aka 'int')}}
 
+  wprintf(L"%l16c", 42.0); // expected-warning {{format specifies type 'char16_t' (aka 'short')}}
+  wprintf(L"%l16s", 42.0); // expected-warning {{format specifies type 'char16_t *' (aka 'short *')}}
+  wprintf(L"%l32c", 42.0); // expected-warning {{format specifies type 'char32_t' (aka 'int')}}
+  wprintf(L"%l32s", 42.0); // expected-warning {{format specifies type 'char32_t *' (aka 'int *')}}
+
   scanf("%jd", 0); // expected-warning {{format specifies type 'intmax_t *' (aka 'long long *')}}
   scanf("%ju", 0); // expected-warning {{format specifies type 'uintmax_t *' (aka 'unsigned long long *')}}
   scanf("%zu", 0); // expected-warning {{format specifies type 'size_t *' (aka 'unsigned long *')}}
   scanf("%td", 0); // expected-warning {{format specifies type 'ptrdiff_t *' (aka 'int *')}}
   scanf("%lc", 0); // expected-warning {{format specifies type 'wchar_t *' (aka 'int *')}}
   scanf("%ls", 0); // expected-warning {{format specifies type 'wchar_t *' (aka 'int *')}}
+  scanf("%l16c", 0); // expected-warning {{format specifies type 'char16_t *' (aka 'int *')}}
+  scanf("%l16s", 0); // expected-warning {{format specifies type 'char16_t *' (aka 'int *')}}
+  scanf("%l32c", 0); // expected-warning {{format specifies type 'char32_t *' (aka 'int *')}}
+  scanf("%l32s", 0); // expected-warning {{format specifies type 'char32_t *' (aka 'int *')}}
   scanf("%S",  0);  // expected-warning {{format specifies type 'wchar_t *' (aka 'int *')}}
   scanf("%C",  0);  // expected-warning {{format specifies type 'wchar_t *' (aka 'int *')}}
 
+  wscanf("%l16c", 0); // expected-warning {{format specifies type 'char16_t *' (aka 'int *')}}
+  wscanf("%l16s", 0); // expected-warning {{format specifies type 'char16_t *' (aka 'int *')}}
+  wscanf("%l32c", 0); // expected-warning {{format specifies type 'char32_t *' (aka 'int *')}}
+  wscanf("%l32s", 0); // expected-warning {{format specifies type 'char32_t *' (aka 'int *')}}
 
   // typedef size_t et al. to something crazy.
   typedef void *size_t;
   typedef void *intmax_t;
   typedef void *uintmax_t;
   typedef void *ptrdiff_t;
 
   // The warning still fires, because it checks the underlying type.
   printf("%jd", (intmax_t)42); // expected-warning {{format specifies type 'intmax_t' (aka 'long long') but the argument has type 'intmax_t' (aka 'void *')}}
   printf("%ju", (uintmax_t)42); // expected-warning {{format specifies type 'uintmax_t' (aka 'unsigned long long') but the argument has type 'uintmax_t' (aka 'void *')}}
   printf("%zu", (size_t)42); // expected-warning {{format specifies type 'size_t' (aka 'unsigned long') but the argument has type 'size_t' (aka 'void *')}}
   printf("%td", (ptrdiff_t)42); // expected-warning {{format specifies type 'ptrdiff_t' (aka 'int') but the argument has type 'ptrdiff_t' (aka 'void *')}}
 }

clang/test/SemaCXX/format-strings.cpp

 #endif
 
   printf("%a", 1.0);
   scanf("%afoobar", fp);
 }
 
 void g() {
   printf("%ls", "foo"); // expected-warning{{format specifies type 'wchar_t *' but the argument has type 'const char *'}}
+  printf("%l16s", "foo"); // expected-warning{{format specifies type 'char16_t *' but the argument has type 'const char *'}}
+  printf("%l32s", "foo"); // expected-warning{{format specifies type 'char32_t *' but the argument has type 'const char *'}}
 }
 
 // Test that we properly handle format_idx on C++ members.
 class Foo {
 public:
   const char *gettext(const char *fmt) __attribute__((format_arg(2)));
 
   int scanf(const char *, ...) __attribute__((format(scanf, 2, 3)));