Extended format string checking to wprintf/wscanf

[MarcusJohnson91]

Summary

Split from D103426

[MarcusJohnson91]

This comment was removed by MarcusJohnson91.

[MarcusJohnson91]

Clang-formatted the diff.

[efriedma]

clang/lib/AST/Expr.cpp

1081 ↗

(On Diff #362446)

getStringAsChar16?

clang/lib/AST/Type.cpp

1979 ↗

(On Diff #362446)

(The discussion of the typedefs on D103426 didn't really finish.)

clang/lib/Lex/LiteralSupport.cpp

1923 ↗

(On Diff #362446)

The math here probably needs to change for wide strings.

clang/lib/Sema/SemaChecking.cpp

9609 ↗

(On Diff #362446)

The math here looks suspicious... TypeSize is not in the same units as StrRef.size().

[MarcusJohnson91]

Rebased

[floam]

This comment was removed by floam.

llvm/include/llvm/Support/ConvertUTF.h

 #define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
 #define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
 
 #define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4
 
 #define UNI_UTF16_BYTE_ORDER_MARK_NATIVE  0xFEFF
 #define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE
 
+#define UNI_UTF32_BYTE_ORDER_MARK_NATIVE 0x0000FEFF
+#define UNI_UTF32_BYTE_ORDER_MARK_SWAPPED 0x0000FFFE
+
 typedef enum {
   conversionOK,           /* conversion successful */
   sourceExhausted,        /* partial character in source, but hit end */
   targetExhausted,        /* insuff. room in target for conversion */
   sourceIllegal           /* source sequence is illegal/malformed */
 } ConversionResult;
 
 typedef enum {

 
 /**
  * Returns true if a blob of text starts with a UTF-16 big or little endian byte
  * order mark.
  */
 bool hasUTF16ByteOrderMark(ArrayRef<char> SrcBytes);
 
 /**
+ * Returns true if a blob of text starts with a UTF-32 big or little endian byte
+ * order mark.
+ */
+bool hasUTF32ByteOrderMark(ArrayRef<char> SrcBytes);
+
+/**
  * Converts a stream of raw bytes assumed to be UTF16 into a UTF8 std::string.
  *
  * \param [in] SrcBytes A buffer of what is assumed to be UTF-16 encoded text.
  * \param [out] Out Converted UTF-8 is stored here on success.
  * \returns true on success
  */
 bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out);
 
 /**
 * Converts a UTF16 string into a UTF8 std::string.
 *
 * \param [in] Src A buffer of UTF-16 encoded text.
 * \param [out] Out Converted UTF-8 is stored here on success.
 * \returns true on success
 */
 bool convertUTF16ToUTF8String(ArrayRef<UTF16> Src, std::string &Out);
 
 /**
+ * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string.
+ *
+ * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text.
+ * \param [out] Out Converted UTF-8 is stored here on success.
+ * \returns true on success
+ */
+bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out);
+
+/**
+ * Converts a UTF32 string into a UTF8 std::string.
+ *
+ * \param [in] Src A buffer of UTF-32 encoded text.
+ * \param [out] Out Converted UTF-8 is stored here on success.
+ * \returns true on success
+ */
+bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out);
+
+/**
  * Converts a UTF-8 string into a UTF-16 string with native endianness.
  *
  * \returns true on success
  */
 bool convertUTF8ToUTF16String(StringRef SrcUTF8,
                               SmallVectorImpl<UTF16> &DstUTF16);
 
 #if defined(_WIN32)

llvm/lib/Support/ConvertUTFWrapper.cpp

 }
 
 bool hasUTF16ByteOrderMark(ArrayRef<char> S) {
   return (S.size() >= 2 &&
           ((S[0] == '\xff' && S[1] == '\xfe') ||
            (S[0] == '\xfe' && S[1] == '\xff')));
 }
 
+bool hasUTF32ByteOrderMark(ArrayRef<char> S) {
+  return (
+      S.size() >= 4 &&
+      ((S[0] == '\x00' && S[1] == '\x00' && S[2] == '\xfe' && S[3] == '\xff') ||
+       (S[0] == '\xff' && S[1] == '\xfe' && S[2] == '\x00' && S[3] == '\x00')));
+}
+
 bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {
   assert(Out.empty());
 
   // Error out on an uneven byte count.
   if (SrcBytes.size() % 2)
     return false;
 
   // Avoid OOB by returning early on empty input.

 
 bool convertUTF16ToUTF8String(ArrayRef<UTF16> Src, std::string &Out)
 {
   return convertUTF16ToUTF8String(
       llvm::ArrayRef<char>(reinterpret_cast<const char *>(Src.data()),
       Src.size() * sizeof(UTF16)), Out);
 }
 
+bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {
+  assert(Out.empty());
+
+  // Error out on an uneven byte count.
+  if (SrcBytes.size() % 4)
+    return false;
+
+  // Avoid OOB by returning early on empty input.
+  if (SrcBytes.empty())
+    return true;
+
+  const UTF32 *Src = reinterpret_cast<const UTF32 *>(SrcBytes.begin());
+  const UTF32 *SrcEnd = reinterpret_cast<const UTF32 *>(SrcBytes.end());
+
+  assert((uintptr_t)Src % sizeof(UTF32) == 0);
+
+  // Byteswap if necessary.
+  std::vector<UTF32> ByteSwapped;
+  if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_SWAPPED) {
+    ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd);
+    for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I)
+      ByteSwapped[I] = llvm::ByteSwap_32(ByteSwapped[I]);
+    Src = &ByteSwapped[0];
+    SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1;
+  }
+
+  // Skip the BOM for conversion.
+  if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_NATIVE)
+    Src++;
+
+  // Just allocate enough space up front.  We'll shrink it later.  Allocate
+  // enough that we can fit a null terminator without reallocating.
+  Out.resize(SrcBytes.size() * UNI_MAX_UTF8_BYTES_PER_CODE_POINT + 1);
+  UTF8 *Dst = reinterpret_cast<UTF8 *>(&Out[0]);
+  UTF8 *DstEnd = Dst + Out.size();
+
+  ConversionResult CR =
+      ConvertUTF32toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion);
+  assert(CR != targetExhausted);
+
+  if (CR != conversionOK) {
+    Out.clear();
+    return false;
+  }
+
+  Out.resize(reinterpret_cast<char *>(Dst) - &Out[0]);
+  Out.push_back(0);
+  Out.pop_back();
+  return true;
+}
+
+bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out) {
+  return convertUTF32ToUTF8String(
+      llvm::ArrayRef<char>(reinterpret_cast<const char *>(Src.data()),
+                           Src.size() * sizeof(UTF32)),
+      Out);
+}
+
 bool convertUTF8ToUTF16String(StringRef SrcUTF8,
                               SmallVectorImpl<UTF16> &DstUTF16) {
   assert(DstUTF16.empty());
 
   // Avoid OOB by returning early on empty input.
   if (SrcUTF8.empty()) {
     DstUTF16.push_back(0);
     DstUTF16.pop_back();

llvm/unittests/Support/ConvertUTFTest.cpp

TEST(ConvertUTFTest, ConvertUTF16BigEndianToUTF8String) {
   ArrayRef<char> Ref(Src, sizeof(Src) - 1);
   std::string Result;
   bool Success = convertUTF16ToUTF8String(Ref, Result);
   EXPECT_TRUE(Success);
   std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
   EXPECT_EQ(Expected, Result);
 }
 
+TEST(ConvertUTFTest, ConvertUTF32LittleEndianToUTF8String) {
+  // Src is the look of disapproval.
+  alignas(UTF32) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";
+  ArrayRef<char> Ref(Src, sizeof(Src) - 1);
+  std::string Result;
+  bool Success = convertUTF32ToUTF8String(Ref, Result);
+  EXPECT_TRUE(Success);
+  std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+  EXPECT_EQ(Expected, Result);
+}
+
+TEST(ConvertUTFTest, ConvertUTF32BigEndianToUTF8String) {
+  // Src is the look of disapproval.
+  alignas(UTF32) static const char Src[] = "\xfe\xff\x0c\xa0\x00_\x0c\xa0";
+  ArrayRef<char> Ref(Src, sizeof(Src) - 1);
+  std::string Result;
+  bool Success = convertUTF32ToUTF8String(Ref, Result);
+  EXPECT_TRUE(Success);
+  std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+  EXPECT_EQ(Expected, Result);
+}
+
 TEST(ConvertUTFTest, ConvertUTF8ToUTF16String) {
   // Src is the look of disapproval.
   static const char Src[] = "\xe0\xb2\xa0_\xe0\xb2\xa0";
   StringRef Ref(Src, sizeof(Src) - 1);
   SmallVector<UTF16, 5> Result;
   bool Success = convertUTF8ToUTF16String(Ref, Result);
   EXPECT_TRUE(Success);
   static const UTF16 Expected[] = {0x0CA0, 0x005f, 0x0CA0, 0};
TEST(ConvertUTFTest, HasUTF16BOM) {
   EXPECT_TRUE(HasBOM);
 
   HasBOM = hasUTF16ByteOrderMark(None);
   EXPECT_FALSE(HasBOM);
   HasBOM = hasUTF16ByteOrderMark(makeArrayRef("\xfe", 1));
   EXPECT_FALSE(HasBOM);
 }
 
+TEST(ConvertUTFTest, HasUTF32BOM) {
+  bool HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\x00\x00\xfe\xff", 4));
+  EXPECT_TRUE(HasBOM);
+  HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\xff\xfe\x00\x00", 4));
+  EXPECT_TRUE(HasBOM);
+  HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\x00\x00\xfe\xff ", 5));
+  EXPECT_TRUE(HasBOM); // Don't care about odd lengths.
+  HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\x00\x00\xfe\xff\x00asdf", 9));
+  EXPECT_TRUE(HasBOM);
+
+  HasBOM = hasUTF32ByteOrderMark(None);
+  EXPECT_FALSE(HasBOM);
+  HasBOM = hasUTF32ByteOrderMark(makeArrayRef("\xfe", 1));
+  EXPECT_FALSE(HasBOM);
+}
+
+TEST(ConvertUTFTest, UTF32WrappersForConvertUTF32ToUTF8String) {
+  // Src is the look of disapproval.
+  alignas(UTF32) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";
+  ArrayRef<UTF32> SrcRef = makeArrayRef((const UTF32 *)Src, 4);
+  std::string Result;
+  bool Success = convertUTF32ToUTF8String(SrcRef, Result);
+  EXPECT_TRUE(Success);
+  std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+  EXPECT_EQ(Expected, Result);
+}
+
 TEST(ConvertUTFTest, UTF16WrappersForConvertUTF16ToUTF8String) {
   // Src is the look of disapproval.
   alignas(UTF16) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";
   ArrayRef<UTF16> SrcRef = makeArrayRef((const UTF16 *)Src, 4);
   std::string Result;
   bool Success = convertUTF16ToUTF8String(SrcRef, Result);
   EXPECT_TRUE(Success);
   std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");