Index: llvm/include/llvm/Support/ConvertUTF.h =================================================================== --- llvm/include/llvm/Support/ConvertUTF.h +++ llvm/include/llvm/Support/ConvertUTF.h @@ -122,6 +122,9 @@ #define UNI_UTF16_BYTE_ORDER_MARK_NATIVE 0xFEFF #define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE + +#define UNI_UTF32_BYTE_ORDER_MARK_NATIVE 0x0000FEFF +#define UNI_UTF32_BYTE_ORDER_MARK_SWAPPED 0x0000FFFE typedef enum { conversionOK, /* conversion successful */ @@ -277,6 +280,24 @@ * \returns true on success */ bool convertUTF16ToUTF8String(ArrayRef Src, std::string &Out); + +/** + * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string. + * + * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text. + * \param [out] Out Converted UTF-8 is stored here on success. + * \returns true on success + */ +bool convertUTF32ToUTF8String(ArrayRef SrcBytes, std::string &Out); + +/** +* Converts a UTF32 string into a UTF8 std::string. +* +* \param [in] Src A buffer of UTF-32 encoded text. +* \param [out] Out Converted UTF-8 is stored here on success. +* \returns true on success +*/ +bool convertUTF32ToUTF8String(ArrayRef Src, std::string &Out); /** * Converts a UTF-8 string into a UTF-16 string with native endianness. Index: llvm/lib/Support/ConvertUTFWrapper.cpp =================================================================== --- llvm/lib/Support/ConvertUTFWrapper.cpp +++ llvm/lib/Support/ConvertUTFWrapper.cpp @@ -140,6 +140,64 @@ llvm::ArrayRef(reinterpret_cast(Src.data()), Src.size() * sizeof(UTF16)), Out); } + +bool convertUTF32ToUTF8String(ArrayRef SrcBytes, std::string &Out) { + assert(Out.empty()); + + // Error out on an uneven byte count. + if (SrcBytes.size() % 2) + return false; + + // Avoid OOB by returning early on empty input. + if (SrcBytes.empty()) + return true; + + const UTF32 *Src = reinterpret_cast(SrcBytes.begin()); + const UTF32 *SrcEnd = reinterpret_cast(SrcBytes.end()); + + assert((uintptr_t)Src % sizeof(UTF32) == 0); + + // Byteswap if necessary. + std::vector ByteSwapped; + if (Src[0] == UNI_UTF16_BYTE_ORDER_MARK_SWAPPED) { + ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd); + for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I) + ByteSwapped[I] = llvm::ByteSwap_32(ByteSwapped[I]); + Src = &ByteSwapped[0]; + SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1; + } + + // Skip the BOM for conversion. + if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_NATIVE) + Src++; + + // Just allocate enough space up front. We'll shrink it later. Allocate + // enough that we can fit a null terminator without reallocating. + Out.resize(SrcBytes.size() * UNI_MAX_UTF8_BYTES_PER_CODE_POINT + 1); + UTF8 *Dst = reinterpret_cast(&Out[0]); + UTF8 *DstEnd = Dst + Out.size(); + + ConversionResult CR = + ConvertUTF32toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion); + assert(CR != targetExhausted); + + if (CR != conversionOK) { + Out.clear(); + return false; + } + + Out.resize(reinterpret_cast(Dst) - &Out[0]); + Out.push_back(0); + Out.pop_back(); + return true; +} + +bool convertUTF32ToUTF8String(ArrayRef Src, std::string &Out) +{ + return convertUTF16ToUTF8String( + llvm::ArrayRef(reinterpret_cast(Src.data()), + Src.size() * sizeof(UTF32)), Out); +} bool convertUTF8ToUTF16String(StringRef SrcUTF8, SmallVectorImpl &DstUTF16) { Index: llvm/unittests/Support/ConvertUTFTest.cpp =================================================================== --- llvm/unittests/Support/ConvertUTFTest.cpp +++ llvm/unittests/Support/ConvertUTFTest.cpp @@ -36,6 +36,28 @@ EXPECT_EQ(Expected, Result); } +TEST(ConvertUTFTest, ConvertUTF32LittleEndianToUTF8String) { + // Src is the look of disapproval. + alignas(UTF32) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c"; + ArrayRef Ref(Src, sizeof(Src) - 1); + std::string Result; + bool Success = convertUTF32ToUTF8String(Ref, Result); + EXPECT_TRUE(Success); + std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0"); + EXPECT_EQ(Expected, Result); +} + +TEST(ConvertUTFTest, ConvertUTF32BigEndianToUTF8String) { + // Src is the look of disapproval. + alignas(UTF32) static const char Src[] = "\xfe\xff\x0c\xa0\x00_\x0c\xa0"; + ArrayRef Ref(Src, sizeof(Src) - 1); + std::string Result; + bool Success = convertUTF32ToUTF8String(Ref, Result); + EXPECT_TRUE(Success); + std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0"); + EXPECT_EQ(Expected, Result); +} + TEST(ConvertUTFTest, ConvertUTF8ToUTF16String) { // Src is the look of disapproval. static const char Src[] = "\xe0\xb2\xa0_\xe0\xb2\xa0";