Index: llvm/include/llvm/Support/ConvertUTF.h =================================================================== --- llvm/include/llvm/Support/ConvertUTF.h +++ llvm/include/llvm/Support/ConvertUTF.h @@ -278,6 +278,24 @@ */ bool convertUTF16ToUTF8String(ArrayRef Src, std::string &Out); +/** + * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string. + * + * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text. + * \param [out] Out Converted UTF-8 is stored here on success. + * \returns true on success + */ +bool convertUTF32ToUTF8String(ArrayRef SrcBytes, std::string &Out); + +/** + * Converts a UTF32 string into a UTF8 std::string. + * + * \param [in] Src A buffer of UTF-32 encoded text. + * \param [out] Out Converted UTF-8 is stored here on success. + * \returns true on success + */ +bool convertUTF32ToUTF8String(ArrayRef Src, std::string &Out); + /** * Converts a UTF-8 string into a UTF-16 string with native endianness. * Index: llvm/lib/Support/ConvertUTFWrapper.cpp =================================================================== --- llvm/lib/Support/ConvertUTFWrapper.cpp +++ llvm/lib/Support/ConvertUTFWrapper.cpp @@ -141,6 +141,64 @@ Src.size() * sizeof(UTF16)), Out); } +bool convertUTF32ToUTF8String(ArrayRef SrcBytes, std::string &Out) { + assert(Out.empty()); + + // Error out on an uneven byte count. + if (SrcBytes.size() % 4) + return false; + + // Avoid OOB by returning early on empty input. + if (SrcBytes.empty()) + return true; + + const UTF32 *Src = reinterpret_cast(SrcBytes.begin()); + const UTF32 *SrcEnd = reinterpret_cast(SrcBytes.end()); + + assert((uintptr_t)Src % sizeof(UTF32) == 0); + + // Byteswap if necessary. + std::vector ByteSwapped; + if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_SWAPPED) { + ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd); + for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I) + ByteSwapped[I] = llvm::ByteSwap_32(ByteSwapped[I]); + Src = &ByteSwapped[0]; + SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1; + } + + // Skip the BOM for conversion. + if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_NATIVE) + Src++; + + // Just allocate enough space up front. We'll shrink it later. Allocate + // enough that we can fit a null terminator without reallocating. + Out.resize(SrcBytes.size() * UNI_MAX_UTF8_BYTES_PER_CODE_POINT + 1); + UTF8 *Dst = reinterpret_cast(&Out[0]); + UTF8 *DstEnd = Dst + Out.size(); + + ConversionResult CR = + ConvertUTF32toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion); + assert(CR != targetExhausted); + + if (CR != conversionOK) { + Out.clear(); + return false; + } + + Out.resize(reinterpret_cast(Dst) - &Out[0]); + Out.push_back(0); + Out.pop_back(); + return true; +} + +bool convertUTF32ToUTF8String(ArrayRef Src, std::string &Out) { + return convertUTF32ToUTF8String( + llvm::ArrayRef(reinterpret_cast(Src.data()), + Src.size() * sizeof(UTF32)), + Out); +} + bool convertUTF8ToUTF16String(StringRef SrcUTF8, SmallVectorImpl &DstUTF16) { assert(DstUTF16.empty()); Index: llvm/unittests/Support/ConvertUTFTest.cpp =================================================================== --- llvm/unittests/Support/ConvertUTFTest.cpp +++ llvm/unittests/Support/ConvertUTFTest.cpp @@ -36,6 +36,28 @@ EXPECT_EQ(Expected, Result); } +TEST(ConvertUTFTest, ConvertUTF32LittleEndianToUTF8String) { + // Src is the look of disapproval. + alignas(UTF32) static const char Src[] = "\xA0\x0C\x00\x00_\xA0\x0C\x00\x00"; + ArrayRef Ref(Src, sizeof(Src) - 1); + std::string Result; + bool Success = convertUTF32ToUTF8String(Ref, Result); + EXPECT_TRUE(Success); + std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0"); + EXPECT_EQ(Expected, Result); +} + +TEST(ConvertUTFTest, ConvertUTF32BigEndianToUTF8String) { + // Src is the look of disapproval. + alignas(UTF32) static const char Src[] = "\x00\x00\x0C\xA0_\x00\x00\x0C\xA0"; + ArrayRef Ref(Src, sizeof(Src) - 1); + std::string Result; + bool Success = convertUTF32ToUTF8String(Ref, Result); + EXPECT_TRUE(Success); + std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0"); + EXPECT_EQ(Expected, Result); +} + TEST(ConvertUTFTest, ConvertUTF8ToUTF16String) { // Src is the look of disapproval. static const char Src[] = "\xe0\xb2\xa0_\xe0\xb2\xa0"; @@ -78,6 +100,17 @@ EXPECT_FALSE(HasBOM); } +TEST(ConvertUTFTest, UTF32WrappersForConvertUTF32ToUTF8String) { + // Src is the look of disapproval. + alignas(UTF32) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c"; + ArrayRef SrcRef = makeArrayRef((const UTF32 *)Src, 4); + std::string Result; + bool Success = convertUTF32ToUTF8String(SrcRef, Result); + EXPECT_TRUE(Success); + std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0"); + EXPECT_EQ(Expected, Result); +} + TEST(ConvertUTFTest, UTF16WrappersForConvertUTF16ToUTF8String) { // Src is the look of disapproval. alignas(UTF16) static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";