diff --git a/clang-tools-extra/clangd/SourceCode.cpp b/clang-tools-extra/clangd/SourceCode.cpp --- a/clang-tools-extra/clangd/SourceCode.cpp +++ b/clang-tools-extra/clangd/SourceCode.cpp @@ -55,8 +55,14 @@ // Iterates over unicode codepoints in the (UTF-8) string. For each, // invokes CB(UTF-8 length, UTF-16 length), and breaks if it returns true. // Returns true if CB returned true, false if we hit the end of string. +// +// If the string is not valid UTF-8, we log this error and "decode" the +// text in some arbitrary way. This is pretty sad, but this tends to happen deep +// within indexing of headers where clang misdetected the encoding, and +// propagating the error all the way back up is (probably?) not be worth it. template static bool iterateCodepoints(llvm::StringRef U8, const Callback &CB) { + bool LoggedInvalid = false; // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). // Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx. for (size_t I = 0; I < U8.size();) { @@ -70,9 +76,20 @@ // This convenient property of UTF-8 holds for all non-ASCII characters. size_t UTF8Length = llvm::countLeadingOnes(C); // 0xxx is ASCII, handled above. 10xxx is a trailing byte, invalid here. - // 11111xxx is not valid UTF-8 at all. Assert because it's probably our bug. - assert((UTF8Length >= 2 && UTF8Length <= 4) && - "Invalid UTF-8, or transcoding bug?"); + // 11111xxx is not valid UTF-8 at all, maybe some ISO-8859-*. + if (LLVM_UNLIKELY(UTF8Length < 2 || UTF8Length > 4)) { + if (!LoggedInvalid) { + elog("File has invalid UTF-8 near offset {0}: {1}", I, llvm::toHex(U8)); + LoggedInvalid = true; + } + // We can't give a correct result, but avoid returning something wild. + // Pretend this is a valid ASCII byte, for lack of better options. + // (Too late to get ISO-8859-* right, we've skipped some bytes already). + if (CB(1, 1)) + return true; + ++I; + continue; + } I += UTF8Length; // Skip over all trailing bytes. // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). // Astral codepoints are encoded as 4 bytes in UTF-8 (11110xxx ...) diff --git a/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp b/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp --- a/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp +++ b/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp @@ -71,6 +71,21 @@ EXPECT_EQ(lspLength("😂"), 1UL); } +TEST(SourceCodeTests, lspLengthBadUTF8) { + // Results are not well-defined if source file isn't valid UTF-8. + // However we shouldn't crash or return something totally wild. + const char *BadUTF8[] = {"\xa0", "\xff\xff\xff\xff\xff"}; + + for (OffsetEncoding Encoding : + {OffsetEncoding::UTF8, OffsetEncoding::UTF16, OffsetEncoding::UTF32}) { + WithContextValue UTF32(kCurrentOffsetEncoding, Encoding); + for (const char *Bad : BadUTF8) { + EXPECT_GE(lspLength(Bad), 0u); + EXPECT_LE(lspLength(Bad), strlen(Bad)); + } + } +} + // The = → 🡆 below are ASCII (1 byte), BMP (3 bytes), and astral (4 bytes). const char File[] = R"(0:0 = 0 1:0 → 8 diff --git a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp --- a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp +++ b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp @@ -1490,6 +1490,20 @@ EXPECT_THAT(Symbols, Contains(QName("operator delete"))); } +TEST_F(SymbolCollectorTest, BadUTF8) { + // Extracted from boost/spirit/home/support/char_encoding/iso8859_1.hpp + // This looks like UTF-8 and fools clang, but has high-ISO-8859-1 comments. + const char *Header = "int PUNCT = 0;\n" + "int types[] = { /* \xa1 */PUNCT };"; + CollectorOpts.RefFilter = RefKind::All; + CollectorOpts.RefsInHeaders = true; + runSymbolCollector(Header, ""); + EXPECT_THAT(Symbols, Contains(QName("types"))); + EXPECT_THAT(Symbols, Contains(QName("PUNCT"))); + // Reference is stored, although offset within line is not reliable. + EXPECT_THAT(Refs, Contains(Pair(findSymbol(Symbols, "PUNCT").ID, _))); +} + } // namespace } // namespace clangd } // namespace clang