(Part 1/2) non-Unicode response file on Windows: UTF-8 BOM

[ygao]

Summary

Hi all,
This is spun off from D7133, based on Rafael's suggestion that the UTF-8 BOM changes should be in its separate patch.
In the process of writing a regression test case, I looked at what was done for UTF-16 BOM, and tried to add a similar unit test for UTF-8 BOM in the same file, but this also means that hasUTF8ByteOrderMark() needs be exposed as an external function instead of a static helper function. I hope that is okay.

• Gao

[rnk]

lgtm

[• rafael]

Pleas use a test with one of the existing tools instead of a unit test.

In fact, you should be able to just add this to test/Other/ResponseFile.ll.

llvm/include/llvm/Support/ConvertUTF.h

 bool hasUTF16ByteOrderMark(ArrayRef<char> SrcBytes);
 
 /**
+ * Returns true if a blob of text starts with a UTF-8 byte order mark.
+ * UTF-8 BOM is a sequence of bytes on Windows and is not affected by the host
+ * system's endianness.
+ */
+bool hasUTF8ByteOrderMark(ArrayRef<char> SrcBytes);
+
+/**
  * Converts a stream of raw bytes assumed to be UTF16 into a UTF8 std::string.
  *
  * \param [in] SrcBytes A buffer of what is assumed to be UTF-16 encoded text.

llvm/lib/Support/CommandLine.cpp

       return false;
     Str = StringRef(UTF8Buf);
   }
+  // If we see UTF-8 BOM sequence at the beginning of a file, we shall remove
+  // these bytes before parsing.
+  // Reference: http://en.wikipedia.org/wiki/UTF-8#Byte_order_mark
+  else if (hasUTF8ByteOrderMark(BufRef))
+    Str = StringRef(BufRef.data() + 3, BufRef.size() - 3);
 
   // Tokenize the contents into NewArgv.
   Tokenizer(Str, Saver, NewArgv, MarkEOLs);

llvm/lib/Support/ConvertUTFWrapper.cpp

            (S[0] == '\xfe' && S[1] == '\xff')));
 }
 
+// It is called byte order marker but the UTF-8 BOM is actually not affected
+// by the host system's endianness.
+bool hasUTF8ByteOrderMark(ArrayRef<char> S) {
+  return (S.size() >= 3 &&
+          S[0] == '\xef' && S[1] == '\xbb' && S[2] == '\xbf');
+}
+
 bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {
   assert(Out.empty());
 

llvm/test/Other/ResponseFile.ll

 ; RUN: llvm-as @%t.list2 -o %t.bc
 ; RUN: llvm-nm %t.bc 2>&1 | FileCheck %s
 
+; When the response file begins with UTF8 BOM sequence, we shall remove them.
+; RUN: echo -e "\xef\xbb\xbf" > %t.list3
+; RUN: echo %s >> %t.list3
+; RUN: echo -e "\xef\xbb\xbf-time-passes @%t.list3" > %t.list4
+; RUN: llvm-as @%t.list4 -o %t.bc
+; RUN: llvm-nm %t.bc 2>&1 | FileCheck %s
+
 ; CHECK: T foobar
 
 define void @foobar() {

llvm/unittests/Support/ConvertUTFTest.cpp

   EXPECT_FALSE(HasBOM);
 }
 
+TEST(ConvertUTFTest, HasUTF8BOM) {
+  bool HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef\xbb\xbf", 3));
+  EXPECT_TRUE(HasBOM);
+  HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef\xbb\xbf ", 4));
+  EXPECT_TRUE(HasBOM);
+  HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef\xbb\xbf\x00asdf", 7));
+  EXPECT_TRUE(HasBOM);
+
+  HasBOM = hasUTF8ByteOrderMark(None);
+  EXPECT_FALSE(HasBOM);
+  HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef", 1));
+  EXPECT_FALSE(HasBOM);
+}
+
 struct ConvertUTFResultContainer {
   ConversionResult ErrorCode;
   std::vector<unsigned> UnicodeScalars;