[libcxx] [test] Read files as bytestrings to fix py3 encoding issues

[mgorny]

Summary

Use binary mode to read test files in libcxx LibcxxTestFormat class.
This ensures that tests are read correctly independently of encoding,
and therefore fixes UnicodeDecodeError when file is opened in Python 3
that defaults to pure ASCII encoding.

Technically this could be also fixed via conditionally appending
encoding argument when opening the file in Python 3. However, since
the code in question only searches for fixed ASCII substrings reading
it in binary mode is simpler and more universal.

[EricWF]

Is this backwards compatible as well?

[mgorny]

In D63346#1552271, @EricWF wrote:

Is this backwards compatible as well?

With Python 2.7? Yes, it has forward compatibility with py3 bytestrings. I'll do one more test run to confirm.

[mgorny]

…and confirmed.

[EricWF]

Thanks. Sorry for the delay.

LGTM.

libcxx/trunk/utils/libcxx/test/format.py

def _execute(self, test, lit_config):
         if '-fmodules' in test.config.available_features:
             test_cxx.compile_flags += [('-D%s' % mdef.strip()) for
                                        mdef in extra_modules_defines]
             test_cxx.addWarningFlagIfSupported('-Wno-macro-redefined')
             # FIXME: libc++ debug tests #define _LIBCPP_ASSERT to override it
             # If we see this we need to build the test against uniquely built
             # modules.
             if is_libcxx_test:
-                with open(test.getSourcePath(), 'r') as f:
+                with open(test.getSourcePath(), 'rb') as f:
                     contents = f.read()
-                if '#define _LIBCPP_ASSERT' in contents:
+                if b'#define _LIBCPP_ASSERT' in contents:
                     test_cxx.useModules(False)
 
         if is_objcxx_test:
             test_cxx.source_lang = 'objective-c++'
             if is_objcxx_arc_test:
                 test_cxx.compile_flags += ['-fobjc-arc']
             else:
                 test_cxx.compile_flags += ['-fno-objc-arc']
def _evaluate_pass_test(self, test, tmpBase, lit_config,
             # Note that cleanup of exec_file happens in `_clean()`. If you
             # override this, cleanup is your reponsibility.
             libcxx.util.cleanFile(object_path)
             self._clean(exec_path)
 
     def _evaluate_fail_test(self, test, test_cxx, parsers):
         source_path = test.getSourcePath()
         # FIXME: lift this detection into LLVM/LIT.
-        with open(source_path, 'r') as f:
+        with open(source_path, 'rb') as f:
             contents = f.read()
-        verify_tags = ['expected-note', 'expected-remark', 'expected-warning',
-                       'expected-error', 'expected-no-diagnostics']
+        verify_tags = [b'expected-note', b'expected-remark',
+                       b'expected-warning', b'expected-error',
+                       b'expected-no-diagnostics']
         use_verify = self.use_verify_for_fail and \
                      any([tag in contents for tag in verify_tags])
         # FIXME(EricWF): GCC 5 does not evaluate static assertions that
         # are dependant on a template parameter when '-fsyntax-only' is passed.
         # This is fixed in GCC 6. However for now we only pass "-fsyntax-only"
         # when using Clang.
         if test_cxx.type != 'gcc':
             test_cxx.flags += ['-fsyntax-only']
         if use_verify:
             test_cxx.useVerify()
             test_cxx.useWarnings()
             if '-Wuser-defined-warnings' in test_cxx.warning_flags:
                 test_cxx.warning_flags += ['-Wno-error=user-defined-warnings']
         else:
             # We still need to enable certain warnings on .fail.cpp test when
             # -verify isn't enabled. Such as -Werror=unused-result. However,
             # we don't want it enabled too liberally, which might incorrectly
             # allow unrelated failure tests to 'pass'.
             #
             # Therefore, we check if the test was expected to fail because of
             # nodiscard before enabling it
-            test_str_list = ['ignoring return value', 'nodiscard', 'NODISCARD']
+            test_str_list = [b'ignoring return value', b'nodiscard',
+                             b'NODISCARD']
             if any(test_str in contents for test_str in test_str_list):
                 test_cxx.flags += ['-Werror=unused-result']
         cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
         expected_rc = 0 if use_verify else 1
         report = libcxx.util.makeReport(cmd, out, err, rc)
         if rc == expected_rc:
             return lit.Test.Result(lit.Test.PASS, report)
         else:
             report += ('Expected compilation to fail!\n' if not use_verify else
                        'Expected compilation using verify to pass!\n')
             return lit.Test.Result(lit.Test.FAIL, report)